From 3c027b691072e0664e67149bddf01dd6066bb0db Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sun, 25 Sep 2022 15:08:35 +1000 Subject: [PATCH 1/2] Fix build scripts targeting the wrong architecture PR #1081 started passing CFLAGS to build scripts, which cargo does too. But unfortunately Bazel's cpp toolchain adds an explicit target to the CFLAGS it provides, and that target uses the host platform instead of the target platform. When building a crate like blake3 or zstd on a Mac with iOS as the target, this ends up in the crate being built for macOS instead of iOS. --- cargo/cargo_build_script.bzl | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/cargo/cargo_build_script.bzl b/cargo/cargo_build_script.bzl index 2539b5534e..2e3961b976 100644 --- a/cargo/cargo_build_script.bzl +++ b/cargo/cargo_build_script.bzl @@ -13,6 +13,31 @@ load("//rust/private:rustc.bzl", "BuildInfo", "get_compilation_mode_opts", "get_ # buildifier: disable=bzl-visibility load("//rust/private:utils.bzl", "dedent", "expand_dict_value_locations", "find_cc_toolchain", "find_toolchain", "name_to_crate_name") +def strip_target(elems): + """Remove '-target xxx' from C(XX)FLAGS. + + The cpp toolchain adds '-target xxx' and '-isysroot xxx' to CFLAGS. If it is not stripped out before + the CFLAGS are provided to build scripts, it can cause the build to take on the host architecture + instead of the target architecture. + + Args: + elems (list): A list of args + + Returns: + list: the modified args + """ + skip_next = False + out_elems = [] + for elem in elems: + if skip_next: + skip_next = False + continue + if elem == "-target" or elem == "-isysroot": + skip_next = True + continue + out_elems.append(elem) + return out_elems + def get_cc_compile_args_and_env(cc_toolchain, feature_configuration): """Gather cc environment variables from the given `cc_toolchain` @@ -30,16 +55,16 @@ def get_cc_compile_args_and_env(cc_toolchain, feature_configuration): feature_configuration = feature_configuration, cc_toolchain = cc_toolchain, ) - cc_c_args = cc_common.get_memory_inefficient_command_line( + cc_c_args = strip_target(cc_common.get_memory_inefficient_command_line( feature_configuration = feature_configuration, action_name = C_COMPILE_ACTION_NAME, variables = compile_variables, - ) - cc_cxx_args = cc_common.get_memory_inefficient_command_line( + )) + cc_cxx_args = strip_target(cc_common.get_memory_inefficient_command_line( feature_configuration = feature_configuration, action_name = CPP_COMPILE_ACTION_NAME, variables = compile_variables, - ) + )) cc_env = cc_common.get_environment_variables( feature_configuration = feature_configuration, action_name = C_COMPILE_ACTION_NAME, From d7fa7dbd15abb1281b5740fdb5dba0cec268eaab Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 7 Oct 2022 09:31:19 +1000 Subject: [PATCH 2/2] Add an end-to-end test to make sure building zstd for iOS works --- .bazelci/presubmit.yml | 8 + examples/.bazelignore | 1 + examples/ios_build/.gitignore | 1 + examples/ios_build/BUILD.bazel | 65 ++++ examples/ios_build/Cargo.Bazel.lock | 63 ++++ examples/ios_build/Cargo.lock | 63 ++++ examples/ios_build/Cargo.toml | 9 + examples/ios_build/WORKSPACE.bazel | 64 ++++ examples/ios_build/cargo-bazel-lock.json | 435 +++++++++++++++++++++++ examples/ios_build/check_arch.sh | 16 + examples/ios_build/src/lib.rs | 2 + 11 files changed, 727 insertions(+) create mode 100644 examples/ios_build/.gitignore create mode 100644 examples/ios_build/BUILD.bazel create mode 100644 examples/ios_build/Cargo.Bazel.lock create mode 100644 examples/ios_build/Cargo.lock create mode 100644 examples/ios_build/Cargo.toml create mode 100644 examples/ios_build/WORKSPACE.bazel create mode 100644 examples/ios_build/cargo-bazel-lock.json create mode 100755 examples/ios_build/check_arch.sh create mode 100644 examples/ios_build/src/lib.rs diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index f3434533e5..c95c63c74f 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -366,6 +366,14 @@ tasks: working_directory: examples/ios build_targets: - "//..." + ios_build: + name: iOS build script cross compile test + platform: macos + working_directory: examples/ios_build + test_flags: + - "--platforms=//:ios_x86_64" + test_targets: + - "//..." buildifier: version: latest diff --git a/examples/.bazelignore b/examples/.bazelignore index 61093694df..1afac096cf 100644 --- a/examples/.bazelignore +++ b/examples/.bazelignore @@ -2,3 +2,4 @@ android cargo_manifest_dir/external_crate crate_universe ios +ios_build diff --git a/examples/ios_build/.gitignore b/examples/ios_build/.gitignore new file mode 100644 index 0000000000..ac51a054d2 --- /dev/null +++ b/examples/ios_build/.gitignore @@ -0,0 +1 @@ +bazel-* diff --git a/examples/ios_build/BUILD.bazel b/examples/ios_build/BUILD.bazel new file mode 100644 index 0000000000..04c2a3d842 --- /dev/null +++ b/examples/ios_build/BUILD.bazel @@ -0,0 +1,65 @@ +load("@rules_rust//rust:defs.bzl", "rust_static_library") + +rust_static_library( + name = "ios_build_lib", + srcs = glob(["**/*.rs"]), + edition = "2018", + deps = [ + "@ios_build//:zstd", + ], +) + +filegroup( + name = "lib", + srcs = ["ios_build_lib"], +) + +sh_test( + name = "check_arch", + srcs = ["check_arch.sh"], + args = ["$(location :lib)"], + data = [":lib"], +) + +platform( + name = "macos_x86_64", + constraint_values = [ + "@platforms//cpu:x86_64", + "@platforms//os:macos", + ], +) + +platform( + name = "macos_arm64", + constraint_values = [ + "@platforms//cpu:arm64", + "@platforms//os:macos", + ], +) + +platform( + name = "ios_x86_64", + constraint_values = [ + "@platforms//cpu:x86_64", + "@platforms//os:ios", + "@build_bazel_apple_support//constraints:simulator", + ], +) + +platform( + name = "ios_sim_arm64", + constraint_values = [ + "@platforms//cpu:arm64", + "@platforms//os:ios", + "@build_bazel_apple_support//constraints:simulator", + ], +) + +platform( + name = "ios_arm64", + constraint_values = [ + "@platforms//cpu:arm64", + "@platforms//os:ios", + "@build_bazel_apple_support//constraints:device", + ], +) diff --git a/examples/ios_build/Cargo.Bazel.lock b/examples/ios_build/Cargo.Bazel.lock new file mode 100644 index 0000000000..919712a9c9 --- /dev/null +++ b/examples/ios_build/Cargo.Bazel.lock @@ -0,0 +1,63 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "cc" +version = "1.0.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +dependencies = [ + "jobserver", +] + +[[package]] +name = "ios_build" +version = "0.1.0" +dependencies = [ + "zstd", +] + +[[package]] +name = "jobserver" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" +dependencies = [ + "libc", +] + +[[package]] +name = "libc" +version = "0.2.134" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "329c933548736bc49fd575ee68c89e8be4d260064184389a5b77517cddd99ffb" + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.1+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b" +dependencies = [ + "cc", + "libc", +] diff --git a/examples/ios_build/Cargo.lock b/examples/ios_build/Cargo.lock new file mode 100644 index 0000000000..919712a9c9 --- /dev/null +++ b/examples/ios_build/Cargo.lock @@ -0,0 +1,63 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "cc" +version = "1.0.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +dependencies = [ + "jobserver", +] + +[[package]] +name = "ios_build" +version = "0.1.0" +dependencies = [ + "zstd", +] + +[[package]] +name = "jobserver" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" +dependencies = [ + "libc", +] + +[[package]] +name = "libc" +version = "0.2.134" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "329c933548736bc49fd575ee68c89e8be4d260064184389a5b77517cddd99ffb" + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.1+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b" +dependencies = [ + "cc", + "libc", +] diff --git a/examples/ios_build/Cargo.toml b/examples/ios_build/Cargo.toml new file mode 100644 index 0000000000..de64c22abe --- /dev/null +++ b/examples/ios_build/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "ios_build" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +zstd = "0.11.2" diff --git a/examples/ios_build/WORKSPACE.bazel b/examples/ios_build/WORKSPACE.bazel new file mode 100644 index 0000000000..68057347cc --- /dev/null +++ b/examples/ios_build/WORKSPACE.bazel @@ -0,0 +1,64 @@ +workspace(name = "examples") + +local_repository( + name = "rules_rust", + path = "../../", +) + +load("@rules_rust//rust:repositories.bzl", "rules_rust_dependencies", "rust_register_toolchains") + +rules_rust_dependencies() + +rust_register_toolchains( + edition = "2018", + extra_target_triples = [ + "aarch64-apple-ios-sim", + "x86_64-apple-ios", + ], +) + +load("@rules_rust//crate_universe:repositories.bzl", "crate_universe_dependencies") + +crate_universe_dependencies(bootstrap = True) + +load("@rules_rust//crate_universe:defs.bzl", "crates_repository", "splicing_config") + +crates_repository( + name = "ios_build", + cargo_lockfile = "//:Cargo.Bazel.lock", + # `generator` is not necessary in official releases. + # See load satement for `cargo_bazel_bootstrap`. + generator = "@cargo_bazel_bootstrap//:cargo-bazel", + lockfile = "//:cargo-bazel-lock.json", + manifests = ["//:Cargo.toml"], + splicing_config = splicing_config( + resolver_version = "2", + ), +) + +load( + "@ios_build//:defs.bzl", + ios_build_crate_repositories = "crate_repositories", +) + +ios_build_crate_repositories() + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +# Used for Bazel CI +http_archive( + name = "bazelci_rules", + sha256 = "eca21884e6f66a88c358e580fd67a6b148d30ab57b1680f62a96c00f9bc6a07e", + strip_prefix = "bazelci_rules-1.0.0", + url = "https://github.com/bazelbuild/continuous-integration/releases/download/rules-1.0.0/bazelci_rules-1.0.0.tar.gz", +) + +load("@bazelci_rules//:rbe_repo.bzl", "rbe_preconfig") + +# Creates a default toolchain config for RBE. +# Use this as is if you are using the rbe_ubuntu16_04 container, +# otherwise refer to RBE docs. +rbe_preconfig( + name = "buildkite_config", + toolchain = "ubuntu1804-bazel-java11", +) diff --git a/examples/ios_build/cargo-bazel-lock.json b/examples/ios_build/cargo-bazel-lock.json new file mode 100644 index 0000000000..4557435a71 --- /dev/null +++ b/examples/ios_build/cargo-bazel-lock.json @@ -0,0 +1,435 @@ +{ + "checksum": "0a1dbbdbe3330575039ff97ce0854718f5c9f99d075e823de9c318e563c342c1", + "crates": { + "cc 1.0.73": { + "name": "cc", + "version": "1.0.73", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/cc/1.0.73/download", + "sha256": "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" + } + }, + "targets": [ + { + "Library": { + "crate_name": "cc", + "crate_root": "src/lib.rs", + "srcs": { + "include": [ + "**/*.rs" + ], + "exclude": [] + } + } + }, + { + "Binary": { + "crate_name": "gcc-shim", + "crate_root": "src/bin/gcc-shim.rs", + "srcs": { + "include": [ + "**/*.rs" + ], + "exclude": [] + } + } + } + ], + "library_target_name": "cc", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": [ + "jobserver", + "parallel" + ], + "deps": { + "common": [ + { + "id": "jobserver 0.1.25", + "target": "jobserver" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "1.0.73" + }, + "license": "MIT/Apache-2.0" + }, + "ios_build 0.1.0": { + "name": "ios_build", + "version": "0.1.0", + "repository": null, + "targets": [ + { + "Library": { + "crate_name": "ios_build", + "crate_root": "src/lib.rs", + "srcs": { + "include": [ + "**/*.rs" + ], + "exclude": [] + } + } + } + ], + "library_target_name": "ios_build", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "zstd 0.11.2+zstd.1.5.2", + "target": "zstd" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.1.0" + }, + "license": null + }, + "jobserver 0.1.25": { + "name": "jobserver", + "version": "0.1.25", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/jobserver/0.1.25/download", + "sha256": "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" + } + }, + "targets": [ + { + "Library": { + "crate_name": "jobserver", + "crate_root": "src/lib.rs", + "srcs": { + "include": [ + "**/*.rs" + ], + "exclude": [] + } + } + } + ], + "library_target_name": "jobserver", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [], + "selects": { + "cfg(unix)": [ + { + "id": "libc 0.2.134", + "target": "libc" + } + ] + } + }, + "edition": "2018", + "version": "0.1.25" + }, + "license": "MIT/Apache-2.0" + }, + "libc 0.2.134": { + "name": "libc", + "version": "0.2.134", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/libc/0.2.134/download", + "sha256": "329c933548736bc49fd575ee68c89e8be4d260064184389a5b77517cddd99ffb" + } + }, + "targets": [ + { + "Library": { + "crate_name": "libc", + "crate_root": "src/lib.rs", + "srcs": { + "include": [ + "**/*.rs" + ], + "exclude": [] + } + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": { + "include": [ + "**/*.rs" + ], + "exclude": [] + } + } + } + ], + "library_target_name": "libc", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": [ + "default", + "std" + ], + "deps": { + "common": [ + { + "id": "libc 0.2.134", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2015", + "version": "0.2.134" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "zstd 0.11.2+zstd.1.5.2": { + "name": "zstd", + "version": "0.11.2+zstd.1.5.2", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/zstd/0.11.2+zstd.1.5.2/download", + "sha256": "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" + } + }, + "targets": [ + { + "Library": { + "crate_name": "zstd", + "crate_root": "src/lib.rs", + "srcs": { + "include": [ + "**/*.rs" + ], + "exclude": [] + } + } + } + ], + "library_target_name": "zstd", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": [ + "arrays", + "default", + "legacy", + "zdict_builder" + ], + "deps": { + "common": [ + { + "id": "zstd-safe 5.0.2+zstd.1.5.2", + "target": "zstd_safe" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.11.2+zstd.1.5.2" + }, + "license": "MIT" + }, + "zstd-safe 5.0.2+zstd.1.5.2": { + "name": "zstd-safe", + "version": "5.0.2+zstd.1.5.2", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/zstd-safe/5.0.2+zstd.1.5.2/download", + "sha256": "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" + } + }, + "targets": [ + { + "Library": { + "crate_name": "zstd_safe", + "crate_root": "src/lib.rs", + "srcs": { + "include": [ + "**/*.rs" + ], + "exclude": [] + } + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": { + "include": [ + "**/*.rs" + ], + "exclude": [] + } + } + } + ], + "library_target_name": "zstd_safe", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": [ + "arrays", + "legacy", + "std", + "zdict_builder" + ], + "deps": { + "common": [ + { + "id": "libc 0.2.134", + "target": "libc" + }, + { + "id": "zstd-safe 5.0.2+zstd.1.5.2", + "target": "build_script_build" + }, + { + "id": "zstd-sys 2.0.1+zstd.1.5.2", + "target": "zstd_sys" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "5.0.2+zstd.1.5.2" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT/Apache-2.0" + }, + "zstd-sys 2.0.1+zstd.1.5.2": { + "name": "zstd-sys", + "version": "2.0.1+zstd.1.5.2", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/zstd-sys/2.0.1+zstd.1.5.2/download", + "sha256": "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b" + } + }, + "targets": [ + { + "Library": { + "crate_name": "zstd_sys", + "crate_root": "src/lib.rs", + "srcs": { + "include": [ + "**/*.rs" + ], + "exclude": [] + } + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": { + "include": [ + "**/*.rs" + ], + "exclude": [] + } + } + } + ], + "library_target_name": "zstd_sys", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": [ + "legacy", + "std", + "zdict_builder" + ], + "deps": { + "common": [ + { + "id": "libc 0.2.134", + "target": "libc" + }, + { + "id": "zstd-sys 2.0.1+zstd.1.5.2", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "2.0.1+zstd.1.5.2" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "cc 1.0.73", + "target": "cc" + } + ], + "selects": {} + }, + "links": "zstd" + }, + "license": "MIT/Apache-2.0" + } + }, + "binary_crates": [ + "cc 1.0.73" + ], + "workspace_members": { + "ios_build 0.1.0": "" + }, + "conditions": { + "cfg(unix)": [ + "aarch64-apple-darwin", + "aarch64-apple-ios", + "aarch64-apple-ios-sim", + "aarch64-linux-android", + "aarch64-unknown-linux-gnu", + "arm-unknown-linux-gnueabi", + "armv7-linux-androideabi", + "armv7-unknown-linux-gnueabi", + "i686-apple-darwin", + "i686-linux-android", + "i686-unknown-freebsd", + "i686-unknown-linux-gnu", + "powerpc-unknown-linux-gnu", + "s390x-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-apple-ios", + "x86_64-linux-android", + "x86_64-unknown-freebsd", + "x86_64-unknown-linux-gnu" + ] + } +} diff --git a/examples/ios_build/check_arch.sh b/examples/ios_build/check_arch.sh new file mode 100755 index 0000000000..8ae6d86157 --- /dev/null +++ b/examples/ios_build/check_arch.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# +# When compiling with --platforms=//:ios_sim_arm64 or --platforms=//:ios_x86_64, +# the library should not contain any references to macOS (platform 1) + +set -e + +if [[ "$OSTYPE" != "darwin"* ]]; then + echo "This test only makes sense on macOS." + exit 0 +fi + +if otool -l $1 | grep 'platform 1'; then + echo "macOS detected." + exit 1 +fi \ No newline at end of file diff --git a/examples/ios_build/src/lib.rs b/examples/ios_build/src/lib.rs new file mode 100644 index 0000000000..5f0cad6448 --- /dev/null +++ b/examples/ios_build/src/lib.rs @@ -0,0 +1,2 @@ +#[allow(unused_imports, clippy::single_component_path_imports)] +use zstd;