diff --git a/.gitignore b/.gitignore index 04f62b9ed..664a85e89 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ MODULE.bazel.lock .DS_Store # third_party dirs and cache dir checked out by update_deps.py +/src/third_party/llvm/ /src/third_party/ndk/ /src/third_party/ninja/ /src/third_party/qt/ diff --git a/src/BUILD.bazel b/src/BUILD.bazel index 8a7972405..87ab1b6dc 100644 --- a/src/BUILD.bazel +++ b/src/BUILD.bazel @@ -97,6 +97,16 @@ selects.config_setting_group( visibility = ["//:__subpackages__"], ) +# Generic Windows platform with clang-cl that automatically matches host CPU +platform( + name = "windows-clang-cl", + constraint_values = [ + "@platforms//os:windows", + "@bazel_tools//tools/cpp:clang-cl", + ], + parents = ["@local_config_platform//:host"], +) + # Special target so as to define special macros for each platforms. # Don't depend on this directly. Use mozc_cc_(library|binary|test) rule instead. cc_library( diff --git a/src/bazel/rules_cc_BUILD.windows.tpl.patch b/src/bazel/rules_cc_BUILD.windows.tpl.patch index 8f062cd17..6a308b60d 100644 --- a/src/bazel/rules_cc_BUILD.windows.tpl.patch +++ b/src/bazel/rules_cc_BUILD.windows.tpl.patch @@ -69,13 +69,13 @@ +toolchain( + name = "cc-toolchain-x64_x86_windows-clang-cl", + exec_compatible_with = [ -+ "//third_party/bazel_platforms/cpu:x86_64", -+ "//third_party/bazel_platforms/os:windows", ++ "@platforms//cpu:x86_64", ++ "@platforms//os:windows", + "@rules_cc//cc/private/toolchain:clang-cl", + ], + target_compatible_with = [ -+ "//third_party/bazel_platforms/cpu:x86_32", -+ "//third_party/bazel_platforms/os:windows", ++ "@platforms//cpu:x86_32", ++ "@platforms//os:windows", + ], + toolchain = ":cc-compiler-x64_x86_windows-clang-cl", + toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", diff --git a/src/build_tools/update_deps.py b/src/build_tools/update_deps.py index 074216d14..cc8554fca 100644 --- a/src/build_tools/update_deps.py +++ b/src/build_tools/update_deps.py @@ -35,6 +35,7 @@ """ import argparse +from collections.abc import Iterator import dataclasses import hashlib import os @@ -42,6 +43,10 @@ import shutil import stat import subprocess +import sys +import tarfile +import time +from typing import Union import zipfile import requests @@ -109,6 +114,11 @@ def __hash__(self): sha256='d0ee3da143211aa447e750085876c9b9d7bcdd637ab5b2c5b41349c617f22f3b', ) +LLVM_WIN = ArchiveInfo( + url='https://github.com/llvm/llvm-project/releases/download/llvmorg-19.1.7/clang+llvm-19.1.7-x86_64-pc-windows-msvc.tar.xz', + size=845236708, + sha256='b4557b4f012161f56a2f5d9e877ab9635cafd7a08f7affe14829bd60c9d357f0', +) def get_sha256(path: pathlib.Path) -> str: """Returns SHA-256 hash digest of the specified file. @@ -182,6 +192,115 @@ def download(archive: ArchiveInfo, dryrun: bool = False) -> None: ) +def llvm_extract_filter( + members: Iterator[tarfile.TarInfo], +) -> Iterator[tarfile.TarInfo]: + """Custom extract filter for the LLVM Tar file. + + This custom filter can be used to adjust directory structure and drop + unnecessary files/directories to save disk space. + + Args: + members: an iterator of TarInfo from the Tar file. + + Yields: + An iterator of TarInfo to be extracted. + """ + with ProgressPrinter() as printer: + for info in members: + paths = info.name.split('/') + if '..' in paths: + continue + if len(paths) < 1: + continue + skipping = True + if ( + len(paths) == 3 + and paths[1] == 'bin' + and paths[2] in ['clang-cl.exe', 'llvm-lib.exe', 'lld-link.exe'] + ): + skipping = False + elif len(paths) >= 2 and paths[1] in ['include', 'lib']: + skipping = False + if skipping: + printer.print_line('skipping ' + info.name) + continue + printer.print_line('extracting ' + info.name) + yield info + + +class StatefulLLVMExtractionFilter: + """A stateful extraction filter for PEP 706. + + See https://peps.python.org/pep-0706/ for details. + """ + + def __enter__(self): + self.printer = ProgressPrinter().__enter__() + return self + + def __exit__(self, *exc): + self.printer.__exit__(exc) + + def __call__( + self, + member: tarfile.TarInfo, + dest_path: Union[str, pathlib.Path], + ) -> Union[tarfile.TarInfo, None]: + data = tarfile.data_filter(member, dest_path) + if data is None: + return None + + skipping = True + paths = member.name.split('/') + if ( + len(paths) == 3 + and paths[1] == 'bin' + and paths[2] in ['clang-cl.exe', 'llvm-lib.exe', 'lld-link.exe'] + ): + skipping = False + elif len(paths) >= 2 and paths[1] in ['include', 'lib']: + skipping = False + if skipping: + self.printer.print_line('skipping ' + member.name) + return None + self.printer.print_line('extracting ' + member.name) + return member + + +def extract_llvm(dryrun: bool = False) -> None: + """Extract LLVM archive. + + Args: + dryrun: True if this is a dry-run. + """ + if not is_windows(): + return + + archive = LLVM_WIN + src = CACHE_DIR.joinpath(archive.filename) + dest = ABS_THIRD_PARTY_DIR.joinpath('llvm').absolute() + + if dest.exists(): + if dryrun: + print(f"dryrun: shutil.rmtree(r'{dest}')") + else: + shutil.rmtree(dest) + + if dryrun: + print(f'dryrun: Extracting {src} into {dest}') + else: + dest.mkdir(parents=True) + with tarfile.open(src, mode='r|xz') as f: + # tarfile.data_filter is available in Python 3.12+. + # See https://peps.python.org/pep-0706/ for details. + if getattr(tarfile, 'data_filter', None): + with StatefulLLVMExtractionFilter() as filter: + f.extractall(path=dest, filter=filter) + else: + f.extractall(path=dest, members=llvm_extract_filter(f)) + + def extract_ninja(dryrun: bool = False) -> None: """Extract ninja-win archive. @@ -329,6 +448,7 @@ def main(): parser.add_argument('--dryrun', action='store_true', default=False) parser.add_argument('--noninja', action='store_true', default=False) parser.add_argument('--noqt', action='store_true', default=False) + parser.add_argument('--nollvm', action='store_true', default=False) parser.add_argument('--nowix', action='store_true', default=False) parser.add_argument('--nondk', action='store_true', default=False) parser.add_argument('--nosubmodules', action='store_true', default=False) @@ -349,6 +469,8 @@ def main(): archives.append(NDK_LINUX) elif is_mac(): archives.append(NDK_MAC) + if (not args.nollvm) and is_windows(): + archives.append(LLVM_WIN) for archive in archives: download(archive, args.dryrun) @@ -356,6 +478,9 @@ def main(): if args.cache_only: return + if (LLVM_WIN in archives): + extract_llvm(args.dryrun) + if (not args.nowix) and is_windows(): restore_dotnet_tools(args.dryrun) diff --git a/src/tools/README.md b/src/tools/README.md new file mode 100644 index 000000000..e645daab4 --- /dev/null +++ b/src/tools/README.md @@ -0,0 +1,5 @@ +# tools + +This file contains a special [wrapper script for bazelisk](https://github.com/bazelbuild/bazelisk/blob/master/README.md#toolsbazel). + + * [bazel.bat](./bazel.bat) diff --git a/src/tools/bazel.bat b/src/tools/bazel.bat new file mode 100644 index 000000000..6b4ee9596 --- /dev/null +++ b/src/tools/bazel.bat @@ -0,0 +1,7 @@ +@echo off +SET selfpath=%~dp0 +set BAZEL_LLVM=%selfpath:~0,-6%\third_party\llvm\clang+llvm-19.1.7-x86_64-pc-windows-msvc +%BAZEL_REAL% %* & call:myexit + +:myexit +exit /b diff --git a/src/windows.bazelrc b/src/windows.bazelrc index d76ed4f81..bb0a82cc9 100644 --- a/src/windows.bazelrc +++ b/src/windows.bazelrc @@ -7,9 +7,8 @@ # bazel --bazelrc=windows.bazelrc build //protocol:commands_proto \ # --config oss_windows -# There has been a long standing path-length issue on building protobuf on -# Windows with bazel. See the following for details. -# https://github.com/protocolbuffers/protobuf/issues/12947 -# https://github.com/bazelbuild/bazel/issues/18683 -# Here is an ugly workaround, which we really hope to get rid of. -startup --output_base=C:/x --windows_enable_symlinks +startup --windows_enable_symlinks + +build --extra_toolchains=@local_config_cc//:cc-toolchain-x64_windows-clang-cl +build --extra_toolchains=@local_config_cc//:cc-toolchain-x64_x86_windows-clang-cl +build --host_platform=//:windows-clang-cl