WIP: Build Mozc with clang-cl

.gitignore

@@ -17,6 +17,7 @@ MODULE.bazel.lock
 .DS_Store
 
 # third_party dirs and cache dir checked out by update_deps.py
+/src/third_party/llvm/
 /src/third_party/ndk/
 /src/third_party/ninja/
 /src/third_party/qt/

src/BUILD.bazel

@@ -97,6 +97,16 @@ selects.config_setting_group(
     visibility = ["//:__subpackages__"],
 )
 
+# Generic Windows platform with clang-cl that automatically matches host CPU
+platform(
+    name = "windows-clang-cl",
+    constraint_values = [
+        "@platforms//os:windows",
+        "@bazel_tools//tools/cpp:clang-cl",
+    ],
+    parents = ["@local_config_platform//:host"],
+)
+
 # Special target so as to define special macros for each platforms.
 # Don't depend on this directly. Use mozc_cc_(library|binary|test) rule instead.
 cc_library(

src/bazel/rules_cc_BUILD.windows.tpl.patch

@@ -69,13 +69,13 @@
 +toolchain(
 +    name = "cc-toolchain-x64_x86_windows-clang-cl",
 +    exec_compatible_with = [
-+        "//third_party/bazel_platforms/cpu:x86_64",
-+        "//third_party/bazel_platforms/os:windows",
++        "@platforms//cpu:x86_64",
++        "@platforms//os:windows",
 +        "@rules_cc//cc/private/toolchain:clang-cl",
 +    ],
 +    target_compatible_with = [
-+        "//third_party/bazel_platforms/cpu:x86_32",
-+        "//third_party/bazel_platforms/os:windows",
++        "@platforms//cpu:x86_32",
++        "@platforms//os:windows",
 +    ],
 +    toolchain = ":cc-compiler-x64_x86_windows-clang-cl",
 +    toolchain_type = "@bazel_tools//tools/cpp:toolchain_type",

src/build_tools/update_deps.py

@@ -35,13 +35,18 @@
 """
 
 import argparse
+from collections.abc import Iterator
 import dataclasses
 import hashlib
 import os
 import pathlib
 import shutil
 import stat
 import subprocess
+import sys
+import tarfile
+import time
+from typing import Union
 import zipfile
 
 import requests
@@ -109,6 +114,11 @@ def __hash__(self):
     sha256='d0ee3da143211aa447e750085876c9b9d7bcdd637ab5b2c5b41349c617f22f3b',
 )
 
+LLVM_WIN = ArchiveInfo(
+    url='https://github.com/llvm/llvm-project/releases/download/llvmorg-19.1.7/clang+llvm-19.1.7-x86_64-pc-windows-msvc.tar.xz',
+    size=845236708,
+    sha256='b4557b4f012161f56a2f5d9e877ab9635cafd7a08f7affe14829bd60c9d357f0',
+)
 
 def get_sha256(path: pathlib.Path) -> str:
   """Returns SHA-256 hash digest of the specified file.
@@ -182,6 +192,115 @@ def download(archive: ArchiveInfo, dryrun: bool = False) -> None:
     )
 
 
+def llvm_extract_filter(
+    members: Iterator[tarfile.TarInfo],
+) -> Iterator[tarfile.TarInfo]:
+  """Custom extract filter for the LLVM Tar file.
+
+  This custom filter can be used to adjust directory structure and drop
+  unnecessary files/directories to save disk space.
+
+  Args:
+    members: an iterator of TarInfo from the Tar file.
+
+  Yields:
+    An iterator of TarInfo to be extracted.
+  """
+  with ProgressPrinter() as printer:
+    for info in members:
+      paths = info.name.split('/')
+      if '..' in paths:
+        continue
+      if len(paths) < 1:
+        continue
+      skipping = True
+      if (
+          len(paths) == 3
+          and paths[1] == 'bin'
+          and paths[2] in ['clang-cl.exe', 'llvm-lib.exe', 'lld-link.exe']
+      ):
+        skipping = False
+      elif len(paths) >= 2 and paths[1] in ['include', 'lib']:
+        skipping = False
+      if skipping:
+        printer.print_line('skipping   ' + info.name)
+        continue
+      printer.print_line('extracting ' + info.name)
+      yield info
+
+
+class StatefulLLVMExtractionFilter:
+  """A stateful extraction filter for PEP 706.
+
+  See https://peps.python.org/pep-0706/ for details.
+  """
+
+  def __enter__(self):
+    self.printer = ProgressPrinter().__enter__()
+    return self
+
+  def __exit__(self, *exc):
+    self.printer.__exit__(exc)
+
+  def __call__(
+      self,
+      member: tarfile.TarInfo,
+      dest_path: Union[str, pathlib.Path],
+  ) -> Union[tarfile.TarInfo, None]:
+    data = tarfile.data_filter(member, dest_path)
+    if data is None:
+      return None
+
+    skipping = True
+    paths = member.name.split('/')
+    if (
+        len(paths) == 3
+        and paths[1] == 'bin'
+        and paths[2] in ['clang-cl.exe', 'llvm-lib.exe', 'lld-link.exe']
+    ):
+      skipping = False
+    elif len(paths) >= 2 and paths[1] in ['include', 'lib']:
+      skipping = False
+    if skipping:
+      self.printer.print_line('skipping   ' + member.name)
+      return None
+    self.printer.print_line('extracting ' + member.name)
+    return member
+
+
+def extract_llvm(dryrun: bool = False) -> None:
+  """Extract LLVM archive.
+
+  Args:
+    dryrun: True if this is a dry-run.
+  """
+  if not is_windows():
+    return
+
+  archive = LLVM_WIN
+  src = CACHE_DIR.joinpath(archive.filename)
+  dest = ABS_THIRD_PARTY_DIR.joinpath('llvm').absolute()
+
+  if dest.exists():
+    if dryrun:
+      print(f"dryrun: shutil.rmtree(r'{dest}')")
+    else:
+      shutil.rmtree(dest)
+
+  if dryrun:
+    print(f'dryrun: Extracting {src} into {dest}')
+  else:
+    dest.mkdir(parents=True)
+    with tarfile.open(src, mode='r|xz') as f:
+      # tarfile.data_filter is available in Python 3.12+.
+      # See https://peps.python.org/pep-0706/ for details.
+      if getattr(tarfile, 'data_filter', None):
+        with StatefulLLVMExtractionFilter() as filter:
+          f.extractall(path=dest, filter=filter)
+      else:
+        f.extractall(path=dest, members=llvm_extract_filter(f))
+
+
 def extract_ninja(dryrun: bool = False) -> None:
   """Extract ninja-win archive.
 
@@ -329,6 +448,7 @@ def main():
   parser.add_argument('--dryrun', action='store_true', default=False)
   parser.add_argument('--noninja', action='store_true', default=False)
   parser.add_argument('--noqt', action='store_true', default=False)
+  parser.add_argument('--nollvm', action='store_true', default=False)
   parser.add_argument('--nowix', action='store_true', default=False)
   parser.add_argument('--nondk', action='store_true', default=False)
   parser.add_argument('--nosubmodules', action='store_true', default=False)
@@ -349,13 +469,18 @@ def main():
       archives.append(NDK_LINUX)
     elif is_mac():
       archives.append(NDK_MAC)
+  if (not args.nollvm) and is_windows():
+    archives.append(LLVM_WIN)
 
   for archive in archives:
     download(archive, args.dryrun)
 
   if args.cache_only:
     return
 
+  if (LLVM_WIN in archives):
+    extract_llvm(args.dryrun)
+
   if (not args.nowix) and is_windows():
     restore_dotnet_tools(args.dryrun)
 

src/tools/README.md

@@ -0,0 +1,5 @@
+# tools
+
+This file contains a special [wrapper script for bazelisk](https://github.com/bazelbuild/bazelisk/blob/master/README.md#toolsbazel).
+
+ * [bazel.bat](./bazel.bat)

src/tools/bazel.bat

@@ -0,0 +1,7 @@
+@echo off
+SET selfpath=%~dp0
+set BAZEL_LLVM=%selfpath:~0,-6%\third_party\llvm\clang+llvm-19.1.7-x86_64-pc-windows-msvc
+%BAZEL_REAL% %* & call:myexit
+
+:myexit
+exit /b

src/windows.bazelrc

@@ -7,9 +7,8 @@
 #   bazel --bazelrc=windows.bazelrc build //protocol:commands_proto \
 #         --config oss_windows
 
-# There has been a long standing path-length issue on building protobuf on
-# Windows with bazel. See the following for details.
-# https://github.com/protocolbuffers/protobuf/issues/12947
-# https://github.com/bazelbuild/bazel/issues/18683
-# Here is an ugly workaround, which we really hope to get rid of.
-startup  --output_base=C:/x --windows_enable_symlinks
+startup  --windows_enable_symlinks
+
+build    --extra_toolchains=@local_config_cc//:cc-toolchain-x64_windows-clang-cl
+build    --extra_toolchains=@local_config_cc//:cc-toolchain-x64_x86_windows-clang-cl
+build    --host_platform=//:windows-clang-cl