diff --git a/.gitignore b/.gitignore index 04f62b9ed..664a85e89 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ MODULE.bazel.lock .DS_Store # third_party dirs and cache dir checked out by update_deps.py +/src/third_party/llvm/ /src/third_party/ndk/ /src/third_party/ninja/ /src/third_party/qt/ diff --git a/docs/build_mozc_in_windows.md b/docs/build_mozc_in_windows.md index abacca0e5..a09795aa1 100644 --- a/docs/build_mozc_in_windows.md +++ b/docs/build_mozc_in_windows.md @@ -68,6 +68,7 @@ python build_tools/update_deps.py In this step, additional build dependencies will be downloaded. + * [LLVM 19.1.7](https://github.com/llvm/llvm-project/releases/tag/llvmorg-19.1.7) * [Ninja 1.11.0](https://github.com/ninja-build/ninja/releases/download/v1.11.0/ninja-win.zip) * [Qt 6.8.0](https://download.qt.io/archive/qt/6.8/6.8.0/submodules/qtbase-everywhere-src-6.8.0.tar.xz) * [.NET tools](../dotnet-tools.json) diff --git a/src/build_tools/update_deps.py b/src/build_tools/update_deps.py index 074216d14..be5b2bb30 100644 --- a/src/build_tools/update_deps.py +++ b/src/build_tools/update_deps.py @@ -35,6 +35,7 @@ """ import argparse +from collections.abc import Iterator import dataclasses import hashlib import os @@ -42,6 +43,10 @@ import shutil import stat import subprocess +import sys +import tarfile +import time +from typing import Union import zipfile import requests @@ -109,6 +114,11 @@ def __hash__(self): sha256='d0ee3da143211aa447e750085876c9b9d7bcdd637ab5b2c5b41349c617f22f3b', ) +LLVM_WIN = ArchiveInfo( + url='https://github.com/llvm/llvm-project/releases/download/llvmorg-19.1.7/clang+llvm-19.1.7-x86_64-pc-windows-msvc.tar.xz', + size=845236708, + sha256='b4557b4f012161f56a2f5d9e877ab9635cafd7a08f7affe14829bd60c9d357f0', +) def get_sha256(path: pathlib.Path) -> str: """Returns SHA-256 hash digest of the specified file. @@ -182,6 +192,78 @@ def download(archive: ArchiveInfo, dryrun: bool = False) -> None: ) +class StatefulLLVMExtractionFilter: + """A stateful extraction filter for PEP 706. + + See https://peps.python.org/pep-0706/ for details. + """ + + def __enter__(self): + self.printer = ProgressPrinter().__enter__() + return self + + def __exit__(self, *exc): + self.printer.__exit__(exc) + + def __call__( + self, + member: tarfile.TarInfo, + dest_path: Union[str, pathlib.Path], + ) -> Union[tarfile.TarInfo, None]: + data = tarfile.data_filter(member, dest_path) + if data is None: + return None + + skipping = True + paths = member.name.split('/') + if ( + len(paths) == 3 + and paths[1] == 'bin' + and paths[2] in ['clang-cl.exe', 'llvm-lib.exe', 'lld-link.exe'] + ): + skipping = False + elif len(paths) >= 2 and paths[1] in ['include', 'lib']: + skipping = False + if skipping: + self.printer.print_line('skipping ' + member.name) + return None + self.printer.print_line('extracting ' + member.name) + return member + + +def extract_llvm(dryrun: bool = False) -> None: + """Extract LLVM archive. + + Args: + dryrun: True if this is a dry-run. + """ + if not is_windows(): + return + + archive = LLVM_WIN + src = CACHE_DIR.joinpath(archive.filename) + dest = ABS_THIRD_PARTY_DIR.joinpath('llvm').absolute() + + if dest.exists(): + if dryrun: + print(f"dryrun: shutil.rmtree(r'{dest}')") + else: + shutil.rmtree(dest) + + if dryrun: + print(f'dryrun: Extracting {src} into {dest}') + else: + dest.mkdir(parents=True) + with tarfile.open(src, mode='r|xz') as f: + # tarfile.data_filter is available in Python 3.12+. + # See https://peps.python.org/pep-0706/ for details. + if getattr(tarfile, 'data_filter', None): + with StatefulLLVMExtractionFilter() as filter: + f.extractall(path=dest, filter=filter) + else: + f.extractall(path=dest, members=llvm_extract_filter(f)) + + def extract_ninja(dryrun: bool = False) -> None: """Extract ninja-win archive. @@ -329,6 +411,7 @@ def main(): parser.add_argument('--dryrun', action='store_true', default=False) parser.add_argument('--noninja', action='store_true', default=False) parser.add_argument('--noqt', action='store_true', default=False) + parser.add_argument('--nollvm', action='store_true', default=False) parser.add_argument('--nowix', action='store_true', default=False) parser.add_argument('--nondk', action='store_true', default=False) parser.add_argument('--nosubmodules', action='store_true', default=False) @@ -349,6 +432,8 @@ def main(): archives.append(NDK_LINUX) elif is_mac(): archives.append(NDK_MAC) + if (not args.nollvm) and is_windows(): + archives.append(LLVM_WIN) for archive in archives: download(archive, args.dryrun) @@ -356,6 +441,9 @@ def main(): if args.cache_only: return + if (LLVM_WIN in archives): + extract_llvm(args.dryrun) + if (not args.nowix) and is_windows(): restore_dotnet_tools(args.dryrun)