diff --git a/Utilities/README.md b/Utilities/README.md new file mode 100644 index 00000000000..c8096af25fe --- /dev/null +++ b/Utilities/README.md @@ -0,0 +1,24 @@ +# Windows Self-Hosted Test + +Here are some steps the worked running on Windows. + +1. Install Docker on a Windows host +2. Launch a `Powershell.exe` session +3. Run the following in power shell to start a container running the nightly toolchain + ``` + docker run --rm --interactive --tty swiftlang/swift:nightly-main-windowsservercore-1809 powershell.exe + ``` +4. When the container start, clone the "merged" PR to `C:\source` + ``` + mkdir C:\source + cd C:\source + git clone https://github.com/swiftlang/swift-package-manager . + # Assign the PR ID to a variable + $PR_ID = "8288" + git fetch origin pull/$PR_ID/merge:ci_merge_$PR_ID + git checkout ci_merge_$PR_ID + ``` +5. Run the CI pipeline script + ``` + python C:\source\Utilities\build-using-self + ``` diff --git a/Utilities/build-using-self b/Utilities/build-using-self index c1277db00e1..a237f38c3d2 100755 --- a/Utilities/build-using-self +++ b/Utilities/build-using-self @@ -1,62 +1,165 @@ -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the Swift open source project -## -## Copyright (c) 2014-2022 Apple Inc. and the Swift project authors -## Licensed under Apache License v2.0 with Runtime Library Exception -## -## See http://swift.org/LICENSE.txt for license information -## See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -## -##===----------------------------------------------------------------------===## - -set -eu - -__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -root_dir="$(cd ${__dir}/.. && pwd)" - -cd "${root_dir}/" -echo "Current directory is ${PWD}" - -CONFIGURATION=debug -export SWIFTCI_IS_SELF_HOSTED=1 - -# Ensure SDKROOT is configured -host=$(uname) -if [ "${host}" == "Darwin" ]; then - export SDKROOT=$(xcrun --show-sdk-path --sdk macosx) -fi - -set -x - -env | sort - -# Display toolchain version -swift --version - -# Perform package update in order to get the latest commits for the dependencies. -swift package update -swift build -c $CONFIGURATION -swift test -c $CONFIGURATION --parallel - -# Run the integration tests with just built SwiftPM. -export SWIFTPM_BIN_DIR=$(swift build -c $CONFIGURATION --show-bin-path) -( - cd ${root_dir}/IntegrationTests - # Perform package update in order to get the latest commits for the dependencies. - swift package update - $SWIFTPM_BIN_DIR/swift-test --parallel +#!/usr/bin/env python3 +# ===----------------------------------------------------------------------===## +# +# This source file is part of the Swift open source project +# +# Copyright (c) 2025 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +# +# ===----------------------------------------------------------------------===## + +import argparse +import logging +import os +import pathlib +import platform +import shlex +import sys +import typing as t + +from helpers import ( + Configuration, + change_directory, + call, + call_output, +) + +logging.basicConfig( + format=" | ".join( + [ + # Prefix script name to the log in an attempt to avoid confusion when parsing logs + f"{pathlib.Path(sys.argv[0]).name}", + "%(asctime)s", + "%(levelname)-8s", + "%(module)s", + "%(funcName)s", + "Line:%(lineno)d", + "%(message)s", + ] + ), + level=logging.INFO, ) -if [ "${host}" == "Darwin" ]; then - echo "Current working directory is ${PWD}" - echo "Bootstrapping with the XCBuild codepath..." - ${root_dir}/Utilities/bootstrap \ - build \ - --release \ - --verbose \ - --cross-compile-hosts macosx-arm64 \ - --skip-cmake-bootstrap \ - --swift-build-path "${SWIFTPM_BIN_DIR}/swift-build" -fi + +REPO_ROOT_PATH = pathlib.Path(__file__).parent.parent.resolve() + + +def get_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "-v", + "--verbose", + dest="is_verbose", + action="store_true", + help="When set, prints verbose information.", + ) + parser.add_argument( + "-c", + "--configuration", + type=Configuration, + dest="config", + default="debug", + choices=[e.value for e in Configuration], + help="The configuraiton to use.", + ) + + args = parser.parse_args() + return args + + +def log_environment() -> None: + logging.info("Environment Variables") + for key, value in sorted(os.environ.items()): + logging.info(" --> %s=%r", key, value) + + +def get_swiftpm_bin_dir(config: Configuration) -> pathlib.Path: + logging.info("Retrieving Swift PM binary directory.") + swiftpm_bin_dir = pathlib.Path( + call_output(["swift", "build", "--configuration", config, "--show-bin-path"]) + ) + logging.info("SwiftPM BIN DIR: %s", swiftpm_bin_dir) + return swiftpm_bin_dir + + +def is_on_darwin() -> bool: + return platform.uname().system == "Darwin" + + +def set_environment(*, swiftpm_bin_dir: pathlib.Path,) -> None: + os.environ["SWIFTCI_IS_SELF_HOSTED"] = "1" + + # Set the SWIFTPM_BIN_DIR path + os.environ["SWIFTPM_BIN_DIR"] = str(swiftpm_bin_dir) + + # Ensure SDKROOT is configure + if is_on_darwin(): + sdk_root = call_output(shlex.split("xcrun --show-sdk-path --sdk macosx")) + logging.debug("macos sdk root = %r", sdk_root) + os.environ["SDKROOT"] = sdk_root + log_environment() + + +def run_bootstrap(swiftpm_bin_dir: pathlib.Path) -> None: + if is_on_darwin(): + logging.info("Current working directory is %s", pathlib.Path.cwd()) + logging.info("Bootstrapping with the XCBuild codepath...") + call( + [ + REPO_ROOT_PATH / "Utilities" / "bootstrap", + "build", + "--release", + "--verbose", + "--cross-compile-hosts", + "macosx-arm64", + "--skip-cmake-bootstrap", + "--swift-build-path", + (swiftpm_bin_dir / "swift-build").resolve(), + ], + ) + + +def main() -> None: + args = get_arguments() + logging.getLogger().setLevel(logging.DEBUG if args.is_verbose else logging.INFO) + logging.debug("Args: %r", args) + + with change_directory(REPO_ROOT_PATH): + swiftpm_bin_dir = get_swiftpm_bin_dir(config=args.config) + set_environment(swiftpm_bin_dir=swiftpm_bin_dir) + + call( + shlex.split("swift --version"), + ) + + call( + shlex.split("swift package update"), + ) + call( + shlex.split(f"swift build --configuration {args.config}"), + ) + call( + shlex.split(f"swift test --configuration {args.config} --parallel"), + ) + + with change_directory(REPO_ROOT_PATH / "IntegrationTests"): + call( + shlex.split("swift package update"), + ) + call( + shlex.split( + f"{swiftpm_bin_dir / 'swift-test'} --parallel", + ), + ) + + run_bootstrap(swiftpm_bin_dir=swiftpm_bin_dir) + + +if __name__ == "__main__": + main() diff --git a/Utilities/helpers.py b/Utilities/helpers.py index f7300614943..971420ee719 100644 --- a/Utilities/helpers.py +++ b/Utilities/helpers.py @@ -3,7 +3,7 @@ ## ## This source file is part of the Swift open source project ## -## Copyright (c) 2014-2020 Apple Inc. and the Swift project authors +## Copyright (c) 2014-2025 Apple Inc. and the Swift project authors ## Licensed under Apache License v2.0 with Runtime Library Exception ## ## See http://swift.org/LICENSE.txt for license information @@ -11,12 +11,37 @@ ## ##===----------------------------------------------------------------------===## -import datetime +import contextlib +import enum +import errno import logging -import subprocess -import sys import os -import errno +import pathlib +import subprocess +import typing as t + + +@contextlib.contextmanager +def change_directory(directory: pathlib.Path) -> t.Iterator[pathlib.Path]: + current_directory = pathlib.Path.cwd() + logging.info("Current directory is %s", current_directory) + logging.info("Changing directory to: %s", directory) + os.chdir(directory) + + try: + yield directory + finally: + logging.debug("Chaning directory back to %s", current_directory) + os.chdir(current_directory) + + +class Configuration(str, enum.Enum): + DEBUG = "debug" + RELEASE = "release" + + def __str__(self) -> str: + return self.value + def symlink_force(source, destination): try: @@ -26,6 +51,7 @@ def symlink_force(source, destination): os.remove(destination) os.symlink(source, destination) + def mkdir_p(path): """Create the given directory, if it does not exist.""" try: @@ -35,13 +61,15 @@ def mkdir_p(path): if e.errno != errno.EEXIST: raise + def call(cmd, cwd=None, verbose=False): """Calls a subprocess.""" - logging.info("executing command >>> %s", ' '.join(cmd)) + cwd = cwd or pathlib.Path.cwd() + logging.info("executing command >>> %r with cwd %s", " ".join([str(c) for c in cmd]), cwd) try: subprocess.check_call(cmd, cwd=cwd) except subprocess.CalledProcessError as cpe: - logging.debug("executing command >>> %s", ' '.join(cmd)) + logging.debug("executing command >>> %r with cwd %s", " ".join([str(c) for c in cmd]), cwd) logging.error( "Process failure: %s\n[---- START OUTPUT ----]\n%s\n[---- END OUTPUT ----]", str(cpe), @@ -49,14 +77,21 @@ def call(cmd, cwd=None, verbose=False): ) raise cpe + def call_output(cmd, cwd=None, stderr=False, verbose=False): """Calls a subprocess for its return data.""" stderr = subprocess.STDOUT if stderr else False - logging.info(' '.join(cmd)) + cwd = cwd or pathlib.Path.cwd() + logging.info("executing command >>> %r with cwd %s", " ".join([str(c) for c in cmd]), cwd) try: - return subprocess.check_output(cmd, cwd=cwd, stderr=stderr, universal_newlines=True).strip() + return subprocess.check_output( + cmd, + cwd=cwd, + stderr=stderr, + universal_newlines=True, + ).strip() except subprocess.CalledProcessError as cpe: - logging.debug(' '.join(cmd)) + logging.debug("executing command >>> %r with cwd %s", " ".join([str(c) for c in cmd]), cwd) logging.error( "%s\n[---- START OUTPUT ----]\n%s\n[---- END OUTPUT ----]", str(cpe),