pipeline: Convert build-using-self to python

Utilities/README.md

@@ -0,0 +1,24 @@
+# Windows Self-Hosted Test
+
+Here are some steps the worked running on Windows.
+
+1. Install Docker on a Windows host
+2. Launch a `Powershell.exe` session
+3. Run the following in power shell to start a container running the nightly toolchain
+    ```
+    docker run --rm --interactive --tty swiftlang/swift:nightly-main-windowsservercore-1809 powershell.exe
+    ```
+4. When the container start, clone the "merged" PR to `C:\source`
+    ```
+    mkdir C:\source
+    cd C:\source
+    git clone https://github.com/swiftlang/swift-package-manager .
+    # Assign the PR ID to a variable
+    $PR_ID = "8288"
+    git fetch origin pull/$PR_ID/merge:ci_merge_$PR_ID
+    git checkout ci_merge_$PR_ID
+    ```
+5. Run the CI pipeline script
+    ```
+    python C:\source\Utilities\build-using-self
+    ```

Utilities/build-using-self

@@ -1,62 +1,165 @@
-#!/bin/bash
-##===----------------------------------------------------------------------===##
-##
-## This source file is part of the Swift open source project
-##
-## Copyright (c) 2014-2022 Apple Inc. and the Swift project authors
-## Licensed under Apache License v2.0 with Runtime Library Exception
-##
-## See http://swift.org/LICENSE.txt for license information
-## See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
-##
-##===----------------------------------------------------------------------===##
-
-set -eu
-
-__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-root_dir="$(cd ${__dir}/.. && pwd)"
-
-cd "${root_dir}/"
-echo "Current directory is ${PWD}"
-
-CONFIGURATION=debug
-export SWIFTCI_IS_SELF_HOSTED=1
-
-# Ensure SDKROOT is configured
-host=$(uname)
-if [ "${host}" == "Darwin" ]; then
-    export SDKROOT=$(xcrun --show-sdk-path --sdk macosx)
-fi
-
-set -x
-
-env | sort
-
-# Display toolchain version
-swift --version
-
-# Perform package update in order to get the latest commits for the dependencies.
-swift package update
-swift build -c $CONFIGURATION
-swift test -c $CONFIGURATION --parallel
-
-# Run the integration tests with just built SwiftPM.
-export SWIFTPM_BIN_DIR=$(swift build -c $CONFIGURATION --show-bin-path)
-(
-    cd ${root_dir}/IntegrationTests
-    # Perform package update in order to get the latest commits for the dependencies.
-    swift package update
-    $SWIFTPM_BIN_DIR/swift-test --parallel
+#!/usr/bin/env python3
+# ===----------------------------------------------------------------------===##
+#
+# This source file is part of the Swift open source project
+#
+# Copyright (c) 2025 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# ===----------------------------------------------------------------------===##
+
+import argparse
+import logging
+import os
+import pathlib
+import platform
+import shlex
+import sys
+import typing as t
+
+from helpers import (
+    Configuration,
+    change_directory,
+    call,
+    call_output,
+)
+
+logging.basicConfig(
+    format=" | ".join(
+        [
+            # Prefix script name to the log in an attempt to avoid confusion when parsing logs
+            f"{pathlib.Path(sys.argv[0]).name}",
+            "%(asctime)s",
+            "%(levelname)-8s",
+            "%(module)s",
+            "%(funcName)s",
+            "Line:%(lineno)d",
+            "%(message)s",
+        ]
+    ),
+    level=logging.INFO,
 )
 
-if [ "${host}" == "Darwin" ]; then
-    echo "Current working directory is ${PWD}"
-    echo "Bootstrapping with the XCBuild codepath..."
-    ${root_dir}/Utilities/bootstrap \
-      build \
-      --release \
-      --verbose \
-      --cross-compile-hosts macosx-arm64 \
-      --skip-cmake-bootstrap \
-      --swift-build-path "${SWIFTPM_BIN_DIR}/swift-build"
-fi
+
+REPO_ROOT_PATH = pathlib.Path(__file__).parent.parent.resolve()
+
+
+def get_arguments() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        dest="is_verbose",
+        action="store_true",
+        help="When set, prints verbose information.",
+    )
+    parser.add_argument(
+        "-c",
+        "--configuration",
+        type=Configuration,
+        dest="config",
+        default="debug",
+        choices=[e.value for e in Configuration],
+        help="The configuraiton to use.",
+    )
+
+    args = parser.parse_args()
+    return args
+
+
+def log_environment() -> None:
+    logging.info("Environment Variables")
+    for key, value in sorted(os.environ.items()):
+        logging.info("  --> %s=%r", key, value)
+
+
+def get_swiftpm_bin_dir(config: Configuration) -> pathlib.Path:
+    logging.info("Retrieving Swift PM binary directory.")
+    swiftpm_bin_dir = pathlib.Path(
+        call_output(["swift", "build", "--configuration", config, "--show-bin-path"])
+    )
+    logging.info("SwiftPM BIN DIR: %s", swiftpm_bin_dir)
+    return swiftpm_bin_dir
+
+
+def is_on_darwin() -> bool:
+    return platform.uname().system == "Darwin"
+
+
+def set_environment(*, swiftpm_bin_dir: pathlib.Path,) -> None:
+    os.environ["SWIFTCI_IS_SELF_HOSTED"] = "1"
+
+    # Set the SWIFTPM_BIN_DIR path
+    os.environ["SWIFTPM_BIN_DIR"] = str(swiftpm_bin_dir)
+
+    # Ensure SDKROOT is configure
+    if is_on_darwin():
+        sdk_root = call_output(shlex.split("xcrun --show-sdk-path --sdk macosx"))
+        logging.debug("macos sdk root = %r", sdk_root)
+        os.environ["SDKROOT"] = sdk_root
+    log_environment()
+
+
+def run_bootstrap(swiftpm_bin_dir: pathlib.Path) -> None:
+    if is_on_darwin():
+        logging.info("Current working directory is %s", pathlib.Path.cwd())
+        logging.info("Bootstrapping with the XCBuild codepath...")
+        call(
+            [
+                REPO_ROOT_PATH / "Utilities" / "bootstrap",
+                "build",
+                "--release",
+                "--verbose",
+                "--cross-compile-hosts",
+                "macosx-arm64",
+                "--skip-cmake-bootstrap",
+                "--swift-build-path",
+                (swiftpm_bin_dir / "swift-build").resolve(),
+            ],
+        )
+
+
+def main() -> None:
+    args = get_arguments()
+    logging.getLogger().setLevel(logging.DEBUG if args.is_verbose else logging.INFO)
+    logging.debug("Args: %r", args)
+
+    with change_directory(REPO_ROOT_PATH):
+        swiftpm_bin_dir = get_swiftpm_bin_dir(config=args.config)
+        set_environment(swiftpm_bin_dir=swiftpm_bin_dir)
+
+        call(
+            shlex.split("swift --version"),
+        )
+
+        call(
+            shlex.split("swift package update"),
+        )
+        call(
+            shlex.split(f"swift build --configuration {args.config}"),
+        )
+        call(
+            shlex.split(f"swift test --configuration {args.config} --parallel"),
+        )
+
+    with change_directory(REPO_ROOT_PATH / "IntegrationTests"):
+        call(
+            shlex.split("swift package update"),
+        )
+        call(
+            shlex.split(
+                f"{swiftpm_bin_dir / 'swift-test'} --parallel",
+            ),
+        )
+
+    run_bootstrap(swiftpm_bin_dir=swiftpm_bin_dir)
+
+
+if __name__ == "__main__":
+    main()

Utilities/helpers.py

@@ -3,20 +3,45 @@
 ##
 ## This source file is part of the Swift open source project
 ##
-## Copyright (c) 2014-2020 Apple Inc. and the Swift project authors
+## Copyright (c) 2014-2025 Apple Inc. and the Swift project authors
 ## Licensed under Apache License v2.0 with Runtime Library Exception
 ##
 ## See http://swift.org/LICENSE.txt for license information
 ## See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
 ##
 ##===----------------------------------------------------------------------===##
 
-import datetime
+import contextlib
+import enum
+import errno
 import logging
-import subprocess
-import sys
 import os
-import errno
+import pathlib
+import subprocess
+import typing as t
+
+
+@contextlib.contextmanager
+def change_directory(directory: pathlib.Path) -> t.Iterator[pathlib.Path]:
+    current_directory = pathlib.Path.cwd()
+    logging.info("Current directory is %s", current_directory)
+    logging.info("Changing directory to: %s", directory)
+    os.chdir(directory)
+
+    try:
+        yield directory
+    finally:
+        logging.debug("Chaning directory back to %s", current_directory)
+        os.chdir(current_directory)
+
+
+class Configuration(str, enum.Enum):
+    DEBUG = "debug"
+    RELEASE = "release"
+
+    def __str__(self) -> str:
+        return self.value
+
 
 def symlink_force(source, destination):
     try:
@@ -26,6 +51,7 @@ def symlink_force(source, destination):
             os.remove(destination)
             os.symlink(source, destination)
 
+
 def mkdir_p(path):
     """Create the given directory, if it does not exist."""
     try:
@@ -35,28 +61,37 @@ def mkdir_p(path):
         if e.errno != errno.EEXIST:
             raise
 
+
 def call(cmd, cwd=None, verbose=False):
     """Calls a subprocess."""
-    logging.info("executing command >>> %s", ' '.join(cmd))
+    cwd = cwd or pathlib.Path.cwd()
+    logging.info("executing command >>> %r with cwd %s", " ".join([str(c) for c in cmd]), cwd)
     try:
         subprocess.check_call(cmd, cwd=cwd)
     except subprocess.CalledProcessError as cpe:
-        logging.debug("executing command >>> %s", ' '.join(cmd))
+        logging.debug("executing command >>> %r with cwd %s", " ".join([str(c) for c in cmd]), cwd)
         logging.error(
             "Process failure: %s\n[---- START OUTPUT ----]\n%s\n[---- END OUTPUT ----]",
             str(cpe),
             cpe.output,
         )
         raise cpe
 
+
 def call_output(cmd, cwd=None, stderr=False, verbose=False):
     """Calls a subprocess for its return data."""
     stderr = subprocess.STDOUT if stderr else False
-    logging.info(' '.join(cmd))
+    cwd = cwd or pathlib.Path.cwd()
+    logging.info("executing command >>> %r with cwd %s", " ".join([str(c) for c in cmd]), cwd)
     try:
-        return subprocess.check_output(cmd, cwd=cwd, stderr=stderr, universal_newlines=True).strip()
+        return subprocess.check_output(
+            cmd,
+            cwd=cwd,
+            stderr=stderr,
+            universal_newlines=True,
+        ).strip()
     except subprocess.CalledProcessError as cpe:
-        logging.debug(' '.join(cmd))
+        logging.debug("executing command >>> %r with cwd %s", " ".join([str(c) for c in cmd]), cwd)
         logging.error(
             "%s\n[---- START OUTPUT ----]\n%s\n[---- END OUTPUT ----]",
             str(cpe),