Skip to content

Commit 797f900

Browse files
bkhouribkhouri
committed
pipeline: Convert build-using-self to python
In preparation for a Windows self-host pipeline, convert the build-using-self from a bash script to a python script to make it platform agnostic.
1 parent 314a274 commit 797f900

File tree

3 files changed

+211
-70
lines changed

3 files changed

+211
-70
lines changed

‎Utilities/build-using-self

+163-60
Original file line numberDiff line numberDiff line change
@@ -1,62 +1,165 @@
1-
#!/bin/bash
2-
##===----------------------------------------------------------------------===##
3-
##
4-
## This source file is part of the Swift open source project
5-
##
6-
## Copyright (c) 2014-2022 Apple Inc. and the Swift project authors
7-
## Licensed under Apache License v2.0 with Runtime Library Exception
8-
##
9-
## See http://swift.org/LICENSE.txt for license information
10-
## See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
11-
##
12-
##===----------------------------------------------------------------------===##
13-
14-
set -eu
15-
16-
__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
17-
root_dir="$(cd ${__dir}/.. && pwd)"
18-
19-
cd "${root_dir}/"
20-
echo "Current directory is ${PWD}"
21-
22-
CONFIGURATION=debug
23-
export SWIFTCI_IS_SELF_HOSTED=1
24-
25-
# Ensure SDKROOT is configured
26-
host=$(uname)
27-
if [ "${host}" == "Darwin" ]; then
28-
export SDKROOT=$(xcrun --show-sdk-path --sdk macosx)
29-
fi
30-
31-
set -x
32-
33-
env | sort
34-
35-
# Display toolchain version
36-
swift --version
37-
38-
# Perform package update in order to get the latest commits for the dependencies.
39-
swift package update
40-
swift build -c $CONFIGURATION
41-
swift test -c $CONFIGURATION --parallel
42-
43-
# Run the integration tests with just built SwiftPM.
44-
export SWIFTPM_BIN_DIR=$(swift build -c $CONFIGURATION --show-bin-path)
45-
(
46-
cd ${root_dir}/IntegrationTests
47-
# Perform package update in order to get the latest commits for the dependencies.
48-
swift package update
49-
$SWIFTPM_BIN_DIR/swift-test --parallel
1+
#!/usr/bin/env python3
2+
# ===----------------------------------------------------------------------===##
3+
#
4+
# This source file is part of the Swift open source project
5+
#
6+
# Copyright (c) 2025 Apple Inc. and the Swift project authors
7+
# Licensed under Apache License v2.0 with Runtime Library Exception
8+
#
9+
# See http://swift.org/LICENSE.txt for license information
10+
# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
11+
#
12+
# ===----------------------------------------------------------------------===##
13+
14+
import argparse
15+
import logging
16+
import os
17+
import pathlib
18+
import platform
19+
import shlex
20+
import sys
21+
import typing as t
22+
23+
from helpers import (
24+
Configuration,
25+
change_directory,
26+
call,
27+
call_output,
28+
)
29+
30+
logging.basicConfig(
31+
format=" | ".join(
32+
[
33+
# Prefix script name to the log in an attempt to avoid confusion when parsing logs
34+
f"{pathlib.Path(sys.argv[0]).name}",
35+
"%(asctime)s",
36+
"%(levelname)-8s",
37+
"%(module)s",
38+
"%(funcName)s",
39+
"Line:%(lineno)d",
40+
"%(message)s",
41+
]
42+
),
43+
level=logging.INFO,
5044
)
5145

52-
if [ "${host}" == "Darwin" ]; then
53-
echo "Current working directory is ${PWD}"
54-
echo "Bootstrapping with the XCBuild codepath..."
55-
${root_dir}/Utilities/bootstrap \
56-
build \
57-
--release \
58-
--verbose \
59-
--cross-compile-hosts macosx-arm64 \
60-
--skip-cmake-bootstrap \
61-
--swift-build-path "${SWIFTPM_BIN_DIR}/swift-build"
62-
fi
46+
47+
REPO_ROOT_PATH = pathlib.Path(__file__).parent.parent.resolve()
48+
49+
50+
def get_arguments() -> argparse.Namespace:
51+
parser = argparse.ArgumentParser(
52+
formatter_class=argparse.ArgumentDefaultsHelpFormatter
53+
)
54+
55+
parser.add_argument(
56+
"-v",
57+
"--verbose",
58+
dest="is_verbose",
59+
action="store_true",
60+
help="When set, prints verbose information.",
61+
)
62+
parser.add_argument(
63+
"-c",
64+
"--configuration",
65+
type=Configuration,
66+
dest="config",
67+
default="debug",
68+
choices=[e.value for e in Configuration],
69+
help="The configuraiton to use.",
70+
)
71+
72+
args = parser.parse_args()
73+
return args
74+
75+
76+
def log_environment() -> None:
77+
logging.info("Environment Variables")
78+
for key, value in sorted(os.environ.items()):
79+
logging.info(" --> %s=%r", key, value)
80+
81+
82+
def get_swiftpm_bin_dir(config: Configuration) -> pathlib.Path:
83+
logging.info("Retrieving Swift PM binary directory.")
84+
swiftpm_bin_dir = pathlib.Path(
85+
call_output(["swift", "build", "--configuration", config, "--show-bin-path"])
86+
)
87+
logging.info("SwiftPM BIN DIR: %s", swiftpm_bin_dir)
88+
return swiftpm_bin_dir
89+
90+
91+
def is_on_darwin() -> bool:
92+
return platform.uname().system == "Darwin"
93+
94+
95+
def set_environment(*, swiftpm_bin_dir: pathlib.Path,) -> None:
96+
os.environ["SWIFTCI_IS_SELF_HOSTED"] = "1"
97+
98+
# Set the SWIFTPM_BIN_DIR path
99+
os.environ["SWIFTPM_BIN_DIR"] = str(swiftpm_bin_dir)
100+
101+
# Ensure SDKROOT is configure
102+
if is_on_darwin():
103+
sdk_root = call_output(shlex.split("xcrun --show-sdk-path --sdk macosx"))
104+
logging.debug("macos sdk root = %r", sdk_root)
105+
os.environ["SDKROOT"] = sdk_root
106+
log_environment()
107+
108+
109+
def run_bootstrap(swiftpm_bin_dir: pathlib.Path) -> None:
110+
if is_on_darwin():
111+
logging.info("Current working directory is %s", pathlib.Path.cwd())
112+
logging.info("Bootstrapping with the XCBuild codepath...")
113+
call(
114+
[
115+
REPO_ROOT_PATH / "Utilities" / "bootstrap",
116+
"build",
117+
"--release",
118+
"--verbose",
119+
"--cross-compile-hosts",
120+
"macosx-arm64",
121+
"--skip-cmake-bootstrap",
122+
"--swift-build-path",
123+
(swiftpm_bin_dir / "swift-build").resolve(),
124+
],
125+
)
126+
127+
128+
def main() -> None:
129+
args = get_arguments()
130+
logging.getLogger().setLevel(logging.DEBUG if args.is_verbose else logging.INFO)
131+
logging.debug("Args: %r", args)
132+
133+
with change_directory(REPO_ROOT_PATH):
134+
swiftpm_bin_dir = get_swiftpm_bin_dir(config=args.config)
135+
set_environment(swiftpm_bin_dir=swiftpm_bin_dir)
136+
137+
call(
138+
shlex.split("swift --version"),
139+
)
140+
141+
call(
142+
shlex.split("swift package update"),
143+
)
144+
call(
145+
shlex.split(f"swift build --configuration {args.config}"),
146+
)
147+
call(
148+
shlex.split(f"swift test --configuration {args.config} --parallel"),
149+
)
150+
151+
with change_directory(REPO_ROOT_PATH / "IntegrationTests"):
152+
call(
153+
shlex.split("swift package update"),
154+
)
155+
call(
156+
shlex.split(
157+
f"{swiftpm_bin_dir / 'swift-test'} --parallel",
158+
),
159+
)
160+
161+
run_bootstrap(swiftpm_bin_dir=swiftpm_bin_dir)
162+
163+
164+
if __name__ == "__main__":
165+
main()

‎Utilities/helpers.py

+45-10
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,37 @@
1111
##
1212
##===----------------------------------------------------------------------===##
1313

14-
import datetime
14+
import contextlib
15+
import enum
16+
import errno
1517
import logging
16-
import subprocess
17-
import sys
1818
import os
19-
import errno
19+
import pathlib
20+
import subprocess
21+
import typing as t
22+
23+
24+
@contextlib.contextmanager
25+
def change_directory(directory: pathlib.Path) -> t.Iterator[pathlib.Path]:
26+
current_directory = pathlib.Path.cwd()
27+
logging.info("Current directory is %s", current_directory)
28+
logging.info("Changing directory to: %s", directory)
29+
os.chdir(directory)
30+
31+
try:
32+
yield directory
33+
finally:
34+
logging.debug("Chaning directory back to %s", current_directory)
35+
os.chdir(current_directory)
36+
37+
38+
class Configuration(str, enum.Enum):
39+
DEBUG = "debug"
40+
RELEASE = "release"
41+
42+
def __str__(self) -> str:
43+
return self.value
44+
2045

2146
def symlink_force(source, destination):
2247
try:
@@ -26,6 +51,7 @@ def symlink_force(source, destination):
2651
os.remove(destination)
2752
os.symlink(source, destination)
2853

54+
2955
def mkdir_p(path):
3056
"""Create the given directory, if it does not exist."""
3157
try:
@@ -35,22 +61,31 @@ def mkdir_p(path):
3561
if e.errno != errno.EEXIST:
3662
raise
3763

64+
3865
def call(cmd, cwd=None, verbose=False):
3966
"""Calls a subprocess."""
40-
logging.info("executing command >>> %s", ' '.join(cmd))
67+
cwd = cwd or pathlib.Path.cwd()
68+
logging.info("executing command >>> %r with cwd %s", " ".join(cmd), cwd)
4169
try:
4270
subprocess.check_call(cmd, cwd=cwd)
4371
except subprocess.CalledProcessError as cpe:
44-
logging.debug("executing command >>> %s", ' '.join(cmd))
72+
logging.debug("executing command >>> %r with cwd %s", " ".join(cmd), cwd)
4573
logging.error("Process failure: %s", str(cpe))
4674
raise cpe
4775

76+
4877
def call_output(cmd, cwd=None, stderr=False, verbose=False):
4978
"""Calls a subprocess for its return data."""
50-
logging.info(' '.join(cmd))
79+
cwd = cwd or pathlib.Path.cwd()
80+
logging.info("executing command >>> %r with cwd %s", " ".join(cmd), cwd)
5181
try:
52-
return subprocess.check_output(cmd, cwd=cwd, stderr=stderr, universal_newlines=True).strip()
82+
return subprocess.check_output(
83+
cmd,
84+
cwd=cwd,
85+
stderr=stderr,
86+
universal_newlines=True,
87+
).strip()
5388
except subprocess.CalledProcessError as cpe:
54-
logging.debug(' '.join(cmd))
55-
logging.error(str(cpe))
89+
logging.debug("executing command >>> %r with cwd %s", " ".join(cmd), cwd)
90+
logging.error("Process failure: %s", str(cpe))
5691
raise cpe

‎Utilities/python/requirements.txt

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
black==24.10.0
2+
flake8==7.1.1
3+
mypy==1.14.1

0 commit comments

Comments
 (0)