Skip to content

Commit d2f62a6

Browse files
bkhouribkhouri
committed
pipeline: Convert build-using-self to python
In preparation for a Windows self-host pipeline, convert the build-using-self from a bash script to a python script to make it platform agnostic.
1 parent 314a274 commit d2f62a6

File tree

3 files changed

+206
-70
lines changed

3 files changed

+206
-70
lines changed

‎Utilities/build-using-self

+161-60
Original file line numberDiff line numberDiff line change
@@ -1,62 +1,163 @@
1-
#!/bin/bash
2-
##===----------------------------------------------------------------------===##
3-
##
4-
## This source file is part of the Swift open source project
5-
##
6-
## Copyright (c) 2014-2022 Apple Inc. and the Swift project authors
7-
## Licensed under Apache License v2.0 with Runtime Library Exception
8-
##
9-
## See http://swift.org/LICENSE.txt for license information
10-
## See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
11-
##
12-
##===----------------------------------------------------------------------===##
13-
14-
set -eu
15-
16-
__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
17-
root_dir="$(cd ${__dir}/.. && pwd)"
18-
19-
cd "${root_dir}/"
20-
echo "Current directory is ${PWD}"
21-
22-
CONFIGURATION=debug
23-
export SWIFTCI_IS_SELF_HOSTED=1
24-
25-
# Ensure SDKROOT is configured
26-
host=$(uname)
27-
if [ "${host}" == "Darwin" ]; then
28-
export SDKROOT=$(xcrun --show-sdk-path --sdk macosx)
29-
fi
30-
31-
set -x
32-
33-
env | sort
34-
35-
# Display toolchain version
36-
swift --version
37-
38-
# Perform package update in order to get the latest commits for the dependencies.
39-
swift package update
40-
swift build -c $CONFIGURATION
41-
swift test -c $CONFIGURATION --parallel
42-
43-
# Run the integration tests with just built SwiftPM.
44-
export SWIFTPM_BIN_DIR=$(swift build -c $CONFIGURATION --show-bin-path)
45-
(
46-
cd ${root_dir}/IntegrationTests
47-
# Perform package update in order to get the latest commits for the dependencies.
48-
swift package update
49-
$SWIFTPM_BIN_DIR/swift-test --parallel
1+
#!/usr/bin/env python3
2+
# ===----------------------------------------------------------------------===##
3+
#
4+
# This source file is part of the Swift open source project
5+
#
6+
# Copyright (c) 2025 Apple Inc. and the Swift project authors
7+
# Licensed under Apache License v2.0 with Runtime Library Exception
8+
#
9+
# See http://swift.org/LICENSE.txt for license information
10+
# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
11+
#
12+
# ===----------------------------------------------------------------------===##
13+
14+
import argparse
15+
import logging
16+
import os
17+
import pathlib
18+
import platform
19+
import sys
20+
import typing as t
21+
22+
from helpers import (
23+
Configuration,
24+
change_directory,
25+
call,
26+
call_output,
27+
)
28+
29+
logging.basicConfig(
30+
format=" | ".join(
31+
[
32+
# Prefix script name to the log in an attempt to avoid confusion when parsing logs
33+
f"{pathlib.Path(sys.argv[0]).name}",
34+
"%(asctime)s",
35+
"%(levelname)-8s",
36+
"%(module)s",
37+
"%(funcName)s",
38+
"Line:%(lineno)d",
39+
"%(message)s",
40+
]
41+
),
42+
level=logging.INFO,
5043
)
5144

52-
if [ "${host}" == "Darwin" ]; then
53-
echo "Current working directory is ${PWD}"
54-
echo "Bootstrapping with the XCBuild codepath..."
55-
${root_dir}/Utilities/bootstrap \
56-
build \
57-
--release \
58-
--verbose \
59-
--cross-compile-hosts macosx-arm64 \
60-
--skip-cmake-bootstrap \
61-
--swift-build-path "${SWIFTPM_BIN_DIR}/swift-build"
62-
fi
45+
46+
REPO_ROOT_PATH = pathlib.Path(__file__).parent.parent.resolve()
47+
48+
49+
def get_arguments() -> argparse.Namespace:
50+
parser = argparse.ArgumentParser(
51+
formatter_class=argparse.ArgumentDefaultsHelpFormatter
52+
)
53+
54+
parser.add_argument(
55+
"-v",
56+
"--verbose",
57+
dest="is_verbose",
58+
action="store_true",
59+
help="When set, prints verbose information.",
60+
)
61+
parser.add_argument(
62+
"-c",
63+
"--configuration",
64+
type=Configuration,
65+
dest="config",
66+
default="debug",
67+
choices=[e.value for e in Configuration],
68+
help="The configuraiton to use.",
69+
)
70+
71+
args = parser.parse_args()
72+
return args
73+
74+
75+
def log_environment() -> None:
76+
logging.info("Environment Variables")
77+
for key, value in sorted(os.environ.items()):
78+
logging.info(" --> %s=%r", key, value)
79+
80+
81+
def get_swiftpm_bin_dir(config: Configuration) -> pathlib.Path:
82+
swiftpm_bin_dir = pathlib.Path(
83+
call_output(["swift", "build", "--configuration", config, "--show-bin-path"])
84+
)
85+
logging.info("SwiftPM BIN DIR: %s", swiftpm_bin_dir)
86+
return swiftpm_bin_dir
87+
88+
89+
def is_on_darwin() -> bool:
90+
return platform.uname().system == "Darwin"
91+
92+
93+
def set_environment(*, config: Configuration) -> None:
94+
os.environ["SWIFTCI_IS_SELF_HOSTED"] = "1"
95+
96+
# Set the SWIFTPM_BIN_DIR path
97+
os.environ["SWIFTPM_BIN_DIR"] = str(get_swiftpm_bin_dir(config=config))
98+
99+
# Ensure SDKROOT is configure
100+
if is_on_darwin():
101+
sdk_root = call_output(["xcrun", "--show-sdk-path", "--sdk", "macosx"])
102+
logging.debug("macos sdk root = %r", sdk_root)
103+
os.environ["SDKROOT"] = sdk_root
104+
log_environment()
105+
106+
107+
def run_bootstrap(swiftpm_bin_dir: pathlib.Path) -> None:
108+
if is_on_darwin():
109+
logging.info("Current working directory is %s", pathlib.Path.cwd())
110+
logging.info("Bootstrapping with the XCBuild codepath...")
111+
call(
112+
[
113+
REPO_ROOT_PATH / "Utilities" / "bootstrap",
114+
"build",
115+
"--release",
116+
"--verbose",
117+
"--cross-compile-hosts",
118+
"macosx-arm64",
119+
"--skip-cmake-bootstrap",
120+
"--swift-build-path",
121+
(swiftpm_bin_dir / "swift-build").resolve(),
122+
],
123+
)
124+
125+
126+
def main() -> None:
127+
args = get_arguments()
128+
logging.getLogger().setLevel(logging.DEBUG if args.is_verbose else logging.INFO)
129+
logging.debug("Args: %r", args)
130+
131+
with change_directory(REPO_ROOT_PATH):
132+
set_environment(config=args.config)
133+
134+
call(
135+
["swift", "--version"],
136+
)
137+
call(
138+
["swift", "package", "update"],
139+
)
140+
call(
141+
["swift", "build", "--configuration", args.config],
142+
)
143+
call(
144+
["swift", "test", "--configuration", args.config, "--parallel"],
145+
)
146+
swiftpm_bin_dir = get_swiftpm_bin_dir(config=args.config)
147+
148+
with change_directory(REPO_ROOT_PATH / "IntegrationTests"):
149+
call(
150+
["swift", "package", "update"],
151+
)
152+
call(
153+
[
154+
str(swiftpm_bin_dir / "swift-test"),
155+
"--parallel",
156+
],
157+
)
158+
159+
run_bootstrap(swiftpm_bin_dir=swiftpm_bin_dir)
160+
161+
162+
if __name__ == "__main__":
163+
main()

‎Utilities/helpers.py

+42-10
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,34 @@
1111
##
1212
##===----------------------------------------------------------------------===##
1313

14-
import datetime
14+
import contextlib
15+
import enum
16+
import errno
1517
import logging
16-
import subprocess
17-
import sys
1818
import os
19-
import errno
19+
import pathlib
20+
import subprocess
21+
import typing as t
22+
23+
24+
@contextlib.contextmanager
25+
def change_directory(directory: pathlib.Path) -> t.Iterator[pathlib.Path]:
26+
current_directory = pathlib.Path.cwd()
27+
logging.info("Current directory is %s", current_directory)
28+
logging.info("Changing directory to: %s", directory)
29+
os.chdir(directory)
30+
31+
try:
32+
yield directory
33+
finally:
34+
logging.debug("Chaning directory back to %s", current_directory)
35+
os.chdir(current_directory)
36+
37+
38+
class Configuration(str, enum.Enum):
39+
DEBUG = "debug"
40+
RELEASE = "release"
41+
2042

2143
def symlink_force(source, destination):
2244
try:
@@ -26,6 +48,7 @@ def symlink_force(source, destination):
2648
os.remove(destination)
2749
os.symlink(source, destination)
2850

51+
2952
def mkdir_p(path):
3053
"""Create the given directory, if it does not exist."""
3154
try:
@@ -35,22 +58,31 @@ def mkdir_p(path):
3558
if e.errno != errno.EEXIST:
3659
raise
3760

61+
3862
def call(cmd, cwd=None, verbose=False):
3963
"""Calls a subprocess."""
40-
logging.info("executing command >>> %s", ' '.join(cmd))
64+
cwd = cwd or pathlib.Path.cwd()
65+
logging.info("executing command >>> %r with cwd %s", " ".join(cmd), cwd)
4166
try:
4267
subprocess.check_call(cmd, cwd=cwd)
4368
except subprocess.CalledProcessError as cpe:
44-
logging.debug("executing command >>> %s", ' '.join(cmd))
69+
logging.debug("executing command >>> %r with cwd %s", " ".join(cmd), cwd)
4570
logging.error("Process failure: %s", str(cpe))
4671
raise cpe
4772

73+
4874
def call_output(cmd, cwd=None, stderr=False, verbose=False):
4975
"""Calls a subprocess for its return data."""
50-
logging.info(' '.join(cmd))
76+
cwd = cwd or pathlib.Path.cwd()
77+
logging.info("executing command >>> %r with cwd %s", " ".join(cmd), cwd)
5178
try:
52-
return subprocess.check_output(cmd, cwd=cwd, stderr=stderr, universal_newlines=True).strip()
79+
return subprocess.check_output(
80+
cmd,
81+
cwd=cwd,
82+
stderr=stderr,
83+
universal_newlines=True,
84+
).strip()
5385
except subprocess.CalledProcessError as cpe:
54-
logging.debug(' '.join(cmd))
55-
logging.error(str(cpe))
86+
logging.debug("executing command >>> %r with cwd %s", " ".join(cmd), cwd)
87+
logging.error("Process failure: %s", str(cpe))
5688
raise cpe

‎Utilities/python/requirements.txt

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
black==24.10.0
2+
flake8==7.1.1
3+
mypy==1.14.1

0 commit comments

Comments
 (0)