Skip to content

Commit f34907c

Browse files
dschaefer2dschaefer2
authored
Support separate modules for static and dynamic exporting symbols for Windows (#8049)
On Windows, there is a limit of around 64K to the number of symbols a DLL/EXE can export. We hit that regularly with large projects like SwiftPM itself. This hides those symbols and only exposes the ones requested for a DLL. For Windows triples only, creates a parallel build graph for Swift modules, one for static linking using -static, and one for exporting linking which is the default. DLL products consume their direct target dependencies as exporting. All other dependencies use the static versions to eliminate unnecessary symbol exports. The bulk of this is managed by the SwiftModuleBuildDescription which will create a duplicate of itself for the exporting case and set its own type to static linking. Both modules are fed to the planner to create llbuild swift command tasks. Code is added for dynamic libraries to hook up the correct inputs for exporting the symbols in the libraries targets. This is WIP as we need to do a lot of testing to ensure we didn't break anything. Ensuring this only affects builds for Windows triples helps mitigate that.
1 parent 7c6da12 commit f34907c

File tree

6 files changed

+262
-4
lines changed

6 files changed

+262
-4
lines changed

‎Sources/Build/BuildDescription/SwiftModuleBuildDescription.swift

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,11 @@ public final class SwiftModuleBuildDescription {
127127

128128
var modulesPath: AbsolutePath {
129129
let suffix = self.buildParameters.suffix
130-
return self.buildParameters.buildPath.appending(component: "Modules\(suffix)")
130+
var path = self.buildParameters.buildPath.appending(component: "Modules\(suffix)")
131+
if self.windowsTargetType == .dynamic {
132+
path = path.appending("dynamic")
133+
}
134+
return path
131135
}
132136

133137
/// The path to the swiftmodule file after compilation.
@@ -264,6 +268,19 @@ public final class SwiftModuleBuildDescription {
264268
/// Whether to disable sandboxing (e.g. for macros).
265269
private let shouldDisableSandbox: Bool
266270

271+
/// For Windows, we default to static objects and but also create objects
272+
/// that export symbols for DLLs. This allows library targets to be used
273+
/// in both contexts
274+
public enum WindowsTargetType {
275+
case `static`
276+
case dynamic
277+
}
278+
/// The target type. Leave nil for non-Windows behavior.
279+
public let windowsTargetType: WindowsTargetType?
280+
281+
/// The corresponding target for dynamic library export (i.e., not -static)
282+
public private(set) var windowsDynamicTarget: SwiftModuleBuildDescription? = nil
283+
267284
/// Create a new target description with target and build parameters.
268285
init(
269286
package: ResolvedPackage,
@@ -319,6 +336,14 @@ public final class SwiftModuleBuildDescription {
319336
observabilityScope: observabilityScope
320337
)
321338

339+
if buildParameters.triple.isWindows() {
340+
// Default to static and add another target for DLLs
341+
self.windowsTargetType = .static
342+
self.windowsDynamicTarget = .init(windowsExportFor: self)
343+
} else {
344+
self.windowsTargetType = nil
345+
}
346+
322347
if self.shouldEmitObjCCompatibilityHeader {
323348
self.moduleMap = try self.generateModuleMap()
324349
}
@@ -340,6 +365,31 @@ public final class SwiftModuleBuildDescription {
340365
try self.generateTestObservation()
341366
}
342367

368+
/// Private init to set up exporting version of this module
369+
private init(windowsExportFor parent: SwiftModuleBuildDescription) {
370+
self.windowsTargetType = .dynamic
371+
self.windowsDynamicTarget = nil
372+
self.tempsPath = parent.tempsPath.appending("dynamic")
373+
374+
// The rest of these are just copied from the parent
375+
self.package = parent.package
376+
self.target = parent.target
377+
self.swiftTarget = parent.swiftTarget
378+
self.toolsVersion = parent.toolsVersion
379+
self.buildParameters = parent.buildParameters
380+
self.macroBuildParameters = parent.macroBuildParameters
381+
self.derivedSources = parent.derivedSources
382+
self.pluginDerivedSources = parent.pluginDerivedSources
383+
self.pluginDerivedResources = parent.pluginDerivedResources
384+
self.testTargetRole = parent.testTargetRole
385+
self.fileSystem = parent.fileSystem
386+
self.buildToolPluginInvocationResults = parent.buildToolPluginInvocationResults
387+
self.prebuildCommandResults = parent.prebuildCommandResults
388+
self.observabilityScope = parent.observabilityScope
389+
self.shouldGenerateTestObservation = parent.shouldGenerateTestObservation
390+
self.shouldDisableSandbox = parent.shouldDisableSandbox
391+
}
392+
343393
private func generateTestObservation() throws {
344394
guard target.type == .test else {
345395
return
@@ -519,6 +569,18 @@ public final class SwiftModuleBuildDescription {
519569
args += ["-parse-as-library"]
520570
}
521571

572+
switch self.windowsTargetType {
573+
case .static:
574+
// Static on Windows
575+
args += ["-static"]
576+
case .dynamic:
577+
// Add the static versions to the include path
578+
// FIXME: need to be much more deliberate about what we're including
579+
args += ["-I", self.modulesPath.parentDirectory.pathString]
580+
case .none:
581+
break
582+
}
583+
522584
// Only add the build path to the framework search path if there are binary frameworks to link against.
523585
if !self.libraryBinaryPaths.isEmpty {
524586
args += ["-F", self.buildParameters.buildPath.pathString]

‎Sources/Build/BuildManifest/LLBuildManifestBuilder+Swift.swift

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,16 @@ extension LLBuildManifestBuilder {
5353
)
5454
} else {
5555
try self.addCmdWithBuiltinSwiftTool(target, inputs: inputs, cmdOutputs: cmdOutputs)
56+
if let dynamicTarget = target.windowsDynamicTarget {
57+
// Generate dynamic module for Windows
58+
let inputs = try self.computeSwiftCompileCmdInputs(dynamicTarget)
59+
let objectNodes = dynamicTarget.buildParameters.prepareForIndexing == .off ? try dynamicTarget.objects.map(Node.file) : []
60+
let moduleNode = Node.file(dynamicTarget.moduleOutputPath)
61+
let cmdOutputs = objectNodes + [moduleNode]
62+
try self.addCmdWithBuiltinSwiftTool(dynamicTarget, inputs: inputs, cmdOutputs: cmdOutputs)
63+
self.addTargetCmd(dynamicTarget, cmdOutputs: cmdOutputs)
64+
try self.addModuleWrapCmd(dynamicTarget)
65+
}
5666
}
5767

5868
self.addTargetCmd(target, cmdOutputs: cmdOutputs)
@@ -532,7 +542,7 @@ extension LLBuildManifestBuilder {
532542
inputs: cmdOutputs,
533543
outputs: [targetOutput]
534544
)
535-
if self.plan.graph.isInRootPackages(target.target, satisfying: target.buildParameters.buildEnvironment) {
545+
if self.plan.graph.isInRootPackages(target.target, satisfying: target.buildParameters.buildEnvironment), target.windowsTargetType != .dynamic {
536546
if !target.isTestTarget {
537547
self.addNode(targetOutput, toTarget: .main)
538548
}
@@ -636,6 +646,11 @@ extension SwiftModuleBuildDescription {
636646
}
637647

638648
public func getLLBuildTargetName() -> String {
639-
self.target.getLLBuildTargetName(buildParameters: self.buildParameters)
649+
let name = self.target.getLLBuildTargetName(buildParameters: self.buildParameters)
650+
if self.windowsTargetType == .dynamic {
651+
return "dynamic." + name
652+
} else {
653+
return name
654+
}
640655
}
641656
}

‎Sources/Build/BuildPlan/BuildPlan+Product.swift

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,18 @@ extension BuildPlan {
101101

102102
buildProduct.staticTargets = dependencies.staticTargets.map(\.module)
103103
buildProduct.dylibs = dependencies.dylibs
104-
buildProduct.objects += try dependencies.staticTargets.flatMap { try $0.objects }
104+
buildProduct.objects += try dependencies.staticTargets.flatMap {
105+
if buildProduct.product.type == .library(.dynamic),
106+
case let .swift(swiftModule) = $0,
107+
let dynamic = swiftModule.windowsDynamicTarget,
108+
buildProduct.product.modules.contains(id: swiftModule.target.id)
109+
{
110+
// On Windows, export symbols from the direct swift targets of the DLL product
111+
return try dynamic.objects
112+
} else {
113+
return try $0.objects
114+
}
115+
}
105116
buildProduct.libraryBinaryPaths = dependencies.libraryBinaryPaths
106117
buildProduct.availableTools = dependencies.availableTools
107118
}

‎Sources/Build/BuildPlan/BuildPlan.swift

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -537,6 +537,9 @@ public class BuildPlan: SPMBuildCore.BuildPlan {
537537
switch buildTarget {
538538
case .swift(let target):
539539
try self.plan(swiftTarget: target)
540+
if let dynamicTarget = target.windowsDynamicTarget {
541+
try self.plan(swiftTarget: dynamicTarget)
542+
}
540543
case .clang(let target):
541544
try self.plan(clangTarget: target)
542545
}

‎Sources/_InternalTestSupport/MockBuildTestHelper.swift

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ extension Basics.Triple {
6666
public static let arm64Linux = try! Self("aarch64-unknown-linux-gnu")
6767
public static let arm64Android = try! Self("aarch64-unknown-linux-android")
6868
public static let windows = try! Self("x86_64-unknown-windows-msvc")
69+
public static let x86_64Windows = try! Self("x86_64-unknown-windows-msvc")
70+
public static let arm64Windows = try! Self("aarch64-unknown-windows-msvc")
6971
public static let wasi = try! Self("wasm32-unknown-wasi")
7072
public static let arm64iOS = try! Self("arm64-apple-ios")
7173
}

‎Tests/BuildTests/WindowsBuildPlanTests.swift

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// This source file is part of the Swift open source project
4+
//
5+
// Copyright (c) 2014-2024 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See http://swift.org/LICENSE.txt for license information
9+
// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
import XCTest
14+
15+
import Basics
16+
@testable import Build
17+
import LLBuildManifest
18+
import _InternalTestSupport
19+
20+
@_spi(DontAdoptOutsideOfSwiftPMExposedForBenchmarksAndTestsOnly)
21+
import PackageGraph
22+
23+
final class WindowsBuildPlanTests: XCTestCase {
24+
// Tests that our build plan is build correctly to handle separation
25+
// of object files that export symbols and ones that don't and to ensure
26+
// DLL products pick up the right ones.
27+
28+
func doTest(triple: Triple) async throws {
29+
let fs = InMemoryFileSystem(emptyFiles: [
30+
"/libPkg/Sources/coreLib/coreLib.swift",
31+
"/libPkg/Sources/dllLib/dllLib.swift",
32+
"/libPkg/Sources/staticLib/staticLib.swift",
33+
"/libPkg/Sources/objectLib/objectLib.swift",
34+
"/exePkg/Sources/exe/main.swift",
35+
])
36+
37+
let observability = ObservabilitySystem.makeForTesting()
38+
39+
let graph = try loadModulesGraph(
40+
fileSystem: fs,
41+
manifests: [
42+
.createFileSystemManifest(
43+
displayName: "libPkg",
44+
path: "/libPkg",
45+
products: [
46+
.init(name: "DLLProduct", type: .library(.dynamic), targets: ["dllLib"]),
47+
.init(name: "StaticProduct", type: .library(.static), targets: ["staticLib"]),
48+
.init(name: "ObjectProduct", type: .library(.automatic), targets: ["objectLib"]),
49+
],
50+
targets: [
51+
.init(name: "coreLib", dependencies: []),
52+
.init(name: "dllLib", dependencies: ["coreLib"]),
53+
.init(name: "staticLib", dependencies: ["coreLib"]),
54+
.init(name: "objectLib", dependencies: ["coreLib"]),
55+
]
56+
),
57+
.createRootManifest(
58+
displayName: "exePkg",
59+
path: "/exePkg",
60+
dependencies: [.fileSystem(path: "/libPkg")],
61+
targets: [
62+
.init(name: "exe", dependencies: [
63+
.product(name: "DLLProduct", package: "libPkg"),
64+
.product(name: "StaticProduct", package: "libPkg"),
65+
.product(name: "ObjectProduct", package: "libPkg"),
66+
]),
67+
]
68+
)
69+
],
70+
observabilityScope: observability.topScope
71+
)
72+
73+
let label: String
74+
let dylibPrefix: String
75+
let dylibExtension: String
76+
let dynamic: String
77+
switch triple {
78+
case Triple.x86_64Windows:
79+
label = "x86_64-unknown-windows-msvc"
80+
dylibPrefix = ""
81+
dylibExtension = "dll"
82+
dynamic = "/dynamic"
83+
case Triple.x86_64MacOS:
84+
label = "x86_64-apple-macosx"
85+
dylibPrefix = "lib"
86+
dylibExtension = "dylib"
87+
dynamic = ""
88+
case Triple.x86_64Linux:
89+
label = "x86_64-unknown-linux-gnu"
90+
dylibPrefix = "lib"
91+
dylibExtension = "so"
92+
dynamic = ""
93+
default:
94+
label = "fixme"
95+
dylibPrefix = ""
96+
dylibExtension = ""
97+
dynamic = ""
98+
}
99+
100+
let tools: [String: [String]] = [
101+
"C.exe-\(label)-debug.exe": [
102+
"/path/to/build/\(label)/debug/coreLib.build/coreLib.swift.o",
103+
"/path/to/build/\(label)/debug/exe.build/main.swift.o",
104+
"/path/to/build/\(label)/debug/objectLib.build/objectLib.swift.o",
105+
"/path/to/build/\(label)/debug/staticLib.build/staticLib.swift.o",
106+
"/path/to/build/\(label)/debug/\(dylibPrefix)DLLProduct.\(dylibExtension)",
107+
"/path/to/build/\(label)/debug/exe.product/Objects.LinkFileList",
108+
] + (triple.isMacOSX ? [] : [
109+
// modulewrap
110+
"/path/to/build/\(label)/debug/coreLib.build/coreLib.swiftmodule.o",
111+
"/path/to/build/\(label)/debug/exe.build/exe.swiftmodule.o",
112+
"/path/to/build/\(label)/debug/objectLib.build/objectLib.swiftmodule.o",
113+
"/path/to/build/\(label)/debug/staticLib.build/staticLib.swiftmodule.o",
114+
]),
115+
"C.DLLProduct-\(label)-debug.dylib": [
116+
"/path/to/build/\(label)/debug/coreLib.build/coreLib.swift.o",
117+
"/path/to/build/\(label)/debug/dllLib.build\(dynamic)/dllLib.swift.o",
118+
"/path/to/build/\(label)/debug/DLLProduct.product/Objects.LinkFileList",
119+
] + (triple.isMacOSX ? [] : [
120+
"/path/to/build/\(label)/debug/coreLib.build/coreLib.swiftmodule.o",
121+
"/path/to/build/\(label)/debug/dllLib.build/dllLib.swiftmodule.o",
122+
])
123+
]
124+
125+
let plan = try await BuildPlan(
126+
destinationBuildParameters: mockBuildParameters(
127+
destination: .target,
128+
triple: triple
129+
),
130+
toolsBuildParameters: mockBuildParameters(
131+
destination: .host,
132+
triple: triple
133+
),
134+
graph: graph,
135+
fileSystem: fs,
136+
observabilityScope: observability.topScope
137+
)
138+
139+
let llbuild = LLBuildManifestBuilder(
140+
plan,
141+
fileSystem: fs,
142+
observabilityScope: observability.topScope
143+
)
144+
try llbuild.generateManifest(at: "/manifest")
145+
146+
for (name, inputNames) in tools {
147+
let command = try XCTUnwrap(llbuild.manifest.commands[name])
148+
XCTAssertEqual(Set(command.tool.inputs), Set(inputNames.map({ Node.file(.init($0)) })))
149+
}
150+
}
151+
152+
func testWindows() async throws {
153+
try await doTest(triple: .x86_64Windows)
154+
}
155+
156+
// Make sure we didn't mess up macOS
157+
func testMacOS() async throws {
158+
try await doTest(triple: .x86_64MacOS)
159+
}
160+
161+
// Make sure we didn't mess up linux
162+
func testLinux() async throws {
163+
try await doTest(triple: .x86_64Linux)
164+
}
165+
}

0 commit comments

Comments
 (0)