SWBBuild SDK support

Author: kabiroberai

Package.swift

@@ -166,6 +166,7 @@ let package = Package(
             name: "PackLib",
             dependencies: [
                 "XUtils",
+                .product(name: "Superutils", package: "xtool-core"),
                 .product(name: "Yams", package: "Yams"),
                 .product(name: "XcodeGenKit", package: "XcodeGen", condition: .when(platforms: [.macOS])),
             ]

Sources/PackLib/BuildSettings.swift

@@ -1,37 +1,73 @@
 import Foundation
 import Subprocess
 import XUtils
+import Superutils
 
 public struct BuildSettings: Sendable {
     private static let customBinDir =
         // this is the same option used by SwiftPM itself for dev builds
         ProcessInfo.processInfo.environment["SWIFTPM_CUSTOM_BIN_DIR"].map { FilePath($0) }
 
-    private static let envURL = URL(fileURLWithPath: "/usr/bin/env")
-
-    public let packagePath: String
+    public var packagePath: String
     public let configuration: BuildConfiguration
     public let triple: String
-    public let sdkOptions: [String]
-    public let options: [String]
+    public let buildSystem: BuildSystem
+    public let customOptions: [String]
+
+    public var sdkOptions: [String]
+    public var sdkEnvironment: [Environment.Key: String?]
+
+    private var configOptions: [String] {
+        return [
+            "--configuration", configuration.rawValue,
+            "--build-system", buildSystem.pmName,
+            "--package-path", packagePath,
+        ]
+    }
+
+    private var resolvedBaseOptions: [String] {
+        configOptions + sdkOptions + customOptions
+    }
 
     public init(
         configuration: BuildConfiguration,
         triple: String,
+        buildSystem: BuildSystem = .default,
         packagePath: String = ".",
         options: [String] = []
     ) async throws {
         self.packagePath = packagePath
         self.configuration = configuration
-        self.options = options
+        self.customOptions = options
         self.triple = triple
+        self.buildSystem = buildSystem
+
+        self.sdkEnvironment = [
+            // xcrun passes an SDKROOT that messes with our sdk configuration
+            "SDKROOT": nil,
+        ]
 
-        // on macOS we don't explicitly install a Swift SDK but
-        // SwiftPM vends "implicit" Darwin SDKs as of Swift 6.1,
-        // i.e. we can pass `--swift-sdk arm64-apple-ios` and it
-        // just works. See:
-        // https://github.com/swiftlang/swift-package-manager/pull/6828
-        self.sdkOptions = ["--swift-sdk", triple]
+        switch buildSystem {
+        case .swiftPM:
+            // on macOS we don't explicitly install a Swift SDK but
+            // SwiftPM vends "implicit" Darwin SDKs as of Swift 6.1,
+            // i.e. we can pass `--swift-sdk arm64-apple-ios` and it
+            // just works. See:
+            // https://github.com/swiftlang/swift-package-manager/pull/6828
+            self.sdkOptions = ["--swift-sdk", triple]
+        case .swiftBuild:
+            self.sdkOptions = ["--triple", triple]
+            #if !os(macOS)
+            let darwinSDK = try await DarwinSDK.current()
+                .orThrow(StringError("No Darwin SDK configured. Please run `xtool setup`."))
+            self.sdkOptions += [
+                "--toolset", "\(darwinSDK.bundle.path)/toolset-swb.json",
+            ]
+            self.sdkEnvironment.merge([
+                "XCODE_EXTRA_PLATFORM_FOLDERS": "\(darwinSDK.bundle.path)/Developer/Platforms",
+            ]) { $1 }
+            #endif
+        }
     }
 
     #if os(macOS)
@@ -44,15 +80,40 @@ public struct BuildSettings: Sendable {
         return result.standardOutput?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
     }
 
-    private static let swiftURL = Task {
+    private static let _swiftURL = Task {
         try await FilePath(xcrun(["-f", "swift"]))
     }
+
+    public static func swiftURL() async throws -> FilePath {
+        try await _swiftURL.value
+    }
+
+    private static let _swiftcURL = Task {
+        try await FilePath(xcrun(["-f", "swiftc"]))
+    }
+
+    public static func swiftcURL() async throws -> FilePath {
+        try await _swiftcURL.value
+    }
+    #else
+    public static func swiftURL() async throws -> FilePath {
+        try await FilePath(ToolRegistry.locate("swift")).orThrow(StringError("Got bad path for swift executable"))
+    }
+
+    public static func swiftcURL() async throws -> FilePath {
+        try await FilePath(ToolRegistry.locate("swiftc")).orThrow(StringError("Got bad path for swiftc executable"))
+    }
     #endif
 
+    public func withPackagePath(_ path: String) -> Self {
+        var copy = self
+        copy.packagePath = path
+        return copy
+    }
+
     public func swiftPMInvocation(
         forTool tool: String,
         arguments: [String],
-        packagePathOverride: String? = nil
     ) async throws -> Subprocess.Configuration {
         let executable: Executable
         let baseArguments: [String]
@@ -65,35 +126,60 @@ public struct BuildSettings: Sendable {
             // to add SDKROOT=.../MacOSX.sdk to our invocations. We avoid this by
             // 1) invoking the real swift executable (located with `xcrun -f`) and
             // 2) explicitly removing SDKROOT from the env, as it may be inherited
-            // through the `swift run pack` invocation.
-            executable = .path(try await Self.swiftURL.value)
+            // through a parent process (e.g. `swift run xtool`).
+            executable = .path(try await Self.swiftURL())
             #else
             executable = .name("swift")
             #endif
             baseArguments = [tool]
         }
 
-        let extraArguments: [String] = [
-            "--package-path", packagePathOverride ?? packagePath,
-            "--configuration", configuration.rawValue,
-        ]
-
-        var rawEnv = ProcessInfo.processInfo.environment
-        rawEnv.removeValue(forKey: "SDKROOT")
-        let env = Dictionary(uniqueKeysWithValues: rawEnv.map {
-            (Environment.Key(rawValue: $0)!, $1)
-        })
-
         return Configuration(
             executable,
-            arguments: .init(baseArguments + extraArguments + sdkOptions + options + arguments),
-            environment: .custom(env),
+            arguments: .init(baseArguments + resolvedBaseOptions + arguments),
+            environment: .inherit.updating(sdkEnvironment),
             platformOptions: .withGracefulShutDown,
         )
     }
+
+    public var buildServerArguments: [String] {
+        return [
+            "package", "experimental-build-server",
+            "--disable-automatic-resolution",
+            // TODO: once https://github.com/swiftlang/swift-package-manager/pull/9819 makes it into a release
+            // (Swift 6.4), pass --experimental-skip-acquiring-lock
+        ] + resolvedBaseOptions
+    }
 }
 
 public enum BuildConfiguration: String, CaseIterable, Sendable {
     case debug
     case release
+
+    var swiftBuildValue: String {
+        switch self {
+        case .debug: "Debug"
+        case .release: "Release"
+        }
+    }
+}
+
+public enum BuildSystem: Sendable {
+    case swiftPM
+    case swiftBuild
+
+    public static var `default`: Self {
+        #if os(macOS)
+        return .swiftBuild
+        #else
+        return .swiftBuild
+        #endif
+    }
+
+    var pmName: String {
+        switch self {
+        case .swiftPM: "native"
+        case .swiftBuild: "swiftbuild"
+        }
+    }
 }

Sources/PackLib/DarwinSDK.swift

@@ -0,0 +1,95 @@
+import Foundation
+import Subprocess
+import XUtils
+
+public struct DarwinSDK {
+    public let bundle: URL
+    public let version: String
+
+    public init?(bundle: URL) {
+        self.bundle = bundle
+        if let version = try? Data(contentsOf: bundle.appendingPathComponent("darwin-sdk-version.txt")) {
+            self.version = String(decoding: version, as: UTF8.self)
+                .trimmingCharacters(in: .whitespacesAndNewlines)
+        } else if ["darwin.xtoolsdk", "darwin.artifactbundle"].contains(bundle.lastPathComponent) {
+            self.version = "unknown"
+        } else {
+            return nil
+        }
+    }
+
+    public static func install(from path: String) async throws {
+        // we can't just move into ~/.swiftpm/swift-sdks because the swiftpm directory
+        // location depends on factors like $XDG_CONFIG_HOME. Rather than replicating
+        // SwiftPM's logic, which may change, it's more reliable to directly invoke
+        // `swift sdk install`. See: https://github.com/xtool-org/xtool/pull/40
+
+        let url = URL(fileURLWithPath: path)
+        guard DarwinSDK(bundle: url) != nil else { throw StringError("Invalid Darwin SDK at '\(path)'")}
+
+        try await addHostClangResourceDir(to: url)
+
+        try await Subprocess.run(
+            .name("swift"),
+            arguments: ["sdk", "install", url.path],
+            output: .discarded
+        )
+        .checkSuccess()
+    }
+
+    private static func addHostClangResourceDir(to sdk: URL) async throws {
+        let clangURL = try await ToolRegistry.locate("clang")
+        let process = try await Subprocess.run(
+            .path(FilePath(clangURL.path)),
+            arguments: ["-print-resource-dir"],
+            output: .string(limit: .max)
+        ).checkSuccess()
+        let output = process.standardOutput ?? ""
+        let hostClangResources = URL(filePath: output.trimmingCharacters(in: .whitespacesAndNewlines))
+        let hostInclude = hostClangResources.appending(path: "include")
+        let sdkInclude = sdk.appending(path: "Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/clang/include")
+        try FileManager.default.copyItem(at: hostInclude, to: sdkInclude)
+    }
+
+    public static func current() async throws -> DarwinSDK? {
+        let outputString: String
+        do {
+            outputString = try await Subprocess.run(
+                .name("swift"),
+                arguments: ["sdk", "configure", "darwin", "arm64-apple-ios", "--show-configuration"],
+                output: .string(limit: .max)
+            )
+            .checkSuccess()
+            .standardOutput
+            ?? ""
+        } catch SubprocessFailure.exited {
+            return nil
+        }
+
+        // should be something like
+        // swiftResourcesPath: /home/user/.swiftpm/swift-sdks/darwin.artifactbundle/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift
+        // swiftlint:disable:previous line_length
+        let resourcesPathPrefix = "swiftResourcesPath: "
+
+        guard let resourcesPath = outputString
+            .split(separator: "\n")
+            .first(where: { $0.hasPrefix(resourcesPathPrefix) })?
+            .dropFirst(resourcesPathPrefix.count)
+            else { return nil }
+
+        var resourcesURL = URL(fileURLWithPath: String(resourcesPath))
+        for _ in 0..<6 {
+            resourcesURL = resourcesURL.deletingLastPathComponent()
+        }
+
+        return DarwinSDK(bundle: resourcesURL)
+    }
+
+    public func isUpToDate() -> Bool {
+        true
+    }
+
+    public func remove() throws {
+        try FileManager.default.removeItem(at: bundle)
+    }
+}

Sources/PackLib/Packer.swift

@@ -55,20 +55,21 @@ public struct Packer: Sendable {
             try Data().write(to: sources.appendingPathComponent("stub.c", isDirectory: false))
         }
 
-        let buildConfig = try await buildSettings.swiftPMInvocation(
-            forTool: "build",
-            arguments: [
-                "--package-path", packageDir.path,
-                "--scratch-path", ".build",
-                // resolving can cause SwiftPM to overwrite the root package deps
-                // with just the deps needed for the builder package (which is to
-                // say, any "dev dependencies" of the root package may be removed.)
-                // fortunately we've already resolved the root package by this point
-                // in order to dump the plan, so we can skip resolution here to skirt
-                // the issue.
-                "--disable-automatic-resolution",
-            ]
-        )
+        let buildConfig = try await buildSettings
+            .withPackagePath(packageDir.path)
+            .swiftPMInvocation(
+                forTool: "build",
+                arguments: [
+                    "--scratch-path", ".build",
+                    // resolving can cause SwiftPM to overwrite the root package deps
+                    // with just the deps needed for the builder package (which is to
+                    // say, any "dev dependencies" of the root package may be removed.)
+                    // fortunately we've already resolved the root package by this point
+                    // in order to dump the plan, so we can skip resolution here to skirt
+                    // the issue.
+                    "--disable-automatic-resolution",
+                ],
+            )
         try await Subprocess.run(
             buildConfig,
             output: .standardError,
@@ -84,8 +85,16 @@ public struct Packer: Sendable {
 
         let outputURL = output.url
 
+        let binPath: String
+        switch buildSettings.buildSystem {
+        case .swiftPM:
+            binPath = "\(buildSettings.triple)/\(buildSettings.configuration.rawValue)"
+        case .swiftBuild:
+            let platformName = buildSettings.triple.contains("simulator") ? "iphonesimulator" : "iphoneos"
+            binPath = "out/Products/\(buildSettings.configuration.swiftBuildValue)-\(platformName)"
+        }
         let binDir = URL(
-            fileURLWithPath: ".build/\(buildSettings.triple)/\(buildSettings.configuration.rawValue)",
+            fileURLWithPath: ".build/\(binPath)",
             isDirectory: true
         )
 

Sources/PackLib/Planner.swift

@@ -244,10 +244,9 @@ public struct Planner: Sendable {
     }
 
     private func _dumpAction(arguments: [String], path: String) async throws -> Data {
-        let dumpConfig = try await buildSettings.swiftPMInvocation(
+        let dumpConfig = try await buildSettings.withPackagePath(path).swiftPMInvocation(
             forTool: "package",
             arguments: arguments,
-            packagePathOverride: path
         )
         return try await Subprocess.run(
             dumpConfig,