diff --git a/Package.swift b/Package.swift index 75b8465..13ab3f4 100644 --- a/Package.swift +++ b/Package.swift @@ -166,6 +166,7 @@ let package = Package( name: "PackLib", dependencies: [ "XUtils", + .product(name: "Superutils", package: "xtool-core"), .product(name: "Yams", package: "Yams"), .product(name: "XcodeGenKit", package: "XcodeGen", condition: .when(platforms: [.macOS])), ] diff --git a/Sources/PackLib/BuildSettings.swift b/Sources/PackLib/BuildSettings.swift index 4ba0efc..9a25967 100644 --- a/Sources/PackLib/BuildSettings.swift +++ b/Sources/PackLib/BuildSettings.swift @@ -1,37 +1,73 @@ import Foundation import Subprocess import XUtils +import Superutils public struct BuildSettings: Sendable { private static let customBinDir = // this is the same option used by SwiftPM itself for dev builds ProcessInfo.processInfo.environment["SWIFTPM_CUSTOM_BIN_DIR"].map { FilePath($0) } - private static let envURL = URL(fileURLWithPath: "/usr/bin/env") - - public let packagePath: String + public var packagePath: String public let configuration: BuildConfiguration public let triple: String - public let sdkOptions: [String] - public let options: [String] + public let buildSystem: BuildSystem + public let customOptions: [String] + + public var sdkOptions: [String] + public var sdkEnvironment: [Environment.Key: String?] + + private var configOptions: [String] { + return [ + "--configuration", configuration.rawValue, + "--build-system", buildSystem.pmName, + "--package-path", packagePath, + ] + } + + private var resolvedBaseOptions: [String] { + configOptions + sdkOptions + customOptions + } public init( configuration: BuildConfiguration, triple: String, + buildSystem: BuildSystem = .default, packagePath: String = ".", options: [String] = [] ) async throws { self.packagePath = packagePath self.configuration = configuration - self.options = options + self.customOptions = options self.triple = triple + self.buildSystem = buildSystem + + self.sdkEnvironment = [ + // xcrun passes an SDKROOT that messes with our sdk configuration + "SDKROOT": nil, + ] - // on macOS we don't explicitly install a Swift SDK but - // SwiftPM vends "implicit" Darwin SDKs as of Swift 6.1, - // i.e. we can pass `--swift-sdk arm64-apple-ios` and it - // just works. See: - // https://github.com/swiftlang/swift-package-manager/pull/6828 - self.sdkOptions = ["--swift-sdk", triple] + switch buildSystem { + case .swiftPM: + // on macOS we don't explicitly install a Swift SDK but + // SwiftPM vends "implicit" Darwin SDKs as of Swift 6.1, + // i.e. we can pass `--swift-sdk arm64-apple-ios` and it + // just works. See: + // https://github.com/swiftlang/swift-package-manager/pull/6828 + self.sdkOptions = ["--swift-sdk", triple] + case .swiftBuild: + self.sdkOptions = ["--triple", triple] + #if !os(macOS) + let darwinSDK = try await DarwinSDK.current() + .orThrow(StringError("No Darwin SDK configured. Please run `xtool setup`.")) + self.sdkOptions += [ + "--toolset", "\(darwinSDK.bundle.path)/toolset-swb.json", + ] + self.sdkEnvironment.merge([ + "XCODE_EXTRA_PLATFORM_FOLDERS": "\(darwinSDK.bundle.path)/Developer/Platforms", + ]) { $1 } + #endif + } } #if os(macOS) @@ -44,15 +80,40 @@ public struct BuildSettings: Sendable { return result.standardOutput?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" } - private static let swiftURL = Task { + private static let _swiftURL = Task { try await FilePath(xcrun(["-f", "swift"])) } + + public static func swiftURL() async throws -> FilePath { + try await _swiftURL.value + } + + private static let _swiftcURL = Task { + try await FilePath(xcrun(["-f", "swiftc"])) + } + + public static func swiftcURL() async throws -> FilePath { + try await _swiftcURL.value + } + #else + public static func swiftURL() async throws -> FilePath { + try await FilePath(ToolRegistry.locate("swift")).orThrow(StringError("Got bad path for swift executable")) + } + + public static func swiftcURL() async throws -> FilePath { + try await FilePath(ToolRegistry.locate("swiftc")).orThrow(StringError("Got bad path for swiftc executable")) + } #endif + public func withPackagePath(_ path: String) -> Self { + var copy = self + copy.packagePath = path + return copy + } + public func swiftPMInvocation( forTool tool: String, arguments: [String], - packagePathOverride: String? = nil ) async throws -> Subprocess.Configuration { let executable: Executable let baseArguments: [String] @@ -65,35 +126,60 @@ public struct BuildSettings: Sendable { // to add SDKROOT=.../MacOSX.sdk to our invocations. We avoid this by // 1) invoking the real swift executable (located with `xcrun -f`) and // 2) explicitly removing SDKROOT from the env, as it may be inherited - // through the `swift run pack` invocation. - executable = .path(try await Self.swiftURL.value) + // through a parent process (e.g. `swift run xtool`). + executable = .path(try await Self.swiftURL()) #else executable = .name("swift") #endif baseArguments = [tool] } - let extraArguments: [String] = [ - "--package-path", packagePathOverride ?? packagePath, - "--configuration", configuration.rawValue, - ] - - var rawEnv = ProcessInfo.processInfo.environment - rawEnv.removeValue(forKey: "SDKROOT") - let env = Dictionary(uniqueKeysWithValues: rawEnv.map { - (Environment.Key(rawValue: $0)!, $1) - }) - return Configuration( executable, - arguments: .init(baseArguments + extraArguments + sdkOptions + options + arguments), - environment: .custom(env), + arguments: .init(baseArguments + resolvedBaseOptions + arguments), + environment: .inherit.updating(sdkEnvironment), platformOptions: .withGracefulShutDown, ) } + + public var buildServerArguments: [String] { + return [ + "package", "experimental-build-server", + "--disable-automatic-resolution", + // TODO: once https://github.com/swiftlang/swift-package-manager/pull/9819 makes it into a release + // (Swift 6.4), pass --experimental-skip-acquiring-lock + ] + resolvedBaseOptions + } } public enum BuildConfiguration: String, CaseIterable, Sendable { case debug case release + + var swiftBuildValue: String { + switch self { + case .debug: "Debug" + case .release: "Release" + } + } +} + +public enum BuildSystem: Sendable { + case swiftPM + case swiftBuild + + public static var `default`: Self { + #if os(macOS) + return .swiftBuild + #else + return .swiftBuild + #endif + } + + var pmName: String { + switch self { + case .swiftPM: "native" + case .swiftBuild: "swiftbuild" + } + } } diff --git a/Sources/PackLib/DarwinSDK.swift b/Sources/PackLib/DarwinSDK.swift new file mode 100644 index 0000000..8f380d2 --- /dev/null +++ b/Sources/PackLib/DarwinSDK.swift @@ -0,0 +1,95 @@ +import Foundation +import Subprocess +import XUtils + +public struct DarwinSDK { + public let bundle: URL + public let version: String + + public init?(bundle: URL) { + self.bundle = bundle + if let version = try? Data(contentsOf: bundle.appendingPathComponent("darwin-sdk-version.txt")) { + self.version = String(decoding: version, as: UTF8.self) + .trimmingCharacters(in: .whitespacesAndNewlines) + } else if ["darwin.xtoolsdk", "darwin.artifactbundle"].contains(bundle.lastPathComponent) { + self.version = "unknown" + } else { + return nil + } + } + + public static func install(from path: String) async throws { + // we can't just move into ~/.swiftpm/swift-sdks because the swiftpm directory + // location depends on factors like $XDG_CONFIG_HOME. Rather than replicating + // SwiftPM's logic, which may change, it's more reliable to directly invoke + // `swift sdk install`. See: https://github.com/xtool-org/xtool/pull/40 + + let url = URL(fileURLWithPath: path) + guard DarwinSDK(bundle: url) != nil else { throw StringError("Invalid Darwin SDK at '\(path)'")} + + try await addHostClangResourceDir(to: url) + + try await Subprocess.run( + .name("swift"), + arguments: ["sdk", "install", url.path], + output: .discarded + ) + .checkSuccess() + } + + private static func addHostClangResourceDir(to sdk: URL) async throws { + let clangURL = try await ToolRegistry.locate("clang") + let process = try await Subprocess.run( + .path(FilePath(clangURL.path)), + arguments: ["-print-resource-dir"], + output: .string(limit: .max) + ).checkSuccess() + let output = process.standardOutput ?? "" + let hostClangResources = URL(filePath: output.trimmingCharacters(in: .whitespacesAndNewlines)) + let hostInclude = hostClangResources.appending(path: "include") + let sdkInclude = sdk.appending(path: "Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/clang/include") + try FileManager.default.copyItem(at: hostInclude, to: sdkInclude) + } + + public static func current() async throws -> DarwinSDK? { + let outputString: String + do { + outputString = try await Subprocess.run( + .name("swift"), + arguments: ["sdk", "configure", "darwin", "arm64-apple-ios", "--show-configuration"], + output: .string(limit: .max) + ) + .checkSuccess() + .standardOutput + ?? "" + } catch SubprocessFailure.exited { + return nil + } + + // should be something like + // swiftResourcesPath: /home/user/.swiftpm/swift-sdks/darwin.artifactbundle/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift + // swiftlint:disable:previous line_length + let resourcesPathPrefix = "swiftResourcesPath: " + + guard let resourcesPath = outputString + .split(separator: "\n") + .first(where: { $0.hasPrefix(resourcesPathPrefix) })? + .dropFirst(resourcesPathPrefix.count) + else { return nil } + + var resourcesURL = URL(fileURLWithPath: String(resourcesPath)) + for _ in 0..<6 { + resourcesURL = resourcesURL.deletingLastPathComponent() + } + + return DarwinSDK(bundle: resourcesURL) + } + + public func isUpToDate() -> Bool { + true + } + + public func remove() throws { + try FileManager.default.removeItem(at: bundle) + } +} diff --git a/Sources/PackLib/Packer.swift b/Sources/PackLib/Packer.swift index 773cdae..b54bbd9 100644 --- a/Sources/PackLib/Packer.swift +++ b/Sources/PackLib/Packer.swift @@ -55,20 +55,21 @@ public struct Packer: Sendable { try Data().write(to: sources.appendingPathComponent("stub.c", isDirectory: false)) } - let buildConfig = try await buildSettings.swiftPMInvocation( - forTool: "build", - arguments: [ - "--package-path", packageDir.path, - "--scratch-path", ".build", - // resolving can cause SwiftPM to overwrite the root package deps - // with just the deps needed for the builder package (which is to - // say, any "dev dependencies" of the root package may be removed.) - // fortunately we've already resolved the root package by this point - // in order to dump the plan, so we can skip resolution here to skirt - // the issue. - "--disable-automatic-resolution", - ] - ) + let buildConfig = try await buildSettings + .withPackagePath(packageDir.path) + .swiftPMInvocation( + forTool: "build", + arguments: [ + "--scratch-path", ".build", + // resolving can cause SwiftPM to overwrite the root package deps + // with just the deps needed for the builder package (which is to + // say, any "dev dependencies" of the root package may be removed.) + // fortunately we've already resolved the root package by this point + // in order to dump the plan, so we can skip resolution here to skirt + // the issue. + "--disable-automatic-resolution", + ], + ) try await Subprocess.run( buildConfig, output: .standardError, @@ -84,8 +85,16 @@ public struct Packer: Sendable { let outputURL = output.url + let binPath: String + switch buildSettings.buildSystem { + case .swiftPM: + binPath = "\(buildSettings.triple)/\(buildSettings.configuration.rawValue)" + case .swiftBuild: + let platformName = buildSettings.triple.contains("simulator") ? "iphonesimulator" : "iphoneos" + binPath = "out/Products/\(buildSettings.configuration.swiftBuildValue)-\(platformName)" + } let binDir = URL( - fileURLWithPath: ".build/\(buildSettings.triple)/\(buildSettings.configuration.rawValue)", + fileURLWithPath: ".build/\(binPath)", isDirectory: true ) diff --git a/Sources/PackLib/Planner.swift b/Sources/PackLib/Planner.swift index 9eea38a..21868f3 100644 --- a/Sources/PackLib/Planner.swift +++ b/Sources/PackLib/Planner.swift @@ -244,10 +244,9 @@ public struct Planner: Sendable { } private func _dumpAction(arguments: [String], path: String) async throws -> Data { - let dumpConfig = try await buildSettings.swiftPMInvocation( + let dumpConfig = try await buildSettings.withPackagePath(path).swiftPMInvocation( forTool: "package", arguments: arguments, - packagePathOverride: path ) return try await Subprocess.run( dumpConfig, diff --git a/Sources/XToolSupport/DevBSPCommand.swift b/Sources/XToolSupport/DevBSPCommand.swift new file mode 100644 index 0000000..8588909 --- /dev/null +++ b/Sources/XToolSupport/DevBSPCommand.swift @@ -0,0 +1,30 @@ +import ArgumentParser +import Foundation +import XKit +import PackLib +import Subprocess + +struct DevBSPCommand: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "build-server", + abstract: "Run build server", + ) + + @Option + var triple: String? + + func run() async throws { + let settings = try await BuildSettings( + configuration: .debug, + triple: triple ?? PackOperation.defaultTriple + ) + try await Subprocess.run( + .path(BuildSettings.swiftURL()), + arguments: .init(settings.buildServerArguments), + input: .standardInput, + output: .standardOutput, + error: .standardError, + ) + .checkSuccess() + } +} diff --git a/Sources/XToolSupport/DevCommand.swift b/Sources/XToolSupport/DevCommand.swift index a827354..6fe6f39 100644 --- a/Sources/XToolSupport/DevCommand.swift +++ b/Sources/XToolSupport/DevCommand.swift @@ -274,6 +274,7 @@ struct DevCommand: AsyncParsableCommand { DevXcodeCommand.self, DevBuildCommand.self, DevRunCommand.self, + DevBSPCommand.self, ], defaultSubcommand: DevRunCommand.self ) diff --git a/Sources/XToolSupport/SDKBuilder.swift b/Sources/XToolSupport/SDKBuilder.swift index c8cfbcd..e4c31cb 100644 --- a/Sources/XToolSupport/SDKBuilder.swift +++ b/Sources/XToolSupport/SDKBuilder.swift @@ -56,6 +56,7 @@ struct SDKBuilder { let output: URL let arch: Arch + // swiftlint:disable:next function_body_length func buildSDK() async throws { // TODO: store relevant info for staleness check let sdkVersion = "develop" @@ -136,6 +137,9 @@ struct SDKBuilder { "linker": { "path": "ld64.lld" }, + "librarian": { + "path": "llvm-lib" + }, "swiftCompiler": { "extraCLIOptions": [ "-Xfrontend", "-enable-cross-import-overlays", @@ -149,6 +153,26 @@ struct SDKBuilder { encoding: .utf8 ) + // this toolset works with the swiftbuild system on Linux + // note that we need Swift 6.4+ because 6.3 has bugs in + // resolving the librarian and linker paths. + try """ + { + "schemaVersion": "1.0", + "rootPath": "toolset/bin", + "linker": { + "path": "ld" + }, + "librarian": { + "path": "llvm-lib" + } + } + """.write( + to: output.appendingPathComponent("toolset-swb.json"), + atomically: false, + encoding: .utf8 + ) + let sdkDefinition = SDKDefinition( schemaVersion: "4.0", targetTriples: [ @@ -215,9 +239,19 @@ struct SDKBuilder { try await input.finish() } .checkSuccess() + + // swift-build assumes we have an Apple-flavored librarian for Apple platforms + // (https://github.com/swiftlang/swift-build/blob/b2433e74e/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift#L1735) + // but allows us to override this assumption if we explicitly provide the name llvm-lib (look a few lines above in that file) + try FileManager.default.moveItem( + at: toolsetDir.appending(path: "bin/libtool"), + to: toolsetDir.appending(path: "bin/llvm-lib") + ) + + // TODO: install cctools ld as well } - // swiftlint:disable:next cyclomatic_complexity function_body_length + // swiftlint:disable:next cyclomatic_complexity private func installDeveloper(in output: URL) async throws -> URL { let dev = output.appendingPathComponent("Developer") @@ -321,10 +355,11 @@ struct SDKBuilder { for platform in ["iPhoneOS", "MacOSX", "iPhoneSimulator"] { let lib = "../../../../../Library" - let dest = dev.appendingPathComponent(""" - Platforms/\(platform).platform/Developer/SDKs/\(platform).sdk\ - /System/Library/Frameworks - """).path + let platformDir = dev.appending(path: "Platforms/\(platform).platform") + + try patchPlatformManifest(path: platformDir.appending(path: "Info.plist")) + + let dest = platformDir.appending(path: "Developer/SDKs/\(platform).sdk/System/Library/Frameworks").path try FileManager.default.createSymbolicLink( atPath: "\(dest)/Testing.framework", @@ -354,6 +389,21 @@ struct SDKBuilder { return dev } + private func patchPlatformManifest(path: URL) throws { + let data = try Data(contentsOf: path) + guard var plist = try PropertyListSerialization.propertyList(from: data, options: [], format: nil) as? [String: Any] else { + throw Console.Error("Could not read '\(path.path)'") + } + + var defaultProperties = (plist["DefaultProperties"] as? [String: Any]) ?? [:] + defaultProperties["SWIFT_RESOURCE_DIR"] = "$(PLATFORM_DIR)/../../Toolchains/XcodeDefault.xctoolchain/usr/lib/swift" + defaultProperties["CLANG_RESOURCE_DIR"] = "$(SWIFT_RESOURCE_DIR)/clang" + plist["DefaultProperties"] = defaultProperties + + let newData = try PropertyListSerialization.data(fromPropertyList: plist, format: .xml, options: 0) + try newData.write(to: path) + } + // returns the number of files we actually want to keep, // useful for computing progress % during fs traversal private func extractXIP(inputPath: String, outDir: String) async throws -> Int { @@ -497,13 +547,16 @@ struct SDKEntry { E("clang"), ]), E("Platforms", ["iPhoneOS", "MacOSX", "iPhoneSimulator"].map { - E("\($0).platform/Developer", [ - E("SDKs"), - E("Library", [ - E("Frameworks"), - E("PrivateFrameworks"), - ]), - E("usr/lib"), + E("\($0).platform", [ + E("Info.plist"), + E("Developer", [ + E("SDKs"), + E("Library", [ + E("Frameworks"), + E("PrivateFrameworks"), + ]), + E("usr/lib"), + ]) ]) }), ]) diff --git a/Sources/XToolSupport/SDKCommand.swift b/Sources/XToolSupport/SDKCommand.swift index cdb0bfc..ddb4404 100644 --- a/Sources/XToolSupport/SDKCommand.swift +++ b/Sources/XToolSupport/SDKCommand.swift @@ -125,98 +125,6 @@ struct DevSDKStatusCommand: AsyncParsableCommand { } } -struct DarwinSDK { - let bundle: URL - let version: String - - init?(bundle: URL) { - self.bundle = bundle - if let version = try? Data(contentsOf: bundle.appendingPathComponent("darwin-sdk-version.txt")) { - self.version = String(decoding: version, as: UTF8.self) - .trimmingCharacters(in: .whitespacesAndNewlines) - } else if ["darwin.xtoolsdk", "darwin.artifactbundle"].contains(bundle.lastPathComponent) { - self.version = "unknown" - } else { - return nil - } - } - - static func install(from path: String) async throws { - // we can't just move into ~/.swiftpm/swift-sdks because the swiftpm directory - // location depends on factors like $XDG_CONFIG_HOME. Rather than replicating - // SwiftPM's logic, which may change, it's more reliable to directly invoke - // `swift sdk install`. See: https://github.com/xtool-org/xtool/pull/40 - - let url = URL(fileURLWithPath: path) - guard DarwinSDK(bundle: url) != nil else { throw Console.Error("Invalid Darwin SDK at '\(path)'")} - - try await addHostClangResourceDir(to: url) - - try await Subprocess.run( - .name("swift"), - arguments: ["sdk", "install", url.path], - output: .discarded - ) - .checkSuccess() - } - - private static func addHostClangResourceDir(to sdk: URL) async throws { - let clangURL = try await ToolRegistry.locate("clang") - let process = try await Subprocess.run( - .path(FilePath(clangURL.path)), - arguments: ["-print-resource-dir"], - output: .string(limit: .max) - ).checkSuccess() - let output = process.standardOutput ?? "" - let hostClangResources = URL(filePath: output.trimmingCharacters(in: .whitespacesAndNewlines)) - let hostInclude = hostClangResources.appending(path: "include") - let sdkInclude = sdk.appending(path: "Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/clang/include") - try FileManager.default.copyItem(at: hostInclude, to: sdkInclude) - } - - static func current() async throws -> DarwinSDK? { - let outputString: String - do { - outputString = try await Subprocess.run( - .name("swift"), - arguments: ["sdk", "configure", "darwin", "arm64-apple-ios", "--show-configuration"], - output: .string(limit: .max) - ) - .checkSuccess() - .standardOutput - ?? "" - } catch SubprocessFailure.exited { - return nil - } - - // should be something like - // swiftResourcesPath: /home/user/.swiftpm/swift-sdks/darwin.artifactbundle/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift - // swiftlint:disable:previous line_length - let resourcesPathPrefix = "swiftResourcesPath: " - - guard let resourcesPath = outputString - .split(separator: "\n") - .first(where: { $0.hasPrefix(resourcesPathPrefix) })? - .dropFirst(resourcesPathPrefix.count) - else { return nil } - - var resourcesURL = URL(fileURLWithPath: String(resourcesPath)) - for _ in 0..<6 { - resourcesURL = resourcesURL.deletingLastPathComponent() - } - - return DarwinSDK(bundle: resourcesURL) - } - - func isUpToDate() -> Bool { - true - } - - func remove() throws { - try FileManager.default.removeItem(at: bundle) - } -} - private enum SwiftVersion {} extension SwiftVersion { static func current() async throws -> Version { diff --git a/Sources/XToolSupport/SetupCommand.swift b/Sources/XToolSupport/SetupCommand.swift index 3546712..cff7c77 100644 --- a/Sources/XToolSupport/SetupCommand.swift +++ b/Sources/XToolSupport/SetupCommand.swift @@ -1,6 +1,7 @@ import ArgumentParser import Foundation import XKit +import PackLib struct SetupCommand: AsyncParsableCommand { static let configuration = CommandConfiguration(