diff --git a/Sources/Harness/Harness.swift b/Sources/Harness/Harness.swift index a1d3e3dd4..f1a8248bf 100644 --- a/Sources/Harness/Harness.swift +++ b/Sources/Harness/Harness.swift @@ -8,16 +8,78 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -@_spi(ForToolsIntegrationOnly) private import Testing -private import ArgumentParser +@_spi(ForToolsIntegrationOnly) import Testing +import ArgumentParser +import Foundation /// The harness' main command (i.e. its entry point). @main struct Harness: Sendable, AsyncParsableCommand { - fileprivate static let configuration = CommandConfiguration( + static let configuration = CommandConfiguration( commandName: "swift-testing-harness" ) +#if !SWT_NO_PROCESS_SPAWNING + @Option(name: "--test-product-path") + var testProductPaths: [String] = [] + +#if SWT_TARGET_OS_APPLE + @Option(name: "--swiftpm-testing-helper-path") + var _swiftPMTestingHelperPath: String? + + var swiftPMTestingHelperPath: String { + get throws { + if let swiftPMTestingHelperPath = _swiftPMTestingHelperPath { + return swiftPMTestingHelperPath + } + + let executablePath = try CommandLine.executablePath + let executableURL = URL(fileURLWithPath: executablePath, isDirectory: false) + let swiftPMTestingHelperURL = executableURL + .deletingLastPathComponent() // - progname + .deletingLastPathComponent() // - "testing" + .appendingPathComponent("pm", isDirectory: true) + .appendingPathComponent("swiftpm-testing-helper", isDirectory: false) + return swiftPMTestingHelperURL.path + } + } +#endif +#endif + +#if !SWT_NO_FILE_IO + @Option(name: "--event-stream-input-path") + var eventStreamInputPaths: [String] = [] +#endif + mutating func run() async throws { - throw ValidationError("This tool is currently unimplemented.") + var grommets = [any Grommet]() + +#if !SWT_NO_PROCESS_SPAWNING +#if SWT_TARGET_OS_APPLE + let swiftPMTestingHelperPath = try swiftPMTestingHelperPath +#endif +#if !SWT_NO_PROCESS_SPAWNING + grommets += try testProductPaths.map { testProductPath in +#if SWT_TARGET_OS_APPLE + let testProductBundle = Bundle(path: testProductPath) + guard let testProductBinaryPath = testProductBundle?.executablePath else { + throw CocoaError(.fileReadNoSuchFile) + } + + return LocalProcessGrommet( + testProductBinaryPath: testProductBinaryPath, + swiftPMTestingHelperPath: swiftPMTestingHelperPath + ) +#else + return LocalProcessGrommet(testProductPath: testProductPath) +#endif + } +#endif +#endif + +#if !SWT_NO_FILE_IO + grommets += try eventStreamInputPaths.map(FileGrommet.init(readingFromFileAtPath:)) +#endif + + try await harnessEntryPoint(running: grommets) as Never } } diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index 102776238..ee84bcf4c 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -8,7 +8,7 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -private import _TestingInternals +internal import _TestingInternals #if canImport(Synchronization) private import Synchronization @@ -310,6 +310,14 @@ public struct __CommandLineArguments_v0: Sendable { /// set the value of this property. var eventStreamVersionNumber: VersionNumber? +#if os(Windows) + /// The value of the `--__harness-event-stream-handle` argument. + nonisolated(unsafe) var harnessEventStreamHANDLE: HANDLE? +#else + /// The value of the `--__harness-event-stream-file-descriptor` argument. + var harnessEventStreamFileDescriptor: CInt? +#endif + /// The value of the `--event-stream-version` or `--experimental-event-stream-version` /// argument, representing the version of the event stream schema to use when /// writing events to ``eventStreamOutput``. @@ -493,6 +501,27 @@ func parseCommandLineArguments(from args: [String]) throws -> __CommandLineArgum result.eventStreamVersionNumber = eventStreamVersion } } + + // Harness event stream file descriptor (file handle on Windows) + var hasHarnessEventStream = false +#if os(Windows) + if let handleString = args.argumentValue(forLabel: "--__harness-event-stream-handle") { + guard let handle = UInt(handleString).flatMap(HANDLE.init(bitPattern:)), + handle != INVALID_HANDLE_VALUE else { + throw _EntryPointError.invalidArgument("--__harness-event-stream-handle", value: handleString) + } + result.harnessEventStreamHANDLE = handle + hasHarnessEventStream = true + } +#else + if let fdString = args.argumentValue(forLabel: "--__harness-event-stream-file-descriptor") { + guard let fd = CInt(fdString) else { + throw _EntryPointError.invalidArgument("--__harness-event-stream-file-descriptor", value: fdString) + } + result.harnessEventStreamFileDescriptor = fd + hasHarnessEventStream = true + } +#endif #endif // XML output @@ -540,6 +569,13 @@ func parseCommandLineArguments(from args: [String]) throws -> __CommandLineArgum if args.contains("--quiet") || args.contains("-q") { result.quiet = true } +#if !SWT_NO_FILE_IO + if hasHarnessEventStream { + // The presence of a harness event stream implies suppression of the normal + // stderr output. + result.verbosity = .min + } +#endif // Filtering func filterValues(forArgumentsWithLabel label: String) -> [String] { @@ -643,6 +679,25 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr oldEventHandler(event, context) } } + + // Harness event stream output +#if os(Windows) + let harnessFile = try args.harnessEventStreamHANDLE.map { try Allocated(FileHandle(unsafeWindowsHANDLE: $0, options: [.writeAccess])) } +#else + let harnessFile = try args.harnessEventStreamFileDescriptor.map { try Allocated(FileHandle(unsafePOSIXFileDescriptor: $0, options: [.writeAccess])) } +#endif + if let harnessFile { + let eventHandler = try eventHandlerForStreamingEvents(withVersionNumber: ABI.HarnessVersion.versionNumber, encodeAsJSONLines: true) { json in + _ = try? harnessFile.value.withLock { + try harnessFile.value.write(json) + try harnessFile.value.write("\n") + } + } + configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in + eventHandler(event, context) + oldEventHandler(event, context) + } + } #endif #endif diff --git a/Sources/Testing/ABI/EntryPoints/FileGrommet.swift b/Sources/Testing/ABI/EntryPoints/FileGrommet.swift new file mode 100644 index 000000000..281d1cafb --- /dev/null +++ b/Sources/Testing/ABI/EntryPoints/FileGrommet.swift @@ -0,0 +1,86 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2026 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +#if !SWT_NO_FILE_IO +package final class FileGrommet: Grommet { + private let _file: FileHandle + private let _filePath: String + + init(readingFrom file: consuming FileHandle, atPath filePath: String? = nil) { + _file = file + _filePath = filePath ?? "" + } + + package convenience init(readingFromFileAtPath filePath: String) throws { + let file = try FileHandle(forReadingAtPath: filePath) + self.init(readingFrom: file, atPath: filePath) + } + + package var grommetName: String { + _filePath + } + + package func run(_ eventHandler: @escaping @Sendable (borrowing Event, borrowing Event.Context) -> Void) async throws { + var context = ABI.Context() + + var terminator: UInt8? + repeat { + let recordJSON: [UInt8] + (recordJSON, terminator) = try _file.read(until: \.isASCIINewline) + + // Allow other tasks to run after we may have blocked for some time on + // I/O with the child process. + await Task.yield() + + if recordJSON.isEmpty { + continue + } + try recordJSON.withUnsafeBytes { recordJSON in + let versionNumber = try ABI.VersionNumber(fromRecordJSON: recordJSON) + guard let abi = ABI._version(forVersionNumber: versionNumber) else { + try? FileHandle.stderr.write("Failed to determine ABI version for JSON record with version number '\(versionNumber)'") + return + } + try _processRecord( + recordJSON, + withABIVersion: abi, + in: &context, + eventHandler: eventHandler + ) + } + } while terminator != nil + } + + private func _processRecord( + _ recordJSON: UnsafeRawBufferPointer, + withABIVersion: V.Type, + in context: inout ABI.Context, + eventHandler: @Sendable (borrowing Event, borrowing Event.Context) -> Void + ) throws where V: ABI._Version { + let record = try JSON.decode(ABI.Record.self, from: recordJSON) + switch record.kind { + case let .test(encodedTest): + _ = Test(decoding: encodedTest, in: &context) + case let .event(encodedEvent): + guard let event = Event(decoding: encodedEvent, in: &context) else { + try? FileHandle.stderr.write("Failed to decode \(encodedEvent)") + return + } + let eventContext = Event.Context( + test: encodedEvent.testID.flatMap(context.test(identifiedBy:)), + testCase: nil, + iteration: encodedEvent.iteration, + configuration: nil + ) + eventHandler(event, eventContext) + } + } +} +#endif diff --git a/Sources/Testing/ABI/EntryPoints/Grommet.swift b/Sources/Testing/ABI/EntryPoints/Grommet.swift new file mode 100644 index 000000000..1c7b3cfff --- /dev/null +++ b/Sources/Testing/ABI/EntryPoints/Grommet.swift @@ -0,0 +1,30 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2026 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +// FIXME: need an actual name for this protocol +package protocol Grommet: Sendable { + var grommetName: String { get } + func run(_ eventHandler: @escaping @Sendable (borrowing Event, borrowing Event.Context) -> Void) async throws +} + +extension Runner: Grommet { + package var grommetName: String { + "" + } + + package func run(_ eventHandler: @escaping @Sendable (borrowing Event, borrowing Event.Context) -> Void) async throws { + var selfCopy = self + selfCopy.configuration.eventHandler = { [oldEventHandler = selfCopy.configuration.eventHandler] event, context in + eventHandler(event, context) + oldEventHandler(event, context) + } + await selfCopy.run() + } +} diff --git a/Sources/Testing/ABI/EntryPoints/HarnessEntryPoint.swift b/Sources/Testing/ABI/EntryPoints/HarnessEntryPoint.swift new file mode 100644 index 000000000..5547f300f --- /dev/null +++ b/Sources/Testing/ABI/EntryPoints/HarnessEntryPoint.swift @@ -0,0 +1,98 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2026 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +private import _TestingInternals + +#if canImport(Synchronization) +private import Synchronization +#endif + +extension ABI { + package typealias HarnessVersion = ExperimentalVersion +} + +package func harnessEntryPoint( + running grommets: [any Grommet] +) async throws -> CInt { + var exitCodes = [CInt]() + +#if !SWT_NO_FILE_IO + let eventRecorder = Event.ConsoleOutputRecorder(options: .for(.stderr)) { string in + try? FileHandle.stderr.write(string) + } +#endif + + let grommetCount = grommets.count + for (i, grommet) in grommets.enumerated() { + let exitCode = Atomic(EXIT_SUCCESS) + + func open(_ grommet: some Grommet) async throws { + try await grommet.run { event, eventContext in + var recordEvent = true + + switch event.kind { + case .testDiscovered: + _ = exitCode.compareExchange(expected: EXIT_SUCCESS, desired: EXIT_NO_TESTS_FOUND, ordering: .sequentiallyConsistent) +#if !SWT_NO_FILE_IO + case .runStarted: + if i > 0 { + recordEvent = false + } + if grommetCount > 1 { + try? FileHandle.stderr.write("Running '\(grommet.grommetName)'...\n") + } + case .runEnded: + if i != grommets.count - 1 { + recordEvent = false + } +#endif + case let .issueRecorded(issue): + if issue.isFailure { + exitCode.store(EXIT_FAILURE, ordering: .sequentiallyConsistent) + } + default: + break + } + +#if !SWT_NO_FILE_IO + if recordEvent { + eventRecorder.record(event, in: eventContext) + } +#endif + } + } + + do { + try await open(grommet) + } catch { + // TODO: handle errors at this layer in an interesting way + exitCode.store(EXIT_FAILURE, ordering: .sequentiallyConsistent) + } + + exitCodes.append(exitCode.load(ordering: .sequentiallyConsistent)) + } + + let noTestsFound = exitCodes.allSatisfy { $0 == EXIT_NO_TESTS_FOUND } + if noTestsFound { + return EXIT_NO_TESTS_FOUND + } + let succeeded = exitCodes.allSatisfy { $0 == EXIT_SUCCESS } + if succeeded { + return EXIT_SUCCESS + } + return EXIT_FAILURE +} + +package func harnessEntryPoint( + running grommets: [any Grommet] +) async throws -> Never { + let exitCode: CInt = try await harnessEntryPoint(running: grommets) + exit(exitCode) +} diff --git a/Sources/Testing/ABI/EntryPoints/LocalProcessGrommet.swift b/Sources/Testing/ABI/EntryPoints/LocalProcessGrommet.swift new file mode 100644 index 000000000..3474d15f0 --- /dev/null +++ b/Sources/Testing/ABI/EntryPoints/LocalProcessGrommet.swift @@ -0,0 +1,107 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2026 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +#if !SWT_NO_PROCESS_SPAWNING +package struct LocalProcessGrommet: Grommet { +#if SWT_TARGET_OS_APPLE + private var _testProductBinaryPath: String + private var _swiftPMTestingHelperPath: String + + package init(testProductBinaryPath: String, swiftPMTestingHelperPath: String) { + _testProductBinaryPath = testProductBinaryPath + _swiftPMTestingHelperPath = swiftPMTestingHelperPath + } +#else + private var _testProductPath: String + + package init(testProductPath: String) { + _testProductPath = testProductPath + } +#endif + + package var grommetName: String { +#if SWT_TARGET_OS_APPLE + _testProductBinaryPath +#else + _testProductPath +#endif + } + + package func run(_ eventHandler: @escaping @Sendable (borrowing Event, borrowing Event.Context) -> Void) async throws { + try await withThrowingTaskGroup(of: Void.self) { taskGroup in + var backChannelReadEnd: FileHandle! + var backChannelWriteEnd: FileHandle! + try FileHandle.makePipe(readEnd: &backChannelReadEnd, writeEnd: &backChannelWriteEnd) + + var arguments = [String]() +#if SWT_TARGET_OS_APPLE + arguments += ["--test-bundle-path", _testProductBinaryPath] +#endif + arguments += [ + "--testing-library", "swift-testing", + ] + arguments += CommandLine.arguments.dropFirst() +#if os(Windows) + backChannelWriteEnd.withUnsafeWindowsHANDLE { handle in + guard let handle else { + return + } + + arguments += [ + "--__harness-event-stream-handle", String(describing: UInt(bitPattern: handle)), + ] + } +#else + backChannelWriteEnd.withUnsafePOSIXFileDescriptor { fd in + guard let fd else { + return + } + + arguments += [ + "--__harness-event-stream-file-descriptor", String(describing: fd), + ] + } +#endif + +#if SWT_TARGET_OS_APPLE + let executablePath = _swiftPMTestingHelperPath +#else + let executablePath = _testProductPath +#endif + + let processID = try withUnsafePointer(to: backChannelWriteEnd) { backChannelWriteEnd in + try spawnExecutable( + atPath: executablePath, + arguments: arguments, + environment: Environment.get(), + standardOutput: .stdout, + standardError: .stderr, + additionalFileHandles: [backChannelWriteEnd] + ) + } + backChannelWriteEnd.close() + + // Wait for the child process to terminate. + taskGroup.addTask(name: decorateTaskName("harness", withAction: "running test process")) { + _ = try await wait(for: processID) + } + + // Read events back out from the back channel. + let fileGrommet = FileGrommet(readingFrom: backChannelReadEnd!) + taskGroup.addTask(name: decorateTaskName("harness", withAction: "reading events from test process")) { + try await fileGrommet.run(eventHandler) + } + + try await taskGroup.waitForAll() + } + + } +} +#endif diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 6542d724e..91cf9cd41 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -705,7 +705,7 @@ extension ExitTest { /// - Returns: A string representation of `fileHandle` that can be converted /// back to a (new) file handle with `_makeFileHandle()`, or `nil` if the /// file handle could not be converted to a string. - private static func _makeEnvironmentVariable(for fileHandle: borrowing FileHandle) -> String? { + static func makeEnvironmentVariable(for fileHandle: borrowing FileHandle) -> String? { #if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) return fileHandle.withUnsafePOSIXFileDescriptor { fd in fd.map(String.init(describing:)) @@ -934,10 +934,10 @@ extension ExitTest { // Let the child process know how to find the back channel and // captured values channel by setting a known environment variable to // the corresponding file descriptor (HANDLE on Windows) for each. - if let backChannelEnvironmentVariable = _makeEnvironmentVariable(for: backChannelWriteEnd) { + if let backChannelEnvironmentVariable = Self.makeEnvironmentVariable(for: backChannelWriteEnd) { childEnvironment["SWT_BACKCHANNEL"] = backChannelEnvironmentVariable } - if let capturedValuesEnvironmentVariable = _makeEnvironmentVariable(for: capturedValuesReadEnd) { + if let capturedValuesEnvironmentVariable = Self.makeEnvironmentVariable(for: capturedValuesReadEnd) { childEnvironment["SWT_CAPTURED_VALUES"] = capturedValuesEnvironmentVariable } diff --git a/Sources/Testing/Support/Additions/CommandLineAdditions.swift b/Sources/Testing/Support/Additions/CommandLineAdditions.swift index fa51c9550..5e19c437f 100644 --- a/Sources/Testing/Support/Additions/CommandLineAdditions.swift +++ b/Sources/Testing/Support/Additions/CommandLineAdditions.swift @@ -26,10 +26,10 @@ enum CommandLine { } #endif -#if !SWT_NO_EXIT_TESTS +#if !SWT_NO_PROCESS_SPAWNING extension CommandLine { /// The path to the current process' executable. - static var executablePath: String { + package static var executablePath: String { get throws { #if SWT_TARGET_OS_APPLE var result: String?