Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 66 additions & 4 deletions Sources/Harness/Harness.swift
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,78 @@
// See https://swift.org/CONTRIBUTORS.txt for Swift project authors
//

@_spi(ForToolsIntegrationOnly) private import Testing
private import ArgumentParser
@_spi(ForToolsIntegrationOnly) import Testing
import ArgumentParser
import Foundation

/// The harness' main command (i.e. its entry point).
@main struct Harness: Sendable, AsyncParsableCommand {
fileprivate static let configuration = CommandConfiguration(
static let configuration = CommandConfiguration(
commandName: "swift-testing-harness"
)

#if !SWT_NO_PROCESS_SPAWNING
@Option(name: "--test-product-path")
var testProductPaths: [String] = []

#if SWT_TARGET_OS_APPLE
@Option(name: "--swiftpm-testing-helper-path")
var _swiftPMTestingHelperPath: String?

var swiftPMTestingHelperPath: String {
get throws {
if let swiftPMTestingHelperPath = _swiftPMTestingHelperPath {
return swiftPMTestingHelperPath
}

let executablePath = try CommandLine.executablePath
let executableURL = URL(fileURLWithPath: executablePath, isDirectory: false)
let swiftPMTestingHelperURL = executableURL
.deletingLastPathComponent() // - progname
.deletingLastPathComponent() // - "testing"
.appendingPathComponent("pm", isDirectory: true)
.appendingPathComponent("swiftpm-testing-helper", isDirectory: false)
return swiftPMTestingHelperURL.path
}
}
#endif
#endif

#if !SWT_NO_FILE_IO
@Option(name: "--event-stream-input-path")
var eventStreamInputPaths: [String] = []
#endif

mutating func run() async throws {
throw ValidationError("This tool is currently unimplemented.")
var grommets = [any Grommet]()

#if !SWT_NO_PROCESS_SPAWNING
#if SWT_TARGET_OS_APPLE
let swiftPMTestingHelperPath = try swiftPMTestingHelperPath
#endif
#if !SWT_NO_PROCESS_SPAWNING
grommets += try testProductPaths.map { testProductPath in
#if SWT_TARGET_OS_APPLE
let testProductBundle = Bundle(path: testProductPath)
guard let testProductBinaryPath = testProductBundle?.executablePath else {
throw CocoaError(.fileReadNoSuchFile)
}

return LocalProcessGrommet(
testProductBinaryPath: testProductBinaryPath,
swiftPMTestingHelperPath: swiftPMTestingHelperPath
)
#else
return LocalProcessGrommet(testProductPath: testProductPath)
#endif
}
#endif
#endif

#if !SWT_NO_FILE_IO
grommets += try eventStreamInputPaths.map(FileGrommet.init(readingFromFileAtPath:))
#endif

try await harnessEntryPoint(running: grommets) as Never
}
}
57 changes: 56 additions & 1 deletion Sources/Testing/ABI/EntryPoints/EntryPoint.swift
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
// See https://swift.org/CONTRIBUTORS.txt for Swift project authors
//

private import _TestingInternals
internal import _TestingInternals

#if canImport(Synchronization)
private import Synchronization
Expand Down Expand Up @@ -310,6 +310,14 @@ public struct __CommandLineArguments_v0: Sendable {
/// set the value of this property.
var eventStreamVersionNumber: VersionNumber?

#if os(Windows)
/// The value of the `--__harness-event-stream-handle` argument.
nonisolated(unsafe) var harnessEventStreamHANDLE: HANDLE?
#else
/// The value of the `--__harness-event-stream-file-descriptor` argument.
var harnessEventStreamFileDescriptor: CInt?
#endif

/// The value of the `--event-stream-version` or `--experimental-event-stream-version`
/// argument, representing the version of the event stream schema to use when
/// writing events to ``eventStreamOutput``.
Expand Down Expand Up @@ -493,6 +501,27 @@ func parseCommandLineArguments(from args: [String]) throws -> __CommandLineArgum
result.eventStreamVersionNumber = eventStreamVersion
}
}

// Harness event stream file descriptor (file handle on Windows)
var hasHarnessEventStream = false
#if os(Windows)
if let handleString = args.argumentValue(forLabel: "--__harness-event-stream-handle") {
guard let handle = UInt(handleString).flatMap(HANDLE.init(bitPattern:)),
handle != INVALID_HANDLE_VALUE else {
throw _EntryPointError.invalidArgument("--__harness-event-stream-handle", value: handleString)
}
result.harnessEventStreamHANDLE = handle
hasHarnessEventStream = true
}
#else
if let fdString = args.argumentValue(forLabel: "--__harness-event-stream-file-descriptor") {
guard let fd = CInt(fdString) else {
throw _EntryPointError.invalidArgument("--__harness-event-stream-file-descriptor", value: fdString)
}
result.harnessEventStreamFileDescriptor = fd
hasHarnessEventStream = true
}
#endif
#endif

// XML output
Expand Down Expand Up @@ -540,6 +569,13 @@ func parseCommandLineArguments(from args: [String]) throws -> __CommandLineArgum
if args.contains("--quiet") || args.contains("-q") {
result.quiet = true
}
#if !SWT_NO_FILE_IO
if hasHarnessEventStream {
// The presence of a harness event stream implies suppression of the normal
// stderr output.
result.verbosity = .min
}
#endif

// Filtering
func filterValues(forArgumentsWithLabel label: String) -> [String] {
Expand Down Expand Up @@ -643,6 +679,25 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr
oldEventHandler(event, context)
}
}

// Harness event stream output
#if os(Windows)
let harnessFile = try args.harnessEventStreamHANDLE.map { try Allocated(FileHandle(unsafeWindowsHANDLE: $0, options: [.writeAccess])) }
#else
let harnessFile = try args.harnessEventStreamFileDescriptor.map { try Allocated(FileHandle(unsafePOSIXFileDescriptor: $0, options: [.writeAccess])) }
#endif
if let harnessFile {
let eventHandler = try eventHandlerForStreamingEvents(withVersionNumber: ABI.HarnessVersion.versionNumber, encodeAsJSONLines: true) { json in
_ = try? harnessFile.value.withLock {
try harnessFile.value.write(json)
try harnessFile.value.write("\n")
}
}
configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in
eventHandler(event, context)
oldEventHandler(event, context)
}
}
#endif
#endif

Expand Down
86 changes: 86 additions & 0 deletions Sources/Testing/ABI/EntryPoints/FileGrommet.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2026 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for Swift project authors
//

#if !SWT_NO_FILE_IO
package final class FileGrommet: Grommet {
private let _file: FileHandle
private let _filePath: String

init(readingFrom file: consuming FileHandle, atPath filePath: String? = nil) {
_file = file
_filePath = filePath ?? ""
}

package convenience init(readingFromFileAtPath filePath: String) throws {
let file = try FileHandle(forReadingAtPath: filePath)
self.init(readingFrom: file, atPath: filePath)
}

package var grommetName: String {
_filePath
}

package func run(_ eventHandler: @escaping @Sendable (borrowing Event, borrowing Event.Context) -> Void) async throws {
var context = ABI.Context()

var terminator: UInt8?
repeat {
let recordJSON: [UInt8]
(recordJSON, terminator) = try _file.read(until: \.isASCIINewline)

// Allow other tasks to run after we may have blocked for some time on
// I/O with the child process.
await Task.yield()

if recordJSON.isEmpty {
continue
}
try recordJSON.withUnsafeBytes { recordJSON in
let versionNumber = try ABI.VersionNumber(fromRecordJSON: recordJSON)
guard let abi = ABI._version(forVersionNumber: versionNumber) else {
try? FileHandle.stderr.write("Failed to determine ABI version for JSON record with version number '\(versionNumber)'")
return
}
try _processRecord(
recordJSON,
withABIVersion: abi,
in: &context,
eventHandler: eventHandler
)
}
} while terminator != nil
}

private func _processRecord<V>(
_ recordJSON: UnsafeRawBufferPointer,
withABIVersion: V.Type,
in context: inout ABI.Context,
eventHandler: @Sendable (borrowing Event, borrowing Event.Context) -> Void
) throws where V: ABI._Version {
let record = try JSON.decode(ABI.Record<ABI.HarnessVersion>.self, from: recordJSON)
switch record.kind {
case let .test(encodedTest):
_ = Test(decoding: encodedTest, in: &context)
case let .event(encodedEvent):
guard let event = Event(decoding: encodedEvent, in: &context) else {
try? FileHandle.stderr.write("Failed to decode \(encodedEvent)")
return
}
let eventContext = Event.Context(
test: encodedEvent.testID.flatMap(context.test(identifiedBy:)),
testCase: nil,
iteration: encodedEvent.iteration,
configuration: nil
)
eventHandler(event, eventContext)
}
}
}
#endif
30 changes: 30 additions & 0 deletions Sources/Testing/ABI/EntryPoints/Grommet.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2026 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for Swift project authors
//

// FIXME: need an actual name for this protocol
package protocol Grommet: Sendable {
var grommetName: String { get }
func run(_ eventHandler: @escaping @Sendable (borrowing Event, borrowing Event.Context) -> Void) async throws
}

extension Runner: Grommet {
package var grommetName: String {
""
}

package func run(_ eventHandler: @escaping @Sendable (borrowing Event, borrowing Event.Context) -> Void) async throws {
var selfCopy = self
selfCopy.configuration.eventHandler = { [oldEventHandler = selfCopy.configuration.eventHandler] event, context in
eventHandler(event, context)
oldEventHandler(event, context)
}
await selfCopy.run()
}
}
98 changes: 98 additions & 0 deletions Sources/Testing/ABI/EntryPoints/HarnessEntryPoint.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2026 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for Swift project authors
//

private import _TestingInternals

#if canImport(Synchronization)
private import Synchronization
#endif

extension ABI {
package typealias HarnessVersion = ExperimentalVersion
}

package func harnessEntryPoint(
running grommets: [any Grommet]
) async throws -> CInt {
var exitCodes = [CInt]()

#if !SWT_NO_FILE_IO
let eventRecorder = Event.ConsoleOutputRecorder(options: .for(.stderr)) { string in
try? FileHandle.stderr.write(string)
}
#endif

let grommetCount = grommets.count
for (i, grommet) in grommets.enumerated() {
let exitCode = Atomic<CInt>(EXIT_SUCCESS)

func open(_ grommet: some Grommet) async throws {
try await grommet.run { event, eventContext in
var recordEvent = true

switch event.kind {
case .testDiscovered:
_ = exitCode.compareExchange(expected: EXIT_SUCCESS, desired: EXIT_NO_TESTS_FOUND, ordering: .sequentiallyConsistent)
#if !SWT_NO_FILE_IO
case .runStarted:
if i > 0 {
recordEvent = false
}
if grommetCount > 1 {
try? FileHandle.stderr.write("Running '\(grommet.grommetName)'...\n")
}
case .runEnded:
if i != grommets.count - 1 {
recordEvent = false
}
#endif
case let .issueRecorded(issue):
if issue.isFailure {
exitCode.store(EXIT_FAILURE, ordering: .sequentiallyConsistent)
}
default:
break
}

#if !SWT_NO_FILE_IO
if recordEvent {
eventRecorder.record(event, in: eventContext)
}
#endif
}
}

do {
try await open(grommet)
} catch {
// TODO: handle errors at this layer in an interesting way
exitCode.store(EXIT_FAILURE, ordering: .sequentiallyConsistent)
}

exitCodes.append(exitCode.load(ordering: .sequentiallyConsistent))
}

let noTestsFound = exitCodes.allSatisfy { $0 == EXIT_NO_TESTS_FOUND }
if noTestsFound {
return EXIT_NO_TESTS_FOUND
}
let succeeded = exitCodes.allSatisfy { $0 == EXIT_SUCCESS }
if succeeded {
return EXIT_SUCCESS
}
return EXIT_FAILURE
}

package func harnessEntryPoint(
running grommets: [any Grommet]
) async throws -> Never {
let exitCode: CInt = try await harnessEntryPoint(running: grommets)
exit(exitCode)
}
Loading
Loading