From a918bd235d794233627c55ab96c2c31bc8d2cc11 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 26 Nov 2025 10:50:50 -0500 Subject: [PATCH] Add an experimental interface for custom test libraries. This PR adds command-line experimental support for custom test libraries that don't sit on top of XCTest or Swift Testing (irony inbound). Developers can specify additional libraries to invoke using the `--experimental-testing-library` argument. In a separate PR, I will teach Swift Testing how to read the `--experimental-testing-library` argument (forwarded as `--testing-library` once per, er, library.) A developer can write a small shim function that gets added to the `__swift5_tests` section as a special test content record and which, when called, invokes its corresponding testing library and translates output to the "language" Swift Testing speaks. The above description is light on details because we haven't designed out the mechanisms in question. In particular, we don't have a stable configuration/input mechanism, so for the moment I'm just forwarding command-line arguments. We also need to teach some part of the stack how to collate results from multiple libraries to produce a single unified test report in various formats. This PR is just one step. --- Sources/Basics/TestingLibrary.swift | 69 +++++++++++++++++++------ Sources/Build/LLBuildCommands.swift | 37 ++++++++----- Sources/Commands/SwiftTestCommand.swift | 65 +++++++++++++---------- Sources/CoreCommands/Options.swift | 63 ++++++++++++++++++---- 4 files changed, 168 insertions(+), 66 deletions(-) diff --git a/Sources/Basics/TestingLibrary.swift b/Sources/Basics/TestingLibrary.swift index 289774ca673..7d3bcaee29d 100644 --- a/Sources/Basics/TestingLibrary.swift +++ b/Sources/Basics/TestingLibrary.swift @@ -11,23 +11,58 @@ //===----------------------------------------------------------------------===// /// The testing libraries supported by the package manager. -public enum TestingLibrary: Sendable, CustomStringConvertible { - /// The XCTest library. - /// - /// This case represents both the open-source swift-corelibs-xctest - /// package and Apple's XCTest framework that ships with Xcode. - case xctest - - /// The swift-testing library. - case swiftTesting - - public var description: String { - switch self { - case .xctest: - "XCTest" - case .swiftTesting: - "Swift Testing" +public enum TestingLibrary: Sendable, Equatable, Hashable, CustomStringConvertible { + /// The XCTest library. + /// + /// This case represents both the open-source swift-corelibs-xctest + /// package and Apple's XCTest framework that ships with Xcode. + case xctest + + /// The swift-testing library. + case swiftTesting + + /// A library specified by name _other than_ XCTest or Swift Testing. + /// + /// - Parameters: + /// - name: The name of the library. The formatting of this string is + /// unspecified. + case other(_ name: String) + + public init(_ name: String) { + switch name.filter(\.isLetter).lowercased() { + case "xctest": + self = .xctest + case "swifttesting": + self = .swiftTesting + default: + self = .other(name) + } + } + + public init(argument: String) { + self.init(argument) + } + + public var shortName: String { + switch self { + case .xctest: + "xctest" + case .swiftTesting: + "swift-testing" + case let .other(name): + name + } + } + + public var description: String { + switch self { + case .xctest: + "XCTest" + case .swiftTesting: + "Swift Testing" + case let .other(name): + name + } } - } } diff --git a/Sources/Build/LLBuildCommands.swift b/Sources/Build/LLBuildCommands.swift index aeb9442eb19..76c4dbb05f8 100644 --- a/Sources/Build/LLBuildCommands.swift +++ b/Sources/Build/LLBuildCommands.swift @@ -273,13 +273,28 @@ final class TestEntryPointCommand: CustomLLBuildCommand, TestBuildCommand { while let argument = iterator.next() { if argument == "--testing-library", let libraryName = iterator.next() { return libraryName.lowercased() - } + } else if let equalsIndex = argument.firstIndex(of: "="), + argument[..(_ arg: A, _ entryPoint: (A?) async -> Never) async -> Never { + await entryPoint(arg) + } + + /// Used if Swift Testing is an older package dependency that does not take a `String` argument. + @_disfavoredOverload private static func enterTestingLibrary(_ testingLibrary: String, _ entryPoint: (A?) async -> Never) async -> Never { + if testingLibrary == "swift-testing" { + await entryPoint(nil) + } + fatalError("Cannot run tests using the '\(testingLibrary)' library: this package has a dependency on a version of Swift Testing that does not support hosting other testing libraries.") + } + #if \#(needsAsyncMainWorkaround) @_silgen_name("$ss13_runAsyncMainyyyyYaKcF") private static func _runAsyncMain(_ asyncFun: @Sendable @escaping () async throws -> ()) @@ -287,23 +302,21 @@ final class TestEntryPointCommand: CustomLLBuildCommand, TestBuildCommand { static func main() \#(needsAsyncMainWorkaround ? "" : "async") { let testingLibrary = Self.testingLibrary() - #if canImport(Testing) - if testingLibrary == "swift-testing" { - #if \#(needsAsyncMainWorkaround) - _runAsyncMain { - await Testing.__swiftPMEntryPoint() as Never - } - #else - await Testing.__swiftPMEntryPoint() as Never - #endif - } - #endif #if \#(isXCTMainAvailable) if testingLibrary == "xctest" { \#(testObservabilitySetup) \#(awaitXCTMainKeyword) XCTMain(__allDiscoveredTests()) as Never } #endif + #if canImport(Testing) + #if \#(needsAsyncMainWorkaround) + _runAsyncMain { + await Self.enterTestingLibrary(testingLibrary) + } + #else + await Self.enterTestingLibrary(testingLibrary, Testing.__swiftPMEntryPoint) + #endif + #endif } } """# diff --git a/Sources/Commands/SwiftTestCommand.swift b/Sources/Commands/SwiftTestCommand.swift index 10c3b280c29..596b3fd9b6d 100644 --- a/Sources/Commands/SwiftTestCommand.swift +++ b/Sources/Commands/SwiftTestCommand.swift @@ -42,9 +42,7 @@ import var TSCBasic.stdoutStream import class TSCBasic.SynchronizedQueue import class TSCBasic.Thread -#if os(Windows) -import WinSDK // for ERROR_NOT_FOUND -#elseif canImport(Android) +#if canImport(Android) import Android #endif @@ -287,8 +285,10 @@ public struct SwiftTestCommand: AsyncSwiftCommand { var results = [TestRunner.Result]() + let enabledLibraries = options.testLibraryOptions.enabledTestingLibraries(swiftCommandState: swiftCommandState) + // Run XCTest. - if options.testLibraryOptions.isEnabled(.xctest, swiftCommandState: swiftCommandState) { + if enabledLibraries.contains(.xctest) { // Validate XCTest is available on Darwin-based systems. If it's not available and we're hitting this code // path, that means the developer must have explicitly passed --enable-xctest (or the toolchain is // corrupt, I suppose.) @@ -357,24 +357,25 @@ public struct SwiftTestCommand: AsyncSwiftCommand { } } - // Run Swift Testing (parallel or not, it has a single entry point.) - if options.testLibraryOptions.isEnabled(.swiftTesting, swiftCommandState: swiftCommandState) { - lazy var testEntryPointPath = testProducts.lazy.compactMap(\.testEntryPointPath).first - if options.testLibraryOptions.isExplicitlyEnabled(.swiftTesting, swiftCommandState: swiftCommandState) || testEntryPointPath == nil { + // Run Swift Testing and third-party libraries (parallel or not, they + // have a single semi-shared entry point.) + lazy var testEntryPointPath = testProducts.lazy.compactMap(\.testEntryPointPath).first + for library in enabledLibraries where library != .xctest { + if options.testLibraryOptions.isExplicitlyEnabled(library, swiftCommandState: swiftCommandState) || testEntryPointPath == nil { results.append( try await runTestProducts( testProducts, additionalArguments: [], productsBuildParameters: buildParameters, swiftCommandState: swiftCommandState, - library: .swiftTesting + library: library ) ) } else if let testEntryPointPath { // Cannot run Swift Testing because an entry point file was used and the developer // didn't explicitly enable Swift Testing. swiftCommandState.observabilityScope.emit( - debug: "Skipping automatic Swift Testing invocation because a test entry point path is present: \(testEntryPointPath)" + debug: "Skipping automatic \(library) invocation because a test entry point path is present: \(testEntryPointPath)" ) } } @@ -490,7 +491,7 @@ public struct SwiftTestCommand: AsyncSwiftCommand { ) async throws -> TestRunner.Result { // Pass through all arguments from the command line to Swift Testing. var additionalArguments = additionalArguments - if library == .swiftTesting { + if library != .xctest { // Reconstruct the arguments list. If an xUnit path was specified, remove it. var commandLineArguments = [String]() var originalCommandLineArguments = CommandLine.arguments.dropFirst().makeIterator() @@ -504,11 +505,12 @@ public struct SwiftTestCommand: AsyncSwiftCommand { additionalArguments += commandLineArguments if var xunitPath = options.xUnitOutput { - if options.testLibraryOptions.isEnabled(.xctest, swiftCommandState: swiftCommandState) { - // We are running Swift Testing, XCTest is also running in this session, and an xUnit path - // was specified. Make sure we don't stomp on XCTest's XML output by having Swift Testing - // write to a different path. - var xunitFileName = "\(xunitPath.basenameWithoutExt)-swift-testing" + let enabledLibraryCount = options.testLibraryOptions.enabledTestingLibraries(swiftCommandState: swiftCommandState).count + if enabledLibraryCount > 1 { + // We are running more than one test library in this session and an xUnit path + // was specified. Make sure the libraries don't stomp on each other's XML output + // by having each of them write to a different path. + var xunitFileName = "\(xunitPath.basenameWithoutExt)-\(library.shortName)" if let ext = xunitPath.extension { xunitFileName = "\(xunitFileName).\(ext)" } @@ -529,7 +531,7 @@ public struct SwiftTestCommand: AsyncSwiftCommand { let runnerPaths: [AbsolutePath] = switch library { case .xctest: testProducts.map(\.bundlePath) - case .swiftTesting: + default: testProducts.map(\.binaryPath) } @@ -812,7 +814,9 @@ extension SwiftTestCommand { library: .swiftTesting ) - if testLibraryOptions.isEnabled(.xctest, swiftCommandState: swiftCommandState) { + let enabledLibraries = testLibraryOptions.enabledTestingLibraries(swiftCommandState: swiftCommandState) + + if enabledLibraries.contains(.xctest) { let testSuites = try TestingSupport.getTestSuites( in: testProducts, swiftCommandState: swiftCommandState, @@ -828,9 +832,9 @@ extension SwiftTestCommand { } } - if testLibraryOptions.isEnabled(.swiftTesting, swiftCommandState: swiftCommandState) { + for library in enabledLibraries where library != .xctest { lazy var testEntryPointPath = testProducts.lazy.compactMap(\.testEntryPointPath).first - if testLibraryOptions.isExplicitlyEnabled(.swiftTesting, swiftCommandState: swiftCommandState) || testEntryPointPath == nil { + if testLibraryOptions.isExplicitlyEnabled(library, swiftCommandState: swiftCommandState) || testEntryPointPath == nil { let additionalArguments = ["--list-tests"] + CommandLine.arguments.dropFirst() let runner = TestRunner( bundlePaths: testProducts.map(\.binaryPath), @@ -839,7 +843,7 @@ extension SwiftTestCommand { toolchain: toolchain, testEnv: testEnv, observabilityScope: swiftCommandState.observabilityScope, - library: .swiftTesting + library: library ) // Finally, run the tests. @@ -1006,7 +1010,7 @@ final class TestRunner { throw TestError.xcodeNotInstalled } args += [xctestPath.pathString] - case .swiftTesting: + default: let helper = try self.toolchain.getSwiftTestingHelper() args += [helper.pathString, "--test-bundle-path", testPath.pathString] } @@ -1018,10 +1022,17 @@ final class TestRunner { #endif } - if library == .swiftTesting { - // HACK: tell the test bundle/executable that we want to run Swift Testing, not XCTest. - // XCTest doesn't understand this argument (yet), so don't pass it there. - args += ["--testing-library", "swift-testing"] + var addLibraryArgument = true +#if os(macOS) + if library == .xctest { + // On macOS, the XCTest runner doesn't understand the --testing-library + // argument, so we must omit it. + addLibraryArgument = false + } +#endif + if addLibraryArgument { + // Tell the test harness process which library to run. + args += ["--testing-library", library.shortName] } return args @@ -1050,7 +1061,7 @@ final class TestRunner { switch result.exitStatus { case .terminated(code: 0): return .success - case .terminated(code: EXIT_NO_TESTS_FOUND) where library == .swiftTesting: + case .terminated(code: EXIT_NO_TESTS_FOUND) where library != .xctest: return .noMatchingTests #if !os(Windows) case .signalled(let signal) where ![SIGINT, SIGKILL, SIGTERM].contains(signal): diff --git a/Sources/CoreCommands/Options.swift b/Sources/CoreCommands/Options.swift index 3b7411d8ce9..8c1673573cd 100644 --- a/Sources/CoreCommands/Options.swift +++ b/Sources/CoreCommands/Options.swift @@ -644,6 +644,8 @@ public struct LinkerOptions: ParsableArguments { public var shouldDisableLocalRpath: Bool = false } +extension TestingLibrary: ExpressibleByArgument {} + /// Which testing libraries to use (and any related options.) @_spi(SwiftPMInternal) public struct TestLibraryOptions: ParsableArguments { @@ -675,24 +677,36 @@ public struct TestLibraryOptions: ParsableArguments { help: .private) public var explicitlyEnableExperimentalSwiftTestingLibrarySupport: Bool? + /// Experimental listing of arbitrary testing libraries by name (rather than + /// just toggling between XCTest and Swift Testing). + @Option(name: .customLong("experimental-testing-library"), + help: .private) + public var explicitlyEnabledLibraries: [TestingLibrary] = [] + /// The common implementation for `isEnabled()` and `isExplicitlyEnabled()`. /// /// It is intentional that `isEnabled()` is not simply this function with a /// default value for the `default` argument. There's no "true" default /// value to use; it depends on the semantics the caller is interested in. private func isEnabled(_ library: TestingLibrary, `default`: Bool, swiftCommandState: SwiftCommandState) -> Bool { - switch library { - case .xctest: - if let explicitlyEnableXCTestSupport { - return explicitlyEnableXCTestSupport - } - if let toolchain = try? swiftCommandState.getHostToolchain(), - toolchain.swiftSDK.xctestSupport == .supported { + if explicitlyEnabledLibraries.isEmpty { + switch library { + case .xctest: + if let explicitlyEnableXCTestSupport { + return explicitlyEnableXCTestSupport + } + if let toolchain = try? swiftCommandState.getHostToolchain(), + toolchain.swiftSDK.xctestSupport == .supported { + return `default` + } + return false + case .swiftTesting: + return explicitlyEnableSwiftTestingLibrarySupport ?? explicitlyEnableExperimentalSwiftTestingLibrarySupport ?? `default` + case .other: return `default` } - return false - case .swiftTesting: - return explicitlyEnableSwiftTestingLibrarySupport ?? explicitlyEnableExperimentalSwiftTestingLibrarySupport ?? `default` + } else { + return explicitlyEnabledLibraries.contains(library) } } @@ -705,6 +719,35 @@ public struct TestLibraryOptions: ParsableArguments { public func isExplicitlyEnabled(_ library: TestingLibrary, swiftCommandState: SwiftCommandState) -> Bool { isEnabled(library, default: false, swiftCommandState: swiftCommandState) } + + /// The set of enabled testing libraries derived from all arguments to the + /// `swift test` command. + /// + /// This function behaves similarly to ``isEnabled(_:swiftCommandState:)`` + /// rather than ``isExplicitlyEnabled(_:swiftCommandState:)``. We don't + /// (currently) need a projection of the latter. + public func enabledTestingLibraries(swiftCommandState: SwiftCommandState) -> [TestingLibrary] { + var result = [TestingLibrary]() + + if isEnabled(.xctest, swiftCommandState: swiftCommandState) { + result.append(.xctest) + } + if isEnabled(.swiftTesting, swiftCommandState: swiftCommandState) { + result.append(.swiftTesting) + } + + // Include all explicitly enabled libraries (other than XCTest and Swift + // Testing which we have special-cased.) Since this is an opt-in list + // rather than a toggle, there's no need to repeatedly call isEnabled(). + // `foundLibraries` is just used to dedup inputs. + var foundLibraries: Set = [.xctest, .swiftTesting] + for library in explicitlyEnabledLibraries where !foundLibraries.contains(library) { + foundLibraries.insert(library) + result.append(library) + } + + return result + } } public struct TraitOptions: ParsableArguments {