diff --git a/Sources/Basics/TestingLibrary.swift b/Sources/Basics/TestingLibrary.swift index 289774ca673..7d3bcaee29d 100644 --- a/Sources/Basics/TestingLibrary.swift +++ b/Sources/Basics/TestingLibrary.swift @@ -11,23 +11,58 @@ //===----------------------------------------------------------------------===// /// The testing libraries supported by the package manager. -public enum TestingLibrary: Sendable, CustomStringConvertible { - /// The XCTest library. - /// - /// This case represents both the open-source swift-corelibs-xctest - /// package and Apple's XCTest framework that ships with Xcode. - case xctest - - /// The swift-testing library. - case swiftTesting - - public var description: String { - switch self { - case .xctest: - "XCTest" - case .swiftTesting: - "Swift Testing" +public enum TestingLibrary: Sendable, Equatable, Hashable, CustomStringConvertible { + /// The XCTest library. + /// + /// This case represents both the open-source swift-corelibs-xctest + /// package and Apple's XCTest framework that ships with Xcode. + case xctest + + /// The swift-testing library. + case swiftTesting + + /// A library specified by name _other than_ XCTest or Swift Testing. + /// + /// - Parameters: + /// - name: The name of the library. The formatting of this string is + /// unspecified. + case other(_ name: String) + + public init(_ name: String) { + switch name.filter(\.isLetter).lowercased() { + case "xctest": + self = .xctest + case "swifttesting": + self = .swiftTesting + default: + self = .other(name) + } + } + + public init(argument: String) { + self.init(argument) + } + + public var shortName: String { + switch self { + case .xctest: + "xctest" + case .swiftTesting: + "swift-testing" + case let .other(name): + name + } + } + + public var description: String { + switch self { + case .xctest: + "XCTest" + case .swiftTesting: + "Swift Testing" + case let .other(name): + name + } } - } } diff --git a/Sources/Build/LLBuildCommands.swift b/Sources/Build/LLBuildCommands.swift index aeb9442eb19..76c4dbb05f8 100644 --- a/Sources/Build/LLBuildCommands.swift +++ b/Sources/Build/LLBuildCommands.swift @@ -273,13 +273,28 @@ final class TestEntryPointCommand: CustomLLBuildCommand, TestBuildCommand { while let argument = iterator.next() { if argument == "--testing-library", let libraryName = iterator.next() { return libraryName.lowercased() - } + } else if let equalsIndex = argument.firstIndex(of: "="), + argument[..(_ arg: A, _ entryPoint: (A?) async -> Never) async -> Never { + await entryPoint(arg) + } + + /// Used if Swift Testing is an older package dependency that does not take a `String` argument. + @_disfavoredOverload private static func enterTestingLibrary(_ testingLibrary: String, _ entryPoint: (A?) async -> Never) async -> Never { + if testingLibrary == "swift-testing" { + await entryPoint(nil) + } + fatalError("Cannot run tests using the '\(testingLibrary)' library: this package has a dependency on a version of Swift Testing that does not support hosting other testing libraries.") + } + #if \#(needsAsyncMainWorkaround) @_silgen_name("$ss13_runAsyncMainyyyyYaKcF") private static func _runAsyncMain(_ asyncFun: @Sendable @escaping () async throws -> ()) @@ -287,23 +302,21 @@ final class TestEntryPointCommand: CustomLLBuildCommand, TestBuildCommand { static func main() \#(needsAsyncMainWorkaround ? "" : "async") { let testingLibrary = Self.testingLibrary() - #if canImport(Testing) - if testingLibrary == "swift-testing" { - #if \#(needsAsyncMainWorkaround) - _runAsyncMain { - await Testing.__swiftPMEntryPoint() as Never - } - #else - await Testing.__swiftPMEntryPoint() as Never - #endif - } - #endif #if \#(isXCTMainAvailable) if testingLibrary == "xctest" { \#(testObservabilitySetup) \#(awaitXCTMainKeyword) XCTMain(__allDiscoveredTests()) as Never } #endif + #if canImport(Testing) + #if \#(needsAsyncMainWorkaround) + _runAsyncMain { + await Self.enterTestingLibrary(testingLibrary) + } + #else + await Self.enterTestingLibrary(testingLibrary, Testing.__swiftPMEntryPoint) + #endif + #endif } } """# diff --git a/Sources/Commands/SwiftTestCommand.swift b/Sources/Commands/SwiftTestCommand.swift index 10c3b280c29..596b3fd9b6d 100644 --- a/Sources/Commands/SwiftTestCommand.swift +++ b/Sources/Commands/SwiftTestCommand.swift @@ -42,9 +42,7 @@ import var TSCBasic.stdoutStream import class TSCBasic.SynchronizedQueue import class TSCBasic.Thread -#if os(Windows) -import WinSDK // for ERROR_NOT_FOUND -#elseif canImport(Android) +#if canImport(Android) import Android #endif @@ -287,8 +285,10 @@ public struct SwiftTestCommand: AsyncSwiftCommand { var results = [TestRunner.Result]() + let enabledLibraries = options.testLibraryOptions.enabledTestingLibraries(swiftCommandState: swiftCommandState) + // Run XCTest. - if options.testLibraryOptions.isEnabled(.xctest, swiftCommandState: swiftCommandState) { + if enabledLibraries.contains(.xctest) { // Validate XCTest is available on Darwin-based systems. If it's not available and we're hitting this code // path, that means the developer must have explicitly passed --enable-xctest (or the toolchain is // corrupt, I suppose.) @@ -357,24 +357,25 @@ public struct SwiftTestCommand: AsyncSwiftCommand { } } - // Run Swift Testing (parallel or not, it has a single entry point.) - if options.testLibraryOptions.isEnabled(.swiftTesting, swiftCommandState: swiftCommandState) { - lazy var testEntryPointPath = testProducts.lazy.compactMap(\.testEntryPointPath).first - if options.testLibraryOptions.isExplicitlyEnabled(.swiftTesting, swiftCommandState: swiftCommandState) || testEntryPointPath == nil { + // Run Swift Testing and third-party libraries (parallel or not, they + // have a single semi-shared entry point.) + lazy var testEntryPointPath = testProducts.lazy.compactMap(\.testEntryPointPath).first + for library in enabledLibraries where library != .xctest { + if options.testLibraryOptions.isExplicitlyEnabled(library, swiftCommandState: swiftCommandState) || testEntryPointPath == nil { results.append( try await runTestProducts( testProducts, additionalArguments: [], productsBuildParameters: buildParameters, swiftCommandState: swiftCommandState, - library: .swiftTesting + library: library ) ) } else if let testEntryPointPath { // Cannot run Swift Testing because an entry point file was used and the developer // didn't explicitly enable Swift Testing. swiftCommandState.observabilityScope.emit( - debug: "Skipping automatic Swift Testing invocation because a test entry point path is present: \(testEntryPointPath)" + debug: "Skipping automatic \(library) invocation because a test entry point path is present: \(testEntryPointPath)" ) } } @@ -490,7 +491,7 @@ public struct SwiftTestCommand: AsyncSwiftCommand { ) async throws -> TestRunner.Result { // Pass through all arguments from the command line to Swift Testing. var additionalArguments = additionalArguments - if library == .swiftTesting { + if library != .xctest { // Reconstruct the arguments list. If an xUnit path was specified, remove it. var commandLineArguments = [String]() var originalCommandLineArguments = CommandLine.arguments.dropFirst().makeIterator() @@ -504,11 +505,12 @@ public struct SwiftTestCommand: AsyncSwiftCommand { additionalArguments += commandLineArguments if var xunitPath = options.xUnitOutput { - if options.testLibraryOptions.isEnabled(.xctest, swiftCommandState: swiftCommandState) { - // We are running Swift Testing, XCTest is also running in this session, and an xUnit path - // was specified. Make sure we don't stomp on XCTest's XML output by having Swift Testing - // write to a different path. - var xunitFileName = "\(xunitPath.basenameWithoutExt)-swift-testing" + let enabledLibraryCount = options.testLibraryOptions.enabledTestingLibraries(swiftCommandState: swiftCommandState).count + if enabledLibraryCount > 1 { + // We are running more than one test library in this session and an xUnit path + // was specified. Make sure the libraries don't stomp on each other's XML output + // by having each of them write to a different path. + var xunitFileName = "\(xunitPath.basenameWithoutExt)-\(library.shortName)" if let ext = xunitPath.extension { xunitFileName = "\(xunitFileName).\(ext)" } @@ -529,7 +531,7 @@ public struct SwiftTestCommand: AsyncSwiftCommand { let runnerPaths: [AbsolutePath] = switch library { case .xctest: testProducts.map(\.bundlePath) - case .swiftTesting: + default: testProducts.map(\.binaryPath) } @@ -812,7 +814,9 @@ extension SwiftTestCommand { library: .swiftTesting ) - if testLibraryOptions.isEnabled(.xctest, swiftCommandState: swiftCommandState) { + let enabledLibraries = testLibraryOptions.enabledTestingLibraries(swiftCommandState: swiftCommandState) + + if enabledLibraries.contains(.xctest) { let testSuites = try TestingSupport.getTestSuites( in: testProducts, swiftCommandState: swiftCommandState, @@ -828,9 +832,9 @@ extension SwiftTestCommand { } } - if testLibraryOptions.isEnabled(.swiftTesting, swiftCommandState: swiftCommandState) { + for library in enabledLibraries where library != .xctest { lazy var testEntryPointPath = testProducts.lazy.compactMap(\.testEntryPointPath).first - if testLibraryOptions.isExplicitlyEnabled(.swiftTesting, swiftCommandState: swiftCommandState) || testEntryPointPath == nil { + if testLibraryOptions.isExplicitlyEnabled(library, swiftCommandState: swiftCommandState) || testEntryPointPath == nil { let additionalArguments = ["--list-tests"] + CommandLine.arguments.dropFirst() let runner = TestRunner( bundlePaths: testProducts.map(\.binaryPath), @@ -839,7 +843,7 @@ extension SwiftTestCommand { toolchain: toolchain, testEnv: testEnv, observabilityScope: swiftCommandState.observabilityScope, - library: .swiftTesting + library: library ) // Finally, run the tests. @@ -1006,7 +1010,7 @@ final class TestRunner { throw TestError.xcodeNotInstalled } args += [xctestPath.pathString] - case .swiftTesting: + default: let helper = try self.toolchain.getSwiftTestingHelper() args += [helper.pathString, "--test-bundle-path", testPath.pathString] } @@ -1018,10 +1022,17 @@ final class TestRunner { #endif } - if library == .swiftTesting { - // HACK: tell the test bundle/executable that we want to run Swift Testing, not XCTest. - // XCTest doesn't understand this argument (yet), so don't pass it there. - args += ["--testing-library", "swift-testing"] + var addLibraryArgument = true +#if os(macOS) + if library == .xctest { + // On macOS, the XCTest runner doesn't understand the --testing-library + // argument, so we must omit it. + addLibraryArgument = false + } +#endif + if addLibraryArgument { + // Tell the test harness process which library to run. + args += ["--testing-library", library.shortName] } return args @@ -1050,7 +1061,7 @@ final class TestRunner { switch result.exitStatus { case .terminated(code: 0): return .success - case .terminated(code: EXIT_NO_TESTS_FOUND) where library == .swiftTesting: + case .terminated(code: EXIT_NO_TESTS_FOUND) where library != .xctest: return .noMatchingTests #if !os(Windows) case .signalled(let signal) where ![SIGINT, SIGKILL, SIGTERM].contains(signal): diff --git a/Sources/CoreCommands/Options.swift b/Sources/CoreCommands/Options.swift index 3b7411d8ce9..8c1673573cd 100644 --- a/Sources/CoreCommands/Options.swift +++ b/Sources/CoreCommands/Options.swift @@ -644,6 +644,8 @@ public struct LinkerOptions: ParsableArguments { public var shouldDisableLocalRpath: Bool = false } +extension TestingLibrary: ExpressibleByArgument {} + /// Which testing libraries to use (and any related options.) @_spi(SwiftPMInternal) public struct TestLibraryOptions: ParsableArguments { @@ -675,24 +677,36 @@ public struct TestLibraryOptions: ParsableArguments { help: .private) public var explicitlyEnableExperimentalSwiftTestingLibrarySupport: Bool? + /// Experimental listing of arbitrary testing libraries by name (rather than + /// just toggling between XCTest and Swift Testing). + @Option(name: .customLong("experimental-testing-library"), + help: .private) + public var explicitlyEnabledLibraries: [TestingLibrary] = [] + /// The common implementation for `isEnabled()` and `isExplicitlyEnabled()`. /// /// It is intentional that `isEnabled()` is not simply this function with a /// default value for the `default` argument. There's no "true" default /// value to use; it depends on the semantics the caller is interested in. private func isEnabled(_ library: TestingLibrary, `default`: Bool, swiftCommandState: SwiftCommandState) -> Bool { - switch library { - case .xctest: - if let explicitlyEnableXCTestSupport { - return explicitlyEnableXCTestSupport - } - if let toolchain = try? swiftCommandState.getHostToolchain(), - toolchain.swiftSDK.xctestSupport == .supported { + if explicitlyEnabledLibraries.isEmpty { + switch library { + case .xctest: + if let explicitlyEnableXCTestSupport { + return explicitlyEnableXCTestSupport + } + if let toolchain = try? swiftCommandState.getHostToolchain(), + toolchain.swiftSDK.xctestSupport == .supported { + return `default` + } + return false + case .swiftTesting: + return explicitlyEnableSwiftTestingLibrarySupport ?? explicitlyEnableExperimentalSwiftTestingLibrarySupport ?? `default` + case .other: return `default` } - return false - case .swiftTesting: - return explicitlyEnableSwiftTestingLibrarySupport ?? explicitlyEnableExperimentalSwiftTestingLibrarySupport ?? `default` + } else { + return explicitlyEnabledLibraries.contains(library) } } @@ -705,6 +719,35 @@ public struct TestLibraryOptions: ParsableArguments { public func isExplicitlyEnabled(_ library: TestingLibrary, swiftCommandState: SwiftCommandState) -> Bool { isEnabled(library, default: false, swiftCommandState: swiftCommandState) } + + /// The set of enabled testing libraries derived from all arguments to the + /// `swift test` command. + /// + /// This function behaves similarly to ``isEnabled(_:swiftCommandState:)`` + /// rather than ``isExplicitlyEnabled(_:swiftCommandState:)``. We don't + /// (currently) need a projection of the latter. + public func enabledTestingLibraries(swiftCommandState: SwiftCommandState) -> [TestingLibrary] { + var result = [TestingLibrary]() + + if isEnabled(.xctest, swiftCommandState: swiftCommandState) { + result.append(.xctest) + } + if isEnabled(.swiftTesting, swiftCommandState: swiftCommandState) { + result.append(.swiftTesting) + } + + // Include all explicitly enabled libraries (other than XCTest and Swift + // Testing which we have special-cased.) Since this is an opt-in list + // rather than a toggle, there's no need to repeatedly call isEnabled(). + // `foundLibraries` is just used to dedup inputs. + var foundLibraries: Set = [.xctest, .swiftTesting] + for library in explicitlyEnabledLibraries where !foundLibraries.contains(library) { + foundLibraries.insert(library) + result.append(library) + } + + return result + } } public struct TraitOptions: ParsableArguments {