diff --git a/Package.swift b/Package.swift index b4c4681fd43..32684b2293c 100644 --- a/Package.swift +++ b/Package.swift @@ -738,6 +738,14 @@ let package = Package( swiftLanguageVersions: [.v5] ) +#if canImport(Darwin) +package.targets.append(contentsOf: [ + .executableTarget( + name: "swiftpm-testing-helper" + ) +]) +#endif + // Workaround SPM's attempt to link in executables which does not work on all // platforms. #if !os(Windows) diff --git a/Sources/Build/BuildManifest/LLBuildManifestBuilder.swift b/Sources/Build/BuildManifest/LLBuildManifestBuilder.swift index a8adc3923e6..7a0c293105d 100644 --- a/Sources/Build/BuildManifest/LLBuildManifestBuilder.swift +++ b/Sources/Build/BuildManifest/LLBuildManifestBuilder.swift @@ -112,9 +112,7 @@ public class LLBuildManifestBuilder { } } - if self.plan.destinationBuildParameters.testingParameters.library == .xctest { - try self.addTestDiscoveryGenerationCommand() - } + try self.addTestDiscoveryGenerationCommand() try self.addTestEntryPointGenerationCommand() // Create command for all products in the plan. @@ -310,9 +308,7 @@ extension LLBuildManifestBuilder { let outputs = testEntryPointTarget.target.sources.paths - let mainFileName = TestEntryPointTool.mainFileName( - for: self.plan.destinationBuildParameters.testingParameters.library - ) + let mainFileName = TestEntryPointTool.mainFileName guard let mainOutput = (outputs.first { $0.basename == mainFileName }) else { throw InternalError("main output (\(mainFileName)) not found") } diff --git a/Sources/Build/BuildPlan/BuildPlan+Test.swift b/Sources/Build/BuildPlan/BuildPlan+Test.swift index 53cce82bc97..35b695cbda0 100644 --- a/Sources/Build/BuildPlan/BuildPlan+Test.swift +++ b/Sources/Build/BuildPlan/BuildPlan+Test.swift @@ -34,13 +34,16 @@ extension BuildPlan { _ fileSystem: FileSystem, _ observabilityScope: ObservabilityScope ) throws -> [(product: ResolvedProduct, discoveryTargetBuildDescription: SwiftModuleBuildDescription?, entryPointTargetBuildDescription: SwiftModuleBuildDescription)] { - guard destinationBuildParameters.testingParameters.testProductStyle.requiresAdditionalDerivedTestTargets, - case .entryPointExecutable(let explicitlyEnabledDiscovery, let explicitlySpecifiedPath) = - destinationBuildParameters.testingParameters.testProductStyle - else { + guard destinationBuildParameters.testingParameters.testProductStyle.requiresAdditionalDerivedTestTargets else { throw InternalError("makeTestManifestTargets should not be used for build plan which does not require additional derived test targets") } + var explicitlyEnabledDiscovery = false + var explicitlySpecifiedPath: AbsolutePath? + if case let .entryPointExecutable(caseExplicitlyEnabledDiscovery, caseExplicitlySpecifiedPath) = destinationBuildParameters.testingParameters.testProductStyle { + explicitlyEnabledDiscovery = caseExplicitlyEnabledDiscovery + explicitlySpecifiedPath = caseExplicitlySpecifiedPath + } let isEntryPointPathSpecifiedExplicitly = explicitlySpecifiedPath != nil var isDiscoveryEnabledRedundantly = explicitlyEnabledDiscovery && !isEntryPointPathSpecifiedExplicitly @@ -116,7 +119,7 @@ extension BuildPlan { resolvedTargetDependencies: [ResolvedModule.Dependency] ) throws -> SwiftModuleBuildDescription { let entryPointDerivedDir = destinationBuildParameters.buildPath.appending(components: "\(testProduct.name).derived") - let entryPointMainFileName = TestEntryPointTool.mainFileName(for: destinationBuildParameters.testingParameters.library) + let entryPointMainFileName = TestEntryPointTool.mainFileName let entryPointMainFile = entryPointDerivedDir.appending(component: entryPointMainFileName) let entryPointSources = Sources(paths: [entryPointMainFile], root: entryPointDerivedDir) @@ -153,16 +156,9 @@ extension BuildPlan { let swiftTargetDependencies: [Module.Dependency] let resolvedTargetDependencies: [ResolvedModule.Dependency] - switch destinationBuildParameters.testingParameters.library { - case .xctest: - discoveryTargets = try generateDiscoveryTargets() - swiftTargetDependencies = [.module(discoveryTargets!.target, conditions: [])] - resolvedTargetDependencies = [.module(discoveryTargets!.resolved, conditions: [])] - case .swiftTesting: - discoveryTargets = nil - swiftTargetDependencies = testProduct.modules.map { .module($0.underlying, conditions: []) } - resolvedTargetDependencies = testProduct.modules.map { .module($0, conditions: []) } - } + discoveryTargets = try generateDiscoveryTargets() + swiftTargetDependencies = [.module(discoveryTargets!.target, conditions: [])] + resolvedTargetDependencies = [.module(discoveryTargets!.resolved, conditions: [])] if let entryPointResolvedTarget = testProduct.testEntryPointModule { if isEntryPointPathSpecifiedExplicitly || explicitlyEnabledDiscovery { diff --git a/Sources/Build/LLBuildCommands.swift b/Sources/Build/LLBuildCommands.swift index a414119c236..3544e64b01e 100644 --- a/Sources/Build/LLBuildCommands.swift +++ b/Sources/Build/LLBuildCommands.swift @@ -50,8 +50,8 @@ extension IndexStore.TestCaseClass.TestMethod { } extension TestEntryPointTool { - public static func mainFileName(for library: BuildParameters.Testing.Library) -> String { - "runner-\(library).swift" + public static var mainFileName: String { + "runner.swift" } } @@ -105,74 +105,76 @@ final class TestDiscoveryCommand: CustomLLBuildCommand, TestBuildCommand { private func execute(fileSystem: Basics.FileSystem, tool: TestDiscoveryTool) throws { let outputs = tool.outputs.compactMap { try? AbsolutePath(validating: $0.name) } - switch self.context.productsBuildParameters.testingParameters.library { - case .swiftTesting: + if case .loadableBundle = context.productsBuildParameters.testingParameters.testProductStyle { + // When building an XCTest bundle, test discovery is handled by the + // test harness process (i.e. this is the Darwin path.) for file in outputs { try fileSystem.writeIfChanged(path: file, string: "") } - case .xctest: - let index = self.context.productsBuildParameters.indexStore - let api = try self.context.indexStoreAPI.get() - let store = try IndexStore.open(store: TSCAbsolutePath(index), api: api) - - // FIXME: We can speed this up by having one llbuild command per object file. - let tests = try store - .listTests(in: tool.inputs.map { try TSCAbsolutePath(AbsolutePath(validating: $0.name)) }) - - let testsByModule = Dictionary(grouping: tests, by: { $0.module.spm_mangledToC99ExtendedIdentifier() }) - - // Find the main file path. - guard let mainFile = outputs.first(where: { path in - path.basename == TestDiscoveryTool.mainFileName - }) else { - throw InternalError("main output (\(TestDiscoveryTool.mainFileName)) not found") - } + return + } - // Write one file for each test module. - // - // We could write everything in one file but that can easily run into type conflicts due - // in complex packages with large number of test modules. - for file in outputs where file != mainFile { - // FIXME: This is relying on implementation detail of the output but passing the - // the context all the way through is not worth it right now. - let module = file.basenameWithoutExt.spm_mangledToC99ExtendedIdentifier() - - guard let tests = testsByModule[module] else { - // This module has no tests so just write an empty file for it. - try fileSystem.writeFileContents(file, bytes: "") - continue - } - try write( - tests: tests, - forModule: module, - fileSystem: fileSystem, - path: file - ) + let index = self.context.productsBuildParameters.indexStore + let api = try self.context.indexStoreAPI.get() + let store = try IndexStore.open(store: TSCAbsolutePath(index), api: api) + + // FIXME: We can speed this up by having one llbuild command per object file. + let tests = try store + .listTests(in: tool.inputs.map { try TSCAbsolutePath(AbsolutePath(validating: $0.name)) }) + + let testsByModule = Dictionary(grouping: tests, by: { $0.module.spm_mangledToC99ExtendedIdentifier() }) + + // Find the main file path. + guard let mainFile = outputs.first(where: { path in + path.basename == TestDiscoveryTool.mainFileName + }) else { + throw InternalError("main output (\(TestDiscoveryTool.mainFileName)) not found") + } + + // Write one file for each test module. + // + // We could write everything in one file but that can easily run into type conflicts due + // in complex packages with large number of test modules. + for file in outputs where file != mainFile { + // FIXME: This is relying on implementation detail of the output but passing the + // the context all the way through is not worth it right now. + let module = file.basenameWithoutExt.spm_mangledToC99ExtendedIdentifier() + + guard let tests = testsByModule[module] else { + // This module has no tests so just write an empty file for it. + try fileSystem.writeFileContents(file, bytes: "") + continue } + try write( + tests: tests, + forModule: module, + fileSystem: fileSystem, + path: file + ) + } - let testsKeyword = tests.isEmpty ? "let" : "var" + let testsKeyword = tests.isEmpty ? "let" : "var" - // Write the main file. - let stream = try LocalFileOutputByteStream(mainFile) + // Write the main file. + let stream = try LocalFileOutputByteStream(mainFile) - stream.send( - #""" - import XCTest + stream.send( + #""" + import XCTest - @available(*, deprecated, message: "Not actually deprecated. Marked as deprecated to allow inclusion of deprecated tests (which test deprecated functionality) without warnings") - @MainActor - public func __allDiscoveredTests() -> [XCTestCaseEntry] { - \#(testsKeyword) tests = [XCTestCaseEntry]() + @available(*, deprecated, message: "Not actually deprecated. Marked as deprecated to allow inclusion of deprecated tests (which test deprecated functionality) without warnings") + @MainActor + public func __allDiscoveredTests() -> [XCTestCaseEntry] { + \#(testsKeyword) tests = [XCTestCaseEntry]() - \#(testsByModule.keys.map { "tests += __\($0)__allTests()" }.joined(separator: "\n ")) + \#(testsByModule.keys.map { "tests += __\($0)__allTests()" }.joined(separator: "\n ")) - return tests - } - """# - ) + return tests + } + """# + ) - stream.flush() - } + stream.flush() } override func execute( @@ -201,9 +203,7 @@ final class TestEntryPointCommand: CustomLLBuildCommand, TestBuildCommand { let outputs = tool.outputs.compactMap { try? AbsolutePath(validating: $0.name) } // Find the main output file - let mainFileName = TestEntryPointTool.mainFileName( - for: self.context.productsBuildParameters.testingParameters.library - ) + let mainFileName = TestEntryPointTool.mainFileName guard let mainFile = outputs.first(where: { path in path.basename == mainFileName }) else { @@ -213,62 +213,99 @@ final class TestEntryPointCommand: CustomLLBuildCommand, TestBuildCommand { // Write the main file. let stream = try LocalFileOutputByteStream(mainFile) - switch self.context.productsBuildParameters.testingParameters.library { - case .swiftTesting: - stream.send( - #""" - #if canImport(Testing) - import Testing - #endif + // Find the inputs, which are the names of the test discovery module(s) + let inputs = tool.inputs.compactMap { try? AbsolutePath(validating: $0.name) } + let discoveryModuleNames = inputs.map(\.basenameWithoutExt) - @main struct Runner { - static func main() async { - #if canImport(Testing) - await Testing.__swiftPMEntryPoint() as Never - #endif + let testObservabilitySetup: String + let buildParameters = self.context.productsBuildParameters + if buildParameters.testingParameters.experimentalTestOutput && buildParameters.triple.supportsTestSummary { + testObservabilitySetup = "_ = SwiftPMXCTestObserver()\n" + } else { + testObservabilitySetup = "" + } + + let swiftTestingImportCondition = "canImport(Testing)" + let xctestImportCondition: String = switch buildParameters.testingParameters.testProductStyle { + case .entryPointExecutable: + "canImport(XCTest)" + case .loadableBundle: + "false" + } + + /// On WASI, we can't block the main thread, so XCTestMain is defined as async. + let awaitXCTMainKeyword = if context.productsBuildParameters.triple.isWASI() { + "await" + } else { + "" + } + + // FIXME: work around crash on Amazon Linux 2 when main function is async (rdar://128303921) + let asyncMainKeyword = if context.productsBuildParameters.triple.isLinux() { + "" + } else { + "async" + } + + stream.send( + #""" + #if \#(swiftTestingImportCondition) + import Testing + #endif + + #if \#(xctestImportCondition) + \#(generateTestObservationCode(buildParameters: buildParameters)) + + import XCTest + \#(discoveryModuleNames.map { "import \($0)" }.joined(separator: "\n")) + #endif + + @main + @available(macOS 10.15.0, iOS 11.0, watchOS 4.0, tvOS 11.0, *) + @available(*, deprecated, message: "Not actually deprecated. Marked as deprecated to allow inclusion of deprecated tests (which test deprecated functionality) without warnings") + struct Runner { + private static func testingLibrary() -> String { + var iterator = CommandLine.arguments.makeIterator() + while let argument = iterator.next() { + if argument == "--testing-library", let libraryName = iterator.next() { + return libraryName.lowercased() + } } - } - """# - ) - case .xctest: - // Find the inputs, which are the names of the test discovery module(s) - let inputs = tool.inputs.compactMap { try? AbsolutePath(validating: $0.name) } - let discoveryModuleNames = inputs.map(\.basenameWithoutExt) - - let testObservabilitySetup: String - let buildParameters = self.context.productsBuildParameters - if buildParameters.testingParameters.experimentalTestOutput && buildParameters.triple.supportsTestSummary { - testObservabilitySetup = "_ = SwiftPMXCTestObserver()\n" - } else { - testObservabilitySetup = "" - } - stream.send( - #""" - \#(generateTestObservationCode(buildParameters: buildParameters)) + // Fallback if not specified: run XCTest (legacy behavior) + return "xctest" + } - import XCTest - \#(discoveryModuleNames.map { "import \($0)" }.joined(separator: "\n")) + #if os(Linux) + // FIXME: work around crash on Amazon Linux 2 when main function is async (rdar://128303921) + @_silgen_name("$ss13_runAsyncMainyyyyYaKcF") + private static func _runAsyncMain(_ asyncFun: @Sendable @escaping () async throws -> ()) + #endif - @main - @available(*, deprecated, message: "Not actually deprecated. Marked as deprecated to allow inclusion of deprecated tests (which test deprecated functionality) without warnings") - struct Runner { - #if os(WASI) - /// On WASI, we can't block the main thread, so XCTestMain is defined as async. - static func main() async { - \#(testObservabilitySetup) - await XCTMain(__allDiscoveredTests()) as Never + static func main() \#(asyncMainKeyword) { + let testingLibrary = Self.testingLibrary() + #if \#(swiftTestingImportCondition) + if testingLibrary == "swift-testing" { + #if os(Linux) + // FIXME: work around crash on Amazon Linux 2 when main function is async (rdar://128303921) + _runAsyncMain { + await Testing.__swiftPMEntryPoint() as Never + } + #else + await Testing.__swiftPMEntryPoint() as Never + #endif } - #else - static func main() { + #endif + #if \#(xctestImportCondition) + if testingLibrary == "xctest" { \#(testObservabilitySetup) - XCTMain(__allDiscoveredTests()) as Never + \#(awaitXCTMainKeyword) XCTMain(__allDiscoveredTests()) as Never } #endif } - """# - ) - } + } + """# + ) stream.flush() } diff --git a/Sources/Build/LLBuildDescription.swift b/Sources/Build/LLBuildDescription.swift index 1576aade5be..b8426d7cb7b 100644 --- a/Sources/Build/LLBuildDescription.swift +++ b/Sources/Build/LLBuildDescription.swift @@ -140,8 +140,7 @@ public struct BuildDescription: Codable { try BuiltTestProduct( productName: desc.product.name, binaryPath: desc.binaryPath, - packagePath: desc.package.path, - library: desc.buildParameters.testingParameters.library + packagePath: desc.package.path ) } self.pluginDescriptions = pluginDescriptions diff --git a/Sources/Commands/PackageCommands/Init.swift b/Sources/Commands/PackageCommands/Init.swift index 6263e84b835..8fd9522df7e 100644 --- a/Sources/Commands/PackageCommands/Init.swift +++ b/Sources/Commands/PackageCommands/Init.swift @@ -54,20 +54,11 @@ extension SwiftPackageCommand { throw InternalError("Could not find the current working directory") } - // NOTE: Do not use testLibraryOptions.enabledTestingLibraries(swiftCommandState:) here - // because the package doesn't exist yet, so there are no dependencies for it to query. - var testingLibraries: Set = [] - if testLibraryOptions.enableXCTestSupport { - testingLibraries.insert(.xctest) - } - if testLibraryOptions.explicitlyEnableSwiftTestingLibrarySupport == true { - testingLibraries.insert(.swiftTesting) - } let packageName = self.packageName ?? cwd.basename let initPackage = try InitPackage( name: packageName, packageType: initMode, - supportedTestingLibraries: testingLibraries, + supportedTestingLibraries: Set(testLibraryOptions.enabledTestingLibraries), destinationPath: cwd, installedSwiftPMConfiguration: swiftCommandState.getHostToolchain().installedSwiftPMConfiguration, fileSystem: swiftCommandState.fileSystem diff --git a/Sources/Commands/SwiftBuildCommand.swift b/Sources/Commands/SwiftBuildCommand.swift index f071a9e599e..cdd6cb14e46 100644 --- a/Sources/Commands/SwiftBuildCommand.swift +++ b/Sources/Commands/SwiftBuildCommand.swift @@ -175,7 +175,7 @@ public struct SwiftBuildCommand: AsyncSwiftCommand { } if case .allIncludingTests = subset { - func updateTestingParameters(of buildParameters: inout BuildParameters, library: BuildParameters.Testing.Library) { + func updateTestingParameters(of buildParameters: inout BuildParameters) { buildParameters.testingParameters = .init( configuration: buildParameters.configuration, targetTriple: buildParameters.triple, @@ -183,15 +183,12 @@ public struct SwiftBuildCommand: AsyncSwiftCommand { enableTestability: buildParameters.testingParameters.enableTestability, experimentalTestOutput: buildParameters.testingParameters.experimentalTestOutput, forceTestDiscovery: globalOptions.build.enableTestDiscovery, - testEntryPointPath: globalOptions.build.testEntryPointPath, - library: library + testEntryPointPath: globalOptions.build.testEntryPointPath ) } - for library in try options.testLibraryOptions.enabledTestingLibraries(swiftCommandState: swiftCommandState) { - updateTestingParameters(of: &productsBuildParameters, library: library) - updateTestingParameters(of: &toolsBuildParameters, library: library) - try build(swiftCommandState, subset: subset, productsBuildParameters: productsBuildParameters, toolsBuildParameters: toolsBuildParameters) - } + updateTestingParameters(of: &productsBuildParameters) + updateTestingParameters(of: &toolsBuildParameters) + try build(swiftCommandState, subset: subset, productsBuildParameters: productsBuildParameters, toolsBuildParameters: toolsBuildParameters) } else { try build(swiftCommandState, subset: subset, productsBuildParameters: productsBuildParameters, toolsBuildParameters: toolsBuildParameters) } diff --git a/Sources/Commands/SwiftTestCommand.swift b/Sources/Commands/SwiftTestCommand.swift index 7840db4f891..b4abba25cb9 100644 --- a/Sources/Commands/SwiftTestCommand.swift +++ b/Sources/Commands/SwiftTestCommand.swift @@ -243,92 +243,91 @@ public struct SwiftTestCommand: AsyncSwiftCommand { @OptionGroup() var options: TestCommandOptions - // MARK: - XCTest - - private func xctestRun(_ swiftCommandState: SwiftCommandState) async throws { - // validate XCTest available on darwin based systems - let toolchain = try swiftCommandState.getTargetToolchain() - if case let .unsupported(reason) = try swiftCommandState.getHostToolchain().swiftSDK.xctestSupport { - if let reason { - throw TestError.xctestNotAvailable(reason: reason) - } else { - throw TestError.xcodeNotInstalled - } - } else if toolchain.targetTriple.isDarwin() && toolchain.xctestPath == nil { - throw TestError.xcodeNotInstalled - } - - let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options, library: .xctest) - + private func run(_ swiftCommandState: SwiftCommandState, buildParameters: BuildParameters, testProducts: [BuiltTestProduct]) async throws { // Remove test output from prior runs and validate priors. - if self.options.enableExperimentalTestOutput && productsBuildParameters.triple.supportsTestSummary { - _ = try? localFileSystem.removeFileTree(productsBuildParameters.testOutputPath) + if self.options.enableExperimentalTestOutput && buildParameters.triple.supportsTestSummary { + _ = try? localFileSystem.removeFileTree(buildParameters.testOutputPath) } - let testProducts = try buildTestsIfNeeded(swiftCommandState: swiftCommandState, library: .xctest) - if !self.options.shouldRunInParallel { - let xctestArgs = try xctestArgs(for: testProducts, swiftCommandState: swiftCommandState) - try await runTestProducts( - testProducts, - additionalArguments: xctestArgs, - productsBuildParameters: productsBuildParameters, - swiftCommandState: swiftCommandState, - library: .xctest - ) - } else { - let testSuites = try TestingSupport.getTestSuites( - in: testProducts, - swiftCommandState: swiftCommandState, - enableCodeCoverage: options.enableCodeCoverage, - shouldSkipBuilding: options.sharedOptions.shouldSkipBuilding, - experimentalTestOutput: options.enableExperimentalTestOutput, - sanitizers: globalOptions.build.sanitizers - ) - let tests = try testSuites - .filteredTests(specifier: options.testCaseSpecifier) - .skippedTests(specifier: options.skippedTests(fileSystem: swiftCommandState.fileSystem)) - - // If there were no matches, emit a warning and exit. - if tests.isEmpty { - swiftCommandState.observabilityScope.emit(.noMatchingTests) - try generateXUnitOutputIfRequested(for: [], swiftCommandState: swiftCommandState) - return - } + var ranSuccessfully = true - // Clean out the code coverage directory that may contain stale - // profraw files from a previous run of the code coverage tool. - if self.options.enableCodeCoverage { - try swiftCommandState.fileSystem.removeFileTree(productsBuildParameters.codeCovPath) + // Run XCTest. + if options.testLibraryOptions.isEnabled(.xctest) { + // validate XCTest available on darwin based systems + let toolchain = try swiftCommandState.getTargetToolchain() + if case let .unsupported(reason) = try swiftCommandState.getHostToolchain().swiftSDK.xctestSupport { + if let reason { + throw TestError.xctestNotAvailable(reason: reason) + } else { + throw TestError.xcodeNotInstalled + } + } else if toolchain.targetTriple.isDarwin() && toolchain.xctestPath == nil { + throw TestError.xcodeNotInstalled } - // Run the tests using the parallel runner. - let runner = ParallelTestRunner( - bundlePaths: testProducts.map { $0.bundlePath }, - cancellator: swiftCommandState.cancellator, - toolchain: toolchain, - numJobs: options.numberOfWorkers ?? ProcessInfo.processInfo.activeProcessorCount, - buildOptions: globalOptions.build, - productsBuildParameters: productsBuildParameters, - shouldOutputSuccess: swiftCommandState.logLevel <= .info, - observabilityScope: swiftCommandState.observabilityScope - ) + if !self.options.shouldRunInParallel { + let xctestArgs = try xctestArgs(for: testProducts, swiftCommandState: swiftCommandState) + ranSuccessfully = try await runTestProducts( + testProducts, + additionalArguments: xctestArgs, + productsBuildParameters: buildParameters, + swiftCommandState: swiftCommandState, + library: .xctest + ) && ranSuccessfully + } else { + let testSuites = try TestingSupport.getTestSuites( + in: testProducts, + swiftCommandState: swiftCommandState, + enableCodeCoverage: options.enableCodeCoverage, + shouldSkipBuilding: options.sharedOptions.shouldSkipBuilding, + experimentalTestOutput: options.enableExperimentalTestOutput, + sanitizers: globalOptions.build.sanitizers + ) + let tests = try testSuites + .filteredTests(specifier: options.testCaseSpecifier) + .skippedTests(specifier: options.skippedTests(fileSystem: swiftCommandState.fileSystem)) + + if tests.isEmpty { + try generateXUnitOutputIfRequested(for: [], swiftCommandState: swiftCommandState) + } else { + // Run the tests using the parallel runner. + let runner = ParallelTestRunner( + bundlePaths: testProducts.map { $0.bundlePath }, + cancellator: swiftCommandState.cancellator, + toolchain: toolchain, + numJobs: options.numberOfWorkers ?? ProcessInfo.processInfo.activeProcessorCount, + buildOptions: globalOptions.build, + productsBuildParameters: buildParameters, + shouldOutputSuccess: swiftCommandState.logLevel <= .info, + observabilityScope: swiftCommandState.observabilityScope + ) - let testResults = try runner.run(tests) + let testResults = try runner.run(tests) - try generateXUnitOutputIfRequested(for: testResults, swiftCommandState: swiftCommandState) + try generateXUnitOutputIfRequested(for: testResults, swiftCommandState: swiftCommandState) - // process code Coverage if request - if self.options.enableCodeCoverage, runner.ranSuccessfully { - try await processCodeCoverage(testProducts, swiftCommandState: swiftCommandState, library: .xctest) + ranSuccessfully = ranSuccessfully && runner.ranSuccessfully + } } + } - if !runner.ranSuccessfully { - swiftCommandState.executionStatus = .failure - } + // Run Swift Testing (parallel or not, it has a single entry point.) + if options.testLibraryOptions.isEnabled(.swiftTesting) { + ranSuccessfully = try await runTestProducts( + testProducts, + additionalArguments: [], + productsBuildParameters: buildParameters, + swiftCommandState: swiftCommandState, + library: .swiftTesting + ) && ranSuccessfully + } - if self.options.enableExperimentalTestOutput, !runner.ranSuccessfully { - try Self.handleTestOutput(productsBuildParameters: productsBuildParameters, packagePath: testProducts[0].packagePath) - } + if !ranSuccessfully { + swiftCommandState.executionStatus = .failure + } + + if self.options.enableExperimentalTestOutput, !ranSuccessfully { + try Self.handleTestOutput(productsBuildParameters: buildParameters, packagePath: testProducts[0].packagePath) } } @@ -360,11 +359,6 @@ public struct SwiftTestCommand: AsyncSwiftCommand { .filteredTests(specifier: options.testCaseSpecifier) .skippedTests(specifier: options.skippedTests(fileSystem: swiftCommandState.fileSystem)) - // If there were no matches, emit a warning. - if tests.isEmpty { - swiftCommandState.observabilityScope.emit(.noMatchingTests) - } - return TestRunner.xctestArguments(forTestSpecifiers: tests.map(\.specifier)) } } @@ -385,21 +379,6 @@ public struct SwiftTestCommand: AsyncSwiftCommand { try generator.generate(at: xUnitOutput) } - // MARK: - swift-testing - - private func swiftTestingRun(_ swiftCommandState: SwiftCommandState) async throws { - let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options, library: .swiftTesting) - let testProducts = try buildTestsIfNeeded(swiftCommandState: swiftCommandState, library: .swiftTesting) - let additionalArguments = Array(CommandLine.arguments.dropFirst()) - try await runTestProducts( - testProducts, - additionalArguments: additionalArguments, - productsBuildParameters: productsBuildParameters, - swiftCommandState: swiftCommandState, - library: .swiftTesting - ) - } - // MARK: - Common implementation public func run(_ swiftCommandState: SwiftCommandState) async throws { @@ -418,12 +397,22 @@ public struct SwiftTestCommand: AsyncSwiftCommand { let command = try List.parse() try command.run(swiftCommandState) } else { - if try options.testLibraryOptions.enableSwiftTestingLibrarySupport(swiftCommandState: swiftCommandState) { - try await swiftTestingRun(swiftCommandState) + let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options) + let testProducts = try buildTestsIfNeeded(swiftCommandState: swiftCommandState) + + // Clean out the code coverage directory that may contain stale + // profraw files from a previous run of the code coverage tool. + if self.options.enableCodeCoverage { + try swiftCommandState.fileSystem.removeFileTree(productsBuildParameters.codeCovPath) } - if options.testLibraryOptions.enableXCTestSupport { - try await xctestRun(swiftCommandState) + + try await run(swiftCommandState, buildParameters: productsBuildParameters, testProducts: testProducts) + + // process code Coverage if request + if self.options.enableCodeCoverage, swiftCommandState.executionStatus != .failure { + try await processCodeCoverage(testProducts, swiftCommandState: swiftCommandState) } + } } @@ -433,11 +422,11 @@ public struct SwiftTestCommand: AsyncSwiftCommand { productsBuildParameters: BuildParameters, swiftCommandState: SwiftCommandState, library: BuildParameters.Testing.Library - ) async throws { - // Clean out the code coverage directory that may contain stale - // profraw files from a previous run of the code coverage tool. - if self.options.enableCodeCoverage { - try swiftCommandState.fileSystem.removeFileTree(productsBuildParameters.codeCovPath) + ) async throws -> Bool { + // Pass through all arguments from the command line to Swift Testing. + var additionalArguments = additionalArguments + if library == .swiftTesting { + additionalArguments += CommandLine.arguments.dropFirst() } let toolchain = try swiftCommandState.getTargetToolchain() @@ -448,8 +437,15 @@ public struct SwiftTestCommand: AsyncSwiftCommand { library: library ) + let runnerPaths: [AbsolutePath] = switch library { + case .xctest: + testProducts.map(\.bundlePath) + case .swiftTesting: + testProducts.map(\.binaryPath) + } + let runner = TestRunner( - bundlePaths: testProducts.map { library == .xctest ? $0.bundlePath : $0.binaryPath }, + bundlePaths: runnerPaths, additionalArguments: additionalArguments, cancellator: swiftCommandState.cancellator, toolchain: toolchain, @@ -459,22 +455,11 @@ public struct SwiftTestCommand: AsyncSwiftCommand { ) // Finally, run the tests. - let ranSuccessfully = runner.test(outputHandler: { + return runner.test(outputHandler: { // command's result output goes on stdout // ie "swift test" should output to stdout print($0, terminator: "") }) - if !ranSuccessfully { - swiftCommandState.executionStatus = .failure - } - - if self.options.enableCodeCoverage, ranSuccessfully { - try await processCodeCoverage(testProducts, swiftCommandState: swiftCommandState, library: library) - } - - if self.options.enableExperimentalTestOutput, !ranSuccessfully { - try Self.handleTestOutput(productsBuildParameters: productsBuildParameters, packagePath: testProducts[0].packagePath) - } } private static func handleTestOutput(productsBuildParameters: BuildParameters, packagePath: AbsolutePath) throws { @@ -511,8 +496,7 @@ public struct SwiftTestCommand: AsyncSwiftCommand { /// Processes the code coverage data and emits a json. private func processCodeCoverage( _ testProducts: [BuiltTestProduct], - swiftCommandState: SwiftCommandState, - library: BuildParameters.Testing.Library + swiftCommandState: SwiftCommandState ) async throws { let workspace = try swiftCommandState.getActiveWorkspace() let root = try swiftCommandState.getWorkspaceRoot() @@ -525,23 +509,23 @@ public struct SwiftTestCommand: AsyncSwiftCommand { } // Merge all the profraw files to produce a single profdata file. - try mergeCodeCovRawDataFiles(swiftCommandState: swiftCommandState, library: library) + try mergeCodeCovRawDataFiles(swiftCommandState: swiftCommandState) - let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options, library: library) + let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options) for product in testProducts { // Export the codecov data as JSON. let jsonPath = productsBuildParameters.codeCovAsJSONPath(packageName: rootManifest.displayName) - try exportCodeCovAsJSON(to: jsonPath, testBinary: product.binaryPath, swiftCommandState: swiftCommandState, library: library) + try exportCodeCovAsJSON(to: jsonPath, testBinary: product.binaryPath, swiftCommandState: swiftCommandState) } } /// Merges all profraw profiles in codecoverage directory into default.profdata file. - private func mergeCodeCovRawDataFiles(swiftCommandState: SwiftCommandState, library: BuildParameters.Testing.Library) throws { + private func mergeCodeCovRawDataFiles(swiftCommandState: SwiftCommandState) throws { // Get the llvm-prof tool. let llvmProf = try swiftCommandState.getTargetToolchain().getLLVMProf() // Get the profraw files. - let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options, library: library) + let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options) let codeCovFiles = try swiftCommandState.fileSystem.getDirectoryContents(productsBuildParameters.codeCovPath) // Construct arguments for invoking the llvm-prof tool. @@ -561,12 +545,11 @@ public struct SwiftTestCommand: AsyncSwiftCommand { private func exportCodeCovAsJSON( to path: AbsolutePath, testBinary: AbsolutePath, - swiftCommandState: SwiftCommandState, - library: BuildParameters.Testing.Library + swiftCommandState: SwiftCommandState ) throws { // Export using the llvm-cov tool. let llvmCov = try swiftCommandState.getTargetToolchain().getLLVMCov() - let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options, library: library) + let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options) let args = [ llvmCov.pathString, "export", @@ -586,10 +569,9 @@ public struct SwiftTestCommand: AsyncSwiftCommand { /// /// - Returns: The paths to the build test products. private func buildTestsIfNeeded( - swiftCommandState: SwiftCommandState, - library: BuildParameters.Testing.Library + swiftCommandState: SwiftCommandState ) throws -> [BuiltTestProduct] { - let (productsBuildParameters, toolsBuildParameters) = try swiftCommandState.buildParametersForTest(options: self.options, library: library) + let (productsBuildParameters, toolsBuildParameters) = try swiftCommandState.buildParametersForTest(options: self.options) return try Commands.buildTestsIfNeeded( swiftCommandState: swiftCommandState, productsBuildParameters: productsBuildParameters, @@ -617,7 +599,7 @@ public struct SwiftTestCommand: AsyncSwiftCommand { throw StringError("'--num-workers' must be greater than zero") } - if !options.testLibraryOptions.enableXCTestSupport { + guard options.testLibraryOptions.isEnabled(.xctest) else { throw StringError("'--num-workers' is only supported when testing with XCTest") } } @@ -644,7 +626,7 @@ extension SwiftTestCommand { guard let rootManifest = rootManifests.values.first else { throw StringError("invalid manifests at \(root.packages)") } - let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(enableCodeCoverage: true, library: .xctest) + let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(enableCodeCoverage: true) print(productsBuildParameters.codeCovAsJSONPath(packageName: rootManifest.displayName)) } } @@ -688,41 +670,10 @@ extension SwiftTestCommand { @Flag(name: [.customLong("list-tests"), .customShort("l")], help: .hidden) var _deprecated_passthrough: Bool = false - // MARK: - XCTest - - private func xctestRun(_ swiftCommandState: SwiftCommandState) throws { - let (productsBuildParameters, toolsBuildParameters) = try swiftCommandState.buildParametersForTest( - enableCodeCoverage: false, - shouldSkipBuilding: sharedOptions.shouldSkipBuilding, - library: .xctest - ) - let testProducts = try buildTestsIfNeeded( - swiftCommandState: swiftCommandState, - productsBuildParameters: productsBuildParameters, - toolsBuildParameters: toolsBuildParameters - ) - let testSuites = try TestingSupport.getTestSuites( - in: testProducts, - swiftCommandState: swiftCommandState, - enableCodeCoverage: false, - shouldSkipBuilding: sharedOptions.shouldSkipBuilding, - experimentalTestOutput: false, - sanitizers: globalOptions.build.sanitizers - ) - - // Print the tests. - for test in testSuites.allTests { - print(test.specifier) - } - } - - // MARK: - swift-testing - - private func swiftTestingRun(_ swiftCommandState: SwiftCommandState) throws { + func run(_ swiftCommandState: SwiftCommandState) throws { let (productsBuildParameters, toolsBuildParameters) = try swiftCommandState.buildParametersForTest( enableCodeCoverage: false, - shouldSkipBuilding: sharedOptions.shouldSkipBuilding, - library: .swiftTesting + shouldSkipBuilding: sharedOptions.shouldSkipBuilding ) let testProducts = try buildTestsIfNeeded( swiftCommandState: swiftCommandState, @@ -738,36 +689,43 @@ extension SwiftTestCommand { library: .swiftTesting ) - let additionalArguments = ["--list-tests"] + CommandLine.arguments.dropFirst() - let runner = TestRunner( - bundlePaths: testProducts.map(\.binaryPath), - additionalArguments: additionalArguments, - cancellator: swiftCommandState.cancellator, - toolchain: toolchain, - testEnv: testEnv, - observabilityScope: swiftCommandState.observabilityScope, - library: .swiftTesting - ) - - // Finally, run the tests. - let ranSuccessfully = runner.test(outputHandler: { - // command's result output goes on stdout - // ie "swift test" should output to stdout - print($0, terminator: "") - }) - if !ranSuccessfully { - swiftCommandState.executionStatus = .failure + if testLibraryOptions.isEnabled(.xctest) { + let testSuites = try TestingSupport.getTestSuites( + in: testProducts, + swiftCommandState: swiftCommandState, + enableCodeCoverage: false, + shouldSkipBuilding: sharedOptions.shouldSkipBuilding, + experimentalTestOutput: false, + sanitizers: globalOptions.build.sanitizers + ) + + // Print the tests. + for test in testSuites.allTests { + print(test.specifier) + } } - } - - // MARK: - Common implementation - func run(_ swiftCommandState: SwiftCommandState) throws { - if try testLibraryOptions.enableSwiftTestingLibrarySupport(swiftCommandState: swiftCommandState) { - try swiftTestingRun(swiftCommandState) - } - if testLibraryOptions.enableXCTestSupport { - try xctestRun(swiftCommandState) + if testLibraryOptions.isEnabled(.swiftTesting) { + let additionalArguments = ["--list-tests"] + CommandLine.arguments.dropFirst() + let runner = TestRunner( + bundlePaths: testProducts.map(\.binaryPath), + additionalArguments: additionalArguments, + cancellator: swiftCommandState.cancellator, + toolchain: toolchain, + testEnv: testEnv, + observabilityScope: swiftCommandState.observabilityScope, + library: .swiftTesting + ) + + // Finally, run the tests. + let ranSuccessfully = runner.test(outputHandler: { + // command's result output goes on stdout + // ie "swift test" should output to stdout + print($0, terminator: "") + }) + if !ranSuccessfully { + swiftCommandState.executionStatus = .failure + } } } @@ -881,20 +839,29 @@ final class TestRunner { /// Constructs arguments to execute XCTest. private func args(forTestAt testPath: AbsolutePath) throws -> [String] { var args: [String] = [] - #if os(macOS) - if library == .xctest { +#if os(macOS) + switch library { + case .xctest: guard let xctestPath = self.toolchain.xctestPath else { throw TestError.xcodeNotInstalled } - args = [xctestPath.pathString] - args += additionalArguments - args += [testPath.pathString] - return args + args += [xctestPath.pathString] + case .swiftTesting: + let helper = try self.toolchain.getSwiftTestingHelper() + args += [helper.pathString, "--test-bundle-path", testPath.pathString] } - #endif - - args += [testPath.description] args += additionalArguments + args += [testPath.pathString] +#else + args += [testPath.pathString] + args += additionalArguments +#endif + + if library == .swiftTesting { + // HACK: tell the test bundle/executable that we want to run Swift Testing, not XCTest. + // XCTest doesn't understand this argument (yet), so don't pass it there. + args += ["--testing-library", "swift-testing"] + } return args } @@ -1068,7 +1035,7 @@ final class ParallelTestRunner { toolchain: self.toolchain, testEnv: testEnv, observabilityScope: self.observabilityScope, - library: .xctest + library: .xctest // swift-testing does not use ParallelTestRunner ) var output = "" let outputLock = NSLock() @@ -1332,21 +1299,14 @@ final class XUnitGenerator { extension SwiftCommandState { func buildParametersForTest( - options: TestCommandOptions, - library: BuildParameters.Testing.Library + options: TestCommandOptions ) throws -> (productsBuildParameters: BuildParameters, toolsBuildParameters: BuildParameters) { - var result = try self.buildParametersForTest( + try self.buildParametersForTest( enableCodeCoverage: options.enableCodeCoverage, enableTestability: options.enableTestableImports, shouldSkipBuilding: options.sharedOptions.shouldSkipBuilding, - experimentalTestOutput: options.enableExperimentalTestOutput, - library: library + experimentalTestOutput: options.enableExperimentalTestOutput ) - if try options.testLibraryOptions.enableSwiftTestingLibrarySupport(swiftCommandState: self) { - result.productsBuildParameters.flags.swiftCompilerFlags += ["-DSWIFT_PM_SUPPORTS_SWIFT_TESTING"] - result.toolsBuildParameters.flags.swiftCompilerFlags += ["-DSWIFT_PM_SUPPORTS_SWIFT_TESTING"] - } - return result } } @@ -1393,12 +1353,6 @@ extension BuildParameters { } } -private extension Basics.Diagnostic { - static var noMatchingTests: Self { - .warning("No matching test cases were run") - } -} - /// Builds the "test" target if enabled in options. /// /// - Returns: The paths to the build test products. diff --git a/Sources/Commands/Utilities/TestingSupport.swift b/Sources/Commands/Utilities/TestingSupport.swift index 1e49d6defec..a29acbde1ae 100644 --- a/Sources/Commands/Utilities/TestingSupport.swift +++ b/Sources/Commands/Utilities/TestingSupport.swift @@ -118,8 +118,7 @@ enum TestingSupport { destinationBuildParameters: swiftCommandState.buildParametersForTest( enableCodeCoverage: enableCodeCoverage, shouldSkipBuilding: shouldSkipBuilding, - experimentalTestOutput: experimentalTestOutput, - library: .xctest + experimentalTestOutput: experimentalTestOutput ).productsBuildParameters, sanitizers: sanitizers, library: .xctest @@ -134,8 +133,7 @@ enum TestingSupport { toolchain: try swiftCommandState.getTargetToolchain(), destinationBuildParameters: swiftCommandState.buildParametersForTest( enableCodeCoverage: enableCodeCoverage, - shouldSkipBuilding: shouldSkipBuilding, - library: .xctest + shouldSkipBuilding: shouldSkipBuilding ).productsBuildParameters, sanitizers: sanitizers, library: .xctest @@ -164,10 +162,6 @@ enum TestingSupport { env["NO_COLOR"] = "1" } - // Set an environment variable to indicate which library's test product - // is being executed. - env["SWIFT_PM_TEST_LIBRARY"] = String(describing: library) - // Add the code coverage related variables. if buildParameters.testingParameters.enableCodeCoverage { // Defines the path at which the profraw files will be written on test execution. @@ -177,7 +171,7 @@ enum TestingSupport { // execution but is required when the tests are running in parallel as // SwiftPM repeatedly invokes the test binary with the test case name as // the filter. - let codecovProfile = buildParameters.buildPath.appending(components: "codecov", "default%m.profraw") + let codecovProfile = buildParameters.buildPath.appending(components: "codecov", "\(library)%m.profraw") env["LLVM_PROFILE_FILE"] = codecovProfile.pathString } #if !os(macOS) @@ -195,6 +189,11 @@ enum TestingSupport { env.appendPath(key: "DYLD_LIBRARY_PATH", value: sdkPlatformFrameworksPath.lib.pathString) } + // We aren't using XCTest's harness logic to run Swift Testing tests. + if library == .xctest { + env["SWIFT_TESTING_ENABLED"] = "0" + } + // Fast path when no sanitizers are enabled. if sanitizers.isEmpty { return env @@ -221,24 +220,21 @@ extension SwiftCommandState { enableCodeCoverage: Bool, enableTestability: Bool? = nil, shouldSkipBuilding: Bool = false, - experimentalTestOutput: Bool = false, - library: BuildParameters.Testing.Library + experimentalTestOutput: Bool = false ) throws -> (productsBuildParameters: BuildParameters, toolsBuildParameters: BuildParameters) { let productsBuildParameters = buildParametersForTest( modifying: try productsBuildParameters, enableCodeCoverage: enableCodeCoverage, enableTestability: enableTestability, shouldSkipBuilding: shouldSkipBuilding, - experimentalTestOutput: experimentalTestOutput, - library: library + experimentalTestOutput: experimentalTestOutput ) let toolsBuildParameters = buildParametersForTest( modifying: try toolsBuildParameters, enableCodeCoverage: enableCodeCoverage, enableTestability: enableTestability, shouldSkipBuilding: shouldSkipBuilding, - experimentalTestOutput: experimentalTestOutput, - library: library + experimentalTestOutput: experimentalTestOutput ) return (productsBuildParameters, toolsBuildParameters) } @@ -248,8 +244,7 @@ extension SwiftCommandState { enableCodeCoverage: Bool, enableTestability: Bool?, shouldSkipBuilding: Bool, - experimentalTestOutput: Bool, - library: BuildParameters.Testing.Library + experimentalTestOutput: Bool ) -> BuildParameters { var parameters = parameters @@ -266,8 +261,7 @@ extension SwiftCommandState { configuration: parameters.configuration, targetTriple: parameters.triple, forceTestDiscovery: explicitlyEnabledDiscovery, - testEntryPointPath: explicitlySpecifiedPath, - library: library + testEntryPointPath: explicitlySpecifiedPath ) parameters.testingParameters.enableCodeCoverage = enableCodeCoverage diff --git a/Sources/CoreCommands/Options.swift b/Sources/CoreCommands/Options.swift index 4e6aab11080..bcb2a4a3566 100644 --- a/Sources/CoreCommands/Options.swift +++ b/Sources/CoreCommands/Options.swift @@ -580,83 +580,36 @@ public struct TestLibraryOptions: ParsableArguments { help: "Enable support for XCTest") public var explicitlyEnableXCTestSupport: Bool? - /// Whether to enable support for XCTest. - public var enableXCTestSupport: Bool { - // Default to enabled. - explicitlyEnableXCTestSupport ?? true - } - - /// Whether to enable support for swift-testing (as explicitly specified by the user.) + /// Whether to enable support for Swift Testing (as explicitly specified by the user.) /// - /// Callers (other than `swift package init`) will generally want to use - /// ``enableSwiftTestingLibrarySupport(swiftCommandState:)`` since it will - /// take into account whether the package has a dependency on swift-testing. - @Flag(name: .customLong("experimental-swift-testing"), + /// Callers will generally want to use ``enableSwiftTestingLibrarySupport`` since it will + /// have the correct default value if the user didn't specify one. + @Flag(name: .customLong("swift-testing"), inversion: .prefixedEnableDisable, - help: "Enable experimental support for swift-testing") + help: "Enable support for Swift Testing") public var explicitlyEnableSwiftTestingLibrarySupport: Bool? - /// Whether to enable support for swift-testing. - public func enableSwiftTestingLibrarySupport( - swiftCommandState: SwiftCommandState - ) throws -> Bool { - // Honor the user's explicit command-line selection, if any. - if let callerSuppliedValue = explicitlyEnableSwiftTestingLibrarySupport { - return callerSuppliedValue - } - - // If the active package has a dependency on swift-testing, automatically enable support for it so that extra steps are not needed. - let workspace = try swiftCommandState.getActiveWorkspace() - let root = try swiftCommandState.getWorkspaceRoot() - let rootManifests = try temp_await { - workspace.loadRootManifests( - packages: root.packages, - observabilityScope: swiftCommandState.observabilityScope, - completion: $0 - ) - } - - // Is swift-testing among the dependencies of the package being built? - // If so, enable support. - let isEnabledByDependency = rootManifests.values.lazy - .flatMap(\.dependencies) - .map(\.identity) - .map(String.init(describing:)) - .contains("swift-testing") - if isEnabledByDependency { - swiftCommandState.observabilityScope.emit(debug: "Enabling swift-testing support due to its presence as a package dependency.") - return true - } - - // Is swift-testing the package being built itself (unlikely)? If so, - // enable support. - let isEnabledByName = root.packages.lazy - .map(PackageIdentity.init(path:)) - .map(String.init(describing:)) - .contains("swift-testing") - if isEnabledByName { - swiftCommandState.observabilityScope.emit(debug: "Enabling swift-testing support because it is a root package.") - return true + /// Legacy experimental equivalent of ``explicitlyEnableSwiftTestingLibrarySupport``. + /// + /// This option will be removed in a future update. + @Flag(name: .customLong("experimental-swift-testing"), + inversion: .prefixedEnableDisable, + help: .private) + public var explicitlyEnableExperimentalSwiftTestingLibrarySupport: Bool? + + /// Test whether or not a given library is enabled. + public func isEnabled(_ library: BuildParameters.Testing.Library) -> Bool { + switch library { + case .xctest: + explicitlyEnableXCTestSupport ?? true + case .swiftTesting: + explicitlyEnableSwiftTestingLibrarySupport ?? explicitlyEnableExperimentalSwiftTestingLibrarySupport ?? true } - - // Default to disabled since swift-testing is experimental (opt-in.) - return false } - /// Get the set of enabled testing libraries. - public func enabledTestingLibraries( - swiftCommandState: SwiftCommandState - ) throws -> Set { - var result = Set() - - if enableXCTestSupport { - result.insert(.xctest) - } - if try enableSwiftTestingLibrarySupport(swiftCommandState: swiftCommandState) { - result.insert(.swiftTesting) - } - - return result + /// The list of enabled testing libraries. + public var enabledTestingLibraries: [BuildParameters.Testing.Library] { + [.xctest, .swiftTesting].filter(isEnabled) } } diff --git a/Sources/PackageModel/UserToolchain.swift b/Sources/PackageModel/UserToolchain.swift index 10a0e995f05..3a9435dc286 100644 --- a/Sources/PackageModel/UserToolchain.swift +++ b/Sources/PackageModel/UserToolchain.swift @@ -387,6 +387,22 @@ public final class UserToolchain: Toolchain { ) } + public func getSwiftTestingHelper() throws -> AbsolutePath { + // The helper would be located in `.build/` directory when + // SwiftPM is built locally and `usr/libexec/swift/pm` directory in + // installed version. + let binDirectories = self.swiftSDK.toolset.rootPaths + + self.swiftSDK.toolset.rootPaths.map { + $0.parentDirectory.appending(components: ["libexec", "swift", "pm"]) + } + + return try UserToolchain.getTool( + "swiftpm-testing-helper", + binDirectories: binDirectories, + fileSystem: self.fileSystem + ) + } + internal static func deriveSwiftCFlags( triple: Triple, swiftSDK: SwiftSDK, diff --git a/Sources/SPMBuildCore/BuildParameters/BuildParameters+Testing.swift b/Sources/SPMBuildCore/BuildParameters/BuildParameters+Testing.swift index bdfb66bb6b8..6ad6108c4a5 100644 --- a/Sources/SPMBuildCore/BuildParameters/BuildParameters+Testing.swift +++ b/Sources/SPMBuildCore/BuildParameters/BuildParameters+Testing.swift @@ -40,13 +40,9 @@ extension BuildParameters { /// Whether this test product style requires additional, derived test targets, i.e. there must be additional test targets, beyond those /// listed explicitly in the package manifest, created in order to add additional behavior (such as entry point logic). + /// FIXME: remove this property since it's always true now. public var requiresAdditionalDerivedTestTargets: Bool { - switch self { - case .loadableBundle: - return false - case .entryPointExecutable: - return true - } + true } /// The explicitly-specified entry point file path, if this style of test product supports it and a path was specified. @@ -113,9 +109,6 @@ extension BuildParameters { } } - /// Which testing library to use for this build. - public var library: Library - public init( configuration: BuildConfiguration, targetTriple: Triple, @@ -123,8 +116,7 @@ extension BuildParameters { enableTestability: Bool? = nil, experimentalTestOutput: Bool = false, forceTestDiscovery: Bool = false, - testEntryPointPath: AbsolutePath? = nil, - library: Library = .xctest + testEntryPointPath: AbsolutePath? = nil ) { self.enableCodeCoverage = enableCodeCoverage self.experimentalTestOutput = experimentalTestOutput @@ -136,11 +128,10 @@ extension BuildParameters { // when building and testing in release mode, one can use the '--disable-testable-imports' flag // to disable testability in `swift test`, but that requires that the tests do not use the testable imports feature self.enableTestability = enableTestability ?? (.debug == configuration) - self.testProductStyle = (targetTriple.isDarwin() && library == .xctest) ? .loadableBundle : .entryPointExecutable( + self.testProductStyle = targetTriple.isDarwin() ? .loadableBundle : .entryPointExecutable( explicitlyEnabledDiscovery: forceTestDiscovery, explicitlySpecifiedPath: testEntryPointPath ) - self.library = library } } } diff --git a/Sources/SPMBuildCore/BuildParameters/BuildParameters.swift b/Sources/SPMBuildCore/BuildParameters/BuildParameters.swift index 5db1b5d6f9f..abade1ddf7d 100644 --- a/Sources/SPMBuildCore/BuildParameters/BuildParameters.swift +++ b/Sources/SPMBuildCore/BuildParameters/BuildParameters.swift @@ -293,16 +293,11 @@ public struct BuildParameters: Encodable { guard !self.triple.isWasm else { return try RelativePath(validating: "\(product.name).wasm") } - switch testingParameters.library { - case .xctest: - let base = "\(product.name).xctest" - if self.triple.isDarwin() { - return try RelativePath(validating: "\(base)/Contents/MacOS/\(product.name)") - } else { - return try RelativePath(validating: base) - } - case .swiftTesting: - return try RelativePath(validating: "\(product.name).swift-testing") + let base = "\(product.name).xctest" + if self.triple.isDarwin() { + return try RelativePath(validating: "\(base)/Contents/MacOS/\(product.name)") + } else { + return try RelativePath(validating: base) } case .macro: #if BUILD_MACROS_AS_DYLIBS diff --git a/Sources/SPMBuildCore/BuiltTestProduct.swift b/Sources/SPMBuildCore/BuiltTestProduct.swift index 881ade7175f..70f31901e3b 100644 --- a/Sources/SPMBuildCore/BuiltTestProduct.swift +++ b/Sources/SPMBuildCore/BuiltTestProduct.swift @@ -28,15 +28,8 @@ public struct BuiltTestProduct: Codable { /// When the test product is not bundled (for instance, when using XCTest on /// non-Darwin targets), this path is equal to ``binaryPath``. public var bundlePath: AbsolutePath { - // Go up the folder hierarchy until we find the .xctest or - // .swift-testing bundle. - let pathExtension: String - switch library { - case .xctest: - pathExtension = ".xctest" - case .swiftTesting: - pathExtension = ".swift-testing" - } + // Go up the folder hierarchy until we find the .xctest bundle. + let pathExtension = ".xctest" let hierarchySequence = sequence(first: binaryPath, next: { $0.isRoot ? nil : $0.parentDirectory }) guard let bundlePath = hierarchySequence.first(where: { $0.basename.hasSuffix(pathExtension) }) else { fatalError("could not find test bundle path from '\(binaryPath)'") @@ -45,18 +38,14 @@ public struct BuiltTestProduct: Codable { return bundlePath } - /// The library used to build this test product. - public var library: BuildParameters.Testing.Library - /// Creates a new instance. /// - Parameters: /// - productName: The test product name. /// - binaryPath: The path of the test binary. /// - packagePath: The path to the package this product was declared in. - public init(productName: String, binaryPath: AbsolutePath, packagePath: AbsolutePath, library: BuildParameters.Testing.Library) { + public init(productName: String, binaryPath: AbsolutePath, packagePath: AbsolutePath) { self.productName = productName self.binaryPath = binaryPath self.packagePath = packagePath - self.library = library } } diff --git a/Sources/XCBuildSupport/XcodeBuildSystem.swift b/Sources/XCBuildSupport/XcodeBuildSystem.swift index 5b8803a0481..bee340282e8 100644 --- a/Sources/XCBuildSupport/XcodeBuildSystem.swift +++ b/Sources/XCBuildSupport/XcodeBuildSystem.swift @@ -58,8 +58,7 @@ public final class XcodeBuildSystem: SPMBuildCore.BuildSystem { BuiltTestProduct( productName: product.name, binaryPath: binaryPath, - packagePath: package.path, - library: buildParameters.testingParameters.library + packagePath: package.path ) ) } diff --git a/Sources/swift-test/Entrypoint.swift b/Sources/swift-test/Entrypoint.swift index eb1c4d1b37a..654d0b881d4 100644 --- a/Sources/swift-test/Entrypoint.swift +++ b/Sources/swift-test/Entrypoint.swift @@ -11,10 +11,44 @@ //===----------------------------------------------------------------------===// import Commands +#if canImport(Darwin.C) +import Darwin.C +#endif @main struct Entrypoint { - static func main() async { + static func main() async throws { +#if canImport(Darwin.C) + // HACK: use the swift-test executable as a host for the .xctest bundle + // when running Swift Testing tests. + let args = CommandLine.arguments + if args.count >= 3, args[1] == "--test-bundle-path" { + let bundlePath = args[2] + guard let image = dlopen(bundlePath, RTLD_LAZY) else { + let errorMessage: String = dlerror().flatMap { +#if compiler(>=6) + String(validatingCString: $0) +#else + String(validatingUTF8: $0) +#endif + } ?? "An unknown error occurred." + fatalError("Failed to open test bundle at path \(bundlePath): \(errorMessage)") + } + defer { + dlclose(image) + } + + // Find and call the main function from the image. This function may + // link to the copy of Swift Testing included with Xcode, or may link to + // a copy that's included as a package dependency. + let main = dlsym(image, "main").map { + unsafeBitCast($0, to: (@convention(c) (CInt, UnsafeMutablePointer?>) -> CInt).self) + } + if let main { + exit(main(CommandLine.argc, CommandLine.unsafeArgv)) + } + } +#endif await SwiftTestCommand.main() } } diff --git a/Sources/swiftpm-testing-helper/Entrypoint.swift b/Sources/swiftpm-testing-helper/Entrypoint.swift new file mode 100644 index 00000000000..8c0df535480 --- /dev/null +++ b/Sources/swiftpm-testing-helper/Entrypoint.swift @@ -0,0 +1,49 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2014-2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Darwin.C + +@main +struct Entrypoint { + static func main() throws { + let args = CommandLine.arguments + if args.count >= 3, args[1] == "--test-bundle-path" { + let bundlePath = args[2] + guard let image = dlopen(bundlePath, RTLD_LAZY) else { + let errorMessage: String = dlerror().flatMap { + #if compiler(>=6) + String(validatingCString: $0) + #else + String(validatingUTF8: $0) + #endif + } ?? "An unknown error occurred." + fatalError("Failed to open test bundle at path \(bundlePath): \(errorMessage)") + } + defer { + dlclose(image) + } + + // Find and call the main function from the image. This function may + // link to the copy of Swift Testing included with Xcode, or may link to + // a copy that's included as a package dependency. + let main = dlsym(image, "main").map { + unsafeBitCast( + $0, + to: (@convention(c) (CInt, UnsafeMutablePointer?>) -> CInt).self + ) + } + if let main { + exit(main(CommandLine.argc, CommandLine.unsafeArgv)) + } + } + } +} diff --git a/Tests/BuildTests/BuildPlanTests.swift b/Tests/BuildTests/BuildPlanTests.swift index bd27cfa7e2f..5520bfd4a37 100644 --- a/Tests/BuildTests/BuildPlanTests.swift +++ b/Tests/BuildTests/BuildPlanTests.swift @@ -1146,9 +1146,6 @@ final class BuildPlanTests: XCTestCase { )) XCTAssertEqual(Set(result.productMap.keys.map(\.productName)), ["APackageTests"]) - #if os(macOS) - XCTAssertEqual(Set(result.targetMap.keys.map(\.moduleName)), ["ATarget", "BTarget", "ATargetTests"]) - #else XCTAssertEqual(Set(result.targetMap.keys.map(\.moduleName)), [ "APackageTests", "APackageDiscoveredTests", @@ -1156,7 +1153,6 @@ final class BuildPlanTests: XCTestCase { "ATargetTests", "BTarget", ]) - #endif } func testBasicReleasePackage() throws { @@ -2213,13 +2209,7 @@ final class BuildPlanTests: XCTestCase { observabilityScope: observability.topScope )) result.checkProductsCount(1) - #if os(macOS) - result.checkTargetsCount(2) - #else - // On non-Apple platforms, when a custom entry point file is present (e.g. XCTMain.swift), there is one - // additional target for the synthesized test entry point. result.checkTargetsCount(3) - #endif let buildPath = result.plan.productsBuildPath @@ -2288,6 +2278,8 @@ final class BuildPlanTests: XCTestCase { buildPath.appending(components: "Modules", "Foo.swiftmodule").pathString, "-Xlinker", "-add_ast_path", "-Xlinker", buildPath.appending(components: "Modules", "FooTests.swiftmodule").pathString, + "-Xlinker", "-add_ast_path", "-Xlinker", + buildPath.appending(components: "PkgPackageTests.build", "PkgPackageTests.swiftmodule").pathString, "-g", ] ) diff --git a/Tests/CommandsTests/TestCommandTests.swift b/Tests/CommandsTests/TestCommandTests.swift index 7b693cabc8b..4c0ef5a494b 100644 --- a/Tests/CommandsTests/TestCommandTests.swift +++ b/Tests/CommandsTests/TestCommandTests.swift @@ -197,7 +197,6 @@ final class TestCommandTests: CommandsTestCase { XCTAssertNoMatch(stdout, .contains("testExample2")) XCTAssertNoMatch(stdout, .contains("testExample3")) XCTAssertNoMatch(stdout, .contains("testExample4")) - XCTAssertMatch(stderr, .contains("No matching test cases were run")) } } diff --git a/Utilities/bootstrap b/Utilities/bootstrap index 63b6aa464d1..62fbbc4b21a 100755 --- a/Utilities/bootstrap +++ b/Utilities/bootstrap @@ -435,7 +435,14 @@ def install(args): def install_swiftpm(prefix, args): # Install the swift-package-manager tool and create symlinks to it. cli_tool_dest = os.path.join(prefix, "bin") + aux_tool_dest = os.path.join(prefix, "libexec", "swift", "pm") + install_binary(args, "swift-package-manager", os.path.join(cli_tool_dest, "swift-package"), destination_is_directory=False) + + # `swiftpm-testing-helper` only exists on Darwin platforms + if os.path.exists(os.path.join(args.bin_dir, "swiftpm-testing-helper")): + install_binary(args, "swiftpm-testing-helper", aux_tool_dest) + for tool in ["swift-build", "swift-test", "swift-run", "swift-package-collection", "swift-package-registry", "swift-sdk", "swift-experimental-sdk"]: src = "swift-package" dest = os.path.join(cli_tool_dest, tool)