From 1d9ea962dde62bb4e33456a411b652246d60b5c2 Mon Sep 17 00:00:00 2001 From: Rachel Brindle Date: Wed, 14 May 2025 23:11:38 -0700 Subject: [PATCH 01/10] Preliminary implementation of polling expectations --- .../Expectations/Expectation+Macro.swift | 167 +++++++++ .../ExpectationChecking+Macro.swift | 96 +++++ Sources/Testing/Polling/CallPolling.swift | 192 ++++++++++ Sources/Testing/Polling/Polling.swift | 293 +++++++++++++++ Sources/Testing/Polling/PollingBehavior.swift | 28 ++ Tests/TestingTests/PollingTests.swift | 352 ++++++++++++++++++ 6 files changed, 1128 insertions(+) create mode 100644 Sources/Testing/Polling/CallPolling.swift create mode 100644 Sources/Testing/Polling/Polling.swift create mode 100644 Sources/Testing/Polling/PollingBehavior.swift create mode 100644 Tests/TestingTests/PollingTests.swift diff --git a/Sources/Testing/Expectations/Expectation+Macro.swift b/Sources/Testing/Expectations/Expectation+Macro.swift index f85c7042b..5230417d4 100644 --- a/Sources/Testing/Expectations/Expectation+Macro.swift +++ b/Sources/Testing/Expectations/Expectation+Macro.swift @@ -572,3 +572,170 @@ public macro require( sourceLocation: SourceLocation = #_sourceLocation, performing expression: @escaping @Sendable @convention(thin) () async throws -> Void ) -> ExitTest.Result = #externalMacro(module: "TestingMacros", type: "ExitTestRequireMacro") + +// MARK: - Polling Expectations + +/// Continuously check an expression until it matches the given PollingBehavior +/// +/// - Parameters: +/// - until: The desired PollingBehavior to check for. +/// - timeout: How long to run poll the expression until stopping. +/// - comment: A comment describing the expectation. +/// - sourceLocation: The source location to which the recorded expectations +/// and issues should be attributed. +/// - expression: The expression to be evaluated. +/// +/// Use this overload of `#expect()` when you wish to poll whether a value +/// changes as the result of activity in another task/queue/thread. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +@freestanding(expression) public macro expect( + until pollingBehavior: PollingBehavior, + timeout: Duration = .seconds(60), + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + expression: @Sendable () async throws -> Bool +) = #externalMacro(module: "TestingMacros", type: "ExpectMacro") + +/// Continuously check an expression until it matches the given PollingBehavior +/// +/// - Parameters: +/// - until: The desired PollingBehavior to check for. +/// - throws: The error the expression should throw. +/// - timeout: How long to run poll the expression until stopping. +/// - comment: A comment describing the expectation. +/// - sourceLocation: The source location to which the recorded expectations +/// and issues should be attributed. +/// - expression: The expression to be evaluated. +/// +/// Use this overload of `#expect()` when you wish to poll whether a value +/// changes as the result of activity in another task/queue/thread. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +@freestanding(expression) public macro expect( + until pollingBehavior: PollingBehavior, + throws error: E, + timeout: Duration = .seconds(60), + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + expression: @Sendable () async throws -> Bool +) = #externalMacro(module: "TestingMacros", type: "ExpectMacro") +where E: Error & Equatable + +/// Continuously check an expression until it matches the given PollingBehavior +/// +/// - Parameters: +/// - until: The desired PollingBehavior to check for. +/// - timeout: How long to run poll the expression until stopping. +/// - comment: A comment describing the expectation. +/// - sourceLocation: The source location to which the recorded expectations +/// and issues should be attributed. +/// - expression: The expression to be evaluated. +/// - throws: A closure to confirm if the expression throws the expected error. +/// +/// Use this overload of `#expect()` when you wish to poll whether a value +/// changes as the result of activity in another task/queue/thread. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +@freestanding(expression) public macro expect( + until pollingBehavior: PollingBehavior, + timeout: Duration = .seconds(60), + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + performing: @Sendable () async throws -> Bool, + throws errorMatcher: @Sendable (any Error) async throws -> Bool +) = #externalMacro(module: "TestingMacros", type: "ExpectMacro") + +/// Continuously check an expression until it matches the given PollingBehavior +/// +/// - Parameters: +/// - until: The desired PollingBehavior to check for. +/// - timeout: How long to run poll the expression until stopping. +/// - comment: A comment describing the expectation. +/// - sourceLocation: The source location to which the recorded expectations +/// and issues should be attributed. +/// - expression: The expression to be evaluated. +/// +/// Use this overload of `#require()` when you wish to poll whether a value +/// changes as the result of activity in another task/queue/thread. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +@freestanding(expression) public macro require( + until pollingBehavior: PollingBehavior, + timeout: Duration = .seconds(60), + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + expression: @Sendable () async throws -> Bool +) = #externalMacro(module: "TestingMacros", type: "RequireMacro") + +/// Continuously check an expression until it matches the given PollingBehavior +/// +/// - Parameters: +/// - until: The desired PollingBehavior to check for. +/// - timeout: How long to run poll the expression until stopping. +/// - comment: A comment describing the expectation. +/// - sourceLocation: The source location to which the recorded expectations +/// and issues should be attributed. +/// - expression: The expression to be evaluated. +/// +/// Use this overload of `#require()` when you wish to poll whether a value +/// changes as the result of activity in another task/queue/thread. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +@freestanding(expression) public macro require( + until pollingBehavior: PollingBehavior, + timeout: Duration = .seconds(60), + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + expression: @Sendable () async throws -> R? +) = #externalMacro(module: "TestingMacros", type: "RequireMacro") +where R: Sendable + +/// Continuously check an expression until it matches the given PollingBehavior +/// +/// - Parameters: +/// - until: The desired PollingBehavior to check for. +/// - throws: The error the expression should throw +/// - timeout: How long to run poll the expression until stopping. +/// - comment: A comment describing the expectation. +/// - sourceLocation: The source location to which the recorded expectations +/// and issues should be attributed. +/// - expression: The expression to be evaluated. +/// +/// Use this overload of `#require()` when you wish to poll whether a value +/// changes as the result of activity in another task/queue/thread. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +@freestanding(expression) public macro require( + until pollingBehavior: PollingBehavior, + throws error: E, + timeout: Duration = .seconds(60), + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + expression: @Sendable () async throws -> Bool +) = #externalMacro(module: "TestingMacros", type: "RequireMacro") +where E: Error & Equatable + +/// Continuously check an expression until it matches the given PollingBehavior +/// +/// - Parameters: +/// - until: The desired PollingBehavior to check for. +/// - timeout: How long to run poll the expression until stopping. +/// - comment: A comment describing the expectation. +/// - sourceLocation: The source location to which the recorded expectations +/// and issues should be attributed. +/// - expression: The expression to be evaluated. +/// - throws: A closure to confirm if the expression throws the expected error. +/// +/// Use this overload of `#require()` when you wish to poll whether a value +/// changes as the result of activity in another task/queue/thread. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +@freestanding(expression) public macro require( + until pollingBehavior: PollingBehavior, + timeout: Duration = .seconds(60), + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + expression: @Sendable () async throws -> Bool, + throws errorMatcher: @Sendable (any Error) async throws -> Bool +) = #externalMacro(module: "TestingMacros", type: "RequireMacro") diff --git a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift index 6d3093f2a..8144a6908 100644 --- a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift +++ b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift @@ -1201,6 +1201,102 @@ public func __checkClosureCall( } #endif +// MARK: - Polling + +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public func __checkClosureCall( + until behavior: PollingBehavior, + timeout: Duration = .seconds(60), + performing closure: @escaping @Sendable () async throws -> Bool, + expression: __Expression, + comments: @autoclosure () -> [Comment], + isRequired: Bool, + sourceLocation: SourceLocation +) async -> Result { + await callPolling( + behavior: behavior, + timeout: timeout, + closure: closure, + expression: expression, + comments: comments(), + isRequired: isRequired, + sourceLocation: sourceLocation + ) +} + +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public func __checkClosureCall( + until behavior: PollingBehavior, + throws error: E, + timeout: Duration = .seconds(60), + performing closure: @escaping @Sendable () async throws -> Bool, + expression: __Expression, + comments: @autoclosure () -> [Comment], + isRequired: Bool, + sourceLocation: SourceLocation +) async -> Result where E: Error & Equatable { + await callPolling( + behavior: behavior, + throws: error, + timeout: timeout, + closure: closure, + expression: expression, + comments: comments(), + isRequired: isRequired, + sourceLocation: sourceLocation + ) +} + +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public func __checkClosureCall( + until behavior: PollingBehavior, + timeout: Duration = .seconds(60), + performing closure: @escaping @Sendable () async throws -> Bool, + throws errorMatcher: @escaping @Sendable (any Error) async throws -> Bool, + expression: __Expression, + comments: @autoclosure () -> [Comment], + isRequired: Bool, + sourceLocation: SourceLocation +) async -> Result { + await callPolling( + behavior: behavior, + timeout: timeout, + closure: closure, + errorMatcher: errorMatcher, + expression: expression, + comments: comments(), + isRequired: isRequired, + sourceLocation: sourceLocation + ) +} + +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public func __checkClosureCall( + until behavior: PollingBehavior, + timeout: Duration = .seconds(60), + performing closure: @escaping @Sendable () async throws -> R, + throws errorMatcher: @escaping @Sendable (any Error) async throws -> Bool, + expression: __Expression, + comments: @autoclosure () -> [Comment], + isRequired: Bool, + sourceLocation: SourceLocation +) async -> Result where R: Sendable { + await callPolling( + behavior: behavior, + timeout: timeout, + closure: closure, + errorMatcher: errorMatcher, + expression: expression, + comments: comments(), + isRequired: isRequired, + sourceLocation: sourceLocation + ) +} + // MARK: - /// Generate a description of an error that includes its type name if not diff --git a/Sources/Testing/Polling/CallPolling.swift b/Sources/Testing/Polling/CallPolling.swift new file mode 100644 index 000000000..4f751f761 --- /dev/null +++ b/Sources/Testing/Polling/CallPolling.swift @@ -0,0 +1,192 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +/// Poll an expression to check that it passes until the given duration +/// +/// - Parameters: +/// - behavior: The PollingBehavior to use. +/// - timeout: How long to poll for until we time out. +/// - closure: The closure to continuously evaluate. +/// - expression: The expression, corresponding to `condition`, that is being +/// evaluated (if available at compile time.) +/// - comments: An array of comments describing the expectation. This array +/// may be empty. +/// - isRequired: Whether or not the expectation is required. The value of +/// this argument does not affect whether or not an error is thrown on +/// failure. +/// - sourceLocation: The source location of the expectation. +/// +/// This function contains the implementation for `#expect(until:)` when no +/// error is expected and no value should be returned. +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +func callPolling( + behavior: PollingBehavior, + timeout: Duration, + closure: @escaping @Sendable () async throws -> Bool, + expression: __Expression, + comments: [Comment], + isRequired: Bool, + sourceLocation: SourceLocation +) async -> Result{ + await Polling.run( + behavior: behavior, + timeout: timeout, + closure: { + do { + return try await closure() + } catch { + return false + } + }, + expression: expression, + comments: comments, + isRequired: isRequired, + sourceLocation: sourceLocation + ) +} + +/// Poll an expression to check that it passes until the given duration +/// +/// - Parameters: +/// - behavior: The PollingBehavior to use. +/// - timeout: How long to poll for until we time out. +/// - closure: The closure to continuously evaluate. +/// - expression: The expression, corresponding to `condition`, that is being +/// evaluated (if available at compile time.) +/// - comments: An array of comments describing the expectation. This array +/// may be empty. +/// - isRequired: Whether or not the expectation is required. The value of +/// this argument does not affect whether or not an error is thrown on +/// failure. +/// - sourceLocation: The source location of the expectation. +/// +/// This function contains the implementation for `#expect(until:)` when an +/// equatable error is expected and no value should be returned. +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +func callPolling( + behavior: PollingBehavior, + throws error: E, + timeout: Duration, + closure: @escaping @Sendable () async throws -> Bool, + expression: __Expression, + comments: [Comment], + isRequired: Bool, + sourceLocation: SourceLocation +) async -> Result where E: Error & Equatable { + await Polling.run( + behavior: behavior, + timeout: timeout, + closure: { + do { + _ = try await closure() + return false + } catch let thrownError as E { + return thrownError == error + } catch { + return false + } + }, + expression: expression, + comments: comments, + isRequired: isRequired, + sourceLocation: sourceLocation + ) +} + +/// Poll an expression to check that it passes until the given duration +/// +/// - Parameters: +/// - behavior: The PollingBehavior to use. +/// - timeout: How long to poll for until we time out. +/// - closure: The closure to continuously evaluate. +/// - expression: The expression, corresponding to `condition`, that is being +/// evaluated (if available at compile time.) +/// - comments: An array of comments describing the expectation. This array +/// may be empty. +/// - isRequired: Whether or not the expectation is required. The value of +/// this argument does not affect whether or not an error is thrown on +/// failure. +/// - sourceLocation: The source location of the expectation. +/// +/// This function contains the implementation for `#expect(until:)` when an +/// error is expected and no value should be returned. +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +func callPolling( + behavior: PollingBehavior, + timeout: Duration, + closure: @escaping @Sendable () async throws -> Bool, + errorMatcher: @escaping @Sendable (any Error) async throws -> Bool, + expression: __Expression, + comments: [Comment], + isRequired: Bool, + sourceLocation: SourceLocation +) async -> Result { + await Polling.run( + behavior: behavior, + timeout: timeout, + closure: { + do { + _ = try await closure() + return false + } catch { + return (try? await errorMatcher(error)) == true + } + }, + expression: expression, + comments: comments, + isRequired: isRequired, + sourceLocation: sourceLocation + ) +} + +/// Poll an expression to check that it passes until the given duration +/// +/// - Parameters: +/// - behavior: The PollingBehavior to use. +/// - timeout: How long to poll for until we time out. +/// - closure: The closure to continuously evaluate. +/// - expression: The expression, corresponding to `condition`, that is being +/// evaluated (if available at compile time.) +/// - comments: An array of comments describing the expectation. This array +/// may be empty. +/// - isRequired: Whether or not the expectation is required. The value of +/// this argument does not affect whether or not an error is thrown on +/// failure. +/// - sourceLocation: The source location of the expectation. +/// +/// This function contains the implementation for `#require(until:)` when no +/// error is expected and a value should be returned. +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +func callPolling( + behavior: PollingBehavior, + timeout: Duration, + closure: @escaping @Sendable () async throws -> R?, + errorMatcher: @escaping @Sendable (any Error) async throws -> Bool, + expression: __Expression, + comments: [Comment], + isRequired: Bool, + sourceLocation: SourceLocation +) async -> Result where R: Sendable { + await Polling.run( + behavior: behavior, + timeout: timeout, + closure: { + do { + return try await closure() + } catch { + return nil + } + }, + expression: expression, + comments: comments, + isRequired: isRequired, + sourceLocation: sourceLocation + ) +} diff --git a/Sources/Testing/Polling/Polling.swift b/Sources/Testing/Polling/Polling.swift new file mode 100644 index 000000000..dd09e1a64 --- /dev/null +++ b/Sources/Testing/Polling/Polling.swift @@ -0,0 +1,293 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +/// A type for managing polling +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +struct Polling { + /// Run polling for a closure that evaluates to a boolean value. + /// + /// - Parameters: + /// - behavior: The PollingBehavior to use. + /// - timeout: How long to poll for until we time out. + /// - closure: The closure to continuously evaluate. + /// - expression: The expression, corresponding to `condition`, that is being + /// evaluated (if available at compile time.) + /// - comments: An array of comments describing the expectation. This array + /// may be empty. + /// - isRequired: Whether or not the expectation is required. The value of + /// this argument does not affect whether or not an error is thrown on + /// failure. + /// - sourceLocation: The source location of the expectation. + static func run( + behavior: PollingBehavior, + timeout: Duration, + closure: @escaping @Sendable () async -> Bool, + expression: __Expression, + comments: [Comment], + isRequired: Bool, + sourceLocation: SourceLocation + ) async -> Result { + var expectation = Expectation( + evaluatedExpression: expression, + isPassing: true, + isRequired: isRequired, + sourceLocation: sourceLocation + ) + + let result = await poll(expression: closure, behavior: behavior, timeout: timeout) + + let sourceContext = SourceContext(backtrace: nil, sourceLocation: sourceLocation) + + switch result { + case .timedOut: + expectation.isPassing = false + Issue( + kind: .expectationFailed(expectation), + comments: comments, + sourceContext: sourceContext + ).record() + case .timedOutWithoutRunning: + expectation.isPassing = false + Issue( + kind: .expectationFailed(expectation), + comments: comments, + sourceContext: sourceContext + ).record() + case .finished: + return __checkValue( + true, + expression: expression, + comments: comments, + isRequired: isRequired, + sourceLocation: sourceLocation + ) + case .failed: + return __checkValue( + false, + expression: expression, + comments: comments, + isRequired: isRequired, + sourceLocation: sourceLocation + ) + case .cancelled: + Issue( + kind: .system, + comments: comments, + sourceContext: sourceContext + ).record() + } + + return .failure(ExpectationFailedError(expectation: expectation)) + } + + /// Run polling for a closure that evaluates to an optional value. + /// + /// - Parameters: + /// - behavior: The PollingBehavior to use. + /// - timeout: How long to poll for until we time out. + /// - closure: The closure to continuously evaluate. + /// - expression: The expression, corresponding to `condition`, that is being + /// evaluated (if available at compile time.) + /// - comments: An array of comments describing the expectation. This array + /// may be empty. + /// - isRequired: Whether or not the expectation is required. The value of + /// this argument does not affect whether or not an error is thrown on + /// failure. + /// - sourceLocation: The source location of the expectation. + static func run( + behavior: PollingBehavior, + timeout: Duration, + closure: @escaping @Sendable () async -> R?, + expression: __Expression, + comments: [Comment], + isRequired: Bool, + sourceLocation: SourceLocation + ) async -> Result where R: Sendable { + var expectation = Expectation( + evaluatedExpression: expression, + isPassing: true, + isRequired: isRequired, + sourceLocation: sourceLocation + ) + + let recorder = Recorder() + + let result = await poll(expression: { + if let value = await closure() { + await recorder.record(value: value) + return true + } + return false + }, behavior: behavior, timeout: timeout) + + let sourceContext = SourceContext(backtrace: nil, sourceLocation: sourceLocation) + + switch result { + case .timedOut: + expectation.isPassing = false + Issue( + kind: .expectationFailed(expectation), + comments: comments, + sourceContext: sourceContext + ).record() + case .timedOutWithoutRunning: + expectation.isPassing = false + Issue( + kind: .expectationFailed(expectation), + comments: comments, + sourceContext: sourceContext + ).record() + case .finished: + return __checkValue( + await recorder.lastValue, + expression: expression, + comments: comments, + isRequired: isRequired, + sourceLocation: sourceLocation + ) + case .failed: + return __checkValue( + nil, + expression: expression, + comments: comments, + isRequired: isRequired, + sourceLocation: sourceLocation + ) + case .cancelled: + Issue( + kind: .system, + comments: comments, + sourceContext: sourceContext + ).record() + } + + return .failure(ExpectationFailedError(expectation: expectation)) + } + + /// A type to record the last value returned by a closure returning an optional + /// This is only used in the `#require(until:)` macro returning an optional. + private actor Recorder { + var lastValue: R? + + /// Record a new value to be returned + func record(value: R) { + self.lastValue = value + } + } + + /// The result of polling expressions + private enum PollResult { + /// The polling timed out, and the expression had run at least once. + case timedOut + /// The polling timed out, but the expression had not finished running in + /// that time. + case timedOutWithoutRunning + /// The expression exited early, and we will report a success status. + case finished + /// The expression returned false under PollingBehavior.passesAlways + case failed + /// The polling was cancelled before polling could finish + case cancelled + } + + /// The poll manager. + /// + /// This function contains the logic for continuously polling an expression, + /// as well as the logic for cancelling the polling once it times out. + /// + /// - Parameters: + /// - expression: An expression to continuously evaluate + /// - behavior: The polling behavior to use + /// - timeout: How long to poll for unitl the timeout triggers. + /// - Returns: The result of this polling. + private static func poll( + expression: @escaping @Sendable () async -> Bool, + behavior: PollingBehavior, + timeout: Duration + ) async -> PollResult { + let pollingProcessor = PollingProcessor(behavior: behavior) + return await withTaskGroup { taskGroup in + taskGroup.addTask { + do { + try await Task.sleep(for: timeout) + } catch {} + // Task.sleep will only throw if it's cancelled, at which point this + // taskgroup has already returned and we don't care about the value + // returned here. + return await pollingProcessor.didTimeout() + } + taskGroup.addTask { + while Task.isCancelled == false { + let expressionPassed = await expression() + if let result = await pollingProcessor.expressionFinished(result: expressionPassed) { + return result + } + } + // The expression was cancelled without having been finished. + // This should end up being reported as a timeout error, due to + // the earlier task added to this task group. + // But there's a chance that the overall task was cancelled. + // in which case, we should report that as a system error. + return PollResult.cancelled + } + + defer { taskGroup.cancelAll() } + return await taskGroup.next() ?? .timedOut + } + } + + /// A type to process events from `Polling.poll`. + private actor PollingProcessor { + let behavior: PollingBehavior + var hasRun = false + + init(behavior: PollingBehavior) { + self.behavior = behavior + } + + /// Record a timeout event from polling. + func didTimeout() -> PollResult { + if !hasRun { + return PollResult.timedOutWithoutRunning + } + switch behavior { + case .passesOnce: + return PollResult.timedOut + case .passesAlways: + return PollResult.finished + } + } + + /// Record that an expression finished running + /// + /// - Parameters: + /// - Result: Whether or not the polled expression passed or not. + /// + /// - Returns: A non-nil PollResult if polling should exit, otherwise nil. + func expressionFinished(result: Bool) -> PollResult? { + hasRun = true + + switch behavior { + case .passesOnce: + if result { + return .finished + } else { + return nil + } + case .passesAlways: + if !result { + return .failed + } else { + return nil + } + } + } + } +} diff --git a/Sources/Testing/Polling/PollingBehavior.swift b/Sources/Testing/Polling/PollingBehavior.swift new file mode 100644 index 000000000..ccfe6045b --- /dev/null +++ b/Sources/Testing/Polling/PollingBehavior.swift @@ -0,0 +1,28 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +/// A type representing the behavior to use while polling. +@_spi(Experimental) +@frozen public enum PollingBehavior { + /// Continuously evaluate the expression until the first time it returns + /// true. + /// If it does not pass once by the time the timeout is reached, then a + /// failure will be reported. + case passesOnce + + /// Continuously evaluate the expression until the first time it returns + /// false. + /// If the expression returns false, then a failure will be reported. + /// If the expression only returns true before the timeout is reached, then + /// no failure will be reported. + /// If the expression does not finish evaluating before the timeout is + /// reached, then a failure will be reported. + case passesAlways +} diff --git a/Tests/TestingTests/PollingTests.swift b/Tests/TestingTests/PollingTests.swift new file mode 100644 index 000000000..79ce2a136 --- /dev/null +++ b/Tests/TestingTests/PollingTests.swift @@ -0,0 +1,352 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing + +@Suite("Polling Tests") +struct PollingTests { + @Suite("PollingBehavior.passesOnce") + struct PassesOnceBehavior { + let delta = Duration.seconds(6) + + @Test("Simple passing expressions") func trivialHappyPath() async { + await #expect(until: .passesOnce) { true } + + await #expect(until: .passesOnce, throws: PollingTestSampleError.ohNo) { + throw PollingTestSampleError.ohNo + } + + await #expect(until: .passesOnce, performing: { + throw PollingTestSampleError.secondCase + }, throws: { error in + (error as? PollingTestSampleError) == .secondCase + }) + + await #expect(until: .passesOnce, throws: PollingTestSampleError.ohNo) { + throw PollingTestSampleError.ohNo + } + } + + @Test("Simple failing expressions") func trivialSadPath() async { + await confirmation("Polling failed", expectedCount: 1) { failed in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case .issueRecorded = event.kind { + failed() + } + } + await Test { + await #expect(until: .passesOnce) { false } + }.run(configuration: configuration) + } + } + + @Test("When the value changes from false to true during execution") func changingFromFail() async { + let incrementor = Incrementor() + + await #expect(until: .passesOnce) { + await incrementor.increment() == 2 + // this will pass only on the second invocation + // This checks that we really are only running the expression until + // the first time it passes. + } + + // and then we check the count just to double check. + #expect(await incrementor.count == 2) + } + + @Test("Unexpected Errors are treated as returning false") + func errorsReported() async { + await confirmation("Polling failed", expectedCount: 1) { failed in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case .issueRecorded = event.kind { + failed() + } + } + await Test { + await #expect(until: .passesOnce) { + throw PollingTestSampleError.ohNo + } + }.run(configuration: configuration) + } + } + } + + @Suite("PollingBehavior.passesAlways") + struct PassesAlwaysBehavior { + // use a very generous delta for CI reasons. + let delta = Duration.seconds(6) + + @Test("Simple passing expressions") func trivialHappyPath() async { + await #expect(until: .passesAlways) { true } + } + + @Test("Simple failing expressions") func trivialSadPath() async { + await confirmation("Polling failed", expectedCount: 1) { failed in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case .issueRecorded = event.kind { + failed() + } + } + await Test { + await #expect(until: .passesAlways) { false } + }.run(configuration: configuration) + } + } + + @Test("if the closures starts off as false, but would become true") + func changingFromFail() async { + let incrementor = Incrementor() + + await confirmation("Polling failed", expectedCount: 1) { failed in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case .issueRecorded = event.kind { + failed() + } + } + await Test { + await #expect(until: .passesAlways) { + await incrementor.increment() == 2 + // this will pass only on the second invocation + // This checks that we fail the test if it immediately returns false + } + }.run(configuration: configuration) + } + + #expect(await incrementor.count == 1) + } + + @Test("if the closure continues to pass") + func continuousCalling() async { + let incrementor = Incrementor() + + await #expect(until: .passesAlways) { + _ = await incrementor.increment() + return true + } + + #expect(await incrementor.count > 1) + } + + @Test("Unexpected Errors will automatically exit & fail") func errorsReported() async { + await confirmation("Polling failed", expectedCount: 1) { failed in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case .issueRecorded = event.kind { + failed() + } + } + await Test { + await #expect(until: .passesAlways) { + throw PollingTestSampleError.ohNo + } + }.run(configuration: configuration) + } + } + } + + @Suite("Duration Tests", .disabled("time-sensitive")) struct DurationTests { + @Suite("PollingBehavior.passesOnce") + struct PassesOnceBehavior { + let delta = Duration.seconds(6) + + @Test("Simple passing expressions") func trivialHappyPath() async { + let duration = await Test.Clock().measure { + await #expect(until: .passesOnce) { true } + + await #expect(until: .passesOnce, throws: PollingTestSampleError.ohNo) { + throw PollingTestSampleError.ohNo + } + + await #expect(until: .passesOnce, performing: { + throw PollingTestSampleError.secondCase + }, throws: { error in + (error as? PollingTestSampleError) == .secondCase + }) + + await #expect(until: .passesOnce, throws: PollingTestSampleError.ohNo) { + throw PollingTestSampleError.ohNo + } + } + #expect(duration.isCloseTo(other: .zero, within: delta)) + } + + @Test("Simple failing expressions") func trivialSadPath() async { + let duration = await Test.Clock().measure { + await confirmation("Polling failed", expectedCount: 1) { failed in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case .issueRecorded = event.kind { + failed() + } + } + await Test { + await #expect(until: .passesOnce) { false } + }.run(configuration: configuration) + } + } + #expect(duration.isCloseTo(other: .seconds(60), within: delta)) + } + + @Test("When the value changes from false to true during execution") func changingFromFail() async { + let incrementor = Incrementor() + + let duration = await Test.Clock().measure { + await #expect(until: .passesOnce) { + await incrementor.increment() == 2 + // this will pass only on the second invocation + // This checks that we really are only running the expression until + // the first time it passes. + } + } + + // and then we check the count just to double check. + #expect(await incrementor.count == 2) + #expect(duration.isCloseTo(other: .zero, within: delta)) + } + + @Test("Unexpected Errors are treated as returning false") + func errorsReported() async { + let duration = await Test.Clock().measure { + await confirmation("Polling failed", expectedCount: 1) { failed in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case .issueRecorded = event.kind { + failed() + } + } + await Test { + await #expect(until: .passesOnce) { + throw PollingTestSampleError.ohNo + } + }.run(configuration: configuration) + } + } + #expect(duration.isCloseTo(other: .seconds(60), within: delta)) + } + } + + @Suite("PollingBehavior.passesAlways") + struct PassesAlwaysBehavior { + // use a very generous delta for CI reasons. + let delta = Duration.seconds(6) + + @Test("Simple passing expressions") func trivialHappyPath() async { + let duration = await Test.Clock().measure { + await #expect(until: .passesAlways) { true } + } + #expect(duration.isCloseTo(other: .seconds(60), within: delta)) + } + + @Test("Simple failing expressions") func trivialSadPath() async { + let duration = await Test.Clock().measure { + await confirmation("Polling failed", expectedCount: 1) { failed in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case .issueRecorded = event.kind { + failed() + } + } + await Test { + await #expect(until: .passesAlways) { false } + }.run(configuration: configuration) + } + } + #expect(duration.isCloseTo(other: .zero, within: delta)) + } + + @Test("if the closures starts off as false, but would become true") + func changingFromFail() async { + let incrementor = Incrementor() + + let duration = await Test.Clock().measure { + await confirmation("Polling failed", expectedCount: 1) { failed in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case .issueRecorded = event.kind { + failed() + } + } + await Test { + await #expect(until: .passesAlways) { + await incrementor.increment() == 2 + } + // this will pass only on the second invocation + // This checks that we fail the test if it immediately returns false + }.run(configuration: configuration) + } + } + + #expect(await incrementor.count == 1) + #expect(duration.isCloseTo(other: .zero, within: delta)) + } + + @Test("if the closure continues to pass") + func continuousCalling() async { + let incrementor = Incrementor() + + let duration = await Test.Clock().measure { + await #expect(until: .passesAlways) { + _ = await incrementor.increment() + return true + } + } + + #expect(await incrementor.count > 1) + #expect(duration.isCloseTo(other: .seconds(60), within: delta)) + } + + @Test("Unexpected Errors will automatically exit & fail") func errorsReported() async { + let duration = await Test.Clock().measure { + await confirmation("Polling failed", expectedCount: 1) { failed in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case .issueRecorded = event.kind { + failed() + } + } + await Test { + await #expect(until: .passesOnce) { + throw PollingTestSampleError.ohNo + } + }.run(configuration: configuration) + } + } + #expect(duration.isCloseTo(other: .zero, within: delta)) + } + } + } +} + +private enum PollingTestSampleError: Error { + case ohNo + case secondCase +} + +extension DurationProtocol { + fileprivate func isCloseTo(other: Self, within delta: Self) -> Bool { + var distance = self - other + if (distance < Self.zero) { + distance *= -1 + } + return distance <= delta + } +} + +private actor Incrementor { + var count = 0 + func increment() -> Int { + count += 1 + return count + } +} From edfe9d45346ab134ae2897e574990ce51a7afdd0 Mon Sep 17 00:00:00 2001 From: Rachel Brindle Date: Mon, 26 May 2025 10:22:13 -0700 Subject: [PATCH 02/10] Make Polling into a function along the lines of Confirmation Additionally, make PollingBehavior an implementation detail of polling, instead of exposed publicly Removes any timeouts involved for polling, as they become increasingly unreliable as the system runs more and more tests --- .../Expectations/Expectation+Macro.swift | 167 ------ .../ExpectationChecking+Macro.swift | 96 ---- Sources/Testing/Issues/Issue.swift | 7 + Sources/Testing/Polling/CallPolling.swift | 192 ------- Sources/Testing/Polling/Polling.swift | 536 ++++++++++-------- Sources/Testing/Polling/PollingBehavior.swift | 28 - Tests/TestingTests/PollingTests.swift | 257 ++------- .../TestSupport/TestingAdditions.swift | 25 + 8 files changed, 382 insertions(+), 926 deletions(-) delete mode 100644 Sources/Testing/Polling/CallPolling.swift delete mode 100644 Sources/Testing/Polling/PollingBehavior.swift diff --git a/Sources/Testing/Expectations/Expectation+Macro.swift b/Sources/Testing/Expectations/Expectation+Macro.swift index 5230417d4..f85c7042b 100644 --- a/Sources/Testing/Expectations/Expectation+Macro.swift +++ b/Sources/Testing/Expectations/Expectation+Macro.swift @@ -572,170 +572,3 @@ public macro require( sourceLocation: SourceLocation = #_sourceLocation, performing expression: @escaping @Sendable @convention(thin) () async throws -> Void ) -> ExitTest.Result = #externalMacro(module: "TestingMacros", type: "ExitTestRequireMacro") - -// MARK: - Polling Expectations - -/// Continuously check an expression until it matches the given PollingBehavior -/// -/// - Parameters: -/// - until: The desired PollingBehavior to check for. -/// - timeout: How long to run poll the expression until stopping. -/// - comment: A comment describing the expectation. -/// - sourceLocation: The source location to which the recorded expectations -/// and issues should be attributed. -/// - expression: The expression to be evaluated. -/// -/// Use this overload of `#expect()` when you wish to poll whether a value -/// changes as the result of activity in another task/queue/thread. -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -@freestanding(expression) public macro expect( - until pollingBehavior: PollingBehavior, - timeout: Duration = .seconds(60), - _ comment: @autoclosure () -> Comment? = nil, - sourceLocation: SourceLocation = #_sourceLocation, - expression: @Sendable () async throws -> Bool -) = #externalMacro(module: "TestingMacros", type: "ExpectMacro") - -/// Continuously check an expression until it matches the given PollingBehavior -/// -/// - Parameters: -/// - until: The desired PollingBehavior to check for. -/// - throws: The error the expression should throw. -/// - timeout: How long to run poll the expression until stopping. -/// - comment: A comment describing the expectation. -/// - sourceLocation: The source location to which the recorded expectations -/// and issues should be attributed. -/// - expression: The expression to be evaluated. -/// -/// Use this overload of `#expect()` when you wish to poll whether a value -/// changes as the result of activity in another task/queue/thread. -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -@freestanding(expression) public macro expect( - until pollingBehavior: PollingBehavior, - throws error: E, - timeout: Duration = .seconds(60), - _ comment: @autoclosure () -> Comment? = nil, - sourceLocation: SourceLocation = #_sourceLocation, - expression: @Sendable () async throws -> Bool -) = #externalMacro(module: "TestingMacros", type: "ExpectMacro") -where E: Error & Equatable - -/// Continuously check an expression until it matches the given PollingBehavior -/// -/// - Parameters: -/// - until: The desired PollingBehavior to check for. -/// - timeout: How long to run poll the expression until stopping. -/// - comment: A comment describing the expectation. -/// - sourceLocation: The source location to which the recorded expectations -/// and issues should be attributed. -/// - expression: The expression to be evaluated. -/// - throws: A closure to confirm if the expression throws the expected error. -/// -/// Use this overload of `#expect()` when you wish to poll whether a value -/// changes as the result of activity in another task/queue/thread. -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -@freestanding(expression) public macro expect( - until pollingBehavior: PollingBehavior, - timeout: Duration = .seconds(60), - _ comment: @autoclosure () -> Comment? = nil, - sourceLocation: SourceLocation = #_sourceLocation, - performing: @Sendable () async throws -> Bool, - throws errorMatcher: @Sendable (any Error) async throws -> Bool -) = #externalMacro(module: "TestingMacros", type: "ExpectMacro") - -/// Continuously check an expression until it matches the given PollingBehavior -/// -/// - Parameters: -/// - until: The desired PollingBehavior to check for. -/// - timeout: How long to run poll the expression until stopping. -/// - comment: A comment describing the expectation. -/// - sourceLocation: The source location to which the recorded expectations -/// and issues should be attributed. -/// - expression: The expression to be evaluated. -/// -/// Use this overload of `#require()` when you wish to poll whether a value -/// changes as the result of activity in another task/queue/thread. -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -@freestanding(expression) public macro require( - until pollingBehavior: PollingBehavior, - timeout: Duration = .seconds(60), - _ comment: @autoclosure () -> Comment? = nil, - sourceLocation: SourceLocation = #_sourceLocation, - expression: @Sendable () async throws -> Bool -) = #externalMacro(module: "TestingMacros", type: "RequireMacro") - -/// Continuously check an expression until it matches the given PollingBehavior -/// -/// - Parameters: -/// - until: The desired PollingBehavior to check for. -/// - timeout: How long to run poll the expression until stopping. -/// - comment: A comment describing the expectation. -/// - sourceLocation: The source location to which the recorded expectations -/// and issues should be attributed. -/// - expression: The expression to be evaluated. -/// -/// Use this overload of `#require()` when you wish to poll whether a value -/// changes as the result of activity in another task/queue/thread. -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -@freestanding(expression) public macro require( - until pollingBehavior: PollingBehavior, - timeout: Duration = .seconds(60), - _ comment: @autoclosure () -> Comment? = nil, - sourceLocation: SourceLocation = #_sourceLocation, - expression: @Sendable () async throws -> R? -) = #externalMacro(module: "TestingMacros", type: "RequireMacro") -where R: Sendable - -/// Continuously check an expression until it matches the given PollingBehavior -/// -/// - Parameters: -/// - until: The desired PollingBehavior to check for. -/// - throws: The error the expression should throw -/// - timeout: How long to run poll the expression until stopping. -/// - comment: A comment describing the expectation. -/// - sourceLocation: The source location to which the recorded expectations -/// and issues should be attributed. -/// - expression: The expression to be evaluated. -/// -/// Use this overload of `#require()` when you wish to poll whether a value -/// changes as the result of activity in another task/queue/thread. -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -@freestanding(expression) public macro require( - until pollingBehavior: PollingBehavior, - throws error: E, - timeout: Duration = .seconds(60), - _ comment: @autoclosure () -> Comment? = nil, - sourceLocation: SourceLocation = #_sourceLocation, - expression: @Sendable () async throws -> Bool -) = #externalMacro(module: "TestingMacros", type: "RequireMacro") -where E: Error & Equatable - -/// Continuously check an expression until it matches the given PollingBehavior -/// -/// - Parameters: -/// - until: The desired PollingBehavior to check for. -/// - timeout: How long to run poll the expression until stopping. -/// - comment: A comment describing the expectation. -/// - sourceLocation: The source location to which the recorded expectations -/// and issues should be attributed. -/// - expression: The expression to be evaluated. -/// - throws: A closure to confirm if the expression throws the expected error. -/// -/// Use this overload of `#require()` when you wish to poll whether a value -/// changes as the result of activity in another task/queue/thread. -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -@freestanding(expression) public macro require( - until pollingBehavior: PollingBehavior, - timeout: Duration = .seconds(60), - _ comment: @autoclosure () -> Comment? = nil, - sourceLocation: SourceLocation = #_sourceLocation, - expression: @Sendable () async throws -> Bool, - throws errorMatcher: @Sendable (any Error) async throws -> Bool -) = #externalMacro(module: "TestingMacros", type: "RequireMacro") diff --git a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift index 8144a6908..6d3093f2a 100644 --- a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift +++ b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift @@ -1201,102 +1201,6 @@ public func __checkClosureCall( } #endif -// MARK: - Polling - -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -public func __checkClosureCall( - until behavior: PollingBehavior, - timeout: Duration = .seconds(60), - performing closure: @escaping @Sendable () async throws -> Bool, - expression: __Expression, - comments: @autoclosure () -> [Comment], - isRequired: Bool, - sourceLocation: SourceLocation -) async -> Result { - await callPolling( - behavior: behavior, - timeout: timeout, - closure: closure, - expression: expression, - comments: comments(), - isRequired: isRequired, - sourceLocation: sourceLocation - ) -} - -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -public func __checkClosureCall( - until behavior: PollingBehavior, - throws error: E, - timeout: Duration = .seconds(60), - performing closure: @escaping @Sendable () async throws -> Bool, - expression: __Expression, - comments: @autoclosure () -> [Comment], - isRequired: Bool, - sourceLocation: SourceLocation -) async -> Result where E: Error & Equatable { - await callPolling( - behavior: behavior, - throws: error, - timeout: timeout, - closure: closure, - expression: expression, - comments: comments(), - isRequired: isRequired, - sourceLocation: sourceLocation - ) -} - -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -public func __checkClosureCall( - until behavior: PollingBehavior, - timeout: Duration = .seconds(60), - performing closure: @escaping @Sendable () async throws -> Bool, - throws errorMatcher: @escaping @Sendable (any Error) async throws -> Bool, - expression: __Expression, - comments: @autoclosure () -> [Comment], - isRequired: Bool, - sourceLocation: SourceLocation -) async -> Result { - await callPolling( - behavior: behavior, - timeout: timeout, - closure: closure, - errorMatcher: errorMatcher, - expression: expression, - comments: comments(), - isRequired: isRequired, - sourceLocation: sourceLocation - ) -} - -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -public func __checkClosureCall( - until behavior: PollingBehavior, - timeout: Duration = .seconds(60), - performing closure: @escaping @Sendable () async throws -> R, - throws errorMatcher: @escaping @Sendable (any Error) async throws -> Bool, - expression: __Expression, - comments: @autoclosure () -> [Comment], - isRequired: Bool, - sourceLocation: SourceLocation -) async -> Result where R: Sendable { - await callPolling( - behavior: behavior, - timeout: timeout, - closure: closure, - errorMatcher: errorMatcher, - expression: expression, - comments: comments(), - isRequired: isRequired, - sourceLocation: sourceLocation - ) -} - // MARK: - /// Generate a description of an error that includes its type name if not diff --git a/Sources/Testing/Issues/Issue.swift b/Sources/Testing/Issues/Issue.swift index 9a2555177..e321a1b9d 100644 --- a/Sources/Testing/Issues/Issue.swift +++ b/Sources/Testing/Issues/Issue.swift @@ -38,6 +38,9 @@ public struct Issue: Sendable { /// confirmed too few or too many times. indirect case confirmationMiscounted(actual: Int, expected: any RangeExpression & Sendable) + @_spi(Experimental) + case confirmationPollingFailed + /// An issue due to an `Error` being thrown by a test function and caught by /// the testing library. /// @@ -286,6 +289,8 @@ extension Issue.Kind: CustomStringConvertible { } } return "Confirmation was confirmed \(actual.counting("time")), but expected to be confirmed \(String(describingForTest: expected)) time(s)" + case .confirmationPollingFailed: + return "Confirmation polling failed" case let .errorCaught(error): return "Caught error: \(error)" case let .timeLimitExceeded(timeLimitComponents: timeLimitComponents): @@ -465,6 +470,8 @@ extension Issue.Kind { .expectationFailed(Expectation.Snapshot(snapshotting: expectation)) case .confirmationMiscounted: .unconditional + case .confirmationPollingFailed: + .unconditional case let .errorCaught(error), let .valueAttachmentFailed(error): .errorCaught(ErrorSnapshot(snapshotting: error)) case let .timeLimitExceeded(timeLimitComponents: timeLimitComponents): diff --git a/Sources/Testing/Polling/CallPolling.swift b/Sources/Testing/Polling/CallPolling.swift deleted file mode 100644 index 4f751f761..000000000 --- a/Sources/Testing/Polling/CallPolling.swift +++ /dev/null @@ -1,192 +0,0 @@ -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2025 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for Swift project authors -// - -/// Poll an expression to check that it passes until the given duration -/// -/// - Parameters: -/// - behavior: The PollingBehavior to use. -/// - timeout: How long to poll for until we time out. -/// - closure: The closure to continuously evaluate. -/// - expression: The expression, corresponding to `condition`, that is being -/// evaluated (if available at compile time.) -/// - comments: An array of comments describing the expectation. This array -/// may be empty. -/// - isRequired: Whether or not the expectation is required. The value of -/// this argument does not affect whether or not an error is thrown on -/// failure. -/// - sourceLocation: The source location of the expectation. -/// -/// This function contains the implementation for `#expect(until:)` when no -/// error is expected and no value should be returned. -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -func callPolling( - behavior: PollingBehavior, - timeout: Duration, - closure: @escaping @Sendable () async throws -> Bool, - expression: __Expression, - comments: [Comment], - isRequired: Bool, - sourceLocation: SourceLocation -) async -> Result{ - await Polling.run( - behavior: behavior, - timeout: timeout, - closure: { - do { - return try await closure() - } catch { - return false - } - }, - expression: expression, - comments: comments, - isRequired: isRequired, - sourceLocation: sourceLocation - ) -} - -/// Poll an expression to check that it passes until the given duration -/// -/// - Parameters: -/// - behavior: The PollingBehavior to use. -/// - timeout: How long to poll for until we time out. -/// - closure: The closure to continuously evaluate. -/// - expression: The expression, corresponding to `condition`, that is being -/// evaluated (if available at compile time.) -/// - comments: An array of comments describing the expectation. This array -/// may be empty. -/// - isRequired: Whether or not the expectation is required. The value of -/// this argument does not affect whether or not an error is thrown on -/// failure. -/// - sourceLocation: The source location of the expectation. -/// -/// This function contains the implementation for `#expect(until:)` when an -/// equatable error is expected and no value should be returned. -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -func callPolling( - behavior: PollingBehavior, - throws error: E, - timeout: Duration, - closure: @escaping @Sendable () async throws -> Bool, - expression: __Expression, - comments: [Comment], - isRequired: Bool, - sourceLocation: SourceLocation -) async -> Result where E: Error & Equatable { - await Polling.run( - behavior: behavior, - timeout: timeout, - closure: { - do { - _ = try await closure() - return false - } catch let thrownError as E { - return thrownError == error - } catch { - return false - } - }, - expression: expression, - comments: comments, - isRequired: isRequired, - sourceLocation: sourceLocation - ) -} - -/// Poll an expression to check that it passes until the given duration -/// -/// - Parameters: -/// - behavior: The PollingBehavior to use. -/// - timeout: How long to poll for until we time out. -/// - closure: The closure to continuously evaluate. -/// - expression: The expression, corresponding to `condition`, that is being -/// evaluated (if available at compile time.) -/// - comments: An array of comments describing the expectation. This array -/// may be empty. -/// - isRequired: Whether or not the expectation is required. The value of -/// this argument does not affect whether or not an error is thrown on -/// failure. -/// - sourceLocation: The source location of the expectation. -/// -/// This function contains the implementation for `#expect(until:)` when an -/// error is expected and no value should be returned. -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -func callPolling( - behavior: PollingBehavior, - timeout: Duration, - closure: @escaping @Sendable () async throws -> Bool, - errorMatcher: @escaping @Sendable (any Error) async throws -> Bool, - expression: __Expression, - comments: [Comment], - isRequired: Bool, - sourceLocation: SourceLocation -) async -> Result { - await Polling.run( - behavior: behavior, - timeout: timeout, - closure: { - do { - _ = try await closure() - return false - } catch { - return (try? await errorMatcher(error)) == true - } - }, - expression: expression, - comments: comments, - isRequired: isRequired, - sourceLocation: sourceLocation - ) -} - -/// Poll an expression to check that it passes until the given duration -/// -/// - Parameters: -/// - behavior: The PollingBehavior to use. -/// - timeout: How long to poll for until we time out. -/// - closure: The closure to continuously evaluate. -/// - expression: The expression, corresponding to `condition`, that is being -/// evaluated (if available at compile time.) -/// - comments: An array of comments describing the expectation. This array -/// may be empty. -/// - isRequired: Whether or not the expectation is required. The value of -/// this argument does not affect whether or not an error is thrown on -/// failure. -/// - sourceLocation: The source location of the expectation. -/// -/// This function contains the implementation for `#require(until:)` when no -/// error is expected and a value should be returned. -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -func callPolling( - behavior: PollingBehavior, - timeout: Duration, - closure: @escaping @Sendable () async throws -> R?, - errorMatcher: @escaping @Sendable (any Error) async throws -> Bool, - expression: __Expression, - comments: [Comment], - isRequired: Bool, - sourceLocation: SourceLocation -) async -> Result where R: Sendable { - await Polling.run( - behavior: behavior, - timeout: timeout, - closure: { - do { - return try await closure() - } catch { - return nil - } - }, - expression: expression, - comments: comments, - isRequired: isRequired, - sourceLocation: sourceLocation - ) -} diff --git a/Sources/Testing/Polling/Polling.swift b/Sources/Testing/Polling/Polling.swift index dd09e1a64..641a4d400 100644 --- a/Sources/Testing/Polling/Polling.swift +++ b/Sources/Testing/Polling/Polling.swift @@ -8,273 +8,224 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -/// A type for managing polling +/// Confirm that some expression eventually returns true +/// +/// - Parameters: +/// - comment: An optional comment to apply to any issues generated by this +/// function. +/// - isolation: The actor to which `body` is isolated, if any. +/// - sourceLocation: The source location to whych any recorded issues should +/// be attributed. +/// - body: The function to invoke. +/// +/// Use polling confirmations to check that an event while a test is running in +/// complex scenarios where other forms of confirmation are insufficient. For +/// example, waiting on some state to change that cannot be easily confirmed +/// through other forms of `confirmation`. +@_spi(Experimental) @available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -struct Polling { - /// Run polling for a closure that evaluates to a boolean value. - /// - /// - Parameters: - /// - behavior: The PollingBehavior to use. - /// - timeout: How long to poll for until we time out. - /// - closure: The closure to continuously evaluate. - /// - expression: The expression, corresponding to `condition`, that is being - /// evaluated (if available at compile time.) - /// - comments: An array of comments describing the expectation. This array - /// may be empty. - /// - isRequired: Whether or not the expectation is required. The value of - /// this argument does not affect whether or not an error is thrown on - /// failure. - /// - sourceLocation: The source location of the expectation. - static func run( - behavior: PollingBehavior, - timeout: Duration, - closure: @escaping @Sendable () async -> Bool, - expression: __Expression, - comments: [Comment], - isRequired: Bool, - sourceLocation: SourceLocation - ) async -> Result { - var expectation = Expectation( - evaluatedExpression: expression, - isPassing: true, - isRequired: isRequired, - sourceLocation: sourceLocation - ) - - let result = await poll(expression: closure, behavior: behavior, timeout: timeout) - - let sourceContext = SourceContext(backtrace: nil, sourceLocation: sourceLocation) - - switch result { - case .timedOut: - expectation.isPassing = false - Issue( - kind: .expectationFailed(expectation), - comments: comments, - sourceContext: sourceContext - ).record() - case .timedOutWithoutRunning: - expectation.isPassing = false - Issue( - kind: .expectationFailed(expectation), - comments: comments, - sourceContext: sourceContext - ).record() - case .finished: - return __checkValue( - true, - expression: expression, - comments: comments, - isRequired: isRequired, - sourceLocation: sourceLocation - ) - case .failed: - return __checkValue( - false, - expression: expression, - comments: comments, - isRequired: isRequired, - sourceLocation: sourceLocation - ) - case .cancelled: - Issue( - kind: .system, - comments: comments, - sourceContext: sourceContext - ).record() +public func confirmPassesEventually( + _ comment: Comment? = nil, + isolation: isolated (any Actor)? = #isolation, + sourceLocation: SourceLocation = #_sourceLocation, + _ body: @escaping () async throws -> Bool +) async { + let poller = Poller( + pollingBehavior: .passesOnce, + comment: comment, + sourceLocation: sourceLocation + ) + await poller.evaluate(isolation: isolation) { + do { + return try await body() + } catch { + return false } - - return .failure(ExpectationFailedError(expectation: expectation)) } +} - /// Run polling for a closure that evaluates to an optional value. - /// - /// - Parameters: - /// - behavior: The PollingBehavior to use. - /// - timeout: How long to poll for until we time out. - /// - closure: The closure to continuously evaluate. - /// - expression: The expression, corresponding to `condition`, that is being - /// evaluated (if available at compile time.) - /// - comments: An array of comments describing the expectation. This array - /// may be empty. - /// - isRequired: Whether or not the expectation is required. The value of - /// this argument does not affect whether or not an error is thrown on - /// failure. - /// - sourceLocation: The source location of the expectation. - static func run( - behavior: PollingBehavior, - timeout: Duration, - closure: @escaping @Sendable () async -> R?, - expression: __Expression, - comments: [Comment], - isRequired: Bool, - sourceLocation: SourceLocation - ) async -> Result where R: Sendable { - var expectation = Expectation( - evaluatedExpression: expression, - isPassing: true, - isRequired: isRequired, - sourceLocation: sourceLocation - ) - - let recorder = Recorder() +/// A type describing an error thrown when polling fails to return a non-nil +/// value +@_spi(Experimental) +public struct PollingFailedError: Error {} - let result = await poll(expression: { - if let value = await closure() { - await recorder.record(value: value) - return true - } +/// Confirm that some expression eventually returns a non-nil value +/// +/// - Parameters: +/// - comment: An optional comment to apply to any issues generated by this +/// function. +/// - isolation: The actor to which `body` is isolated, if any. +/// - sourceLocation: The source location to whych any recorded issues should +/// be attributed. +/// - body: The function to invoke. +/// +/// - Returns: The first non-nil value returned by `body`. +/// +/// - Throws: A `PollingFailedError` will be thrown if `body` never returns a +/// non-optional value +/// +/// Use polling confirmations to check that an event while a test is running in +/// complex scenarios where other forms of confirmation are insufficient. For +/// example, waiting on some state to change that cannot be easily confirmed +/// through other forms of `confirmation`. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public func confirmPassesEventually( + _ comment: Comment? = nil, + isolation: isolated (any Actor)? = #isolation, + sourceLocation: SourceLocation = #_sourceLocation, + _ body: @escaping () async throws -> R? +) async throws -> R where R: Sendable { + let recorder = PollingRecorder() + let poller = Poller( + pollingBehavior: .passesOnce, + comment: comment, + sourceLocation: sourceLocation + ) + await poller.evaluate(isolation: isolation) { + do { + return try await recorder.record(value: body()) + } catch { return false - }, behavior: behavior, timeout: timeout) - - let sourceContext = SourceContext(backtrace: nil, sourceLocation: sourceLocation) - - switch result { - case .timedOut: - expectation.isPassing = false - Issue( - kind: .expectationFailed(expectation), - comments: comments, - sourceContext: sourceContext - ).record() - case .timedOutWithoutRunning: - expectation.isPassing = false - Issue( - kind: .expectationFailed(expectation), - comments: comments, - sourceContext: sourceContext - ).record() - case .finished: - return __checkValue( - await recorder.lastValue, - expression: expression, - comments: comments, - isRequired: isRequired, - sourceLocation: sourceLocation - ) - case .failed: - return __checkValue( - nil, - expression: expression, - comments: comments, - isRequired: isRequired, - sourceLocation: sourceLocation - ) - case .cancelled: - Issue( - kind: .system, - comments: comments, - sourceContext: sourceContext - ).record() } + } - return .failure(ExpectationFailedError(expectation: expectation)) + if let value = await recorder.lastValue { + return value } + throw PollingFailedError() +} - /// A type to record the last value returned by a closure returning an optional - /// This is only used in the `#require(until:)` macro returning an optional. - private actor Recorder { - var lastValue: R? +/// Confirm that some expression always returns true +/// +/// - Parameters: +/// - comment: An optional comment to apply to any issues generated by this +/// function. +/// - isolation: The actor to which `body` is isolated, if any. +/// - sourceLocation: The source location to whych any recorded issues should +/// be attributed. +/// - body: The function to invoke. +/// +/// Use polling confirmations to check that an event while a test is running in +/// complex scenarios where other forms of confirmation are insufficient. For +/// example, confirming that some state does not change. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public func confirmAlwaysPasses( + _ comment: Comment? = nil, + isolation: isolated (any Actor)? = #isolation, + sourceLocation: SourceLocation = #_sourceLocation, + _ body: @escaping () async throws -> Bool +) async { + let poller = Poller( + pollingBehavior: .passesAlways, + comment: comment, + sourceLocation: sourceLocation + ) + await poller.evaluate(isolation: isolation) { + do { + return try await body() + } catch { + return false + } + } +} - /// Record a new value to be returned - func record(value: R) { - self.lastValue = value +/// Confirm that some expression always returns a non-optional value +/// +/// - Parameters: +/// - comment: An optional comment to apply to any issues generated by this +/// function. +/// - isolation: The actor to which `body` is isolated, if any. +/// - sourceLocation: The source location to whych any recorded issues should +/// be attributed. +/// - body: The function to invoke. +/// +/// - Returns: The value from the last time `body` was invoked. +/// +/// - Throws: A `PollingFailedError` will be thrown if `body` ever returns a +/// non-optional value +/// +/// Use polling confirmations to check that an event while a test is running in +/// complex scenarios where other forms of confirmation are insufficient. For +/// example, confirming that some state does not change. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public func confirmAlwaysPasses( + _ comment: Comment? = nil, + isolation: isolated (any Actor)? = #isolation, + sourceLocation: SourceLocation = #_sourceLocation, + _ body: @escaping () async throws -> R? +) async throws -> R where R: Sendable { + let recorder = PollingRecorder() + let poller = Poller( + pollingBehavior: .passesAlways, + comment: comment, + sourceLocation: sourceLocation + ) + await poller.evaluate(isolation: isolation) { + do { + return try await recorder.record(value: body()) + } catch { + return false } } - /// The result of polling expressions - private enum PollResult { - /// The polling timed out, and the expression had run at least once. - case timedOut - /// The polling timed out, but the expression had not finished running in - /// that time. - case timedOutWithoutRunning - /// The expression exited early, and we will report a success status. - case finished - /// The expression returned false under PollingBehavior.passesAlways - case failed - /// The polling was cancelled before polling could finish - case cancelled + if let value = await recorder.lastValue { + return value } + throw PollingFailedError() +} - /// The poll manager. - /// - /// This function contains the logic for continuously polling an expression, - /// as well as the logic for cancelling the polling once it times out. - /// - /// - Parameters: - /// - expression: An expression to continuously evaluate - /// - behavior: The polling behavior to use - /// - timeout: How long to poll for unitl the timeout triggers. - /// - Returns: The result of this polling. - private static func poll( - expression: @escaping @Sendable () async -> Bool, - behavior: PollingBehavior, - timeout: Duration - ) async -> PollResult { - let pollingProcessor = PollingProcessor(behavior: behavior) - return await withTaskGroup { taskGroup in - taskGroup.addTask { - do { - try await Task.sleep(for: timeout) - } catch {} - // Task.sleep will only throw if it's cancelled, at which point this - // taskgroup has already returned and we don't care about the value - // returned here. - return await pollingProcessor.didTimeout() - } - taskGroup.addTask { - while Task.isCancelled == false { - let expressionPassed = await expression() - if let result = await pollingProcessor.expressionFinished(result: expressionPassed) { - return result - } - } - // The expression was cancelled without having been finished. - // This should end up being reported as a timeout error, due to - // the earlier task added to this task group. - // But there's a chance that the overall task was cancelled. - // in which case, we should report that as a system error. - return PollResult.cancelled - } +/// A type to record the last value returned by a closure returning an optional +/// This is only used in the `confirm` polling functions evaluating an optional. +private actor PollingRecorder { + var lastValue: R? - defer { taskGroup.cancelAll() } - return await taskGroup.next() ?? .timedOut - } + /// Record a new value to be returned + func record(value: R) { + self.lastValue = value } - /// A type to process events from `Polling.poll`. - private actor PollingProcessor { - let behavior: PollingBehavior - var hasRun = false - - init(behavior: PollingBehavior) { - self.behavior = behavior + func record(value: R?) -> Bool { + if let value { + self.lastValue = value + return true + } else { + return false } + } +} - /// Record a timeout event from polling. - func didTimeout() -> PollResult { - if !hasRun { - return PollResult.timedOutWithoutRunning - } - switch behavior { - case .passesOnce: - return PollResult.timedOut - case .passesAlways: - return PollResult.finished - } - } +/// A type for managing polling +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +private struct Poller { + enum PollingBehavior { + /// Continuously evaluate the expression until the first time it returns + /// true. + /// If it does not pass once by the time the timeout is reached, then a + /// failure will be reported. + case passesOnce + + /// Continuously evaluate the expression until the first time it returns + /// false. + /// If the expression returns false, then a failure will be reported. + /// If the expression only returns true before the timeout is reached, then + /// no failure will be reported. + /// If the expression does not finish evaluating before the timeout is + /// reached, then a failure will be reported. + case passesAlways - /// Record that an expression finished running + /// Process the result of a polled expression and decide whether to continue polling. /// /// - Parameters: - /// - Result: Whether or not the polled expression passed or not. + /// - expressionResult: The result of the polled expression /// - /// - Returns: A non-nil PollResult if polling should exit, otherwise nil. - func expressionFinished(result: Bool) -> PollResult? { - hasRun = true - - switch behavior { + /// - Returns: A poll result (if polling should stop), or nil (if polling should continue) + func processFinishedExpression( + expressionResult result: Bool + ) -> PollResult? { + switch self { case .passesOnce: if result { return .finished @@ -290,4 +241,105 @@ struct Polling { } } } + + /// The result of polling expressions + enum PollResult { + /// The polling ran for the total number of iterations + case ranToCompletion + /// The expression exited early, and we will report a success status. + case finished + /// The expression returned false under PollingBehavior.passesAlways + case failed + /// The polling was cancelled before polling could finish + case cancelled + + /// Process the poll result into an issue + /// + /// - Parameters: + /// - comment: The comment to record as part of the issue + /// - sourceContext: The source context for the issue + /// - pollingBehavior: The polling behavior used. + /// - Returns: An issue if one should be recorded, otherwise nil. + func issue( + comment: Comment?, + sourceContext: SourceContext, + pollingBehavior: PollingBehavior + ) -> Issue? { + let issueKind: Issue.Kind + switch self { + case .finished, .cancelled: + return nil + case .ranToCompletion: + if case .passesAlways = pollingBehavior { + return nil + } + issueKind = .confirmationPollingFailed + case .failed: + issueKind = .confirmationPollingFailed + } + return Issue( + kind: issueKind, + comments: Array(comment), + sourceContext: sourceContext + ) + } + } + + /// The polling behavior (poll until the expression first passes, or poll + /// while the expression continues to pass) + let pollingBehavior: PollingBehavior + + /// A comment from the test author associated with the polling + let comment: Comment? + + /// The source location that asked for polling. + let sourceLocation: SourceLocation + + /// Evaluate polling, and process the result, raising an issue if necessary. + /// + /// - Parameters: + /// - body: The expression to poll + /// - Side effects: If polling fails (see `PollingBehavior`), then this will + /// record an issue. + func evaluate( + isolation: isolated (any Actor)?, + _ body: @escaping () async -> Bool + ) async { + let result = await poll( + runAmount: 1_000_000, + expression: body + ) + result.issue( + comment: comment, + sourceContext: .init(backtrace: .current(), sourceLocation: sourceLocation), + pollingBehavior: pollingBehavior + )?.record() + } + + /// This function contains the logic for continuously polling an expression, + /// as well as processing the results of that expression + /// + /// - Parameters: + /// - expression: An expression to continuously evaluate + /// - behavior: The polling behavior to use + /// - timeout: How long to poll for unitl the timeout triggers. + /// - Returns: The result of this polling. + private func poll( + runAmount: Int, + isolation: isolated (any Actor)? = #isolation, + expression: @escaping () async -> Bool + ) async -> PollResult { + for _ in 0...none } } + #expect(issues.count == 3) } @Test("When the value changes from false to true during execution") func changingFromFail() async { let incrementor = Incrementor() - await #expect(until: .passesOnce) { + await confirmPassesEventually { await incrementor.increment() == 2 // this will pass only on the second invocation // This checks that we really are only running the expression until @@ -62,75 +45,62 @@ struct PollingTests { #expect(await incrementor.count == 2) } - @Test("Unexpected Errors are treated as returning false") + @Test("Thrown errors are treated as returning false") func errorsReported() async { - await confirmation("Polling failed", expectedCount: 1) { failed in - var configuration = Configuration() - configuration.eventHandler = { event, _ in - if case .issueRecorded = event.kind { - failed() - } + let issues = await runTest { + await confirmPassesEventually { + throw PollingTestSampleError.ohNo } - await Test { - await #expect(until: .passesOnce) { - throw PollingTestSampleError.ohNo - } - }.run(configuration: configuration) } + #expect(issues.count == 1) } } - @Suite("PollingBehavior.passesAlways") + @Suite("confirmAlwaysPasses") struct PassesAlwaysBehavior { // use a very generous delta for CI reasons. let delta = Duration.seconds(6) @Test("Simple passing expressions") func trivialHappyPath() async { - await #expect(until: .passesAlways) { true } + await confirmAlwaysPasses { true } + } + + @Test("Returning value returns the last value from the expression") + func returnsLastValueReturned() async throws { + let incrementor = Incrementor() + let value = try await confirmAlwaysPasses { + await incrementor.increment() + } + #expect(value > 1) } @Test("Simple failing expressions") func trivialSadPath() async { - await confirmation("Polling failed", expectedCount: 1) { failed in - var configuration = Configuration() - configuration.eventHandler = { event, _ in - if case .issueRecorded = event.kind { - failed() - } - } - await Test { - await #expect(until: .passesAlways) { false } - }.run(configuration: configuration) + let issues = await runTest { + await confirmAlwaysPasses { false } + _ = try await confirmAlwaysPasses { Optional.none } } + #expect(issues.count == 3) } - @Test("if the closures starts off as false, but would become true") + @Test("if the closures starts off as true, but becomes false") func changingFromFail() async { let incrementor = Incrementor() - - await confirmation("Polling failed", expectedCount: 1) { failed in - var configuration = Configuration() - configuration.eventHandler = { event, _ in - if case .issueRecorded = event.kind { - failed() - } + let issues = await runTest { + await confirmAlwaysPasses { + await incrementor.increment() == 2 + // this will pass only on the first invocation + // This checks that we fail the test if it starts failing later during + // polling } - await Test { - await #expect(until: .passesAlways) { - await incrementor.increment() == 2 - // this will pass only on the second invocation - // This checks that we fail the test if it immediately returns false - } - }.run(configuration: configuration) } - - #expect(await incrementor.count == 1) + #expect(issues.count == 1) } @Test("if the closure continues to pass") func continuousCalling() async { let incrementor = Incrementor() - await #expect(until: .passesAlways) { + await confirmAlwaysPasses { _ = await incrementor.increment() return true } @@ -138,62 +108,34 @@ struct PollingTests { #expect(await incrementor.count > 1) } - @Test("Unexpected Errors will automatically exit & fail") func errorsReported() async { - await confirmation("Polling failed", expectedCount: 1) { failed in - var configuration = Configuration() - configuration.eventHandler = { event, _ in - if case .issueRecorded = event.kind { - failed() - } + @Test("Thrown errors will automatically exit & fail") func errorsReported() async { + let issues = await runTest { + await confirmAlwaysPasses { + throw PollingTestSampleError.ohNo } - await Test { - await #expect(until: .passesAlways) { - throw PollingTestSampleError.ohNo - } - }.run(configuration: configuration) } + #expect(issues.count == 1) } } @Suite("Duration Tests", .disabled("time-sensitive")) struct DurationTests { - @Suite("PollingBehavior.passesOnce") + @Suite("confirmPassesEventually") struct PassesOnceBehavior { let delta = Duration.seconds(6) @Test("Simple passing expressions") func trivialHappyPath() async { let duration = await Test.Clock().measure { - await #expect(until: .passesOnce) { true } - - await #expect(until: .passesOnce, throws: PollingTestSampleError.ohNo) { - throw PollingTestSampleError.ohNo - } - - await #expect(until: .passesOnce, performing: { - throw PollingTestSampleError.secondCase - }, throws: { error in - (error as? PollingTestSampleError) == .secondCase - }) - - await #expect(until: .passesOnce, throws: PollingTestSampleError.ohNo) { - throw PollingTestSampleError.ohNo - } + await confirmPassesEventually { true } } #expect(duration.isCloseTo(other: .zero, within: delta)) } @Test("Simple failing expressions") func trivialSadPath() async { let duration = await Test.Clock().measure { - await confirmation("Polling failed", expectedCount: 1) { failed in - var configuration = Configuration() - configuration.eventHandler = { event, _ in - if case .issueRecorded = event.kind { - failed() - } - } - await Test { - await #expect(until: .passesOnce) { false } - }.run(configuration: configuration) + let issues = await runTest { + await confirmPassesEventually { false } } + #expect(issues.count == 1) } #expect(duration.isCloseTo(other: .seconds(60), within: delta)) } @@ -202,7 +144,7 @@ struct PollingTests { let incrementor = Incrementor() let duration = await Test.Clock().measure { - await #expect(until: .passesOnce) { + await confirmPassesEventually { await incrementor.increment() == 2 // this will pass only on the second invocation // This checks that we really are only running the expression until @@ -214,113 +156,26 @@ struct PollingTests { #expect(await incrementor.count == 2) #expect(duration.isCloseTo(other: .zero, within: delta)) } - - @Test("Unexpected Errors are treated as returning false") - func errorsReported() async { - let duration = await Test.Clock().measure { - await confirmation("Polling failed", expectedCount: 1) { failed in - var configuration = Configuration() - configuration.eventHandler = { event, _ in - if case .issueRecorded = event.kind { - failed() - } - } - await Test { - await #expect(until: .passesOnce) { - throw PollingTestSampleError.ohNo - } - }.run(configuration: configuration) - } - } - #expect(duration.isCloseTo(other: .seconds(60), within: delta)) - } } - @Suite("PollingBehavior.passesAlways") + @Suite("confirmAlwaysPasses") struct PassesAlwaysBehavior { // use a very generous delta for CI reasons. let delta = Duration.seconds(6) @Test("Simple passing expressions") func trivialHappyPath() async { let duration = await Test.Clock().measure { - await #expect(until: .passesAlways) { true } + await confirmAlwaysPasses { true } } #expect(duration.isCloseTo(other: .seconds(60), within: delta)) } @Test("Simple failing expressions") func trivialSadPath() async { let duration = await Test.Clock().measure { - await confirmation("Polling failed", expectedCount: 1) { failed in - var configuration = Configuration() - configuration.eventHandler = { event, _ in - if case .issueRecorded = event.kind { - failed() - } - } - await Test { - await #expect(until: .passesAlways) { false } - }.run(configuration: configuration) - } - } - #expect(duration.isCloseTo(other: .zero, within: delta)) - } - - @Test("if the closures starts off as false, but would become true") - func changingFromFail() async { - let incrementor = Incrementor() - - let duration = await Test.Clock().measure { - await confirmation("Polling failed", expectedCount: 1) { failed in - var configuration = Configuration() - configuration.eventHandler = { event, _ in - if case .issueRecorded = event.kind { - failed() - } - } - await Test { - await #expect(until: .passesAlways) { - await incrementor.increment() == 2 - } - // this will pass only on the second invocation - // This checks that we fail the test if it immediately returns false - }.run(configuration: configuration) - } - } - - #expect(await incrementor.count == 1) - #expect(duration.isCloseTo(other: .zero, within: delta)) - } - - @Test("if the closure continues to pass") - func continuousCalling() async { - let incrementor = Incrementor() - - let duration = await Test.Clock().measure { - await #expect(until: .passesAlways) { - _ = await incrementor.increment() - return true - } - } - - #expect(await incrementor.count > 1) - #expect(duration.isCloseTo(other: .seconds(60), within: delta)) - } - - @Test("Unexpected Errors will automatically exit & fail") func errorsReported() async { - let duration = await Test.Clock().measure { - await confirmation("Polling failed", expectedCount: 1) { failed in - var configuration = Configuration() - configuration.eventHandler = { event, _ in - if case .issueRecorded = event.kind { - failed() - } - } - await Test { - await #expect(until: .passesOnce) { - throw PollingTestSampleError.ohNo - } - }.run(configuration: configuration) + let issues = await runTest { + await confirmAlwaysPasses { false } } + #expect(issues.count == 1) } #expect(duration.isCloseTo(other: .zero, within: delta)) } diff --git a/Tests/TestingTests/TestSupport/TestingAdditions.swift b/Tests/TestingTests/TestSupport/TestingAdditions.swift index 4648f96af..ed2e481d2 100644 --- a/Tests/TestingTests/TestSupport/TestingAdditions.swift +++ b/Tests/TestingTests/TestSupport/TestingAdditions.swift @@ -94,6 +94,31 @@ func runTestFunction(named name: String, in containingType: Any.Type, configurat await runner.run() } +/// Create a ``Test`` instance for the expression and run it, returning any +/// issues recorded. +/// +/// - Parameters: +/// - expression: The test expression to run +/// +/// - Returns: The list of issues recorded. +func runTest( + testFunction: @escaping @Sendable () async throws -> Void +) async -> [Issue] { + let issues = Locked(rawValue: [Issue]()) + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case let .issueRecorded(issue) = event.kind { + print("issue recorded: \(issue)") + issues.withLock { + $0.append(issue) + } + } + } + await Test(testFunction: testFunction).run(configuration: configuration) + return issues.rawValue +} + extension Runner { /// Initialize an instance of this type that runs the free test function /// named `testName` in the module specified in `fileID`. From 825739c6610e02976cdaea65b81ff8e514a18b9f Mon Sep 17 00:00:00 2001 From: Rachel Brindle Date: Fri, 6 Jun 2025 09:00:46 -0700 Subject: [PATCH 03/10] Polling: Take in configuration arguments, add polling interval --- Sources/Testing/Polling/Polling.swift | 47 ++++++++++++++++++--------- Tests/TestingTests/PollingTests.swift | 13 ++------ 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/Sources/Testing/Polling/Polling.swift b/Sources/Testing/Polling/Polling.swift index 641a4d400..dee08d278 100644 --- a/Sources/Testing/Polling/Polling.swift +++ b/Sources/Testing/Polling/Polling.swift @@ -26,12 +26,16 @@ @available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) public func confirmPassesEventually( _ comment: Comment? = nil, + maxPollingIterations: Int = 1000, + pollingInterval: Duration = .milliseconds(1), isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation = #_sourceLocation, _ body: @escaping () async throws -> Bool ) async { let poller = Poller( pollingBehavior: .passesOnce, + pollingIterations: maxPollingIterations, + pollingInterval: pollingInterval, comment: comment, sourceLocation: sourceLocation ) @@ -70,8 +74,11 @@ public struct PollingFailedError: Error {} /// through other forms of `confirmation`. @_spi(Experimental) @available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +@discardableResult public func confirmPassesEventually( _ comment: Comment? = nil, + maxPollingIterations: Int = 1000, + pollingInterval: Duration = .milliseconds(1), isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation = #_sourceLocation, _ body: @escaping () async throws -> R? @@ -79,6 +86,8 @@ public func confirmPassesEventually( let recorder = PollingRecorder() let poller = Poller( pollingBehavior: .passesOnce, + pollingIterations: maxPollingIterations, + pollingInterval: pollingInterval, comment: comment, sourceLocation: sourceLocation ) @@ -113,12 +122,16 @@ public func confirmPassesEventually( @available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) public func confirmAlwaysPasses( _ comment: Comment? = nil, + maxPollingIterations: Int = 1000, + pollingInterval: Duration = .milliseconds(1), isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation = #_sourceLocation, _ body: @escaping () async throws -> Bool ) async { let poller = Poller( pollingBehavior: .passesAlways, + pollingIterations: maxPollingIterations, + pollingInterval: pollingInterval, comment: comment, sourceLocation: sourceLocation ) @@ -153,28 +166,26 @@ public func confirmAlwaysPasses( @available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) public func confirmAlwaysPasses( _ comment: Comment? = nil, + maxPollingIterations: Int = 1000, + pollingInterval: Duration = .milliseconds(1), isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation = #_sourceLocation, _ body: @escaping () async throws -> R? -) async throws -> R where R: Sendable { - let recorder = PollingRecorder() +) async { let poller = Poller( pollingBehavior: .passesAlways, + pollingIterations: maxPollingIterations, + pollingInterval: pollingInterval, comment: comment, sourceLocation: sourceLocation ) await poller.evaluate(isolation: isolation) { do { - return try await recorder.record(value: body()) + return try await body() != nil } catch { return false } } - - if let value = await recorder.lastValue { - return value - } - throw PollingFailedError() } /// A type to record the last value returned by a closure returning an optional @@ -289,6 +300,11 @@ private struct Poller { /// while the expression continues to pass) let pollingBehavior: PollingBehavior + // How many times to poll + let pollingIterations: Int + // Minimum waiting period between polling + let pollingInterval: Duration + /// A comment from the test author associated with the polling let comment: Comment? @@ -306,7 +322,6 @@ private struct Poller { _ body: @escaping () async -> Bool ) async { let result = await poll( - runAmount: 1_000_000, expression: body ) result.issue( @@ -325,20 +340,22 @@ private struct Poller { /// - timeout: How long to poll for unitl the timeout triggers. /// - Returns: The result of this polling. private func poll( - runAmount: Int, isolation: isolated (any Actor)? = #isolation, expression: @escaping () async -> Bool ) async -> PollResult { - for _ in 0.. 1) - } - @Test("Simple failing expressions") func trivialSadPath() async { let issues = await runTest { await confirmAlwaysPasses { false } - _ = try await confirmAlwaysPasses { Optional.none } + await confirmAlwaysPasses { Optional.none } } - #expect(issues.count == 3) + #expect(issues.count == 2) } @Test("if the closures starts off as true, but becomes false") From 151aae84a0ac3860d200fb508edbe1e0e48b4b55 Mon Sep 17 00:00:00 2001 From: Rachel Brindle Date: Sat, 7 Jun 2025 19:01:43 -0700 Subject: [PATCH 04/10] Add traits for configuring polling --- Sources/Testing/Polling/Polling.swift | 156 ++++++++++++------ .../Traits/PollingConfigurationTrait.swift | 90 ++++++++++ Tests/TestingTests/PollingTests.swift | 123 +++++++++++++- .../TestSupport/TestingAdditions.swift | 28 +++- 4 files changed, 337 insertions(+), 60 deletions(-) create mode 100644 Sources/Testing/Traits/PollingConfigurationTrait.swift diff --git a/Sources/Testing/Polling/Polling.swift b/Sources/Testing/Polling/Polling.swift index dee08d278..cc9a79812 100644 --- a/Sources/Testing/Polling/Polling.swift +++ b/Sources/Testing/Polling/Polling.swift @@ -8,11 +8,29 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +internal let defaultPollingConfiguration = ( + maxPollingIterations: 1000, + pollingInterval: Duration.milliseconds(1) +) + /// Confirm that some expression eventually returns true /// /// - Parameters: /// - comment: An optional comment to apply to any issues generated by this /// function. +/// - maxPollingIterations: The maximum amount of times to attempt polling. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then +/// polling will be attempted 1000 times before recording an issue. +/// `maxPollingIterations` must be greater than 0. +/// - pollingInterval: The minimum amount of time to wait between polling attempts. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then +/// polling will wait at least 1 millisecond between polling attempts. +/// `pollingInterval` must be greater than 0. /// - isolation: The actor to which `body` is isolated, if any. /// - sourceLocation: The source location to whych any recorded issues should /// be attributed. @@ -26,16 +44,24 @@ @available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) public func confirmPassesEventually( _ comment: Comment? = nil, - maxPollingIterations: Int = 1000, - pollingInterval: Duration = .milliseconds(1), + maxPollingIterations: Int? = nil, + pollingInterval: Duration? = nil, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation = #_sourceLocation, _ body: @escaping () async throws -> Bool ) async { let poller = Poller( pollingBehavior: .passesOnce, - pollingIterations: maxPollingIterations, - pollingInterval: pollingInterval, + pollingIterations: getValueFromPollingTrait( + providedValue: maxPollingIterations, + default: defaultPollingConfiguration.maxPollingIterations, + \ConfirmPassesEventuallyConfigurationTrait.maxPollingIterations + ), + pollingInterval: getValueFromPollingTrait( + providedValue: pollingInterval, + default: defaultPollingConfiguration.pollingInterval, + \ConfirmPassesEventuallyConfigurationTrait.pollingInterval + ), comment: comment, sourceLocation: sourceLocation ) @@ -58,6 +84,18 @@ public struct PollingFailedError: Error {} /// - Parameters: /// - comment: An optional comment to apply to any issues generated by this /// function. +/// - maxPollingIterations: The maximum amount of times to attempt polling. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then +/// polling will be attempted 1000 times before recording an issue. +/// `maxPollingIterations` must be greater than 0. +/// - pollingInterval: The minimum amount of time to wait between polling attempts. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then +/// polling will wait at least 1 millisecond between polling attempts. +/// `pollingInterval` must be greater than 0. /// - isolation: The actor to which `body` is isolated, if any. /// - sourceLocation: The source location to whych any recorded issues should /// be attributed. @@ -77,8 +115,8 @@ public struct PollingFailedError: Error {} @discardableResult public func confirmPassesEventually( _ comment: Comment? = nil, - maxPollingIterations: Int = 1000, - pollingInterval: Duration = .milliseconds(1), + maxPollingIterations: Int? = nil, + pollingInterval: Duration? = nil, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation = #_sourceLocation, _ body: @escaping () async throws -> R? @@ -86,8 +124,16 @@ public func confirmPassesEventually( let recorder = PollingRecorder() let poller = Poller( pollingBehavior: .passesOnce, - pollingIterations: maxPollingIterations, - pollingInterval: pollingInterval, + pollingIterations: getValueFromPollingTrait( + providedValue: maxPollingIterations, + default: defaultPollingConfiguration.maxPollingIterations, + \ConfirmPassesEventuallyConfigurationTrait.maxPollingIterations + ), + pollingInterval: getValueFromPollingTrait( + providedValue: pollingInterval, + default: defaultPollingConfiguration.pollingInterval, + \ConfirmPassesEventuallyConfigurationTrait.pollingInterval + ), comment: comment, sourceLocation: sourceLocation ) @@ -110,6 +156,18 @@ public func confirmPassesEventually( /// - Parameters: /// - comment: An optional comment to apply to any issues generated by this /// function. +/// - maxPollingIterations: The maximum amount of times to attempt polling. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmPassesAlwaysConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmPassesAlwaysConfigurationTrait`` has been added, then +/// polling will be attempted 1000 times before recording an issue. +/// `maxPollingIterations` must be greater than 0. +/// - pollingInterval: The minimum amount of time to wait between polling attempts. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmPassesAlwaysConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmPassesAlwaysConfigurationTrait`` has been added, then +/// polling will wait at least 1 millisecond between polling attempts. +/// `pollingInterval` must be greater than 0. /// - isolation: The actor to which `body` is isolated, if any. /// - sourceLocation: The source location to whych any recorded issues should /// be attributed. @@ -122,16 +180,24 @@ public func confirmPassesEventually( @available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) public func confirmAlwaysPasses( _ comment: Comment? = nil, - maxPollingIterations: Int = 1000, - pollingInterval: Duration = .milliseconds(1), + maxPollingIterations: Int? = nil, + pollingInterval: Duration? = nil, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation = #_sourceLocation, _ body: @escaping () async throws -> Bool ) async { let poller = Poller( pollingBehavior: .passesAlways, - pollingIterations: maxPollingIterations, - pollingInterval: pollingInterval, + pollingIterations: getValueFromPollingTrait( + providedValue: maxPollingIterations, + default: defaultPollingConfiguration.maxPollingIterations, + \ConfirmPassesAlwaysConfigurationTrait.maxPollingIterations + ), + pollingInterval: getValueFromPollingTrait( + providedValue: pollingInterval, + default: defaultPollingConfiguration.pollingInterval, + \ConfirmPassesAlwaysConfigurationTrait.pollingInterval + ), comment: comment, sourceLocation: sourceLocation ) @@ -144,48 +210,34 @@ public func confirmAlwaysPasses( } } -/// Confirm that some expression always returns a non-optional value +/// A helper function to de-duplicate the logic of grabbing configuration from +/// either the passed-in value (if given), the hardcoded default, and the +/// appropriate configuration trait. /// -/// - Parameters: -/// - comment: An optional comment to apply to any issues generated by this -/// function. -/// - isolation: The actor to which `body` is isolated, if any. -/// - sourceLocation: The source location to whych any recorded issues should -/// be attributed. -/// - body: The function to invoke. +/// The provided value, if non-nil is returned. Otherwise, this looks for +/// the last `TraitKind` specified, and if one exists, returns the value +/// as determined by `keyPath`. +/// If no configuration trait has been applied, then this returns the `default`. /// -/// - Returns: The value from the last time `body` was invoked. -/// -/// - Throws: A `PollingFailedError` will be thrown if `body` ever returns a -/// non-optional value -/// -/// Use polling confirmations to check that an event while a test is running in -/// complex scenarios where other forms of confirmation are insufficient. For -/// example, confirming that some state does not change. -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -public func confirmAlwaysPasses( - _ comment: Comment? = nil, - maxPollingIterations: Int = 1000, - pollingInterval: Duration = .milliseconds(1), - isolation: isolated (any Actor)? = #isolation, - sourceLocation: SourceLocation = #_sourceLocation, - _ body: @escaping () async throws -> R? -) async { - let poller = Poller( - pollingBehavior: .passesAlways, - pollingIterations: maxPollingIterations, - pollingInterval: pollingInterval, - comment: comment, - sourceLocation: sourceLocation - ) - await poller.evaluate(isolation: isolation) { - do { - return try await body() != nil - } catch { - return false - } +/// - Parameters: +/// - providedValue: The value provided by the test author when calling +/// `confirmPassesEventually` or `confirmAlwaysPasses`. +/// - default: The harded coded default value, as defined in +/// `defaultPollingConfiguration` +/// - keyPath: The keyPath mapping from `TraitKind` to the desired value type. +private func getValueFromPollingTrait( + providedValue: Value?, + default: Value, + _ keyPath: KeyPath +) -> Value { + if let providedValue { return providedValue } + guard let test = Test.current else { return `default` } + guard let trait = test.traits.compactMap({ $0 as? TraitKind }).last else { + print("No traits of type \(TraitKind.self) found. Returning default.") + print("Traits: \(test.traits)") + return `default` } + return trait[keyPath: keyPath] } /// A type to record the last value returned by a closure returning an optional @@ -321,6 +373,8 @@ private struct Poller { isolation: isolated (any Actor)?, _ body: @escaping () async -> Bool ) async { + precondition(pollingIterations > 0) + precondition(pollingInterval > Duration.zero) let result = await poll( expression: body ) diff --git a/Sources/Testing/Traits/PollingConfigurationTrait.swift b/Sources/Testing/Traits/PollingConfigurationTrait.swift new file mode 100644 index 000000000..56e6ba968 --- /dev/null +++ b/Sources/Testing/Traits/PollingConfigurationTrait.swift @@ -0,0 +1,90 @@ +// +// PollingConfiguration.swift +// swift-testing +// +// Created by Rachel Brindle on 6/6/25. +// + +/// A trait to provide a default polling configuration to all usages of +/// ``confirmPassesEventually`` within a test or suite. +/// +/// To add this trait to a test, use the ``Trait/pollingConfirmationEventually`` +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public struct ConfirmPassesEventuallyConfigurationTrait: TestTrait, SuiteTrait { + public var maxPollingIterations: Int + public var pollingInterval: Duration + + public var isRecursive: Bool { true } + + public init(maxPollingIterations: Int?, pollingInterval: Duration?) { + self.maxPollingIterations = maxPollingIterations ?? defaultPollingConfiguration.maxPollingIterations + self.pollingInterval = pollingInterval ?? defaultPollingConfiguration.pollingInterval + } +} + +/// A trait to provide a default polling configuration to all usages of +/// ``confirmPassesAlways`` within a test or suite. +/// +/// To add this trait to a test, use the ``Trait/pollingConfirmationAlways`` +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public struct ConfirmPassesAlwaysConfigurationTrait: TestTrait, SuiteTrait { + public var maxPollingIterations: Int + public var pollingInterval: Duration + + public var isRecursive: Bool { true } + + public init(maxPollingIterations: Int?, pollingInterval: Duration?) { + self.maxPollingIterations = maxPollingIterations ?? defaultPollingConfiguration.maxPollingIterations + self.pollingInterval = pollingInterval ?? defaultPollingConfiguration.pollingInterval + } +} + +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +extension Trait where Self == ConfirmPassesEventuallyConfigurationTrait { + /// Specifies defaults for ``confirmPassesEventually`` in the test or suite. + /// + /// - Parameters: + /// - maxPollingIterations: The maximum amount of times to attempt polling. + /// If nil, polling will be attempted up to 1000 times. + /// `maxPollingIterations` must be greater than 0. + /// - pollingInterval: The minimum amount of time to wait between polling + /// attempts. + /// If nil, polling will wait at least 1 millisecond between polling attempts. + /// `pollingInterval` must be greater than 0. + public static func confirmPassesEventuallyDefaults( + maxPollingIterations: Int? = nil, + pollingInterval: Duration? = nil + ) -> Self { + ConfirmPassesEventuallyConfigurationTrait( + maxPollingIterations: maxPollingIterations, + pollingInterval: pollingInterval + ) + } +} + +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +extension Trait where Self == ConfirmPassesAlwaysConfigurationTrait { + /// Specifies defaults for ``confirmPassesAlways`` in the test or suite. + /// + /// - Parameters: + /// - maxPollingIterations: The maximum amount of times to attempt polling. + /// If nil, polling will be attempted up to 1000 times. + /// `maxPollingIterations` must be greater than 0. + /// - pollingInterval: The minimum amount of time to wait between polling + /// attempts. + /// If nil, polling will wait at least 1 millisecond between polling attempts. + /// `pollingInterval` must be greater than 0. + public static func confirmPassesAlwaysDefaults( + maxPollingIterations: Int? = nil, + pollingInterval: Duration? = nil + ) -> Self { + ConfirmPassesAlwaysConfigurationTrait( + maxPollingIterations: maxPollingIterations, + pollingInterval: pollingInterval + ) + } +} diff --git a/Tests/TestingTests/PollingTests.swift b/Tests/TestingTests/PollingTests.swift index cd7aed967..d6330372a 100644 --- a/Tests/TestingTests/PollingTests.swift +++ b/Tests/TestingTests/PollingTests.swift @@ -14,8 +14,6 @@ struct PollingTests { @Suite("confirmPassesEventually") struct PassesOnceBehavior { - let delta = Duration.seconds(6) - @Test("Simple passing expressions") func trivialHappyPath() async throws { await confirmPassesEventually { true } @@ -54,13 +52,76 @@ struct PollingTests { } #expect(issues.count == 1) } + + @Test("Waits up to 1000 times before failing") + func defaultPollingCount() async { + let incrementor = Incrementor() + _ = await runTest { + // this test will intentionally fail. + await confirmPassesEventually(pollingInterval: .nanoseconds(1)) { + await incrementor.increment() == 0 + } + } + #expect(await incrementor.count == 1000) + } + + @Suite( + "Configuration traits", + .confirmPassesEventuallyDefaults(maxPollingIterations: 100) + ) + struct WithConfigurationTraits { + @Test("When no test or callsite configuration provided, uses the suite configuration") + func testUsesSuiteConfiguration() async throws { + let incrementor = Incrementor() + var test = Test { + await confirmPassesEventually(pollingInterval: .nanoseconds(1)) { + await incrementor.increment() == 0 + } + } + test.traits = Test.current?.traits ?? [] + await runTest(test: test) + let count = await incrementor.count + #expect(count == 100) + } + + @Test( + "When test configuration porvided, uses the test configuration", + .confirmPassesEventuallyDefaults(maxPollingIterations: 10) + ) + func testUsesTestConfigurationOverSuiteConfiguration() async { + let incrementor = Incrementor() + var test = Test { + // this test will intentionally fail. + await confirmPassesEventually(pollingInterval: .nanoseconds(1)) { + await incrementor.increment() == 0 + } + } + test.traits = Test.current?.traits ?? [] + await runTest(test: test) + #expect(await incrementor.count == 10) + } + + @Test( + "When callsite configuration provided, uses that", + .confirmPassesEventuallyDefaults(maxPollingIterations: 10) + ) + func testUsesCallsiteConfiguration() async { + let incrementor = Incrementor() + var test = Test { + // this test will intentionally fail. + await confirmPassesEventually(maxPollingIterations: 50, pollingInterval: .nanoseconds(1)) { + await incrementor.increment() == 0 + } + } + test.traits = Test.current?.traits ?? [] + await runTest(test: test) + #expect(await incrementor.count == 50) + } + } } @Suite("confirmAlwaysPasses") struct PassesAlwaysBehavior { - // use a very generous delta for CI reasons. - let delta = Duration.seconds(6) - @Test("Simple passing expressions") func trivialHappyPath() async { await confirmAlwaysPasses { true } } @@ -68,9 +129,8 @@ struct PollingTests { @Test("Simple failing expressions") func trivialSadPath() async { let issues = await runTest { await confirmAlwaysPasses { false } - await confirmAlwaysPasses { Optional.none } } - #expect(issues.count == 2) + #expect(issues.count == 1) } @Test("if the closures starts off as true, but becomes false") @@ -107,6 +167,55 @@ struct PollingTests { } #expect(issues.count == 1) } + + @Test("Waits up to 1000 times before passing") + func defaultPollingCount() async { + let incrementor = Incrementor() + await confirmAlwaysPasses(pollingInterval: .nanoseconds(1)) { + await incrementor.increment() != 0 + } + #expect(await incrementor.count == 1000) + } + + @Suite( + "Configuration traits", + .confirmPassesAlwaysDefaults(maxPollingIterations: 100) + ) + struct WithConfigurationTraits { + @Test("When no test or callsite configuration provided, uses the suite configuration") + func testUsesSuiteConfiguration() async throws { + let incrementor = Incrementor() + await confirmAlwaysPasses(pollingInterval: .nanoseconds(1)) { + await incrementor.increment() != 0 + } + let count = await incrementor.count + #expect(count == 100) + } + + @Test( + "When test configuration porvided, uses the test configuration", + .confirmPassesAlwaysDefaults(maxPollingIterations: 10) + ) + func testUsesTestConfigurationOverSuiteConfiguration() async { + let incrementor = Incrementor() + await confirmAlwaysPasses(pollingInterval: .nanoseconds(1)) { + await incrementor.increment() != 0 + } + #expect(await incrementor.count == 10) + } + + @Test( + "When callsite configuration provided, uses that", + .confirmPassesAlwaysDefaults(maxPollingIterations: 10) + ) + func testUsesCallsiteConfiguration() async { + let incrementor = Incrementor() + await confirmAlwaysPasses(maxPollingIterations: 50, pollingInterval: .nanoseconds(1)) { + await incrementor.increment() != 0 + } + #expect(await incrementor.count == 50) + } + } } @Suite("Duration Tests", .disabled("time-sensitive")) struct DurationTests { diff --git a/Tests/TestingTests/TestSupport/TestingAdditions.swift b/Tests/TestingTests/TestSupport/TestingAdditions.swift index ed2e481d2..56455a710 100644 --- a/Tests/TestingTests/TestSupport/TestingAdditions.swift +++ b/Tests/TestingTests/TestSupport/TestingAdditions.swift @@ -98,9 +98,10 @@ func runTestFunction(named name: String, in containingType: Any.Type, configurat /// issues recorded. /// /// - Parameters: -/// - expression: The test expression to run +/// - testFunction: The test expression to run /// /// - Returns: The list of issues recorded. +@discardableResult func runTest( testFunction: @escaping @Sendable () async throws -> Void ) async -> [Issue] { @@ -109,7 +110,6 @@ func runTest( var configuration = Configuration() configuration.eventHandler = { event, _ in if case let .issueRecorded(issue) = event.kind { - print("issue recorded: \(issue)") issues.withLock { $0.append(issue) } @@ -119,6 +119,30 @@ func runTest( return issues.rawValue } +/// Runs the passed-in `Test`, returning any issues recorded. +/// +/// - Parameters: +/// - test: The test to run +/// +/// - Returns: The list of issues recorded. +@discardableResult +func runTest( + test: Test +) async -> [Issue] { + let issues = Locked(rawValue: [Issue]()) + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case let .issueRecorded(issue) = event.kind { + issues.withLock { + $0.append(issue) + } + } + } + await test.run(configuration: configuration) + return issues.rawValue +} + extension Runner { /// Initialize an instance of this type that runs the free test function /// named `testName` in the module specified in `fileID`. From 2dce5112822d23590370a030941fcfed52ebf523 Mon Sep 17 00:00:00 2001 From: Rachel Brindle Date: Sat, 7 Jun 2025 21:22:26 -0700 Subject: [PATCH 05/10] Use consistent naming between confirmAlwaysPasses and the related configuration trait Stop unnecessarily waiting after the last polling attempt has finished. Allow for subsequent polling configuration traits which specified nil for a value to fall back to earlier polling configuration traits before falling back to the default. --- Sources/Testing/Polling/Polling.swift | 44 ++++++++------- .../Traits/PollingConfigurationTrait.swift | 38 +++++++------ Tests/TestingTests/PollingTests.swift | 56 +++++++++++++++---- 3 files changed, 92 insertions(+), 46 deletions(-) diff --git a/Sources/Testing/Polling/Polling.swift b/Sources/Testing/Polling/Polling.swift index cc9a79812..ffd3978f8 100644 --- a/Sources/Testing/Polling/Polling.swift +++ b/Sources/Testing/Polling/Polling.swift @@ -21,11 +21,13 @@ internal let defaultPollingConfiguration = ( /// function. /// - maxPollingIterations: The maximum amount of times to attempt polling. /// If nil, this uses whatever value is specified under the last -/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or suite. +/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or +/// suite. /// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then /// polling will be attempted 1000 times before recording an issue. /// `maxPollingIterations` must be greater than 0. -/// - pollingInterval: The minimum amount of time to wait between polling attempts. +/// - pollingInterval: The minimum amount of time to wait between polling +/// attempts. /// If nil, this uses whatever value is specified under the last /// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or suite. /// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then @@ -86,11 +88,13 @@ public struct PollingFailedError: Error {} /// function. /// - maxPollingIterations: The maximum amount of times to attempt polling. /// If nil, this uses whatever value is specified under the last -/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or suite. +/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or +/// suite. /// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then /// polling will be attempted 1000 times before recording an issue. /// `maxPollingIterations` must be greater than 0. -/// - pollingInterval: The minimum amount of time to wait between polling attempts. +/// - pollingInterval: The minimum amount of time to wait between polling +/// attempts. /// If nil, this uses whatever value is specified under the last /// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or suite. /// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then @@ -158,14 +162,15 @@ public func confirmPassesEventually( /// function. /// - maxPollingIterations: The maximum amount of times to attempt polling. /// If nil, this uses whatever value is specified under the last -/// ``ConfirmPassesAlwaysConfigurationTrait`` added to the test or suite. -/// If no ``ConfirmPassesAlwaysConfigurationTrait`` has been added, then +/// ``ConfirmAlwaysPassesConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmAlwaysPassesConfigurationTrait`` has been added, then /// polling will be attempted 1000 times before recording an issue. /// `maxPollingIterations` must be greater than 0. -/// - pollingInterval: The minimum amount of time to wait between polling attempts. +/// - pollingInterval: The minimum amount of time to wait between polling +/// attempts. /// If nil, this uses whatever value is specified under the last -/// ``ConfirmPassesAlwaysConfigurationTrait`` added to the test or suite. -/// If no ``ConfirmPassesAlwaysConfigurationTrait`` has been added, then +/// ``ConfirmAlwaysPassesConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmAlwaysPassesConfigurationTrait`` has been added, then /// polling will wait at least 1 millisecond between polling attempts. /// `pollingInterval` must be greater than 0. /// - isolation: The actor to which `body` is isolated, if any. @@ -191,12 +196,12 @@ public func confirmAlwaysPasses( pollingIterations: getValueFromPollingTrait( providedValue: maxPollingIterations, default: defaultPollingConfiguration.maxPollingIterations, - \ConfirmPassesAlwaysConfigurationTrait.maxPollingIterations + \ConfirmAlwaysPassesConfigurationTrait.maxPollingIterations ), pollingInterval: getValueFromPollingTrait( providedValue: pollingInterval, default: defaultPollingConfiguration.pollingInterval, - \ConfirmPassesAlwaysConfigurationTrait.pollingInterval + \ConfirmAlwaysPassesConfigurationTrait.pollingInterval ), comment: comment, sourceLocation: sourceLocation @@ -228,16 +233,13 @@ public func confirmAlwaysPasses( private func getValueFromPollingTrait( providedValue: Value?, default: Value, - _ keyPath: KeyPath + _ keyPath: KeyPath ) -> Value { if let providedValue { return providedValue } guard let test = Test.current else { return `default` } - guard let trait = test.traits.compactMap({ $0 as? TraitKind }).last else { - print("No traits of type \(TraitKind.self) found. Returning default.") - print("Traits: \(test.traits)") - return `default` - } - return trait[keyPath: keyPath] + let possibleTraits = test.traits.compactMap { $0 as? TraitKind } + let traitValues = possibleTraits.compactMap { $0[keyPath: keyPath] } + return traitValues.last ?? `default` } /// A type to record the last value returned by a closure returning an optional @@ -397,12 +399,16 @@ private struct Poller { isolation: isolated (any Actor)? = #isolation, expression: @escaping () async -> Bool ) async -> PollResult { - for _ in 0.. Self { - ConfirmPassesAlwaysConfigurationTrait( + ConfirmAlwaysPassesConfigurationTrait( maxPollingIterations: maxPollingIterations, pollingInterval: pollingInterval ) diff --git a/Tests/TestingTests/PollingTests.swift b/Tests/TestingTests/PollingTests.swift index d6330372a..173ac8ecf 100644 --- a/Tests/TestingTests/PollingTests.swift +++ b/Tests/TestingTests/PollingTests.swift @@ -179,7 +179,7 @@ struct PollingTests { @Suite( "Configuration traits", - .confirmPassesAlwaysDefaults(maxPollingIterations: 100) + .confirmAlwaysPassesDefaults(maxPollingIterations: 100) ) struct WithConfigurationTraits { @Test("When no test or callsite configuration provided, uses the suite configuration") @@ -194,7 +194,7 @@ struct PollingTests { @Test( "When test configuration porvided, uses the test configuration", - .confirmPassesAlwaysDefaults(maxPollingIterations: 10) + .confirmAlwaysPassesDefaults(maxPollingIterations: 10) ) func testUsesTestConfigurationOverSuiteConfiguration() async { let incrementor = Incrementor() @@ -206,7 +206,7 @@ struct PollingTests { @Test( "When callsite configuration provided, uses that", - .confirmPassesAlwaysDefaults(maxPollingIterations: 10) + .confirmAlwaysPassesDefaults(maxPollingIterations: 10) ) func testUsesCallsiteConfiguration() async { let incrementor = Incrementor() @@ -218,10 +218,11 @@ struct PollingTests { } } - @Suite("Duration Tests", .disabled("time-sensitive")) struct DurationTests { + @Suite("Duration Tests", .disabled("time-sensitive")) + struct DurationTests { @Suite("confirmPassesEventually") struct PassesOnceBehavior { - let delta = Duration.seconds(6) + let delta = Duration.milliseconds(100) @Test("Simple passing expressions") func trivialHappyPath() async { let duration = await Test.Clock().measure { @@ -237,10 +238,11 @@ struct PollingTests { } #expect(issues.count == 1) } - #expect(duration.isCloseTo(other: .seconds(60), within: delta)) + #expect(duration.isCloseTo(other: .seconds(2), within: delta)) } - @Test("When the value changes from false to true during execution") func changingFromFail() async { + @Test("When the value changes from false to true during execution") + func changingFromFail() async { let incrementor = Incrementor() let duration = await Test.Clock().measure { @@ -256,18 +258,36 @@ struct PollingTests { #expect(await incrementor.count == 2) #expect(duration.isCloseTo(other: .zero, within: delta)) } + + @Test("Doesn't wait after the last iteration") + func lastIteration() async { + let duration = await Test.Clock().measure { + let issues = await runTest { + await confirmPassesEventually( + maxPollingIterations: 10, + pollingInterval: .seconds(1) // Wait a long time to handle jitter. + ) { false } + } + #expect(issues.count == 1) + } + #expect( + duration.isCloseTo( + other: .seconds(9), + within: .milliseconds(500) + ) + ) + } } @Suite("confirmAlwaysPasses") struct PassesAlwaysBehavior { - // use a very generous delta for CI reasons. - let delta = Duration.seconds(6) + let delta = Duration.milliseconds(100) @Test("Simple passing expressions") func trivialHappyPath() async { let duration = await Test.Clock().measure { await confirmAlwaysPasses { true } } - #expect(duration.isCloseTo(other: .seconds(60), within: delta)) + #expect(duration.isCloseTo(other: .seconds(1), within: delta)) } @Test("Simple failing expressions") func trivialSadPath() async { @@ -279,6 +299,22 @@ struct PollingTests { } #expect(duration.isCloseTo(other: .zero, within: delta)) } + + @Test("Doesn't wait after the last iteration") + func lastIteration() async { + let duration = await Test.Clock().measure { + await confirmAlwaysPasses( + maxPollingIterations: 10, + pollingInterval: .seconds(1) // Wait a long time to handle jitter. + ) { true } + } + #expect( + duration.isCloseTo( + other: .seconds(9), + within: .milliseconds(500) + ) + ) + } } } } From bc01e1bc84f1e79d88a01e6580fbd6f727d059ec Mon Sep 17 00:00:00 2001 From: Rachel Brindle Date: Sat, 7 Jun 2025 22:11:04 -0700 Subject: [PATCH 06/10] Add requirePassesEventually and requireAlwaysPasses These two mirror their confirm counterparts, only throwing an error (instead of recording an issue) when they fail. --- Sources/Testing/Polling/Polling.swift | 164 ++++++++++++++++++++++++-- Tests/TestingTests/PollingTests.swift | 11 +- 2 files changed, 166 insertions(+), 9 deletions(-) diff --git a/Sources/Testing/Polling/Polling.swift b/Sources/Testing/Polling/Polling.swift index ffd3978f8..851e68ecd 100644 --- a/Sources/Testing/Polling/Polling.swift +++ b/Sources/Testing/Polling/Polling.swift @@ -14,6 +14,10 @@ internal let defaultPollingConfiguration = ( pollingInterval: Duration.milliseconds(1) ) +/// A type describing an error thrown when polling fails. +@_spi(Experimental) +public struct PollingFailedError: Error, Equatable {} + /// Confirm that some expression eventually returns true /// /// - Parameters: @@ -76,10 +80,73 @@ public func confirmPassesEventually( } } -/// A type describing an error thrown when polling fails to return a non-nil -/// value +/// Require that some expression eventually returns true +/// +/// - Parameters: +/// - comment: An optional comment to apply to any issues generated by this +/// function. +/// - maxPollingIterations: The maximum amount of times to attempt polling. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or +/// suite. +/// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then +/// polling will be attempted 1000 times before recording an issue. +/// `maxPollingIterations` must be greater than 0. +/// - pollingInterval: The minimum amount of time to wait between polling +/// attempts. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then +/// polling will wait at least 1 millisecond between polling attempts. +/// `pollingInterval` must be greater than 0. +/// - isolation: The actor to which `body` is isolated, if any. +/// - sourceLocation: The source location to whych any recorded issues should +/// be attributed. +/// - body: The function to invoke. +/// +/// - Throws: A `PollingFailedError` will be thrown if the expression never +/// returns true. +/// +/// Use polling confirmations to check that an event while a test is running in +/// complex scenarios where other forms of confirmation are insufficient. For +/// example, waiting on some state to change that cannot be easily confirmed +/// through other forms of `confirmation`. @_spi(Experimental) -public struct PollingFailedError: Error {} +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public func requirePassesEventually( + _ comment: Comment? = nil, + maxPollingIterations: Int? = nil, + pollingInterval: Duration? = nil, + isolation: isolated (any Actor)? = #isolation, + sourceLocation: SourceLocation = #_sourceLocation, + _ body: @escaping () async throws -> Bool +) async throws { + let poller = Poller( + pollingBehavior: .passesOnce, + pollingIterations: getValueFromPollingTrait( + providedValue: maxPollingIterations, + default: defaultPollingConfiguration.maxPollingIterations, + \ConfirmPassesEventuallyConfigurationTrait.maxPollingIterations + ), + pollingInterval: getValueFromPollingTrait( + providedValue: pollingInterval, + default: defaultPollingConfiguration.pollingInterval, + \ConfirmPassesEventuallyConfigurationTrait.pollingInterval + ), + comment: comment, + sourceLocation: sourceLocation + ) + let passed = await poller.evaluate(raiseIssue: false, isolation: isolation) { + do { + return try await body() + } catch { + return false + } + } + if !passed { + throw PollingFailedError() + } +} /// Confirm that some expression eventually returns a non-nil value /// @@ -108,7 +175,7 @@ public struct PollingFailedError: Error {} /// - Returns: The first non-nil value returned by `body`. /// /// - Throws: A `PollingFailedError` will be thrown if `body` never returns a -/// non-optional value +/// non-optional value. /// /// Use polling confirmations to check that an event while a test is running in /// complex scenarios where other forms of confirmation are insufficient. For @@ -215,6 +282,72 @@ public func confirmAlwaysPasses( } } +/// Require that some expression always returns true +/// +/// - Parameters: +/// - comment: An optional comment to apply to any issues generated by this +/// function. +/// - maxPollingIterations: The maximum amount of times to attempt polling. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmAlwaysPassesConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmAlwaysPassesConfigurationTrait`` has been added, then +/// polling will be attempted 1000 times before recording an issue. +/// `maxPollingIterations` must be greater than 0. +/// - pollingInterval: The minimum amount of time to wait between polling +/// attempts. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmAlwaysPassesConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmAlwaysPassesConfigurationTrait`` has been added, then +/// polling will wait at least 1 millisecond between polling attempts. +/// `pollingInterval` must be greater than 0. +/// - isolation: The actor to which `body` is isolated, if any. +/// - sourceLocation: The source location to whych any recorded issues should +/// be attributed. +/// - body: The function to invoke. +/// +/// - Throws: A `PollingFailedError` will be thrown if the expression ever +/// returns false. +/// +/// Use polling confirmations to check that an event while a test is running in +/// complex scenarios where other forms of confirmation are insufficient. For +/// example, confirming that some state does not change. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public func requireAlwaysPasses( + _ comment: Comment? = nil, + maxPollingIterations: Int? = nil, + pollingInterval: Duration? = nil, + isolation: isolated (any Actor)? = #isolation, + sourceLocation: SourceLocation = #_sourceLocation, + _ body: @escaping () async throws -> Bool +) async throws { + let poller = Poller( + pollingBehavior: .passesAlways, + pollingIterations: getValueFromPollingTrait( + providedValue: maxPollingIterations, + default: defaultPollingConfiguration.maxPollingIterations, + \ConfirmAlwaysPassesConfigurationTrait.maxPollingIterations + ), + pollingInterval: getValueFromPollingTrait( + providedValue: pollingInterval, + default: defaultPollingConfiguration.pollingInterval, + \ConfirmAlwaysPassesConfigurationTrait.pollingInterval + ), + comment: comment, + sourceLocation: sourceLocation + ) + let passed = await poller.evaluate(raiseIssue: false, isolation: isolation) { + do { + return try await body() + } catch { + return false + } + } + if !passed { + throw PollingFailedError() + } +} + /// A helper function to de-duplicate the logic of grabbing configuration from /// either the passed-in value (if given), the hardcoded default, and the /// appropriate configuration trait. @@ -368,23 +501,38 @@ private struct Poller { /// Evaluate polling, and process the result, raising an issue if necessary. /// /// - Parameters: + /// - raiseIssue: Whether or not to raise an issue. + /// This should only be false for `requirePassesEventually` or + /// `requireAlwaysPasses`. + /// - isolation: The isolation to use /// - body: The expression to poll + /// + /// - Returns: Whether or not polling passed. + /// /// - Side effects: If polling fails (see `PollingBehavior`), then this will /// record an issue. - func evaluate( + @discardableResult func evaluate( + raiseIssue: Bool = true, isolation: isolated (any Actor)?, _ body: @escaping () async -> Bool - ) async { + ) async -> Bool { precondition(pollingIterations > 0) precondition(pollingInterval > Duration.zero) let result = await poll( expression: body ) - result.issue( + if let issue = result.issue( comment: comment, sourceContext: .init(backtrace: .current(), sourceLocation: sourceLocation), pollingBehavior: pollingBehavior - )?.record() + ) { + if raiseIssue { + issue.record() + } + return false + } else { + return true + } } /// This function contains the logic for continuously polling an expression, diff --git a/Tests/TestingTests/PollingTests.swift b/Tests/TestingTests/PollingTests.swift index 173ac8ecf..4f97e15bd 100644 --- a/Tests/TestingTests/PollingTests.swift +++ b/Tests/TestingTests/PollingTests.swift @@ -16,8 +16,10 @@ struct PollingTests { struct PassesOnceBehavior { @Test("Simple passing expressions") func trivialHappyPath() async throws { await confirmPassesEventually { true } + try await requirePassesEventually { true } let value = try await confirmPassesEventually { 1 } + #expect(value == 1) } @@ -25,6 +27,9 @@ struct PollingTests { let issues = await runTest { await confirmPassesEventually { false } _ = try await confirmPassesEventually { Optional.none } + await #expect(throws: PollingFailedError()) { + try await requirePassesEventually { false } + } } #expect(issues.count == 3) } @@ -122,13 +127,17 @@ struct PollingTests { @Suite("confirmAlwaysPasses") struct PassesAlwaysBehavior { - @Test("Simple passing expressions") func trivialHappyPath() async { + @Test("Simple passing expressions") func trivialHappyPath() async throws { await confirmAlwaysPasses { true } + try await requireAlwaysPasses { true } } @Test("Simple failing expressions") func trivialSadPath() async { let issues = await runTest { await confirmAlwaysPasses { false } + await #expect(throws: PollingFailedError()) { + try await requireAlwaysPasses { false } + } } #expect(issues.count == 1) } From ef109b0bccfa80df492a4f1824cb93ea3ef144ca Mon Sep 17 00:00:00 2001 From: Rachel Brindle Date: Tue, 1 Jul 2025 23:01:04 -0700 Subject: [PATCH 07/10] Rewrite confirmPassesEventually when returning an optional to remove the PollingRecorder actor. Now, this uses a separate method for evaluating polling to remove that actor. --- Sources/Testing/Polling/Polling.swift | 111 +++++++++++++++++++------- 1 file changed, 84 insertions(+), 27 deletions(-) diff --git a/Sources/Testing/Polling/Polling.swift b/Sources/Testing/Polling/Polling.swift index 851e68ecd..d855cbcd7 100644 --- a/Sources/Testing/Polling/Polling.swift +++ b/Sources/Testing/Polling/Polling.swift @@ -190,9 +190,8 @@ public func confirmPassesEventually( pollingInterval: Duration? = nil, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation = #_sourceLocation, - _ body: @escaping () async throws -> R? -) async throws -> R where R: Sendable { - let recorder = PollingRecorder() + _ body: @escaping () async throws -> sending R? +) async throws -> R { let poller = Poller( pollingBehavior: .passesOnce, pollingIterations: getValueFromPollingTrait( @@ -208,15 +207,15 @@ public func confirmPassesEventually( comment: comment, sourceLocation: sourceLocation ) - await poller.evaluate(isolation: isolation) { + let recordedValue = await poller.evaluate(isolation: isolation) { do { - return try await recorder.record(value: body()) + return try await body() } catch { - return false + return nil } } - if let value = await recorder.lastValue { + if let value = recordedValue { return value } throw PollingFailedError() @@ -375,26 +374,6 @@ private func getValueFromPollingTrait( return traitValues.last ?? `default` } -/// A type to record the last value returned by a closure returning an optional -/// This is only used in the `confirm` polling functions evaluating an optional. -private actor PollingRecorder { - var lastValue: R? - - /// Record a new value to be returned - func record(value: R) { - self.lastValue = value - } - - func record(value: R?) -> Bool { - if let value { - self.lastValue = value - return true - } else { - return false - } - } -} - /// A type for managing polling @available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) private struct Poller { @@ -567,4 +546,82 @@ private struct Poller { } return .ranToCompletion } + + /// Evaluate polling, and process the result, raising an issue if necessary. + /// + /// - Note: This method is only intended to be used when pollingBehavior is + /// `.passesOnce` + /// + /// - Parameters: + /// - raiseIssue: Whether or not to raise an issue. + /// This should only be false for `requirePassesEventually` or + /// `requireAlwaysPasses`. + /// - isolation: The isolation to use + /// - body: The expression to poll + /// + /// - Returns: the value if polling passed, nil otherwise. + /// + /// - Side effects: If polling fails (see `PollingBehavior`), then this will + /// record an issue. + @discardableResult func evaluate( + raiseIssue: Bool = true, + isolation: isolated (any Actor)?, + _ body: @escaping () async -> sending R? + ) async -> R? { + precondition(pollingIterations > 0) + precondition(pollingInterval > Duration.zero) + let (result, value) = await poll( + expression: body + ) + if let issue = result.issue( + comment: comment, + sourceContext: .init(backtrace: .current(), sourceLocation: sourceLocation), + pollingBehavior: pollingBehavior + ) { + if raiseIssue { + issue.record() + } + return value + } else { + return value + } + } + + /// This function contains the logic for continuously polling an expression, + /// as well as processing the results of that expression + /// + /// - Note: This method is only intended to be used when pollingBehavior is + /// `.passesOnce` + /// + /// - Parameters: + /// - expression: An expression to continuously evaluate + /// - behavior: The polling behavior to use + /// - timeout: How long to poll for unitl the timeout triggers. + /// - Returns: The result of this polling and the most recent value if the + /// result is .finished, otherwise nil. + private func poll( + isolation: isolated (any Actor)? = #isolation, + expression: @escaping () async -> sending R? + ) async -> (PollResult, R?) { + for iteration in 0.. Date: Wed, 2 Jul 2025 07:37:49 -0700 Subject: [PATCH 08/10] Clean up the duplicate Poller.evaluate/Poller.poll methods Removed the duplicate poll method, and made evaluate-returning-bool into a wrapper for evaluate-returning-optional --- Sources/Testing/Polling/Polling.swift | 71 ++++++--------------------- 1 file changed, 15 insertions(+), 56 deletions(-) diff --git a/Sources/Testing/Polling/Polling.swift b/Sources/Testing/Polling/Polling.swift index d855cbcd7..5145ff4b6 100644 --- a/Sources/Testing/Polling/Polling.swift +++ b/Sources/Testing/Polling/Polling.swift @@ -71,7 +71,7 @@ public func confirmPassesEventually( comment: comment, sourceLocation: sourceLocation ) - await poller.evaluate(isolation: isolation) { + await poller.evaluateBool(isolation: isolation) { do { return try await body() } catch { @@ -136,7 +136,7 @@ public func requirePassesEventually( comment: comment, sourceLocation: sourceLocation ) - let passed = await poller.evaluate(raiseIssue: false, isolation: isolation) { + let passed = await poller.evaluateBool(raiseIssue: false, isolation: isolation) { do { return try await body() } catch { @@ -272,7 +272,7 @@ public func confirmAlwaysPasses( comment: comment, sourceLocation: sourceLocation ) - await poller.evaluate(isolation: isolation) { + await poller.evaluateBool(isolation: isolation) { do { return try await body() } catch { @@ -335,7 +335,7 @@ public func requireAlwaysPasses( comment: comment, sourceLocation: sourceLocation ) - let passed = await poller.evaluate(raiseIssue: false, isolation: isolation) { + let passed = await poller.evaluateBool(raiseIssue: false, isolation: isolation) { do { return try await body() } catch { @@ -490,61 +490,19 @@ private struct Poller { /// /// - Side effects: If polling fails (see `PollingBehavior`), then this will /// record an issue. - @discardableResult func evaluate( + @discardableResult func evaluateBool( raiseIssue: Bool = true, isolation: isolated (any Actor)?, _ body: @escaping () async -> Bool ) async -> Bool { - precondition(pollingIterations > 0) - precondition(pollingInterval > Duration.zero) - let result = await poll( - expression: body - ) - if let issue = result.issue( - comment: comment, - sourceContext: .init(backtrace: .current(), sourceLocation: sourceLocation), - pollingBehavior: pollingBehavior - ) { - if raiseIssue { - issue.record() - } - return false - } else { - return true - } - } - - /// This function contains the logic for continuously polling an expression, - /// as well as processing the results of that expression - /// - /// - Parameters: - /// - expression: An expression to continuously evaluate - /// - behavior: The polling behavior to use - /// - timeout: How long to poll for unitl the timeout triggers. - /// - Returns: The result of this polling. - private func poll( - isolation: isolated (any Actor)? = #isolation, - expression: @escaping () async -> Bool - ) async -> PollResult { - for iteration in 0.. sending R? ) async -> (PollResult, R?) { + var lastResult: R? for iteration in 0.. Date: Thu, 3 Jul 2025 00:04:46 -0700 Subject: [PATCH 09/10] Configure polling confirmations as timeout & polling interval This is less direct, but much more intuitive for test authors. Also add exit tests confirming that these values are non-negative --- Sources/Testing/Polling/Polling.swift | 97 +++++++++------- .../Traits/PollingConfigurationTrait.swift | 36 +++--- Tests/TestingTests/PollingTests.swift | 105 ++++++++++++++---- 3 files changed, 161 insertions(+), 77 deletions(-) diff --git a/Sources/Testing/Polling/Polling.swift b/Sources/Testing/Polling/Polling.swift index 5145ff4b6..efa2cef8c 100644 --- a/Sources/Testing/Polling/Polling.swift +++ b/Sources/Testing/Polling/Polling.swift @@ -10,7 +10,7 @@ @available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) internal let defaultPollingConfiguration = ( - maxPollingIterations: 1000, + pollingDuration: Duration.seconds(1), pollingInterval: Duration.milliseconds(1) ) @@ -23,12 +23,14 @@ public struct PollingFailedError: Error, Equatable {} /// - Parameters: /// - comment: An optional comment to apply to any issues generated by this /// function. -/// - maxPollingIterations: The maximum amount of times to attempt polling. +/// - pollingDuration: The expected length of time to continue polling for. +/// This value may not correspond to the wall-clock time that polling lasts for, especially +/// on highly-loaded systems with a lot of tests running. /// If nil, this uses whatever value is specified under the last /// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or /// suite. /// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then -/// polling will be attempted 1000 times before recording an issue. +/// polling will be attempted for about 1 second before recording an issue. /// `maxPollingIterations` must be greater than 0. /// - pollingInterval: The minimum amount of time to wait between polling /// attempts. @@ -50,7 +52,7 @@ public struct PollingFailedError: Error, Equatable {} @available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) public func confirmPassesEventually( _ comment: Comment? = nil, - maxPollingIterations: Int? = nil, + pollingDuration: Duration? = nil, pollingInterval: Duration? = nil, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation = #_sourceLocation, @@ -58,10 +60,10 @@ public func confirmPassesEventually( ) async { let poller = Poller( pollingBehavior: .passesOnce, - pollingIterations: getValueFromPollingTrait( - providedValue: maxPollingIterations, - default: defaultPollingConfiguration.maxPollingIterations, - \ConfirmPassesEventuallyConfigurationTrait.maxPollingIterations + pollingDuration: getValueFromPollingTrait( + providedValue: pollingDuration, + default: defaultPollingConfiguration.pollingDuration, + \ConfirmPassesEventuallyConfigurationTrait.pollingDuration ), pollingInterval: getValueFromPollingTrait( providedValue: pollingInterval, @@ -85,12 +87,14 @@ public func confirmPassesEventually( /// - Parameters: /// - comment: An optional comment to apply to any issues generated by this /// function. -/// - maxPollingIterations: The maximum amount of times to attempt polling. +/// - pollingDuration: The expected length of time to continue polling for. +/// This value may not correspond to the wall-clock time that polling lasts for, especially +/// on highly-loaded systems with a lot of tests running. /// If nil, this uses whatever value is specified under the last /// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or /// suite. /// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then -/// polling will be attempted 1000 times before recording an issue. +/// polling will be attempted for about 1 second before recording an issue. /// `maxPollingIterations` must be greater than 0. /// - pollingInterval: The minimum amount of time to wait between polling /// attempts. @@ -115,7 +119,7 @@ public func confirmPassesEventually( @available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) public func requirePassesEventually( _ comment: Comment? = nil, - maxPollingIterations: Int? = nil, + pollingDuration: Duration? = nil, pollingInterval: Duration? = nil, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation = #_sourceLocation, @@ -123,10 +127,10 @@ public func requirePassesEventually( ) async throws { let poller = Poller( pollingBehavior: .passesOnce, - pollingIterations: getValueFromPollingTrait( - providedValue: maxPollingIterations, - default: defaultPollingConfiguration.maxPollingIterations, - \ConfirmPassesEventuallyConfigurationTrait.maxPollingIterations + pollingDuration: getValueFromPollingTrait( + providedValue: pollingDuration, + default: defaultPollingConfiguration.pollingDuration, + \ConfirmPassesEventuallyConfigurationTrait.pollingDuration ), pollingInterval: getValueFromPollingTrait( providedValue: pollingInterval, @@ -153,12 +157,14 @@ public func requirePassesEventually( /// - Parameters: /// - comment: An optional comment to apply to any issues generated by this /// function. -/// - maxPollingIterations: The maximum amount of times to attempt polling. +/// - pollingDuration: The expected length of time to continue polling for. +/// This value may not correspond to the wall-clock time that polling lasts for, especially +/// on highly-loaded systems with a lot of tests running. /// If nil, this uses whatever value is specified under the last /// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or /// suite. /// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then -/// polling will be attempted 1000 times before recording an issue. +/// polling will be attempted for about 1 second before recording an issue. /// `maxPollingIterations` must be greater than 0. /// - pollingInterval: The minimum amount of time to wait between polling /// attempts. @@ -186,7 +192,7 @@ public func requirePassesEventually( @discardableResult public func confirmPassesEventually( _ comment: Comment? = nil, - maxPollingIterations: Int? = nil, + pollingDuration: Duration? = nil, pollingInterval: Duration? = nil, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation = #_sourceLocation, @@ -194,10 +200,10 @@ public func confirmPassesEventually( ) async throws -> R { let poller = Poller( pollingBehavior: .passesOnce, - pollingIterations: getValueFromPollingTrait( - providedValue: maxPollingIterations, - default: defaultPollingConfiguration.maxPollingIterations, - \ConfirmPassesEventuallyConfigurationTrait.maxPollingIterations + pollingDuration: getValueFromPollingTrait( + providedValue: pollingDuration, + default: defaultPollingConfiguration.pollingDuration, + \ConfirmPassesEventuallyConfigurationTrait.pollingDuration ), pollingInterval: getValueFromPollingTrait( providedValue: pollingInterval, @@ -226,11 +232,13 @@ public func confirmPassesEventually( /// - Parameters: /// - comment: An optional comment to apply to any issues generated by this /// function. -/// - maxPollingIterations: The maximum amount of times to attempt polling. +/// - pollingDuration: The expected length of time to continue polling for. +/// This value may not correspond to the wall-clock time that polling lasts for, especially +/// on highly-loaded systems with a lot of tests running. /// If nil, this uses whatever value is specified under the last /// ``ConfirmAlwaysPassesConfigurationTrait`` added to the test or suite. /// If no ``ConfirmAlwaysPassesConfigurationTrait`` has been added, then -/// polling will be attempted 1000 times before recording an issue. +/// polling will be attempted for about 1 second before recording an issue. /// `maxPollingIterations` must be greater than 0. /// - pollingInterval: The minimum amount of time to wait between polling /// attempts. @@ -251,7 +259,7 @@ public func confirmPassesEventually( @available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) public func confirmAlwaysPasses( _ comment: Comment? = nil, - maxPollingIterations: Int? = nil, + pollingDuration: Duration? = nil, pollingInterval: Duration? = nil, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation = #_sourceLocation, @@ -259,10 +267,10 @@ public func confirmAlwaysPasses( ) async { let poller = Poller( pollingBehavior: .passesAlways, - pollingIterations: getValueFromPollingTrait( - providedValue: maxPollingIterations, - default: defaultPollingConfiguration.maxPollingIterations, - \ConfirmAlwaysPassesConfigurationTrait.maxPollingIterations + pollingDuration: getValueFromPollingTrait( + providedValue: pollingDuration, + default: defaultPollingConfiguration.pollingDuration, + \ConfirmAlwaysPassesConfigurationTrait.pollingDuration ), pollingInterval: getValueFromPollingTrait( providedValue: pollingInterval, @@ -286,11 +294,13 @@ public func confirmAlwaysPasses( /// - Parameters: /// - comment: An optional comment to apply to any issues generated by this /// function. -/// - maxPollingIterations: The maximum amount of times to attempt polling. +/// - pollingDuration: The expected length of time to continue polling for. +/// This value may not correspond to the wall-clock time that polling lasts for, especially +/// on highly-loaded systems with a lot of tests running. /// If nil, this uses whatever value is specified under the last /// ``ConfirmAlwaysPassesConfigurationTrait`` added to the test or suite. /// If no ``ConfirmAlwaysPassesConfigurationTrait`` has been added, then -/// polling will be attempted 1000 times before recording an issue. +/// polling will be attempted for about 1 second before recording an issue. /// `maxPollingIterations` must be greater than 0. /// - pollingInterval: The minimum amount of time to wait between polling /// attempts. @@ -314,7 +324,7 @@ public func confirmAlwaysPasses( @available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) public func requireAlwaysPasses( _ comment: Comment? = nil, - maxPollingIterations: Int? = nil, + pollingDuration: Duration? = nil, pollingInterval: Duration? = nil, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation = #_sourceLocation, @@ -322,10 +332,10 @@ public func requireAlwaysPasses( ) async throws { let poller = Poller( pollingBehavior: .passesAlways, - pollingIterations: getValueFromPollingTrait( - providedValue: maxPollingIterations, - default: defaultPollingConfiguration.maxPollingIterations, - \ConfirmAlwaysPassesConfigurationTrait.maxPollingIterations + pollingDuration: getValueFromPollingTrait( + providedValue: pollingDuration, + default: defaultPollingConfiguration.pollingDuration, + \ConfirmAlwaysPassesConfigurationTrait.pollingDuration ), pollingInterval: getValueFromPollingTrait( providedValue: pollingInterval, @@ -466,8 +476,8 @@ private struct Poller { /// while the expression continues to pass) let pollingBehavior: PollingBehavior - // How many times to poll - let pollingIterations: Int + // Approximately how long to poll for + let pollingDuration: Duration // Minimum waiting period between polling let pollingInterval: Duration @@ -526,9 +536,16 @@ private struct Poller { isolation: isolated (any Actor)?, _ body: @escaping () async -> sending R? ) async -> R? { - precondition(pollingIterations > 0) + precondition(pollingDuration > Duration.zero) precondition(pollingInterval > Duration.zero) + precondition(pollingDuration > pollingInterval) + let durationSeconds = Double(pollingDuration.components.seconds) + Double(pollingDuration.components.attoseconds) * 1e-18 + let intervalSeconds = Double(pollingInterval.components.seconds) + Double(pollingInterval.components.attoseconds) * 1e-18 + + let pollingIterations = max(Int(durationSeconds / intervalSeconds), 1) + let (result, value) = await poll( + pollingIterations: pollingIterations, expression: body ) if let issue = result.issue( @@ -552,12 +569,14 @@ private struct Poller { /// `.passesOnce` /// /// - Parameters: + /// - pollingIterations: The maximum amount of times to continue polling. /// - expression: An expression to continuously evaluate /// - behavior: The polling behavior to use /// - timeout: How long to poll for unitl the timeout triggers. /// - Returns: The result of this polling and the most recent value if the /// result is .finished, otherwise nil. private func poll( + pollingIterations: Int, isolation: isolated (any Actor)? = #isolation, expression: @escaping () async -> sending R? ) async -> (PollResult, R?) { diff --git a/Sources/Testing/Traits/PollingConfigurationTrait.swift b/Sources/Testing/Traits/PollingConfigurationTrait.swift index 10ca73bb0..40c08bf0c 100644 --- a/Sources/Testing/Traits/PollingConfigurationTrait.swift +++ b/Sources/Testing/Traits/PollingConfigurationTrait.swift @@ -13,13 +13,13 @@ @_spi(Experimental) @available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) public struct ConfirmPassesEventuallyConfigurationTrait: TestTrait, SuiteTrait { - public var maxPollingIterations: Int? + public var pollingDuration: Duration? public var pollingInterval: Duration? public var isRecursive: Bool { true } - public init(maxPollingIterations: Int?, pollingInterval: Duration?) { - self.maxPollingIterations = maxPollingIterations + public init(pollingDuration: Duration?, pollingInterval: Duration?) { + self.pollingDuration = pollingDuration self.pollingInterval = pollingInterval } } @@ -32,13 +32,13 @@ public struct ConfirmPassesEventuallyConfigurationTrait: TestTrait, SuiteTrait { @_spi(Experimental) @available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) public struct ConfirmAlwaysPassesConfigurationTrait: TestTrait, SuiteTrait { - public var maxPollingIterations: Int? + public var pollingDuration: Duration? public var pollingInterval: Duration? public var isRecursive: Bool { true } - public init(maxPollingIterations: Int?, pollingInterval: Duration?) { - self.maxPollingIterations = maxPollingIterations + public init(pollingDuration: Duration?, pollingInterval: Duration?) { + self.pollingDuration = pollingDuration self.pollingInterval = pollingInterval } } @@ -49,20 +49,22 @@ extension Trait where Self == ConfirmPassesEventuallyConfigurationTrait { /// Specifies defaults for ``confirmPassesEventually`` in the test or suite. /// /// - Parameters: - /// - maxPollingIterations: The maximum amount of times to attempt polling. - /// If nil, polling will be attempted up to 1000 times. - /// `maxPollingIterations` must be greater than 0. + /// - pollingDuration: The expected amount of times to continue polling for. + /// This value may not correspond to the wall-clock time that polling lasts for, especially + /// on highly-loaded systems with a lot of tests running. + /// if nil, polling will be attempted for approximately 1 second. + /// `pollingDuration` must be greater than 0. /// - pollingInterval: The minimum amount of time to wait between polling /// attempts. /// If nil, polling will wait at least 1 millisecond between polling /// attempts. /// `pollingInterval` must be greater than 0. public static func confirmPassesEventuallyDefaults( - maxPollingIterations: Int? = nil, + pollingDuration: Duration? = nil, pollingInterval: Duration? = nil ) -> Self { ConfirmPassesEventuallyConfigurationTrait( - maxPollingIterations: maxPollingIterations, + pollingDuration: pollingDuration, pollingInterval: pollingInterval ) } @@ -74,20 +76,22 @@ extension Trait where Self == ConfirmAlwaysPassesConfigurationTrait { /// Specifies defaults for ``confirmPassesAlways`` in the test or suite. /// /// - Parameters: - /// - maxPollingIterations: The maximum amount of times to attempt polling. - /// If nil, polling will be attempted up to 1000 times. - /// `maxPollingIterations` must be greater than 0. + /// - pollingDuration: The expected amount of times to continue polling for. + /// This value may not correspond to the wall-clock time that polling lasts for, especially + /// on highly-loaded systems with a lot of tests running. + /// if nil, polling will be attempted for approximately 1 second. + /// `pollingDuration` must be greater than 0. /// - pollingInterval: The minimum amount of time to wait between polling /// attempts. /// If nil, polling will wait at least 1 millisecond between polling /// attempts. /// `pollingInterval` must be greater than 0. public static func confirmAlwaysPassesDefaults( - maxPollingIterations: Int? = nil, + pollingDuration: Duration? = nil, pollingInterval: Duration? = nil ) -> Self { ConfirmAlwaysPassesConfigurationTrait( - maxPollingIterations: maxPollingIterations, + pollingDuration: pollingDuration, pollingInterval: pollingInterval ) } diff --git a/Tests/TestingTests/PollingTests.swift b/Tests/TestingTests/PollingTests.swift index 4f97e15bd..0229b4d86 100644 --- a/Tests/TestingTests/PollingTests.swift +++ b/Tests/TestingTests/PollingTests.swift @@ -58,12 +58,12 @@ struct PollingTests { #expect(issues.count == 1) } - @Test("Waits up to 1000 times before failing") + @Test("Calculates how many times to poll based on the duration & interval") func defaultPollingCount() async { let incrementor = Incrementor() _ = await runTest { // this test will intentionally fail. - await confirmPassesEventually(pollingInterval: .nanoseconds(1)) { + await confirmPassesEventually(pollingInterval: .milliseconds(1)) { await incrementor.increment() == 0 } } @@ -72,14 +72,14 @@ struct PollingTests { @Suite( "Configuration traits", - .confirmPassesEventuallyDefaults(maxPollingIterations: 100) + .confirmPassesEventuallyDefaults(pollingDuration: .milliseconds(100)) ) struct WithConfigurationTraits { @Test("When no test or callsite configuration provided, uses the suite configuration") func testUsesSuiteConfiguration() async throws { let incrementor = Incrementor() var test = Test { - await confirmPassesEventually(pollingInterval: .nanoseconds(1)) { + await confirmPassesEventually(pollingInterval: .milliseconds(1)) { await incrementor.increment() == 0 } } @@ -90,14 +90,14 @@ struct PollingTests { } @Test( - "When test configuration porvided, uses the test configuration", - .confirmPassesEventuallyDefaults(maxPollingIterations: 10) + "When test configuration provided, uses the test configuration", + .confirmPassesEventuallyDefaults(pollingDuration: .milliseconds(10)) ) func testUsesTestConfigurationOverSuiteConfiguration() async { let incrementor = Incrementor() var test = Test { // this test will intentionally fail. - await confirmPassesEventually(pollingInterval: .nanoseconds(1)) { + await confirmPassesEventually(pollingInterval: .milliseconds(1)) { await incrementor.increment() == 0 } } @@ -108,13 +108,16 @@ struct PollingTests { @Test( "When callsite configuration provided, uses that", - .confirmPassesEventuallyDefaults(maxPollingIterations: 10) + .confirmPassesEventuallyDefaults(pollingDuration: .milliseconds(10)) ) func testUsesCallsiteConfiguration() async { let incrementor = Incrementor() var test = Test { // this test will intentionally fail. - await confirmPassesEventually(maxPollingIterations: 50, pollingInterval: .nanoseconds(1)) { + await confirmPassesEventually( + pollingDuration: .milliseconds(50), + pollingInterval: .milliseconds(1) + ) { await incrementor.increment() == 0 } } @@ -122,6 +125,32 @@ struct PollingTests { await runTest(test: test) #expect(await incrementor.count == 50) } + +#if !SWT_NO_EXIT_TESTS + @Test("Requires duration be greater than interval") + func testRequiresDurationGreaterThanInterval() async { + await #expect(processExitsWith: .failure) { + await confirmPassesEventually( + pollingDuration: .seconds(1), + pollingInterval: .milliseconds(1100) + ) { true } + } + } + + @Test("Requires duration be greater than 0") + func testRequiresDurationGreaterThan0() async { + await #expect(processExitsWith: .failure) { + await confirmPassesEventually(pollingDuration: .seconds(0)) { true } + } + } + + @Test("Requires interval be greater than 0") + func testRequiresIntervalGreaterThan0() async { + await #expect(processExitsWith: .failure) { + await confirmPassesEventually(pollingInterval: .seconds(0)) { true } + } + } +#endif } } @@ -168,7 +197,8 @@ struct PollingTests { #expect(await incrementor.count > 1) } - @Test("Thrown errors will automatically exit & fail") func errorsReported() async { + @Test("Thrown errors will automatically exit & fail") + func errorsReported() async { let issues = await runTest { await confirmAlwaysPasses { throw PollingTestSampleError.ohNo @@ -177,10 +207,10 @@ struct PollingTests { #expect(issues.count == 1) } - @Test("Waits up to 1000 times before passing") + @Test("Calculates how many times to poll based on the duration & interval") func defaultPollingCount() async { let incrementor = Incrementor() - await confirmAlwaysPasses(pollingInterval: .nanoseconds(1)) { + await confirmAlwaysPasses(pollingInterval: .milliseconds(1)) { await incrementor.increment() != 0 } #expect(await incrementor.count == 1000) @@ -188,13 +218,15 @@ struct PollingTests { @Suite( "Configuration traits", - .confirmAlwaysPassesDefaults(maxPollingIterations: 100) + .confirmAlwaysPassesDefaults(pollingDuration: .milliseconds(100)) ) struct WithConfigurationTraits { - @Test("When no test or callsite configuration provided, uses the suite configuration") + @Test( + "When no test/callsite configuration, it uses the suite configuration" + ) func testUsesSuiteConfiguration() async throws { let incrementor = Incrementor() - await confirmAlwaysPasses(pollingInterval: .nanoseconds(1)) { + await confirmAlwaysPasses(pollingInterval: .milliseconds(1)) { await incrementor.increment() != 0 } let count = await incrementor.count @@ -203,11 +235,11 @@ struct PollingTests { @Test( "When test configuration porvided, uses the test configuration", - .confirmAlwaysPassesDefaults(maxPollingIterations: 10) + .confirmAlwaysPassesDefaults(pollingDuration: .milliseconds(10)) ) func testUsesTestConfigurationOverSuiteConfiguration() async { let incrementor = Incrementor() - await confirmAlwaysPasses(pollingInterval: .nanoseconds(1)) { + await confirmAlwaysPasses(pollingInterval: .milliseconds(1)) { await incrementor.increment() != 0 } #expect(await incrementor.count == 10) @@ -215,15 +247,44 @@ struct PollingTests { @Test( "When callsite configuration provided, uses that", - .confirmAlwaysPassesDefaults(maxPollingIterations: 10) + .confirmAlwaysPassesDefaults(pollingDuration: .milliseconds(10)) ) func testUsesCallsiteConfiguration() async { let incrementor = Incrementor() - await confirmAlwaysPasses(maxPollingIterations: 50, pollingInterval: .nanoseconds(1)) { + await confirmAlwaysPasses( + pollingDuration: .milliseconds(50), + pollingInterval: .milliseconds(1) + ) { await incrementor.increment() != 0 } #expect(await incrementor.count == 50) } + +#if !SWT_NO_EXIT_TESTS + @Test("Requires duration be greater than interval") + func testRequiresDurationGreaterThanInterval() async { + await #expect(processExitsWith: .failure) { + await confirmAlwaysPasses( + pollingDuration: .seconds(1), + pollingInterval: .milliseconds(1100) + ) { true } + } + } + + @Test("Requires duration be greater than 0") + func testRequiresDurationGreaterThan0() async { + await #expect(processExitsWith: .failure) { + await confirmAlwaysPasses(pollingDuration: .seconds(0)) { true } + } + } + + @Test("Requires interval be greater than 0") + func testRequiresIntervalGreaterThan0() async { + await #expect(processExitsWith: .failure) { + await confirmAlwaysPasses(pollingInterval: .seconds(0)) { true } + } + } +#endif } } @@ -273,7 +334,7 @@ struct PollingTests { let duration = await Test.Clock().measure { let issues = await runTest { await confirmPassesEventually( - maxPollingIterations: 10, + pollingDuration: .seconds(10), pollingInterval: .seconds(1) // Wait a long time to handle jitter. ) { false } } @@ -296,7 +357,7 @@ struct PollingTests { let duration = await Test.Clock().measure { await confirmAlwaysPasses { true } } - #expect(duration.isCloseTo(other: .seconds(1), within: delta)) + #expect(duration.isCloseTo(other: .seconds(2), within: delta)) } @Test("Simple failing expressions") func trivialSadPath() async { @@ -313,7 +374,7 @@ struct PollingTests { func lastIteration() async { let duration = await Test.Clock().measure { await confirmAlwaysPasses( - maxPollingIterations: 10, + pollingDuration: .seconds(10), pollingInterval: .seconds(1) // Wait a long time to handle jitter. ) { true } } From 3328a8c26d5c75a6312a4de6dc4d5ce46142337b Mon Sep 17 00:00:00 2001 From: Rachel Brindle Date: Mon, 21 Jul 2025 00:15:17 -0700 Subject: [PATCH 10/10] Polling Confirmations: Rename to actually use the confirmation name Follow more english-sentence-like guidance for function naming Simplify the polling confirmation API down to just 2 public functions, 1 enum, and 1 error type. Always throw an error when polling fails, get rid of the separate issue recording. --- Sources/Testing/Issues/Issue.swift | 30 +- Sources/Testing/Polling/Polling.swift | 680 +++++++----------- .../Traits/PollingConfigurationTrait.swift | 106 +-- Tests/TestingTests/PollingTests.swift | 255 ++++--- 4 files changed, 494 insertions(+), 577 deletions(-) diff --git a/Sources/Testing/Issues/Issue.swift b/Sources/Testing/Issues/Issue.swift index e321a1b9d..26831a017 100644 --- a/Sources/Testing/Issues/Issue.swift +++ b/Sources/Testing/Issues/Issue.swift @@ -38,8 +38,14 @@ public struct Issue: Sendable { /// confirmed too few or too many times. indirect case confirmationMiscounted(actual: Int, expected: any RangeExpression & Sendable) + /// An issue due to a polling confirmation having failed. + /// + /// This issue can occur when calling ``confirmation(_:until:within:pollingEvery:isolation:sourceLocation:_:)-455gr`` + /// or + /// ``confirmation(_:until:within:pollingEvery:isolation:sourceLocation:_:)-5tnlk`` + /// whenever the polling fails, as described in ``PollingStopCondition``. @_spi(Experimental) - case confirmationPollingFailed + case pollingConfirmationFailed /// An issue due to an `Error` being thrown by a test function and caught by /// the testing library. @@ -289,8 +295,8 @@ extension Issue.Kind: CustomStringConvertible { } } return "Confirmation was confirmed \(actual.counting("time")), but expected to be confirmed \(String(describingForTest: expected)) time(s)" - case .confirmationPollingFailed: - return "Confirmation polling failed" + case .pollingConfirmationFailed: + return "Polling confirmation failed" case let .errorCaught(error): return "Caught error: \(error)" case let .timeLimitExceeded(timeLimitComponents: timeLimitComponents): @@ -427,6 +433,15 @@ extension Issue.Kind { /// too few or too many times. indirect case confirmationMiscounted(actual: Int, expected: Int) + /// An issue due to a polling confirmation having failed. + /// + /// This issue can occur when calling ``confirmation(_:until:within:pollingEvery:isolation:sourceLocation:_:)-455gr`` + /// or + /// ``confirmation(_:until:within:pollingEvery:isolation:sourceLocation:_:)-5tnlk`` + /// whenever the polling fails, as described in ``PollingStopCondition``. + @_spi(Experimental) + case pollingConfirmationFailed + /// An issue due to an `Error` being thrown by a test function and caught by /// the testing library. /// @@ -470,8 +485,8 @@ extension Issue.Kind { .expectationFailed(Expectation.Snapshot(snapshotting: expectation)) case .confirmationMiscounted: .unconditional - case .confirmationPollingFailed: - .unconditional + case .pollingConfirmationFailed: + .pollingConfirmationFailed case let .errorCaught(error), let .valueAttachmentFailed(error): .errorCaught(ErrorSnapshot(snapshotting: error)) case let .timeLimitExceeded(timeLimitComponents: timeLimitComponents): @@ -490,6 +505,7 @@ extension Issue.Kind { case unconditional case expectationFailed case confirmationMiscounted + case pollingConfirmationFailed case errorCaught case timeLimitExceeded case knownIssueNotRecorded @@ -562,6 +578,8 @@ extension Issue.Kind { forKey: .confirmationMiscounted) try confirmationMiscountedContainer.encode(actual, forKey: .actual) try confirmationMiscountedContainer.encode(expected, forKey: .expected) + case .pollingConfirmationFailed: + try container.encode(true, forKey: .pollingConfirmationFailed) case let .errorCaught(error): var errorCaughtContainer = container.nestedContainer(keyedBy: _CodingKeys._ErrorCaughtKeys.self, forKey: .errorCaught) try errorCaughtContainer.encode(error, forKey: .error) @@ -617,6 +635,8 @@ extension Issue.Kind.Snapshot: CustomStringConvertible { } case let .confirmationMiscounted(actual: actual, expected: expected): "Confirmation was confirmed \(actual.counting("time")), but expected to be confirmed \(expected.counting("time"))" + case .pollingConfirmationFailed: + "Polling confirmation failed" case let .errorCaught(error): "Caught error: \(error)" case let .timeLimitExceeded(timeLimitComponents: timeLimitComponents): diff --git a/Sources/Testing/Polling/Polling.swift b/Sources/Testing/Polling/Polling.swift index efa2cef8c..e91c30953 100644 --- a/Sources/Testing/Polling/Polling.swift +++ b/Sources/Testing/Polling/Polling.swift @@ -8,7 +8,8 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +/// Default values for polling confirmations. +@available(_clockAPI, *) internal let defaultPollingConfiguration = ( pollingDuration: Duration.seconds(1), pollingInterval: Duration.milliseconds(1) @@ -16,345 +17,187 @@ internal let defaultPollingConfiguration = ( /// A type describing an error thrown when polling fails. @_spi(Experimental) -public struct PollingFailedError: Error, Equatable {} +public struct PollingFailedError: Error, Sendable, Codable { + /// A user-specified comment describing this confirmation + public var comment: Comment? -/// Confirm that some expression eventually returns true -/// -/// - Parameters: -/// - comment: An optional comment to apply to any issues generated by this -/// function. -/// - pollingDuration: The expected length of time to continue polling for. -/// This value may not correspond to the wall-clock time that polling lasts for, especially -/// on highly-loaded systems with a lot of tests running. -/// If nil, this uses whatever value is specified under the last -/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or -/// suite. -/// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then -/// polling will be attempted for about 1 second before recording an issue. -/// `maxPollingIterations` must be greater than 0. -/// - pollingInterval: The minimum amount of time to wait between polling -/// attempts. -/// If nil, this uses whatever value is specified under the last -/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or suite. -/// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then -/// polling will wait at least 1 millisecond between polling attempts. -/// `pollingInterval` must be greater than 0. -/// - isolation: The actor to which `body` is isolated, if any. -/// - sourceLocation: The source location to whych any recorded issues should -/// be attributed. -/// - body: The function to invoke. -/// -/// Use polling confirmations to check that an event while a test is running in -/// complex scenarios where other forms of confirmation are insufficient. For -/// example, waiting on some state to change that cannot be easily confirmed -/// through other forms of `confirmation`. -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -public func confirmPassesEventually( - _ comment: Comment? = nil, - pollingDuration: Duration? = nil, - pollingInterval: Duration? = nil, - isolation: isolated (any Actor)? = #isolation, - sourceLocation: SourceLocation = #_sourceLocation, - _ body: @escaping () async throws -> Bool -) async { - let poller = Poller( - pollingBehavior: .passesOnce, - pollingDuration: getValueFromPollingTrait( - providedValue: pollingDuration, - default: defaultPollingConfiguration.pollingDuration, - \ConfirmPassesEventuallyConfigurationTrait.pollingDuration - ), - pollingInterval: getValueFromPollingTrait( - providedValue: pollingInterval, - default: defaultPollingConfiguration.pollingInterval, - \ConfirmPassesEventuallyConfigurationTrait.pollingInterval - ), - comment: comment, - sourceLocation: sourceLocation - ) - await poller.evaluateBool(isolation: isolation) { - do { - return try await body() - } catch { - return false + /// A ``SourceContext`` indicating where and how this confirmation was called + @_spi(ForToolsIntegrationOnly) + public var sourceContext: SourceContext + + /// Initialize an instance of this type with the specified details + /// + /// - Parameters: + /// - comment: A user-specified comment describing this confirmation. + /// Defaults to `nil`. + /// - sourceContext: A ``SourceContext`` indicating where and how this + /// confirmation was called. + public init( + comment: Comment? = nil, + sourceContext: SourceContext + ) { + self.comment = comment + self.sourceContext = sourceContext + } +} + +extension PollingFailedError: CustomIssueRepresentable { + func customize(_ issue: consuming Issue) -> Issue { + if let comment { + issue.comments.append(comment) } + issue.kind = .pollingConfirmationFailed + issue.sourceContext = sourceContext + return issue } } -/// Require that some expression eventually returns true +/// A type defining when to stop polling early. +/// This also determines what happens if the duration elapses during polling. +public enum PollingStopCondition: Sendable { + /// Evaluates the expression until the first time it returns true. + /// If it does not pass once by the time the timeout is reached, then a + /// failure will be reported. + case firstPass + + /// Evaluates the expression until the first time it returns false. + /// If the expression returns false, then a failure will be reported. + /// If the expression only returns true before the timeout is reached, then + /// no failure will be reported. + /// If the expression does not finish evaluating before the timeout is + /// reached, then a failure will be reported. + case stopsPassing +} + +/// Poll expression within the duration based on the given stop condition /// /// - Parameters: -/// - comment: An optional comment to apply to any issues generated by this -/// function. -/// - pollingDuration: The expected length of time to continue polling for. -/// This value may not correspond to the wall-clock time that polling lasts for, especially -/// on highly-loaded systems with a lot of tests running. +/// - comment: A user-specified comment describing this confirmation. +/// - stopCondition: When to stop polling. +/// - duration: The expected length of time to continue polling for. +/// This value may not correspond to the wall-clock time that polling lasts +/// for, especially on highly-loaded systems with a lot of tests running. /// If nil, this uses whatever value is specified under the last -/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or +/// ``PollingUntilFirstPassConfigurationTrait`` or +/// ``PollingUntilStopsPassingConfigurationTrait`` added to the test or /// suite. -/// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then -/// polling will be attempted for about 1 second before recording an issue. -/// `maxPollingIterations` must be greater than 0. -/// - pollingInterval: The minimum amount of time to wait between polling -/// attempts. +/// If no such trait has been added, then polling will be attempted for +/// about 1 second before recording an issue. +/// `duration` must be greater than 0. +/// - interval: The minimum amount of time to wait between polling attempts. /// If nil, this uses whatever value is specified under the last -/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or suite. -/// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then -/// polling will wait at least 1 millisecond between polling attempts. -/// `pollingInterval` must be greater than 0. +/// ``PollingUntilFirstPassConfigurationTrait`` or +/// ``PollingUntilStopsPassingConfigurationTrait`` added to the test or +/// suite. +/// If no such trait has been added, then polling will wait at least +/// 1 millisecond between polling attempts. +/// `interval` must be greater than 0. /// - isolation: The actor to which `body` is isolated, if any. -/// - sourceLocation: The source location to whych any recorded issues should -/// be attributed. +/// - sourceLocation: The location in source where the confirmation was called. /// - body: The function to invoke. /// -/// - Throws: A `PollingFailedError` will be thrown if the expression never -/// returns true. +/// - Throws: A `PollingFailedError` if the `body` does not return true within +/// the polling duration. /// /// Use polling confirmations to check that an event while a test is running in /// complex scenarios where other forms of confirmation are insufficient. For /// example, waiting on some state to change that cannot be easily confirmed /// through other forms of `confirmation`. @_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -public func requirePassesEventually( +@available(_clockAPI, *) +public func confirmation( _ comment: Comment? = nil, - pollingDuration: Duration? = nil, - pollingInterval: Duration? = nil, + until stopCondition: PollingStopCondition, + within duration: Duration? = nil, + pollingEvery interval: Duration? = nil, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation = #_sourceLocation, _ body: @escaping () async throws -> Bool ) async throws { let poller = Poller( - pollingBehavior: .passesOnce, - pollingDuration: getValueFromPollingTrait( - providedValue: pollingDuration, - default: defaultPollingConfiguration.pollingDuration, - \ConfirmPassesEventuallyConfigurationTrait.pollingDuration - ), - pollingInterval: getValueFromPollingTrait( - providedValue: pollingInterval, - default: defaultPollingConfiguration.pollingInterval, - \ConfirmPassesEventuallyConfigurationTrait.pollingInterval - ), + stopCondition: stopCondition, + duration: stopCondition.duration(with: duration), + interval: stopCondition.interval(with: interval), comment: comment, - sourceLocation: sourceLocation + sourceContext: SourceContext( + backtrace: .current(), + sourceLocation: sourceLocation + ) ) - let passed = await poller.evaluateBool(raiseIssue: false, isolation: isolation) { + try await poller.evaluate(isolation: isolation) { do { return try await body() } catch { return false } } - if !passed { - throw PollingFailedError() - } } /// Confirm that some expression eventually returns a non-nil value /// /// - Parameters: -/// - comment: An optional comment to apply to any issues generated by this -/// function. -/// - pollingDuration: The expected length of time to continue polling for. -/// This value may not correspond to the wall-clock time that polling lasts for, especially -/// on highly-loaded systems with a lot of tests running. +/// - comment: A user-specified comment describing this confirmation. +/// - stopCondition: When to stop polling. +/// - duration: The expected length of time to continue polling for. +/// This value may not correspond to the wall-clock time that polling lasts +/// for, especially on highly-loaded systems with a lot of tests running. /// If nil, this uses whatever value is specified under the last -/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or +/// ``PollingUntilFirstPassConfigurationTrait`` or +/// ``PollingUntilStopsPassingConfigurationTrait`` added to the test or /// suite. -/// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then -/// polling will be attempted for about 1 second before recording an issue. -/// `maxPollingIterations` must be greater than 0. -/// - pollingInterval: The minimum amount of time to wait between polling -/// attempts. +/// If no such trait has been added, then polling will be attempted for +/// about 1 second before recording an issue. +/// `duration` must be greater than 0. +/// - interval: The minimum amount of time to wait between polling attempts. /// If nil, this uses whatever value is specified under the last -/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or suite. -/// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then -/// polling will wait at least 1 millisecond between polling attempts. -/// `pollingInterval` must be greater than 0. +/// ``PollingUntilFirstPassConfigurationTrait`` or +/// ``PollingUntilStopsPassingConfigurationTrait`` added to the test or +/// suite. +/// If no such trait has been added, then polling will wait at least +/// 1 millisecond between polling attempts. +/// `interval` must be greater than 0. /// - isolation: The actor to which `body` is isolated, if any. -/// - sourceLocation: The source location to whych any recorded issues should -/// be attributed. +/// - sourceLocation: The location in source where the confirmation was called. /// - body: The function to invoke. /// -/// - Returns: The first non-nil value returned by `body`. +/// - Throws: A `PollingFailedError` if the `body` does not return true within +/// the polling duration. /// -/// - Throws: A `PollingFailedError` will be thrown if `body` never returns a -/// non-optional value. +/// - Returns: The last non-nil value returned by `body`. /// /// Use polling confirmations to check that an event while a test is running in /// complex scenarios where other forms of confirmation are insufficient. For /// example, waiting on some state to change that cannot be easily confirmed /// through other forms of `confirmation`. @_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +@available(_clockAPI, *) @discardableResult -public func confirmPassesEventually( +public func confirmation( _ comment: Comment? = nil, - pollingDuration: Duration? = nil, - pollingInterval: Duration? = nil, + until stopCondition: PollingStopCondition, + within duration: Duration? = nil, + pollingEvery interval: Duration? = nil, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation = #_sourceLocation, _ body: @escaping () async throws -> sending R? ) async throws -> R { let poller = Poller( - pollingBehavior: .passesOnce, - pollingDuration: getValueFromPollingTrait( - providedValue: pollingDuration, - default: defaultPollingConfiguration.pollingDuration, - \ConfirmPassesEventuallyConfigurationTrait.pollingDuration - ), - pollingInterval: getValueFromPollingTrait( - providedValue: pollingInterval, - default: defaultPollingConfiguration.pollingInterval, - \ConfirmPassesEventuallyConfigurationTrait.pollingInterval - ), + stopCondition: stopCondition, + duration: stopCondition.duration(with: duration), + interval: stopCondition.interval(with: interval), comment: comment, - sourceLocation: sourceLocation + sourceContext: SourceContext( + backtrace: .current(), + sourceLocation: sourceLocation + ) ) - let recordedValue = await poller.evaluate(isolation: isolation) { + return try await poller.evaluateOptional(isolation: isolation) { do { return try await body() } catch { return nil } } - - if let value = recordedValue { - return value - } - throw PollingFailedError() -} - -/// Confirm that some expression always returns true -/// -/// - Parameters: -/// - comment: An optional comment to apply to any issues generated by this -/// function. -/// - pollingDuration: The expected length of time to continue polling for. -/// This value may not correspond to the wall-clock time that polling lasts for, especially -/// on highly-loaded systems with a lot of tests running. -/// If nil, this uses whatever value is specified under the last -/// ``ConfirmAlwaysPassesConfigurationTrait`` added to the test or suite. -/// If no ``ConfirmAlwaysPassesConfigurationTrait`` has been added, then -/// polling will be attempted for about 1 second before recording an issue. -/// `maxPollingIterations` must be greater than 0. -/// - pollingInterval: The minimum amount of time to wait between polling -/// attempts. -/// If nil, this uses whatever value is specified under the last -/// ``ConfirmAlwaysPassesConfigurationTrait`` added to the test or suite. -/// If no ``ConfirmAlwaysPassesConfigurationTrait`` has been added, then -/// polling will wait at least 1 millisecond between polling attempts. -/// `pollingInterval` must be greater than 0. -/// - isolation: The actor to which `body` is isolated, if any. -/// - sourceLocation: The source location to whych any recorded issues should -/// be attributed. -/// - body: The function to invoke. -/// -/// Use polling confirmations to check that an event while a test is running in -/// complex scenarios where other forms of confirmation are insufficient. For -/// example, confirming that some state does not change. -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -public func confirmAlwaysPasses( - _ comment: Comment? = nil, - pollingDuration: Duration? = nil, - pollingInterval: Duration? = nil, - isolation: isolated (any Actor)? = #isolation, - sourceLocation: SourceLocation = #_sourceLocation, - _ body: @escaping () async throws -> Bool -) async { - let poller = Poller( - pollingBehavior: .passesAlways, - pollingDuration: getValueFromPollingTrait( - providedValue: pollingDuration, - default: defaultPollingConfiguration.pollingDuration, - \ConfirmAlwaysPassesConfigurationTrait.pollingDuration - ), - pollingInterval: getValueFromPollingTrait( - providedValue: pollingInterval, - default: defaultPollingConfiguration.pollingInterval, - \ConfirmAlwaysPassesConfigurationTrait.pollingInterval - ), - comment: comment, - sourceLocation: sourceLocation - ) - await poller.evaluateBool(isolation: isolation) { - do { - return try await body() - } catch { - return false - } - } -} - -/// Require that some expression always returns true -/// -/// - Parameters: -/// - comment: An optional comment to apply to any issues generated by this -/// function. -/// - pollingDuration: The expected length of time to continue polling for. -/// This value may not correspond to the wall-clock time that polling lasts for, especially -/// on highly-loaded systems with a lot of tests running. -/// If nil, this uses whatever value is specified under the last -/// ``ConfirmAlwaysPassesConfigurationTrait`` added to the test or suite. -/// If no ``ConfirmAlwaysPassesConfigurationTrait`` has been added, then -/// polling will be attempted for about 1 second before recording an issue. -/// `maxPollingIterations` must be greater than 0. -/// - pollingInterval: The minimum amount of time to wait between polling -/// attempts. -/// If nil, this uses whatever value is specified under the last -/// ``ConfirmAlwaysPassesConfigurationTrait`` added to the test or suite. -/// If no ``ConfirmAlwaysPassesConfigurationTrait`` has been added, then -/// polling will wait at least 1 millisecond between polling attempts. -/// `pollingInterval` must be greater than 0. -/// - isolation: The actor to which `body` is isolated, if any. -/// - sourceLocation: The source location to whych any recorded issues should -/// be attributed. -/// - body: The function to invoke. -/// -/// - Throws: A `PollingFailedError` will be thrown if the expression ever -/// returns false. -/// -/// Use polling confirmations to check that an event while a test is running in -/// complex scenarios where other forms of confirmation are insufficient. For -/// example, confirming that some state does not change. -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -public func requireAlwaysPasses( - _ comment: Comment? = nil, - pollingDuration: Duration? = nil, - pollingInterval: Duration? = nil, - isolation: isolated (any Actor)? = #isolation, - sourceLocation: SourceLocation = #_sourceLocation, - _ body: @escaping () async throws -> Bool -) async throws { - let poller = Poller( - pollingBehavior: .passesAlways, - pollingDuration: getValueFromPollingTrait( - providedValue: pollingDuration, - default: defaultPollingConfiguration.pollingDuration, - \ConfirmAlwaysPassesConfigurationTrait.pollingDuration - ), - pollingInterval: getValueFromPollingTrait( - providedValue: pollingInterval, - default: defaultPollingConfiguration.pollingInterval, - \ConfirmAlwaysPassesConfigurationTrait.pollingInterval - ), - comment: comment, - sourceLocation: sourceLocation - ) - let passed = await poller.evaluateBool(raiseIssue: false, isolation: isolation) { - do { - return try await body() - } catch { - return false - } - } - if !passed { - throw PollingFailedError() - } } /// A helper function to de-duplicate the logic of grabbing configuration from @@ -364,15 +207,18 @@ public func requireAlwaysPasses( /// The provided value, if non-nil is returned. Otherwise, this looks for /// the last `TraitKind` specified, and if one exists, returns the value /// as determined by `keyPath`. -/// If no configuration trait has been applied, then this returns the `default`. +/// If the provided value is nil, and no configuration trait has been applied, +/// then this returns the value specified in `default`. /// /// - Parameters: /// - providedValue: The value provided by the test author when calling /// `confirmPassesEventually` or `confirmAlwaysPasses`. /// - default: The harded coded default value, as defined in -/// `defaultPollingConfiguration` -/// - keyPath: The keyPath mapping from `TraitKind` to the desired value type. -private func getValueFromPollingTrait( +/// `defaultPollingConfiguration`. +/// - keyPath: The keyPath mapping from `TraitKind` to the value type. +/// +/// - Returns: The value to use. +private func getValueFromTrait( providedValue: Value?, default: Value, _ keyPath: KeyPath @@ -384,128 +230,107 @@ private func getValueFromPollingTrait( return traitValues.last ?? `default` } -/// A type for managing polling -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -private struct Poller { - enum PollingBehavior { - /// Continuously evaluate the expression until the first time it returns - /// true. - /// If it does not pass once by the time the timeout is reached, then a - /// failure will be reported. - case passesOnce - - /// Continuously evaluate the expression until the first time it returns - /// false. - /// If the expression returns false, then a failure will be reported. - /// If the expression only returns true before the timeout is reached, then - /// no failure will be reported. - /// If the expression does not finish evaluating before the timeout is - /// reached, then a failure will be reported. - case passesAlways - - /// Process the result of a polled expression and decide whether to continue polling. - /// - /// - Parameters: - /// - expressionResult: The result of the polled expression - /// - /// - Returns: A poll result (if polling should stop), or nil (if polling should continue) - func processFinishedExpression( - expressionResult result: Bool - ) -> PollResult? { - switch self { - case .passesOnce: - if result { - return .finished - } else { - return nil - } - case .passesAlways: - if !result { - return .failed - } else { - return nil - } - } +extension PollingStopCondition { + /// Process the result of a polled expression and decide whether to continue + /// polling. + /// + /// - Parameters: + /// - expressionResult: The result of the polled expression. + /// + /// - Returns: A poll result (if polling should stop), or nil (if polling + /// should continue). + @available(_clockAPI, *) + fileprivate func shouldStopPolling( + expressionResult result: Bool + ) -> Bool { + switch self { + case .firstPass: + return result + case .stopsPassing: + return !result } } - /// The result of polling expressions - enum PollResult { - /// The polling ran for the total number of iterations - case ranToCompletion - /// The expression exited early, and we will report a success status. - case finished - /// The expression returned false under PollingBehavior.passesAlways - case failed - /// The polling was cancelled before polling could finish - case cancelled + /// Determine the polling duration to use for the given provided value. + /// Based on ``getValueFromTrait``, this falls back using + /// ``defaultPollingConfiguration.pollingInterval`` and + /// ``PollingUntilFirstPassConfigurationTrait``. + @available(_clockAPI, *) + fileprivate func duration(with provided: Duration?) -> Duration { + switch self { + case .firstPass: + getValueFromTrait( + providedValue: provided, + default: defaultPollingConfiguration.pollingDuration, + \PollingUntilFirstPassConfigurationTrait.duration + ) + case .stopsPassing: + getValueFromTrait( + providedValue: provided, + default: defaultPollingConfiguration.pollingDuration, + \PollingUntilStopsPassingConfigurationTrait.duration + ) + } + } - /// Process the poll result into an issue - /// - /// - Parameters: - /// - comment: The comment to record as part of the issue - /// - sourceContext: The source context for the issue - /// - pollingBehavior: The polling behavior used. - /// - Returns: An issue if one should be recorded, otherwise nil. - func issue( - comment: Comment?, - sourceContext: SourceContext, - pollingBehavior: PollingBehavior - ) -> Issue? { - let issueKind: Issue.Kind - switch self { - case .finished, .cancelled: - return nil - case .ranToCompletion: - if case .passesAlways = pollingBehavior { - return nil - } - issueKind = .confirmationPollingFailed - case .failed: - issueKind = .confirmationPollingFailed - } - return Issue( - kind: issueKind, - comments: Array(comment), - sourceContext: sourceContext + /// Determine the polling interval to use for the given provided value. + /// Based on ``getValueFromTrait``, this falls back using + /// ``defaultPollingConfiguration.pollingInterval`` and + /// ``PollingUntilFirstPassConfigurationTrait``. + @available(_clockAPI, *) + fileprivate func interval(with provided: Duration?) -> Duration { + switch self { + case .firstPass: + getValueFromTrait( + providedValue: provided, + default: defaultPollingConfiguration.pollingInterval, + \PollingUntilFirstPassConfigurationTrait.interval + ) + case .stopsPassing: + getValueFromTrait( + providedValue: provided, + default: defaultPollingConfiguration.pollingInterval, + \PollingUntilStopsPassingConfigurationTrait.interval ) } } +} - /// The polling behavior (poll until the expression first passes, or poll - /// while the expression continues to pass) - let pollingBehavior: PollingBehavior +/// A type for managing polling +@available(_clockAPI, *) +private struct Poller { + /// The stop condition to follow + let stopCondition: PollingStopCondition - // Approximately how long to poll for - let pollingDuration: Duration - // Minimum waiting period between polling - let pollingInterval: Duration + /// Approximately how long to poll for + let duration: Duration - /// A comment from the test author associated with the polling + /// The minimum waiting period between polling + let interval: Duration + + /// A user-specified comment describing this confirmation let comment: Comment? - /// The source location that asked for polling. - let sourceLocation: SourceLocation + /// A ``SourceContext`` indicating where and how this confirmation was called + let sourceContext: SourceContext - /// Evaluate polling, and process the result, raising an issue if necessary. + /// Evaluate polling, throwing an error if polling fails. /// /// - Parameters: - /// - raiseIssue: Whether or not to raise an issue. - /// This should only be false for `requirePassesEventually` or - /// `requireAlwaysPasses`. - /// - isolation: The isolation to use - /// - body: The expression to poll + /// - isolation: The isolation to use. + /// - body: The expression to poll. + /// + /// - Throws: A ``PollingFailedError`` if polling doesn't pass. /// /// - Returns: Whether or not polling passed. /// - /// - Side effects: If polling fails (see `PollingBehavior`), then this will - /// record an issue. - @discardableResult func evaluateBool( - raiseIssue: Bool = true, + /// - Side effects: If polling fails (see ``PollingStopCondition``), then + /// this will record an issue. + @discardableResult func evaluate( isolation: isolated (any Actor)?, _ body: @escaping () async -> Bool - ) async -> Bool { - await evaluate(raiseIssue: raiseIssue, isolation: isolation) { + ) async throws -> Bool { + try await evaluateOptional(isolation: isolation) { if await body() { // return any non-nil value. return true @@ -515,91 +340,76 @@ private struct Poller { } != nil } - /// Evaluate polling, and process the result, raising an issue if necessary. - /// - /// - Note: This method is only intended to be used when pollingBehavior is - /// `.passesOnce` + /// Evaluate polling, throwing an error if polling fails. /// /// - Parameters: - /// - raiseIssue: Whether or not to raise an issue. - /// This should only be false for `requirePassesEventually` or - /// `requireAlwaysPasses`. - /// - isolation: The isolation to use - /// - body: The expression to poll + /// - isolation: The isolation to use. + /// - body: The expression to poll. + /// + /// - Throws: A ``PollingFailedError`` if polling doesn't pass. /// - /// - Returns: the value if polling passed, nil otherwise. + /// - Returns: the last non-nil value returned by `body`. /// - /// - Side effects: If polling fails (see `PollingBehavior`), then this will - /// record an issue. - @discardableResult func evaluate( - raiseIssue: Bool = true, + /// - Side effects: If polling fails (see ``PollingStopCondition``), then + /// this will record an issue. + @discardableResult func evaluateOptional( isolation: isolated (any Actor)?, _ body: @escaping () async -> sending R? - ) async -> R? { - precondition(pollingDuration > Duration.zero) - precondition(pollingInterval > Duration.zero) - precondition(pollingDuration > pollingInterval) - let durationSeconds = Double(pollingDuration.components.seconds) + Double(pollingDuration.components.attoseconds) * 1e-18 - let intervalSeconds = Double(pollingInterval.components.seconds) + Double(pollingInterval.components.attoseconds) * 1e-18 + ) async throws -> R { + precondition(duration > Duration.zero) + precondition(interval > Duration.zero) + precondition(duration > interval) - let pollingIterations = max(Int(durationSeconds / intervalSeconds), 1) + let iterations = max(Int(duration.seconds() / interval.seconds()), 1) - let (result, value) = await poll( - pollingIterations: pollingIterations, - expression: body - ) - if let issue = result.issue( - comment: comment, - sourceContext: .init(backtrace: .current(), sourceLocation: sourceLocation), - pollingBehavior: pollingBehavior - ) { - if raiseIssue { - issue.record() - } + if let value = await poll(iterations: iterations, expression: body) { return value } else { - return value + throw PollingFailedError(comment: comment, sourceContext: sourceContext) } } /// This function contains the logic for continuously polling an expression, - /// as well as processing the results of that expression - /// - /// - Note: This method is only intended to be used when pollingBehavior is - /// `.passesOnce` + /// as well as processing the results of that expression. /// /// - Parameters: - /// - pollingIterations: The maximum amount of times to continue polling. - /// - expression: An expression to continuously evaluate - /// - behavior: The polling behavior to use - /// - timeout: How long to poll for unitl the timeout triggers. - /// - Returns: The result of this polling and the most recent value if the - /// result is .finished, otherwise nil. + /// - iterations: The maximum amount of times to continue polling. + /// - expression: An expression to continuously evaluate. + /// + /// - Returns: The most recent value if the polling succeeded, else nil. private func poll( - pollingIterations: Int, + iterations: Int, isolation: isolated (any Actor)? = #isolation, expression: @escaping () async -> sending R? - ) async -> (PollResult, R?) { + ) async -> R? { var lastResult: R? - for iteration in 0.. Double { + let secondsComponent = Double(components.seconds) + let attosecondsComponent = Double(components.attoseconds) * 1e-18 + return secondsComponent + attosecondsComponent } } diff --git a/Sources/Testing/Traits/PollingConfigurationTrait.swift b/Sources/Testing/Traits/PollingConfigurationTrait.swift index 40c08bf0c..f269480d3 100644 --- a/Sources/Testing/Traits/PollingConfigurationTrait.swift +++ b/Sources/Testing/Traits/PollingConfigurationTrait.swift @@ -6,93 +6,107 @@ // /// A trait to provide a default polling configuration to all usages of -/// ``confirmPassesEventually`` within a test or suite. +/// ``confirmation(_:until:within:pollingEvery:isolation:sourceLocation:_:)-455gr`` +/// and +/// ``confirmation(_:until:within:pollingEvery:isolation:sourceLocation:_:)-5tnlk`` +/// within a test or suite for the ``PollingStopCondition.firstPass`` +/// stop condition. /// /// To add this trait to a test, use the -/// ``Trait/confirmPassesEventuallyDefaults`` function. +/// ``Trait/pollingUntilFirstPassDefaults`` function. @_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -public struct ConfirmPassesEventuallyConfigurationTrait: TestTrait, SuiteTrait { - public var pollingDuration: Duration? - public var pollingInterval: Duration? +@available(_clockAPI, *) +public struct PollingUntilFirstPassConfigurationTrait: TestTrait, SuiteTrait { + /// How long to continue polling for + public var duration: Duration? + /// The minimum amount of time to wait between polling attempts + public var interval: Duration? public var isRecursive: Bool { true } - public init(pollingDuration: Duration?, pollingInterval: Duration?) { - self.pollingDuration = pollingDuration - self.pollingInterval = pollingInterval + public init(duration: Duration?, interval: Duration?) { + self.duration = duration + self.interval = interval } } /// A trait to provide a default polling configuration to all usages of -/// ``confirmAlwaysPasses`` within a test or suite. +/// ``confirmation(_:until:within:pollingEvery:isolation:sourceLocation:_:)-455gr`` +/// and +/// ``confirmation(_:until:within:pollingEvery:isolation:sourceLocation:_:)-5tnlk`` +/// within a test or suite for the ``PollingStopCondition.stopsPassing`` +/// stop condition. /// -/// To add this trait to a test, use the ``Trait/confirmAlwaysPassesDefaults`` +/// To add this trait to a test, use the ``Trait/pollingUntilStopsPassingDefaults`` /// function. @_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -public struct ConfirmAlwaysPassesConfigurationTrait: TestTrait, SuiteTrait { - public var pollingDuration: Duration? - public var pollingInterval: Duration? +@available(_clockAPI, *) +public struct PollingUntilStopsPassingConfigurationTrait: TestTrait, SuiteTrait { + /// How long to continue polling for + public var duration: Duration? + /// The minimum amount of time to wait between polling attempts + public var interval: Duration? public var isRecursive: Bool { true } - public init(pollingDuration: Duration?, pollingInterval: Duration?) { - self.pollingDuration = pollingDuration - self.pollingInterval = pollingInterval + public init(duration: Duration?, interval: Duration?) { + self.duration = duration + self.interval = interval } } @_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -extension Trait where Self == ConfirmPassesEventuallyConfigurationTrait { +@available(_clockAPI, *) +extension Trait where Self == PollingUntilFirstPassConfigurationTrait { /// Specifies defaults for ``confirmPassesEventually`` in the test or suite. /// /// - Parameters: - /// - pollingDuration: The expected amount of times to continue polling for. - /// This value may not correspond to the wall-clock time that polling lasts for, especially - /// on highly-loaded systems with a lot of tests running. + /// - duration: The expected length of time to continue polling for. + /// This value may not correspond to the wall-clock time that polling + /// lasts for, especially on highly-loaded systems with a lot of tests + /// running. /// if nil, polling will be attempted for approximately 1 second. - /// `pollingDuration` must be greater than 0. - /// - pollingInterval: The minimum amount of time to wait between polling + /// `duration` must be greater than 0. + /// - interval: The minimum amount of time to wait between polling /// attempts. /// If nil, polling will wait at least 1 millisecond between polling /// attempts. - /// `pollingInterval` must be greater than 0. - public static func confirmPassesEventuallyDefaults( - pollingDuration: Duration? = nil, - pollingInterval: Duration? = nil + /// `interval` must be greater than 0. + public static func pollingUntilFirstPassDefaults( + until duration: Duration? = nil, + pollingEvery interval: Duration? = nil ) -> Self { - ConfirmPassesEventuallyConfigurationTrait( - pollingDuration: pollingDuration, - pollingInterval: pollingInterval + PollingUntilFirstPassConfigurationTrait( + duration: duration, + interval: interval ) } } @_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -extension Trait where Self == ConfirmAlwaysPassesConfigurationTrait { +@available(_clockAPI, *) +extension Trait where Self == PollingUntilStopsPassingConfigurationTrait { /// Specifies defaults for ``confirmPassesAlways`` in the test or suite. /// /// - Parameters: - /// - pollingDuration: The expected amount of times to continue polling for. - /// This value may not correspond to the wall-clock time that polling lasts for, especially - /// on highly-loaded systems with a lot of tests running. + /// - duration: The expected length of time to continue polling for. + /// This value may not correspond to the wall-clock time that polling + /// lasts for, especially on highly-loaded systems with a lot of tests + /// running. /// if nil, polling will be attempted for approximately 1 second. - /// `pollingDuration` must be greater than 0. - /// - pollingInterval: The minimum amount of time to wait between polling + /// `duration` must be greater than 0. + /// - interval: The minimum amount of time to wait between polling /// attempts. /// If nil, polling will wait at least 1 millisecond between polling /// attempts. - /// `pollingInterval` must be greater than 0. - public static func confirmAlwaysPassesDefaults( - pollingDuration: Duration? = nil, - pollingInterval: Duration? = nil + /// `interval` must be greater than 0. + public static func pollingUntilStopsPassingDefaults( + until duration: Duration? = nil, + pollingEvery interval: Duration? = nil ) -> Self { - ConfirmAlwaysPassesConfigurationTrait( - pollingDuration: pollingDuration, - pollingInterval: pollingInterval + PollingUntilStopsPassingConfigurationTrait( + duration: duration, + interval: interval ) } } diff --git a/Tests/TestingTests/PollingTests.swift b/Tests/TestingTests/PollingTests.swift index 0229b4d86..6dd009047 100644 --- a/Tests/TestingTests/PollingTests.swift +++ b/Tests/TestingTests/PollingTests.swift @@ -10,34 +10,45 @@ @testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing -@Suite("Polling Tests") -struct PollingTests { - @Suite("confirmPassesEventually") - struct PassesOnceBehavior { +@Suite("Polling Confirmation Tests") +struct PollingConfirmationTests { + @Suite("with PollingStopCondition.firstPass") + struct StopConditionFirstPass { + let stop = PollingStopCondition.firstPass + + @available(_clockAPI, *) @Test("Simple passing expressions") func trivialHappyPath() async throws { - await confirmPassesEventually { true } - try await requirePassesEventually { true } + try await confirmation(until: stop) { true } - let value = try await confirmPassesEventually { 1 } + let value = try await confirmation(until: stop) { 1 } #expect(value == 1) } + @available(_clockAPI, *) @Test("Simple failing expressions") func trivialSadPath() async throws { - let issues = await runTest { - await confirmPassesEventually { false } - _ = try await confirmPassesEventually { Optional.none } - await #expect(throws: PollingFailedError()) { - try await requirePassesEventually { false } - } + var issues = await runTest { + try await confirmation(until: stop) { false } + } + issues += await runTest { + _ = try await confirmation(until: stop) { Optional.none } } - #expect(issues.count == 3) + #expect(issues.count == 2) + #expect(issues.allSatisfy { + if case .pollingConfirmationFailed = $0.kind { + return true + } else { + return false + } + }) } - @Test("When the value changes from false to true during execution") func changingFromFail() async { + @available(_clockAPI, *) + @Test("When the value changes from false to true during execution") + func changingFromFail() async throws { let incrementor = Incrementor() - await confirmPassesEventually { + try await confirmation(until: stop) { await incrementor.increment() == 2 // this will pass only on the second invocation // This checks that we really are only running the expression until @@ -48,22 +59,24 @@ struct PollingTests { #expect(await incrementor.count == 2) } + @available(_clockAPI, *) @Test("Thrown errors are treated as returning false") - func errorsReported() async { + func errorsReported() async throws { let issues = await runTest { - await confirmPassesEventually { + try await confirmation(until: stop) { throw PollingTestSampleError.ohNo } } #expect(issues.count == 1) } + @available(_clockAPI, *) @Test("Calculates how many times to poll based on the duration & interval") func defaultPollingCount() async { let incrementor = Incrementor() _ = await runTest { // this test will intentionally fail. - await confirmPassesEventually(pollingInterval: .milliseconds(1)) { + try await confirmation(until: stop, pollingEvery: .milliseconds(1)) { await incrementor.increment() == 0 } } @@ -72,14 +85,17 @@ struct PollingTests { @Suite( "Configuration traits", - .confirmPassesEventuallyDefaults(pollingDuration: .milliseconds(100)) + .pollingUntilFirstPassDefaults(until: .milliseconds(100)) ) struct WithConfigurationTraits { + let stop = PollingStopCondition.firstPass + + @available(_clockAPI, *) @Test("When no test or callsite configuration provided, uses the suite configuration") func testUsesSuiteConfiguration() async throws { let incrementor = Incrementor() var test = Test { - await confirmPassesEventually(pollingInterval: .milliseconds(1)) { + try await confirmation(until: stop, pollingEvery: .milliseconds(1)) { await incrementor.increment() == 0 } } @@ -89,15 +105,16 @@ struct PollingTests { #expect(count == 100) } + @available(_clockAPI, *) @Test( "When test configuration provided, uses the test configuration", - .confirmPassesEventuallyDefaults(pollingDuration: .milliseconds(10)) + .pollingUntilFirstPassDefaults(until: .milliseconds(10)) ) func testUsesTestConfigurationOverSuiteConfiguration() async { let incrementor = Incrementor() var test = Test { // this test will intentionally fail. - await confirmPassesEventually(pollingInterval: .milliseconds(1)) { + try await confirmation(until: stop, pollingEvery: .milliseconds(1)) { await incrementor.increment() == 0 } } @@ -106,17 +123,19 @@ struct PollingTests { #expect(await incrementor.count == 10) } + @available(_clockAPI, *) @Test( "When callsite configuration provided, uses that", - .confirmPassesEventuallyDefaults(pollingDuration: .milliseconds(10)) + .pollingUntilFirstPassDefaults(until: .milliseconds(10)) ) func testUsesCallsiteConfiguration() async { let incrementor = Incrementor() var test = Test { // this test will intentionally fail. - await confirmPassesEventually( - pollingDuration: .milliseconds(50), - pollingInterval: .milliseconds(1) + try await confirmation( + until: stop, + within: .milliseconds(50), + pollingEvery: .milliseconds(1) ) { await incrementor.increment() == 0 } @@ -127,69 +146,93 @@ struct PollingTests { } #if !SWT_NO_EXIT_TESTS + @available(_clockAPI, *) @Test("Requires duration be greater than interval") func testRequiresDurationGreaterThanInterval() async { await #expect(processExitsWith: .failure) { - await confirmPassesEventually( - pollingDuration: .seconds(1), - pollingInterval: .milliseconds(1100) + try await confirmation( + until: .stopsPassing, + within: .seconds(1), + pollingEvery: .milliseconds(1100) ) { true } } } + @available(_clockAPI, *) @Test("Requires duration be greater than 0") func testRequiresDurationGreaterThan0() async { await #expect(processExitsWith: .failure) { - await confirmPassesEventually(pollingDuration: .seconds(0)) { true } + try await confirmation( + until: .stopsPassing, + within: .seconds(0) + ) { true } } } + @available(_clockAPI, *) @Test("Requires interval be greater than 0") func testRequiresIntervalGreaterThan0() async { await #expect(processExitsWith: .failure) { - await confirmPassesEventually(pollingInterval: .seconds(0)) { true } + try await confirmation( + until: .stopsPassing, + pollingEvery: .seconds(0) + ) { true } } } #endif } } - @Suite("confirmAlwaysPasses") - struct PassesAlwaysBehavior { + @Suite("with PollingStopCondition.stopsPassing") + struct StopConditionStopsPassing { + let stop = PollingStopCondition.stopsPassing + @available(_clockAPI, *) @Test("Simple passing expressions") func trivialHappyPath() async throws { - await confirmAlwaysPasses { true } - try await requireAlwaysPasses { true } + try await confirmation(until: stop) { true } + let value = try await confirmation(until: stop) { 1 } + + #expect(value == 1) } + @available(_clockAPI, *) @Test("Simple failing expressions") func trivialSadPath() async { - let issues = await runTest { - await confirmAlwaysPasses { false } - await #expect(throws: PollingFailedError()) { - try await requireAlwaysPasses { false } - } + var issues = await runTest { + try await confirmation(until: stop) { false } } - #expect(issues.count == 1) + issues += await runTest { + _ = try await confirmation(until: stop) { Optional.none } + } + #expect(issues.count == 2) + #expect(issues.allSatisfy { + if case .pollingConfirmationFailed = $0.kind { + return true + } else { + return false + } + }) } + @available(_clockAPI, *) @Test("if the closures starts off as true, but becomes false") func changingFromFail() async { let incrementor = Incrementor() let issues = await runTest { - await confirmAlwaysPasses { + try await confirmation(until: stop) { await incrementor.increment() == 2 // this will pass only on the first invocation - // This checks that we fail the test if it starts failing later during - // polling + // This checks that we fail the test if it starts failing later + // during polling } } #expect(issues.count == 1) } + @available(_clockAPI, *) @Test("if the closure continues to pass") - func continuousCalling() async { + func continuousCalling() async throws { let incrementor = Incrementor() - await confirmAlwaysPasses { + try await confirmation(until: stop) { _ = await incrementor.increment() return true } @@ -197,20 +240,22 @@ struct PollingTests { #expect(await incrementor.count > 1) } + @available(_clockAPI, *) @Test("Thrown errors will automatically exit & fail") func errorsReported() async { let issues = await runTest { - await confirmAlwaysPasses { + try await confirmation(until: stop) { throw PollingTestSampleError.ohNo } } #expect(issues.count == 1) } + @available(_clockAPI, *) @Test("Calculates how many times to poll based on the duration & interval") - func defaultPollingCount() async { + func defaultPollingCount() async throws { let incrementor = Incrementor() - await confirmAlwaysPasses(pollingInterval: .milliseconds(1)) { + try await confirmation(until: stop, pollingEvery: .milliseconds(1)) { await incrementor.increment() != 0 } #expect(await incrementor.count == 1000) @@ -218,42 +263,49 @@ struct PollingTests { @Suite( "Configuration traits", - .confirmAlwaysPassesDefaults(pollingDuration: .milliseconds(100)) + .pollingUntilStopsPassingDefaults(until: .milliseconds(100)) ) struct WithConfigurationTraits { + let stop = PollingStopCondition.stopsPassing + + @available(_clockAPI, *) @Test( "When no test/callsite configuration, it uses the suite configuration" ) func testUsesSuiteConfiguration() async throws { let incrementor = Incrementor() - await confirmAlwaysPasses(pollingInterval: .milliseconds(1)) { + try await confirmation(until: stop, pollingEvery: .milliseconds(1)) { await incrementor.increment() != 0 } let count = await incrementor.count #expect(count == 100) } + @available(_clockAPI, *) @Test( "When test configuration porvided, uses the test configuration", - .confirmAlwaysPassesDefaults(pollingDuration: .milliseconds(10)) + .pollingUntilStopsPassingDefaults(until: .milliseconds(10)) ) - func testUsesTestConfigurationOverSuiteConfiguration() async { + func testUsesTestConfigurationOverSuiteConfiguration() async throws { let incrementor = Incrementor() - await confirmAlwaysPasses(pollingInterval: .milliseconds(1)) { + try await confirmation(until: stop, pollingEvery: .milliseconds(1)) { await incrementor.increment() != 0 } - #expect(await incrementor.count == 10) + let count = await incrementor.count + #expect(await count == 10) } + @available(_clockAPI, *) @Test( "When callsite configuration provided, uses that", - .confirmAlwaysPassesDefaults(pollingDuration: .milliseconds(10)) + .pollingUntilStopsPassingDefaults(until: .milliseconds(10)) ) - func testUsesCallsiteConfiguration() async { + func testUsesCallsiteConfiguration() async throws { let incrementor = Incrementor() - await confirmAlwaysPasses( - pollingDuration: .milliseconds(50), - pollingInterval: .milliseconds(1) + try await confirmation( + until: stop, + within: .milliseconds(50), + pollingEvery: .milliseconds(1) ) { await incrementor.increment() != 0 } @@ -261,27 +313,37 @@ struct PollingTests { } #if !SWT_NO_EXIT_TESTS + @available(_clockAPI, *) @Test("Requires duration be greater than interval") func testRequiresDurationGreaterThanInterval() async { await #expect(processExitsWith: .failure) { - await confirmAlwaysPasses( - pollingDuration: .seconds(1), - pollingInterval: .milliseconds(1100) + try await confirmation( + until: .firstPass, + within: .seconds(1), + pollingEvery: .milliseconds(1100) ) { true } } } + @available(_clockAPI, *) @Test("Requires duration be greater than 0") func testRequiresDurationGreaterThan0() async { await #expect(processExitsWith: .failure) { - await confirmAlwaysPasses(pollingDuration: .seconds(0)) { true } + try await confirmation( + until: .firstPass, + within: .seconds(0) + ) { true } } } + @available(_clockAPI, *) @Test("Requires interval be greater than 0") func testRequiresIntervalGreaterThan0() async { await #expect(processExitsWith: .failure) { - await confirmAlwaysPasses(pollingInterval: .seconds(0)) { true } + try await confirmation( + until: .firstPass, + pollingEvery: .seconds(0) + ) { true } } } #endif @@ -290,33 +352,37 @@ struct PollingTests { @Suite("Duration Tests", .disabled("time-sensitive")) struct DurationTests { - @Suite("confirmPassesEventually") - struct PassesOnceBehavior { + @Suite("with PollingStopCondition.firstPass") + struct StopConditionFirstPass { + let stop = PollingStopCondition.firstPass let delta = Duration.milliseconds(100) - @Test("Simple passing expressions") func trivialHappyPath() async { - let duration = await Test.Clock().measure { - await confirmPassesEventually { true } + @available(_clockAPI, *) + @Test("Simple passing expressions") func trivialHappyPath() async throws { + let duration = try await Test.Clock().measure { + try await confirmation(until: stop) { true } } #expect(duration.isCloseTo(other: .zero, within: delta)) } + @available(_clockAPI, *) @Test("Simple failing expressions") func trivialSadPath() async { let duration = await Test.Clock().measure { let issues = await runTest { - await confirmPassesEventually { false } + try await confirmation(until: stop) { false } } #expect(issues.count == 1) } #expect(duration.isCloseTo(other: .seconds(2), within: delta)) } + @available(_clockAPI, *) @Test("When the value changes from false to true during execution") - func changingFromFail() async { + func changingFromFail() async throws { let incrementor = Incrementor() - let duration = await Test.Clock().measure { - await confirmPassesEventually { + let duration = try await Test.Clock().measure { + try await confirmation(until: stop) { await incrementor.increment() == 2 // this will pass only on the second invocation // This checks that we really are only running the expression until @@ -329,13 +395,15 @@ struct PollingTests { #expect(duration.isCloseTo(other: .zero, within: delta)) } + @available(_clockAPI, *) @Test("Doesn't wait after the last iteration") func lastIteration() async { let duration = await Test.Clock().measure { let issues = await runTest { - await confirmPassesEventually( - pollingDuration: .seconds(10), - pollingInterval: .seconds(1) // Wait a long time to handle jitter. + try await confirmation( + until: stop, + within: .seconds(10), + pollingEvery: .seconds(1) // Wait a long time to handle jitter. ) { false } } #expect(issues.count == 1) @@ -349,33 +417,37 @@ struct PollingTests { } } - @Suite("confirmAlwaysPasses") - struct PassesAlwaysBehavior { + @Suite("with PollingStopCondition.stopsPassing") + struct StopConditionStopsPassing { + let stop = PollingStopCondition.stopsPassing let delta = Duration.milliseconds(100) - @Test("Simple passing expressions") func trivialHappyPath() async { - let duration = await Test.Clock().measure { - await confirmAlwaysPasses { true } + @available(_clockAPI, *) + @Test("Simple passing expressions") func trivialHappyPath() async throws { + let duration = try await Test.Clock().measure { + try await confirmation(until: stop) { true } } #expect(duration.isCloseTo(other: .seconds(2), within: delta)) } + @available(_clockAPI, *) @Test("Simple failing expressions") func trivialSadPath() async { let duration = await Test.Clock().measure { - let issues = await runTest { - await confirmAlwaysPasses { false } + _ = await runTest { + try await confirmation(until: stop) { false } } - #expect(issues.count == 1) } #expect(duration.isCloseTo(other: .zero, within: delta)) } + @available(_clockAPI, *) @Test("Doesn't wait after the last iteration") - func lastIteration() async { - let duration = await Test.Clock().measure { - await confirmAlwaysPasses( - pollingDuration: .seconds(10), - pollingInterval: .seconds(1) // Wait a long time to handle jitter. + func lastIteration() async throws { + let duration = try await Test.Clock().measure { + try await confirmation( + until: stop, + within: .seconds(10), + pollingEvery: .seconds(1) // Wait a long time to handle jitter. ) { true } } #expect( @@ -394,6 +466,7 @@ private enum PollingTestSampleError: Error { case secondCase } +@available(_clockAPI, *) extension DurationProtocol { fileprivate func isCloseTo(other: Self, within delta: Self) -> Bool { var distance = self - other