diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt index d02ba6ea5..bf192e843 100644 --- a/Sources/Testing/CMakeLists.txt +++ b/Sources/Testing/CMakeLists.txt @@ -29,6 +29,7 @@ add_library(Testing Events/TimeValue.swift ExitTests/ExitCondition.swift ExitTests/ExitTest.swift + ExitTests/ExitTest.Result.swift ExitTests/SpawnProcess.swift ExitTests/WaitFor.swift Expectations/Expectation.swift diff --git a/Sources/Testing/ExitTests/ExitTest.Result.swift b/Sources/Testing/ExitTests/ExitTest.Result.swift new file mode 100644 index 000000000..8780300d4 --- /dev/null +++ b/Sources/Testing/ExitTests/ExitTest.Result.swift @@ -0,0 +1,86 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +#if SWT_NO_EXIT_TESTS +@available(*, unavailable, message: "Exit tests are not available on this platform.") +#endif +extension ExitTest { + /// A type representing the result of an exit test after it has exited and + /// returned control to the calling test function. + /// + /// Both ``expect(exitsWith:_:sourceLocation:performing:)`` and + /// ``require(exitsWith:_:sourceLocation:performing:)`` return instances of + /// this type. + public struct Result: Sendable { + /// The exit condition the exit test exited with. + /// + /// When the exit test passes, the value of this property is equal to the + /// value of the `expectedExitCondition` argument passed to + /// ``expect(exitsWith:_:sourceLocation:performing:)`` or to + /// ``require(exitsWith:_:sourceLocation:performing:)``. You can compare two + /// instances of ``ExitCondition`` with ``ExitCondition/==(lhs:rhs:)``. + public var exitCondition: ExitCondition + + /// Whatever error might have been thrown when trying to invoke the exit + /// test that produced this result. + /// + /// This property is not part of the public interface of the testing + /// library. + var caughtError: (any Error)? + + @_spi(ForToolsIntegrationOnly) + public init(exitCondition: ExitCondition) { + self.exitCondition = exitCondition + } + + /// Initialize an instance of this type representing the result of an exit + /// test that failed to run due to a system error or other failure. + /// + /// - Parameters: + /// - exitCondition: The exit condition the exit test exited with, if + /// available. The default value of this argument is + /// ``ExitCondition/failure`` for lack of a more accurate one. + /// - error: The error associated with the exit test on failure, if any. + /// + /// If an error (e.g. a failure calling `posix_spawn()`) occurs in the exit + /// test handler configured by the exit test's host environment, the exit + /// test handler should throw that error. The testing library will then + /// record it appropriately. + /// + /// When used with `#require(exitsWith:)`, an instance initialized with this + /// initializer will throw `error`. + /// + /// This initializer is not part of the public interface of the testing + /// library. + init(exitCondition: ExitCondition = .failure, catching error: any Error) { + self.exitCondition = exitCondition + self.caughtError = error + } + + /// Handle this instance as if it were returned from a call to `#expect()`. + /// + /// - Warning: This function is used to implement the `#expect()` and + /// `#require()` macros. Do not call it directly. + @inlinable public func __expected() -> Self { + self + } + + /// Handle this instance as if it were returned from a call to `#require()`. + /// + /// - Warning: This function is used to implement the `#expect()` and + /// `#require()` macros. Do not call it directly. + public func __required() throws -> Self { + if let caughtError { + throw caughtError + } + return self + } + } +} diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index 8923429ad..9cc95cbfb 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -10,18 +10,18 @@ private import _TestingInternals -#if !SWT_NO_EXIT_TESTS -#if SWT_NO_PIPES -#error("Support for exit tests requires support for (anonymous) pipes.") -#endif - /// A type describing an exit test. /// /// Instances of this type describe an exit test defined by the test author and /// discovered or called at runtime. -@_spi(Experimental) @_spi(ForToolsIntegrationOnly) +@_spi(Experimental) +#if SWT_NO_EXIT_TESTS +@available(*, unavailable, message: "Exit tests are not available on this platform.") +#endif public struct ExitTest: Sendable, ~Copyable { +#if !SWT_NO_EXIT_TESTS /// The expected exit condition of the exit test. + @_spi(ForToolsIntegrationOnly) public var expectedExitCondition: ExitCondition /// The body closure of the exit test. @@ -31,6 +31,7 @@ public struct ExitTest: Sendable, ~Copyable { /// /// The source location is unique to each exit test and is consistent between /// processes, so it can be used to uniquely identify an exit test at runtime. + @_spi(ForToolsIntegrationOnly) public var sourceLocation: SourceLocation /// Disable crash reporting, crash logging, or core dumps for the current @@ -83,6 +84,7 @@ public struct ExitTest: Sendable, ~Copyable { /// to terminate the process; if it does not, the testing library will /// terminate the process in a way that causes the corresponding expectation /// to fail. + @_spi(ForToolsIntegrationOnly) public consuming func callAsFunction() async -> Never { Self._disableCrashReporting() @@ -98,8 +100,13 @@ public struct ExitTest: Sendable, ~Copyable { let expectingFailure = expectedExitCondition == .failure exit(expectingFailure ? EXIT_SUCCESS : EXIT_FAILURE) } +#endif } +#if !SWT_NO_EXIT_TESTS +#if SWT_NO_PIPES +#error("Support for exit tests requires support for (anonymous) pipes.") +#endif // MARK: - Discovery /// A protocol describing a type that contains an exit test. @@ -131,6 +138,7 @@ extension ExitTest { /// /// - Returns: The specified exit test function, or `nil` if no such exit test /// could be found. + @_spi(ForToolsIntegrationOnly) public static func find(at sourceLocation: SourceLocation) -> Self? { var result: Self? @@ -176,35 +184,47 @@ func callExitTest( isRequired: Bool, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation -) async -> Result { +) async -> ExitTest.Result { guard let configuration = Configuration.current ?? Configuration.all.first else { preconditionFailure("A test must be running on the current task to use #expect(exitsWith:).") } - let actualExitCondition: ExitCondition + var result: ExitTest.Result do { let exitTest = ExitTest(expectedExitCondition: expectedExitCondition, sourceLocation: sourceLocation) - actualExitCondition = try await configuration.exitTestHandler(exitTest) + result = try await configuration.exitTestHandler(exitTest) } catch { // An error here would indicate a problem in the exit test handler such as a // failure to find the process' path, to construct arguments to the - // subprocess, or to spawn the subprocess. These are not expected to be - // common issues, however they would constitute a failure of the test - // infrastructure rather than the test itself and perhaps should not cause - // the test to terminate early. - let issue = Issue(kind: .errorCaught(error), comments: comments(), sourceContext: .init(backtrace: .current(), sourceLocation: sourceLocation)) + // subprocess, or to spawn the subprocess. Such failures are system issues, + // not test issues, because they constitute failures of the test + // infrastructure rather than the test itself. + // + // But here's a philosophical question: should the exit test also fail with + // an expectation failure? Arguably no, because the issue is a systemic one + // and (presumably) not a bug in the test. But also arguably yes, because + // the exit test did not do what the test author expected it to do. + let backtrace = Backtrace(forFirstThrowOf: error) ?? .current() + let issue = Issue( + kind: .system, + comments: comments() + CollectionOfOne(Comment(rawValue: String(describingForTest: error))), + sourceContext: SourceContext(backtrace: backtrace, sourceLocation: sourceLocation) + ) issue.record(configuration: configuration) - return __checkValue( - false, - expression: expression, - comments: comments(), - isRequired: isRequired, - sourceLocation: sourceLocation - ) + // For lack of a better way to handle an exit test failing in this way, + // we record the system issue above, then let the expectation fail below by + // reporting an exit condition that's the inverse of the expected one. + result = ExitTest.Result(exitCondition: expectedExitCondition == .failure ? .success : .failure) } - return __checkValue( + // How did the exit test actually exit? + let actualExitCondition = result.exitCondition + + // Plumb the resulting exit condition through the general expectation + // machinery. If the expectation failed, capture the ExpectationFailedError + // instance so that calls to #require(exitsWith:) throw it correctly. + let checkResult = __checkValue( expectedExitCondition == actualExitCondition, expression: expression, expressionWithCapturedRuntimeValues: expression.capturingRuntimeValues(actualExitCondition), @@ -213,6 +233,11 @@ func callExitTest( isRequired: isRequired, sourceLocation: sourceLocation ) + if case let .failure(error) = checkResult { + result.caughtError = error + } + + return result } // MARK: - SwiftPM/tools integration @@ -223,7 +248,8 @@ extension ExitTest { /// - Parameters: /// - exitTest: The exit test that is starting. /// - /// - Returns: The condition under which the exit test exited. + /// - Returns: The result of the exit test including the condition under which + /// it exited. /// /// - Throws: Any error that prevents the normal invocation or execution of /// the exit test. @@ -239,7 +265,8 @@ extension ExitTest { /// are available or the child environment is otherwise terminated. The parent /// environment is then responsible for interpreting those results and /// recording any issues that occur. - public typealias Handler = @Sendable (_ exitTest: borrowing ExitTest) async throws -> ExitCondition + @_spi(ForToolsIntegrationOnly) + public typealias Handler = @Sendable (_ exitTest: borrowing ExitTest) async throws -> ExitTest.Result /// The back channel file handle set up by the parent process. /// @@ -334,7 +361,7 @@ extension ExitTest { // or unsetenv(), so we need to recompute the child environment each time. // The executable and XCTest bundle paths should not change over time, so we // can precompute them. - let childProcessExecutablePath = Result { try CommandLine.executablePath } + let childProcessExecutablePath = Swift.Result { try CommandLine.executablePath } // Construct appropriate arguments for the child process. Generally these // arguments are going to be whatever's necessary to respawn the current @@ -415,7 +442,7 @@ extension ExitTest { childEnvironment["SWT_EXPERIMENTAL_EXIT_TEST_SOURCE_LOCATION"] = String(decoding: json, as: UTF8.self) } - return try await withThrowingTaskGroup(of: ExitCondition?.self) { taskGroup in + return try await withThrowingTaskGroup(of: ExitTest.Result?.self) { taskGroup in // Create a "back channel" pipe to handle events from the child process. let backChannel = try FileHandle.Pipe() @@ -450,7 +477,8 @@ extension ExitTest { // Await termination of the child process. taskGroup.addTask { - try await wait(for: processID) + let exitCondition = try await wait(for: processID) + return ExitTest.Result(exitCondition: exitCondition) } // Read back all data written to the back channel by the child process diff --git a/Sources/Testing/Expectations/Expectation+Macro.swift b/Sources/Testing/Expectations/Expectation+Macro.swift index 695dc7411..08cfbf908 100644 --- a/Sources/Testing/Expectations/Expectation+Macro.swift +++ b/Sources/Testing/Expectations/Expectation+Macro.swift @@ -422,6 +422,10 @@ public macro require( /// issues should be attributed. /// - expression: The expression to be evaluated. /// +/// - Returns: An instance of ``ExitTest/Result`` describing the state of the +/// exit test when it exited including the actual exit condition that it +/// reported to the testing library. +/// /// Use this overload of `#expect()` when an expression will cause the current /// process to terminate and the nature of that termination will determine if /// the test passes or fails. For example, to test that calling `fatalError()` @@ -479,12 +483,13 @@ public macro require( #if SWT_NO_EXIT_TESTS @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif +@discardableResult @freestanding(expression) public macro expect( exitsWith expectedExitCondition: ExitCondition, _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, performing expression: @convention(thin) () async throws -> Void -) = #externalMacro(module: "TestingMacros", type: "ExitTestExpectMacro") +) -> ExitTest.Result = #externalMacro(module: "TestingMacros", type: "ExitTestExpectMacro") /// Check that an expression causes the process to terminate in a given fashion /// and throw an error if it did not. @@ -496,6 +501,10 @@ public macro require( /// issues should be attributed. /// - expression: The expression to be evaluated. /// +/// - Returns: An instance of ``ExitTest/Result`` describing the state of the +/// exit test when it exited including the actual exit condition that it +/// reported to the testing library. +/// /// - Throws: An instance of ``ExpectationFailedError`` if the exit condition of /// the child process does not equal `expectedExitCondition`. /// @@ -556,9 +565,10 @@ public macro require( #if SWT_NO_EXIT_TESTS @available(*, unavailable, message: "Exit tests are not available on this platform.") #endif +@discardableResult @freestanding(expression) public macro require( exitsWith expectedExitCondition: ExitCondition, _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, performing expression: @convention(thin) () async throws -> Void -) = #externalMacro(module: "TestingMacros", type: "ExitTestRequireMacro") +) -> ExitTest.Result = #externalMacro(module: "TestingMacros", type: "ExitTestRequireMacro") diff --git a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift index 0c65dee1a..252d96954 100644 --- a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift +++ b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift @@ -1148,7 +1148,7 @@ public func __checkClosureCall( isRequired: Bool, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation -) async -> Result { +) async -> ExitTest.Result { await callExitTest( exitsWith: expectedExitCondition, expression: expression, diff --git a/Sources/Testing/Issues/Issue+Recording.swift b/Sources/Testing/Issues/Issue+Recording.swift index 2a6facefb..e13099eaf 100644 --- a/Sources/Testing/Issues/Issue+Recording.swift +++ b/Sources/Testing/Issues/Issue+Recording.swift @@ -32,7 +32,7 @@ extension Issue { if case let .errorCaught(error) = kind, let error = error as? SystemError { var selfCopy = self selfCopy.kind = .system - selfCopy.comments.append(Comment(rawValue: String(describing: error))) + selfCopy.comments.append(Comment(rawValue: String(describingForTest: error))) return selfCopy.record(configuration: configuration) } diff --git a/Tests/TestingTests/ExitTestTests.swift b/Tests/TestingTests/ExitTestTests.swift index 60f708957..286d01f7b 100644 --- a/Tests/TestingTests/ExitTestTests.swift +++ b/Tests/TestingTests/ExitTestTests.swift @@ -89,7 +89,7 @@ private import _TestingInternals // Mock an exit test where the process exits successfully. configuration.exitTestHandler = { _ in - return .exitCode(EXIT_SUCCESS) + return ExitTest.Result(exitCondition: .exitCode(EXIT_SUCCESS)) } await Test { await #expect(exitsWith: .success) {} @@ -97,7 +97,7 @@ private import _TestingInternals // Mock an exit test where the process exits with a generic failure. configuration.exitTestHandler = { _ in - return .failure + return ExitTest.Result(exitCondition: .failure) } await Test { await #expect(exitsWith: .failure) {} @@ -113,7 +113,7 @@ private import _TestingInternals // Mock an exit test where the process exits with a particular error code. configuration.exitTestHandler = { _ in - return .exitCode(123) + return ExitTest.Result(exitCondition: .exitCode(123)) } await Test { await #expect(exitsWith: .failure) {} @@ -122,7 +122,7 @@ private import _TestingInternals #if !os(Windows) // Mock an exit test where the process exits with a signal. configuration.exitTestHandler = { _ in - return .signal(SIGABRT) + return ExitTest.Result(exitCondition: .signal(SIGABRT)) } await Test { await #expect(exitsWith: .signal(SIGABRT)) {} @@ -151,7 +151,7 @@ private import _TestingInternals // Mock exit tests that were expected to fail but passed. configuration.exitTestHandler = { _ in - return .exitCode(EXIT_SUCCESS) + return ExitTest.Result(exitCondition: .exitCode(EXIT_SUCCESS)) } await Test { await #expect(exitsWith: .failure) {} @@ -168,7 +168,7 @@ private import _TestingInternals #if !os(Windows) // Mock exit tests that unexpectedly signalled. configuration.exitTestHandler = { _ in - return .signal(SIGABRT) + return ExitTest.Result(exitCondition: .signal(SIGABRT)) } await Test { await #expect(exitsWith: .exitCode(EXIT_SUCCESS)) {} @@ -294,6 +294,108 @@ private import _TestingInternals exit(EXIT_SUCCESS) } } + + @Test("Result is set correctly (success)") + func exitTestResultOnSuccess() async throws { + // Test that basic passing exit tests produce the correct results (#expect) + var result = await #expect(exitsWith: .success) { + exit(EXIT_SUCCESS) + } + #expect(result.exitCondition === .success) + result = await #expect(exitsWith: .exitCode(123)) { + exit(123) + } + #expect(result.exitCondition === .exitCode(123)) + + // Test that basic passing exit tests produce the correct results (#require) + result = try await #require(exitsWith: .success) { + exit(EXIT_SUCCESS) + } + #expect(result.exitCondition === .success) + result = try await #require(exitsWith: .exitCode(123)) { + exit(123) + } + #expect(result.exitCondition === .exitCode(123)) + } + + @Test("Result is set correctly (failure)") + func exitTestResultOnFailure() async { + // Test that an exit test that produces the wrong exit condition reports it + // as an expectation failure, but also returns the exit condition (#expect) + await confirmation("Expectation failed") { expectationFailed in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case let .issueRecorded(issue) = event.kind { + if case .expectationFailed = issue.kind { + expectationFailed() + } else { + issue.record() + } + } + } + configuration.exitTestHandler = { _ in + ExitTest.Result(exitCondition: .exitCode(123)) + } + + await Test { + let result = await #expect(exitsWith: .success) {} + #expect(result.exitCondition === .exitCode(123)) + }.run(configuration: configuration) + } + + // Test that an exit test that produces the wrong exit condition throws an + // ExpectationFailedError (#require) + await confirmation("Expectation failed") { expectationFailed in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case let .issueRecorded(issue) = event.kind { + if case .expectationFailed = issue.kind { + expectationFailed() + } else { + issue.record() + } + } + } + configuration.exitTestHandler = { _ in + ExitTest.Result(exitCondition: .failure) + } + + await Test { + try await #require(exitsWith: .success) {} + fatalError("Unreachable") + }.run(configuration: configuration) + } + } + + @Test("Result is set correctly (system failure)") + func exitTestResultOnSystemFailure() async { + // Test that an exit test that fails to start due to a system error produces + // a .system issue and reports .failure as its exit condition. + await confirmation("System issue recorded") { systemIssueRecorded in + await confirmation("Expectation failed") { expectationFailed in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case let .issueRecorded(issue) = event.kind { + if case .system = issue.kind { + systemIssueRecorded() + } else if case .expectationFailed = issue.kind { + expectationFailed() + } else { + issue.record() + } + } + } + configuration.exitTestHandler = { _ in + throw MyError() + } + + await Test { + let result = await #expect(exitsWith: .success) {} + #expect(result.exitCondition === .failure) + }.run(configuration: configuration) + } + } + } } // MARK: - Fixtures