From fdf9fa0a541aa6c158fe7e5866daa82e4f2b1a39 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Wed, 29 Mar 2023 14:48:57 +0100 Subject: [PATCH 1/5] Allow DNS override MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sometimes it can be useful to connect to one host e.g. `x.example.com` but request and validate the certificate chain as if we would connect to `y.example.com`. This is what this PR adds support for by adding a `dnsOverride` configuration to `HTTPClient.Configuration`. This is similar to curls `—resolve-to` option but only allows overriding host and not ports for now. --- Package.swift | 2 + .../AsyncAwait/HTTPClient+execute.swift | 4 +- .../HTTPClientRequest+Prepared.swift | 4 +- Sources/AsyncHTTPClient/ConnectionPool.swift | 54 +++++++++++++++--- .../HTTPConnectionPool+Factory.swift | 4 +- .../ConnectionPool/RequestOptions.swift | 8 ++- Sources/AsyncHTTPClient/HTTPClient.swift | 11 ++++ .../TLSConfiguration.swift | 12 +++- Sources/AsyncHTTPClient/RequestBag.swift | 8 +-- .../AsyncAwaitEndToEndTests.swift | 54 ++++++++++++++++++ .../HTTPClientNIOTSTests.swift | 2 +- .../HTTPClientRequestTests.swift | 39 ++++++++----- .../HTTPClientTestUtils.swift | 56 ++++++++++--------- .../RequestBagTests.swift | 8 ++- .../Resources/example.com.cert.pem | 12 ++++ .../Resources/example.com.private-key.pem | 6 ++ 16 files changed, 219 insertions(+), 65 deletions(-) create mode 100644 Tests/AsyncHTTPClientTests/Resources/example.com.cert.pem create mode 100644 Tests/AsyncHTTPClientTests/Resources/example.com.private-key.pem diff --git a/Package.swift b/Package.swift index f72de842e..83d5c65bc 100644 --- a/Package.swift +++ b/Package.swift @@ -68,6 +68,8 @@ let package = Package( resources: [ .copy("Resources/self_signed_cert.pem"), .copy("Resources/self_signed_key.pem"), + .copy("Resources/example.com.cert.pem"), + .copy("Resources/example.com.private-key.pem"), ] ), ] diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift index 5328b7688..5f0a5f7c5 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift @@ -77,7 +77,7 @@ extension HTTPClient { // this loop is there to follow potential redirects while true { - let preparedRequest = try HTTPClientRequest.Prepared(currentRequest) + let preparedRequest = try HTTPClientRequest.Prepared(currentRequest, dnsOverride: configuration.dnsOverride) let response = try await executeCancellable(preparedRequest, deadline: deadline, logger: logger) guard var redirectState = currentRedirectState else { @@ -131,7 +131,7 @@ extension HTTPClient { return try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) -> Void in let transaction = Transaction( request: request, - requestOptions: .init(idleReadTimeout: nil), + requestOptions: .fromClientConfiguration(self.configuration), logger: logger, connectionDeadline: .now() + (self.configuration.timeout.connectionCreationTimeout), preferredEventLoop: eventLoop, diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift index bd7417725..489ba5626 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift @@ -42,7 +42,7 @@ extension HTTPClientRequest { @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension HTTPClientRequest.Prepared { - init(_ request: HTTPClientRequest) throws { + init(_ request: HTTPClientRequest, dnsOverride: [String: String] = [:]) throws { guard let url = URL(string: request.url) else { throw HTTPClientError.invalidURL } @@ -58,7 +58,7 @@ extension HTTPClientRequest.Prepared { self.init( url: url, - poolKey: .init(url: deconstructedURL, tlsConfiguration: nil), + poolKey: .init(url: deconstructedURL, tlsConfiguration: nil, dnsOverride: dnsOverride), requestFramingMetadata: metadata, head: .init( version: .http1_1, diff --git a/Sources/AsyncHTTPClient/ConnectionPool.swift b/Sources/AsyncHTTPClient/ConnectionPool.swift index 0dac50e5f..bd24af3da 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool.swift @@ -14,6 +14,25 @@ import NIOSSL +#if canImport(Darwin) +import Darwin.C +#elseif os(Linux) || os(FreeBSD) || os(Android) +import Glibc +#else +#error("unsupported target operating system") +#endif + +extension String { + var isIPAddress: Bool { + var ipv4Address = in_addr() + var ipv6Address = in6_addr() + return self.withCString { host in + inet_pton(AF_INET, host, &ipv4Address) == 1 || + inet_pton(AF_INET6, host, &ipv6Address) == 1 + } + } +} + enum ConnectionPool { /// Used by the `ConnectionPool` to index its `HTTP1ConnectionProvider`s /// @@ -24,15 +43,18 @@ enum ConnectionPool { var scheme: Scheme var connectionTarget: ConnectionTarget private var tlsConfiguration: BestEffortHashableTLSConfiguration? + var serverNameIndicatorOverride: String? init( scheme: Scheme, connectionTarget: ConnectionTarget, - tlsConfiguration: BestEffortHashableTLSConfiguration? = nil + tlsConfiguration: BestEffortHashableTLSConfiguration? = nil, + serverNameIndicatorOverride: String? ) { self.scheme = scheme self.connectionTarget = connectionTarget self.tlsConfiguration = tlsConfiguration + self.serverNameIndicatorOverride = serverNameIndicatorOverride } var description: String { @@ -48,26 +70,44 @@ enum ConnectionPool { case .unixSocket(let socketPath): hostDescription = socketPath } - return "\(self.scheme)://\(hostDescription) TLS-hash: \(hash)" + return "\(self.scheme)://\(hostDescription)\(self.serverNameIndicatorOverride.map { " SNI: \($0)" } ?? "") TLS-hash: \(hash) " } } } +extension DeconstructedURL { + func applyDNSOverride(_ dnsOverride: [String: String]) -> (ConnectionTarget, serverNameIndicatorOverride: String?) { + guard + let originalHost = self.connectionTarget.host, + let hostOverride = dnsOverride[originalHost] + else { + return (self.connectionTarget, nil) + } + return ( + .init(remoteHost: hostOverride, port: self.connectionTarget.port ?? self.scheme.defaultPort), + serverNameIndicatorOverride: originalHost.isIPAddress ? nil : originalHost + ) + } +} + extension ConnectionPool.Key { - init(url: DeconstructedURL, tlsConfiguration: TLSConfiguration?) { + init(url: DeconstructedURL, tlsConfiguration: TLSConfiguration?, dnsOverride: [String: String]) { + let (connectionTarget, serverNameIndicatorOverride) = url.applyDNSOverride(dnsOverride) self.init( scheme: url.scheme, - connectionTarget: url.connectionTarget, + connectionTarget: connectionTarget, tlsConfiguration: tlsConfiguration.map { BestEffortHashableTLSConfiguration(wrapping: $0) - } + }, + serverNameIndicatorOverride: serverNameIndicatorOverride ) } - init(_ request: HTTPClient.Request) { + init(_ request: HTTPClient.Request, dnsOverride: [String: String] = [:]) { self.init( url: request.deconstructedURL, - tlsConfiguration: request.tlsConfiguration + tlsConfiguration: request.tlsConfiguration, + dnsOverride: dnsOverride ) } } diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift index 60338f615..7c0e2736a 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift @@ -409,7 +409,7 @@ extension HTTPConnectionPool.ConnectionFactory { #if canImport(Network) if #available(OSX 10.14, iOS 12.0, tvOS 12.0, watchOS 6.0, *), let tsBootstrap = NIOTSConnectionBootstrap(validatingGroup: eventLoop) { // create NIOClientTCPBootstrap with NIOTS TLS provider - let bootstrapFuture = tlsConfig.getNWProtocolTLSOptions(on: eventLoop).map { + let bootstrapFuture = tlsConfig.getNWProtocolTLSOptions(on: eventLoop, serverNameIndicatorOverride: key.serverNameIndicatorOverride).map { options -> NIOClientTCPBootstrapProtocol in tsBootstrap @@ -434,7 +434,7 @@ extension HTTPConnectionPool.ConnectionFactory { } #endif - let sslServerHostname = self.key.connectionTarget.sslServerHostname + let sslServerHostname = self.key.serverNameIndicatorOverride ?? self.key.connectionTarget.sslServerHostname let sslContextFuture = sslContextCache.sslContext( tlsConfiguration: tlsConfig, eventLoop: eventLoop, diff --git a/Sources/AsyncHTTPClient/ConnectionPool/RequestOptions.swift b/Sources/AsyncHTTPClient/ConnectionPool/RequestOptions.swift index 2092498d8..ffd5d2758 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/RequestOptions.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/RequestOptions.swift @@ -17,16 +17,20 @@ import NIOCore struct RequestOptions { /// The maximal `TimeAmount` that is allowed to pass between `channelRead`s from the Channel. var idleReadTimeout: TimeAmount? + + var dnsOverride: [String: String] - init(idleReadTimeout: TimeAmount?) { + init(idleReadTimeout: TimeAmount?, dnsOverride: [String: String]) { self.idleReadTimeout = idleReadTimeout + self.dnsOverride = dnsOverride } } extension RequestOptions { static func fromClientConfiguration(_ configuration: HTTPClient.Configuration) -> Self { RequestOptions( - idleReadTimeout: configuration.timeout.read + idleReadTimeout: configuration.timeout.read, + dnsOverride: configuration.dnsOverride ) } } diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index 2f4368402..9afb7b5ef 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -711,6 +711,17 @@ public class HTTPClient { public struct Configuration { /// TLS configuration, defaults to `TLSConfiguration.makeClientConfiguration()`. public var tlsConfiguration: Optional + + /// Sometimes it can be useful to connect to one host e.g. `x.example.com` but + /// request and validate the certificate chain as if we would connect to `y.example.com`. + /// ``dnsOverride`` allows to do just that by mapping host names which we will request and validate the certificate chain, to a different + /// host name which will be used to actually connect to. + /// + /// **Example:** if ``dnsOverride`` is set to `["example.com": "localhost"]` and we execute a request with a + /// `url` of `https://example.com/`, the ``HTTPClient`` will actually open a connection to `localhost` instead of `example.com`. + /// ``HTTPClient`` will still request certificates from the server for `example.com` and validate them as if we would connect to `example.com`. + public var dnsOverride: [String: String] = [:] + /// Enables following 3xx redirects automatically. /// /// Following redirects are supported: diff --git a/Sources/AsyncHTTPClient/NIOTransportServices/TLSConfiguration.swift b/Sources/AsyncHTTPClient/NIOTransportServices/TLSConfiguration.swift index e20f52634..6fe830a8b 100644 --- a/Sources/AsyncHTTPClient/NIOTransportServices/TLSConfiguration.swift +++ b/Sources/AsyncHTTPClient/NIOTransportServices/TLSConfiguration.swift @@ -66,11 +66,11 @@ extension TLSConfiguration { /// /// - Parameter eventLoop: EventLoop to wait for creation of options on /// - Returns: Future holding NWProtocolTLS Options - func getNWProtocolTLSOptions(on eventLoop: EventLoop) -> EventLoopFuture { + func getNWProtocolTLSOptions(on eventLoop: EventLoop, serverNameIndicatorOverride: String?) -> EventLoopFuture { let promise = eventLoop.makePromise(of: NWProtocolTLS.Options.self) Self.tlsDispatchQueue.async { do { - let options = try self.getNWProtocolTLSOptions() + let options = try self.getNWProtocolTLSOptions(serverNameIndicatorOverride: serverNameIndicatorOverride) promise.succeed(options) } catch { promise.fail(error) @@ -82,7 +82,7 @@ extension TLSConfiguration { /// create NWProtocolTLS.Options for use with NIOTransportServices from the NIOSSL TLSConfiguration /// /// - Returns: Equivalent NWProtocolTLS Options - func getNWProtocolTLSOptions() throws -> NWProtocolTLS.Options { + func getNWProtocolTLSOptions(serverNameIndicatorOverride: String?) throws -> NWProtocolTLS.Options { let options = NWProtocolTLS.Options() let useMTELGExplainer = """ @@ -91,6 +91,12 @@ extension TLSConfiguration { will make AsyncHTTPClient use NIO on BSD Sockets and not Network.framework (which is the preferred \ platform networking stack). """ + + if let serverNameIndicatorOverride = serverNameIndicatorOverride { + serverNameIndicatorOverride.withCString { serverNameIndicatorOverride in + sec_protocol_options_set_tls_server_name(options.securityProtocolOptions, serverNameIndicatorOverride) + } + } // minimum TLS protocol if #available(macOS 10.15, iOS 13.0, tvOS 13.0, watchOS 6.0, *) { diff --git a/Sources/AsyncHTTPClient/RequestBag.swift b/Sources/AsyncHTTPClient/RequestBag.swift index 1119236fb..ac5d500ad 100644 --- a/Sources/AsyncHTTPClient/RequestBag.swift +++ b/Sources/AsyncHTTPClient/RequestBag.swift @@ -26,7 +26,9 @@ final class RequestBag { private static var maxConsumeBodyPartStackDepth: Int { 50 } - + + let poolKey: ConnectionPool.Key + let task: HTTPClient.Task var eventLoop: EventLoop { self.task.eventLoop @@ -63,6 +65,7 @@ final class RequestBag { connectionDeadline: NIODeadline, requestOptions: RequestOptions, delegate: Delegate) throws { + self.poolKey = .init(request, dnsOverride: requestOptions.dnsOverride) self.eventLoopPreference = eventLoopPreference self.task = task self.state = .init(redirectHandler: redirectHandler) @@ -392,9 +395,6 @@ final class RequestBag { } extension RequestBag: HTTPSchedulableRequest { - var poolKey: ConnectionPool.Key { - ConnectionPool.Key(self.request) - } var tlsConfiguration: TLSConfiguration? { self.request.tlsConfiguration diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index 3d0e709d6..49e4e1054 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -491,6 +491,60 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } } } + + func testDnsOverride() { + guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } + XCTAsyncTest(timeout: 5) { + /// key + cert was created with the following code (depends on swift-certificates) + /// ``` + /// let privateKey = P384.Signing.PrivateKey() + /// let name = try DistinguishedName { + /// OrganizationName("Self Signed") + /// CommonName("localhost") + /// } + /// let certificate = try Certificate( + /// version: .v3, + /// serialNumber: .init(), + /// publicKey: .init(privateKey.publicKey), + /// notValidBefore: Date(), + /// notValidAfter: Date() + .days(365), + /// issuer: name, + /// subject: name, + /// signatureAlgorithm: .ecdsaWithSHA384, + /// extensions: try .init { + /// SubjectAlternativeNames([.dnsName("example.com")]) + /// ExtendedKeyUsage([.serverAuth]) + /// }, + /// issuerPrivateKey: .init(privateKey) + /// ) + ///``` + let certPath = Bundle.module.path(forResource: "example.com.cert", ofType: "pem")! + let keyPath = Bundle.module.path(forResource: "example.com.private-key", ofType: "pem")! + let localhostCert = try NIOSSLCertificate.fromPEMFile(certPath) + let configuration = TLSConfiguration.makeServerConfiguration( + certificateChain: localhostCert.map { .certificate($0) }, + privateKey: .file(keyPath) + ) + let bin = HTTPBin(.http2(tlsConfiguration: configuration)) + defer { XCTAssertNoThrow(try bin.shutdown()) } + + var config = HTTPClient.Configuration() + .enableFastFailureModeForTesting() + var tlsConfig = TLSConfiguration.makeClientConfiguration() + + tlsConfig.trustRoots = .certificates(localhostCert) + config.tlsConfiguration = tlsConfig + // this is the actual configuration under test + config.dnsOverride = ["example.com": "localhost"] + + let localClient = HTTPClient(eventLoopGroupProvider: .createNew, configuration: config) + defer { XCTAssertNoThrow(try localClient.syncShutdown()) } + let request = HTTPClientRequest(url: "https://example.com:\(bin.port)/echo-headers") + let response = await XCTAssertNoThrowWithResult(try await localClient.execute(request, deadline: .now() + .seconds(2))) + XCTAssertEqual(response?.status, .ok) + XCTAssertEqual(response?.version, .http2) + } + } func testInvalidURL() { guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift index 9727746cc..3db4385cd 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift @@ -165,7 +165,7 @@ class HTTPClientNIOTSTests: XCTestCase { var tlsConfig = TLSConfiguration.makeClientConfiguration() tlsConfig.trustRoots = .file("not/a/certificate") - XCTAssertThrowsError(try tlsConfig.getNWProtocolTLSOptions()) { error in + XCTAssertThrowsError(try tlsConfig.getNWProtocolTLSOptions(serverNameIndicatorOverride: nil)) { error in switch error { case let error as NIOSSL.NIOSSLError where error == .failedToLoadCertificate: break diff --git a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift index fa424b042..aa1071de6 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift @@ -37,7 +37,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .https, connectionTarget: .domain(name: "example.com", port: 443), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, @@ -69,7 +70,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .unix, connectionTarget: .unixSocket(path: "/some_path"), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, @@ -98,7 +100,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .httpUnix, connectionTarget: .unixSocket(path: "/example/folder.sock"), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, @@ -127,7 +130,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .httpsUnix, connectionTarget: .unixSocket(path: "/example/folder.sock"), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, @@ -155,7 +159,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .https, connectionTarget: .domain(name: "example.com", port: 443), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, @@ -184,7 +189,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .http, connectionTarget: .domain(name: "example.com", port: 80), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, @@ -218,7 +224,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .http, connectionTarget: .domain(name: "example.com", port: 80), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, @@ -252,7 +259,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .http, connectionTarget: .domain(name: "example.com", port: 80), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, @@ -286,7 +294,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .http, connectionTarget: .domain(name: "example.com", port: 80), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, @@ -321,7 +330,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .http, connectionTarget: .domain(name: "example.com", port: 80), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, @@ -355,7 +365,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .http, connectionTarget: .domain(name: "example.com", port: 80), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, @@ -394,7 +405,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .http, connectionTarget: .domain(name: "example.com", port: 80), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, @@ -433,7 +445,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .http, connectionTarget: .domain(name: "example.com", port: 80), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index ca24cba1c..86bb07659 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -312,6 +312,10 @@ enum TemporaryFileHelpers { enum TestTLS { static let certificate = try! NIOSSLCertificate(bytes: Array(cert.utf8), format: .pem) static let privateKey = try! NIOSSLPrivateKey(bytes: Array(key.utf8), format: .pem) + static let serverConfiguration: TLSConfiguration = .makeServerConfiguration( + certificateChain: [.certificate(TestTLS.certificate)], + privateKey: .privateKey(TestTLS.privateKey) + ) } internal final class HTTPBin where @@ -327,34 +331,54 @@ internal final class HTTPBin where // refuses all connections case refuse // supports http1.1 connections only, which can be either plain text or encrypted - case http1_1(ssl: Bool = false, compress: Bool = false) + case http1_1( + tlsConfiguration: TLSConfiguration? = nil, + compress: Bool = false + ) // supports http1.1 and http2 connections which must be always encrypted case http2( + tlsConfiguration: TLSConfiguration = TestTLS.serverConfiguration, compress: Bool = false, settings: HTTP2Settings? = nil ) + + static func http1_1(ssl: Bool, compress: Bool = false) -> Self { + .http1_1(tlsConfiguration: ssl ? nil : TestTLS.serverConfiguration, compress: compress) + } // supports request decompression and http response compression var compress: Bool { switch self { case .refuse: return false - case .http1_1(ssl: _, compress: let compress), .http2(compress: let compress, _): + case .http1_1(_, let compress), .http2(_, let compress, _): return compress } } var httpSettings: HTTP2Settings { switch self { - case .http1_1, .http2(_, nil), .refuse: + case .http1_1, .http2(_, _, nil), .refuse: return [ HTTP2Setting(parameter: .maxConcurrentStreams, value: 10), HTTP2Setting(parameter: .maxHeaderListSize, value: HPACKDecoder.defaultMaxHeaderListSize), ] - case .http2(_, .some(let customSettings)): + case .http2(_, _, .some(let customSettings)): return customSettings } } + + var tlsConfiguration: TLSConfiguration? { + switch self { + case .refuse: + return nil + case .http1_1(let tlsConfiguration, _): + return tlsConfiguration + case .http2(var tlsConfiguration, _, _): + tlsConfiguration.applicationProtocols = NIOHTTP2SupportedALPNProtocols + return tlsConfiguration + } + } } enum Proxy { @@ -540,30 +564,8 @@ internal final class HTTPBin where } } - private static func tlsConfiguration(for mode: Mode) -> TLSConfiguration? { - var configuration: TLSConfiguration? - - switch mode { - case .refuse, .http1_1(ssl: false, compress: _): - break - case .http2: - configuration = .makeServerConfiguration( - certificateChain: [.certificate(TestTLS.certificate)], - privateKey: .privateKey(TestTLS.privateKey) - ) - configuration!.applicationProtocols = NIOHTTP2SupportedALPNProtocols - case .http1_1(ssl: true, compress: _): - configuration = .makeServerConfiguration( - certificateChain: [.certificate(TestTLS.certificate)], - privateKey: .privateKey(TestTLS.privateKey) - ) - } - - return configuration - } - private static func sslContext(for mode: Mode) -> NIOSSLContext? { - if let tlsConfiguration = self.tlsConfiguration(for: mode) { + if let tlsConfiguration = mode.tlsConfiguration { return try! NIOSSLContext(configuration: tlsConfiguration) } return nil diff --git a/Tests/AsyncHTTPClientTests/RequestBagTests.swift b/Tests/AsyncHTTPClientTests/RequestBagTests.swift index 43134d453..36efee949 100644 --- a/Tests/AsyncHTTPClientTests/RequestBagTests.swift +++ b/Tests/AsyncHTTPClientTests/RequestBagTests.swift @@ -972,9 +972,13 @@ class MockTaskQueuer: HTTPRequestScheduler { } extension RequestOptions { - static func forTests(idleReadTimeout: TimeAmount? = nil) -> Self { + static func forTests( + idleReadTimeout: TimeAmount? = nil, + dnsOverride: [String: String] = [:] + ) -> Self { RequestOptions( - idleReadTimeout: idleReadTimeout + idleReadTimeout: idleReadTimeout, + dnsOverride: dnsOverride ) } } diff --git a/Tests/AsyncHTTPClientTests/Resources/example.com.cert.pem b/Tests/AsyncHTTPClientTests/Resources/example.com.cert.pem new file mode 100644 index 000000000..69af76e77 --- /dev/null +++ b/Tests/AsyncHTTPClientTests/Resources/example.com.cert.pem @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIBwzCCAUmgAwIBAgIVAIFK2HEjRjd9rH6Szp3jT52U4wYjMAoGCCqGSM49BAMD +MCoxFDASBgNVBAoMC1NlbGYgU2lnbmVkMRIwEAYDVQQDDAlsb2NhbGhvc3QwHhcN +MjMwMzI5MTE1ODQwWhcNMjQwMzI4MTE1ODQwWjAqMRQwEgYDVQQKDAtTZWxmIFNp +Z25lZDESMBAGA1UEAwwJbG9jYWxob3N0MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAE +SiOrOD8CbOyvj0yg+ArayukRCjw9AAaW3lsrsiRsSaqRxDcZ7+uR5nt2FUXc25mD +Ap+adz4g5gigpIUaVQc69AgMavYFCHF3Tb0TF1D4yAFLk8GFuWqxHDuqCQaGoyS5 +oy8wLTAWBgNVHREEDzANggtleGFtcGxlLmNvbTATBgNVHSUEDDAKBggrBgEFBQcD +ATAKBggqhkjOPQQDAwNoADBlAjALdKj7fq0Hvv69KUdMGvpHBaqRq+4+X4T1gAm/ +Z09XPB3BAd9z3Ov7fMnc65iKRwICMQCxxu0rBJUmR9v1BINxA4S1EPH0S/U5ysTp +Wu1n1LZ3C5ooxMiO50cPuWupaB2LElY= +-----END CERTIFICATE----- \ No newline at end of file diff --git a/Tests/AsyncHTTPClientTests/Resources/example.com.private-key.pem b/Tests/AsyncHTTPClientTests/Resources/example.com.private-key.pem new file mode 100644 index 000000000..775a5ea56 --- /dev/null +++ b/Tests/AsyncHTTPClientTests/Resources/example.com.private-key.pem @@ -0,0 +1,6 @@ +-----BEGIN PRIVATE KEY----- +MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDAbqzPBHiy/SoUXTlYl +F0q3AK+N5wvpb93vS8jdRYAY2BIKIQOurw4WLp0qVxKgYGqhZANiAARKI6s4PwJs +7K+PTKD4CtrK6REKPD0ABpbeWyuyJGxJqpHENxnv65Hme3YVRdzbmYMCn5p3PiDm +CKCkhRpVBzr0CAxq9gUIcXdNvRMXUPjIAUuTwYW5arEcO6oJBoajJLk= +-----END PRIVATE KEY----- \ No newline at end of file From dffcb48b29734add2f95a23ad25b508888607547 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Wed, 29 Mar 2023 15:00:02 +0100 Subject: [PATCH 2/5] soundness --- Sources/AsyncHTTPClient/ConnectionPool.swift | 2 +- .../AsyncHTTPClient/ConnectionPool/RequestOptions.swift | 2 +- Sources/AsyncHTTPClient/HTTPClient.swift | 4 ++-- .../NIOTransportServices/TLSConfiguration.swift | 2 +- Sources/AsyncHTTPClient/RequestBag.swift | 5 ++--- .../AsyncAwaitEndToEndTests+XCTest.swift | 1 + Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift | 8 ++++---- Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift | 4 ++-- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool.swift b/Sources/AsyncHTTPClient/ConnectionPool.swift index bd24af3da..b27e3fb97 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool.swift @@ -28,7 +28,7 @@ extension String { var ipv6Address = in6_addr() return self.withCString { host in inet_pton(AF_INET, host, &ipv4Address) == 1 || - inet_pton(AF_INET6, host, &ipv6Address) == 1 + inet_pton(AF_INET6, host, &ipv6Address) == 1 } } } diff --git a/Sources/AsyncHTTPClient/ConnectionPool/RequestOptions.swift b/Sources/AsyncHTTPClient/ConnectionPool/RequestOptions.swift index ffd5d2758..c46f1289c 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/RequestOptions.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/RequestOptions.swift @@ -17,7 +17,7 @@ import NIOCore struct RequestOptions { /// The maximal `TimeAmount` that is allowed to pass between `channelRead`s from the Channel. var idleReadTimeout: TimeAmount? - + var dnsOverride: [String: String] init(idleReadTimeout: TimeAmount?, dnsOverride: [String: String]) { diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index 9afb7b5ef..beb2ea458 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -711,7 +711,7 @@ public class HTTPClient { public struct Configuration { /// TLS configuration, defaults to `TLSConfiguration.makeClientConfiguration()`. public var tlsConfiguration: Optional - + /// Sometimes it can be useful to connect to one host e.g. `x.example.com` but /// request and validate the certificate chain as if we would connect to `y.example.com`. /// ``dnsOverride`` allows to do just that by mapping host names which we will request and validate the certificate chain, to a different @@ -721,7 +721,7 @@ public class HTTPClient { /// `url` of `https://example.com/`, the ``HTTPClient`` will actually open a connection to `localhost` instead of `example.com`. /// ``HTTPClient`` will still request certificates from the server for `example.com` and validate them as if we would connect to `example.com`. public var dnsOverride: [String: String] = [:] - + /// Enables following 3xx redirects automatically. /// /// Following redirects are supported: diff --git a/Sources/AsyncHTTPClient/NIOTransportServices/TLSConfiguration.swift b/Sources/AsyncHTTPClient/NIOTransportServices/TLSConfiguration.swift index 6fe830a8b..f79954da7 100644 --- a/Sources/AsyncHTTPClient/NIOTransportServices/TLSConfiguration.swift +++ b/Sources/AsyncHTTPClient/NIOTransportServices/TLSConfiguration.swift @@ -91,7 +91,7 @@ extension TLSConfiguration { will make AsyncHTTPClient use NIO on BSD Sockets and not Network.framework (which is the preferred \ platform networking stack). """ - + if let serverNameIndicatorOverride = serverNameIndicatorOverride { serverNameIndicatorOverride.withCString { serverNameIndicatorOverride in sec_protocol_options_set_tls_server_name(options.securityProtocolOptions, serverNameIndicatorOverride) diff --git a/Sources/AsyncHTTPClient/RequestBag.swift b/Sources/AsyncHTTPClient/RequestBag.swift index ac5d500ad..2b20193b4 100644 --- a/Sources/AsyncHTTPClient/RequestBag.swift +++ b/Sources/AsyncHTTPClient/RequestBag.swift @@ -26,9 +26,9 @@ final class RequestBag { private static var maxConsumeBodyPartStackDepth: Int { 50 } - + let poolKey: ConnectionPool.Key - + let task: HTTPClient.Task var eventLoop: EventLoop { self.task.eventLoop @@ -395,7 +395,6 @@ final class RequestBag { } extension RequestBag: HTTPSchedulableRequest { - var tlsConfiguration: TLSConfiguration? { self.request.tlsConfiguration } diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift index 20538c43a..ce0e2846d 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift @@ -40,6 +40,7 @@ extension AsyncAwaitEndToEndTests { ("testImmediateDeadline", testImmediateDeadline), ("testConnectTimeout", testConnectTimeout), ("testSelfSignedCertificateIsRejectedWithCorrectErrorIfRequestDeadlineIsExceeded", testSelfSignedCertificateIsRejectedWithCorrectErrorIfRequestDeadlineIsExceeded), + ("testDnsOverride", testDnsOverride), ("testInvalidURL", testInvalidURL), ("testRedirectChangesHostHeader", testRedirectChangesHostHeader), ("testShutdown", testShutdown), diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index 49e4e1054..535d0714d 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -491,7 +491,7 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } } } - + func testDnsOverride() { guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest(timeout: 5) { @@ -517,7 +517,7 @@ final class AsyncAwaitEndToEndTests: XCTestCase { /// }, /// issuerPrivateKey: .init(privateKey) /// ) - ///``` + /// ``` let certPath = Bundle.module.path(forResource: "example.com.cert", ofType: "pem")! let keyPath = Bundle.module.path(forResource: "example.com.private-key", ofType: "pem")! let localhostCert = try NIOSSLCertificate.fromPEMFile(certPath) @@ -531,12 +531,12 @@ final class AsyncAwaitEndToEndTests: XCTestCase { var config = HTTPClient.Configuration() .enableFastFailureModeForTesting() var tlsConfig = TLSConfiguration.makeClientConfiguration() - + tlsConfig.trustRoots = .certificates(localhostCert) config.tlsConfiguration = tlsConfig // this is the actual configuration under test config.dnsOverride = ["example.com": "localhost"] - + let localClient = HTTPClient(eventLoopGroupProvider: .createNew, configuration: config) defer { XCTAssertNoThrow(try localClient.syncShutdown()) } let request = HTTPClientRequest(url: "https://example.com:\(bin.port)/echo-headers") diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index 86bb07659..a7cb3280d 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -341,7 +341,7 @@ internal final class HTTPBin where compress: Bool = false, settings: HTTP2Settings? = nil ) - + static func http1_1(ssl: Bool, compress: Bool = false) -> Self { .http1_1(tlsConfiguration: ssl ? nil : TestTLS.serverConfiguration, compress: compress) } @@ -367,7 +367,7 @@ internal final class HTTPBin where return customSettings } } - + var tlsConfiguration: TLSConfiguration? { switch self { case .refuse: From e2faaeeaca352f0b201e1bf91eaa37bc47e09050 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Wed, 29 Mar 2023 16:27:36 +0100 Subject: [PATCH 3/5] Fix tests --- Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift | 6 +++++- Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index 535d0714d..e80957079 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -539,10 +539,14 @@ final class AsyncAwaitEndToEndTests: XCTestCase { let localClient = HTTPClient(eventLoopGroupProvider: .createNew, configuration: config) defer { XCTAssertNoThrow(try localClient.syncShutdown()) } - let request = HTTPClientRequest(url: "https://example.com:\(bin.port)/echo-headers") + let request = HTTPClientRequest(url: "https://example.com:\(bin.port)/echohostheader") let response = await XCTAssertNoThrowWithResult(try await localClient.execute(request, deadline: .now() + .seconds(2))) XCTAssertEqual(response?.status, .ok) XCTAssertEqual(response?.version, .http2) + var body = try await response?.body.collect(upTo: 1024) + let readableBytes = body?.readableBytes ?? 0 + let responseInfo = try body?.readJSONDecodable(RequestInfo.self, length: readableBytes) + XCTAssertEqual(responseInfo?.data, "example.com\(bin.port == 443 ? "" : ":\(bin.port)")") } } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index a7cb3280d..c617555c6 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -343,7 +343,7 @@ internal final class HTTPBin where ) static func http1_1(ssl: Bool, compress: Bool = false) -> Self { - .http1_1(tlsConfiguration: ssl ? nil : TestTLS.serverConfiguration, compress: compress) + .http1_1(tlsConfiguration: ssl ? TestTLS.serverConfiguration : nil, compress: compress) } // supports request decompression and http response compression From f296aa4cd01a074ff580791dcf7f6717f3c6a93a Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Wed, 29 Mar 2023 16:33:06 +0100 Subject: [PATCH 4/5] Refactor logic into `ConnectionPool.Key.serverNameIndicator` --- .../ConnectionPool/HTTPConnectionPool+Factory.swift | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift index 7c0e2736a..48aedfd8e 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift @@ -281,7 +281,7 @@ extension HTTPConnectionPool.ConnectionFactory { } let tlsEventHandler = TLSEventsHandler(deadline: deadline) - let sslServerHostname = self.key.connectionTarget.sslServerHostname + let sslServerHostname = self.key.serverNameIndicator let sslContextFuture = self.sslContextCache.sslContext( tlsConfiguration: tlsConfig, eventLoop: channel.eventLoop, @@ -434,7 +434,6 @@ extension HTTPConnectionPool.ConnectionFactory { } #endif - let sslServerHostname = self.key.serverNameIndicatorOverride ?? self.key.connectionTarget.sslServerHostname let sslContextFuture = sslContextCache.sslContext( tlsConfiguration: tlsConfig, eventLoop: eventLoop, @@ -449,7 +448,7 @@ extension HTTPConnectionPool.ConnectionFactory { let sync = channel.pipeline.syncOperations let sslHandler = try NIOSSLClientHandler( context: sslContext, - serverHostname: sslServerHostname + serverHostname: self.key.serverNameIndicator ) let tlsEventHandler = TLSEventsHandler(deadline: deadline) @@ -488,6 +487,12 @@ extension Scheme { } } +extension ConnectionPool.Key { + var serverNameIndicator: String? { + serverNameIndicatorOverride ?? connectionTarget.sslServerHostname + } +} + extension ConnectionTarget { fileprivate var sslServerHostname: String? { switch self { From 281e1864ce38664908b7b393ccfedeb034d84a65 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Wed, 29 Mar 2023 16:42:04 +0100 Subject: [PATCH 5/5] Fix Swift 5.5 --- Package@swift-5.5.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Package@swift-5.5.swift b/Package@swift-5.5.swift index 8ae20ed9c..02c91c7ef 100644 --- a/Package@swift-5.5.swift +++ b/Package@swift-5.5.swift @@ -67,6 +67,8 @@ let package = Package( resources: [ .copy("Resources/self_signed_cert.pem"), .copy("Resources/self_signed_key.pem"), + .copy("Resources/example.com.cert.pem"), + .copy("Resources/example.com.private-key.pem"), ] ), ]