enhancement(tls settings): support of SNI when connecting to remote server (#21365)

* support of SNI when connecting to remote server

* incorporate comments

* bubble up error

* change unwrap to expect and other comment incorporation

* refactor to remove expect

* fix change log file

Author: anil-db

.github/actions/spelling/expect.txt

@@ -1280,3 +1280,4 @@ zulip
 Zunstable
 zup
 zurp
+sni

changelog.d/21365_tls_sni_support.enhancement.md

@@ -0,0 +1,3 @@
+Add support for providing Server Name Indication in the TLS handshake when connecting to a server.
+
+authors: anil-db

lib/vector-core/src/tls/mod.rs

@@ -100,6 +100,8 @@ pub enum TlsError {
     AddCertToStore { source: ErrorStack },
     #[snafu(display("Error setting up the verification certificate: {}", source))]
     SetVerifyCert { source: ErrorStack },
+    #[snafu(display("Error setting SNI: {}", source))]
+    SetSni { source: ErrorStack },
     #[snafu(display("Error setting ALPN protocols: {}", source))]
     SetAlpnProtocols { source: ErrorStack },
     #[snafu(display(
@@ -183,13 +185,15 @@ pub fn tls_connector_builder(settings: &MaybeTlsSettings) -> Result<SslConnector
 }
 
 fn tls_connector(settings: &MaybeTlsSettings) -> Result<ConnectConfiguration> {
-    let verify_hostname = settings
-        .tls()
-        .map_or(true, |settings| settings.verify_hostname);
-    let configure = tls_connector_builder(settings)?
+    let mut configure = tls_connector_builder(settings)?
         .build()
         .configure()
-        .context(TlsBuildConnectorSnafu)?
-        .verify_hostname(verify_hostname);
+        .context(TlsBuildConnectorSnafu)?;
+    let tls_setting = settings.tls().cloned();
+    if let Some(tls_setting) = &tls_setting {
+        tls_setting
+            .apply_connect_configuration(&mut configure)
+            .context(SetSniSnafu)?;
+    }
     Ok(configure)
 }

lib/vector-core/src/tls/settings.rs

@@ -148,6 +148,14 @@ pub struct TlsConfig {
     #[configurable(metadata(docs::examples = "PassWord1"))]
     #[configurable(metadata(docs::human_name = "Key File Password"))]
     pub key_pass: Option<String>,
+
+    /// Server name to use when using Server Name Indication (SNI).
+    ///
+    /// Only relevant for outgoing connections.
+    #[serde(alias = "server_name")]
+    #[configurable(metadata(docs::examples = "www.example.com"))]
+    #[configurable(metadata(docs::human_name = "Server Name"))]
+    pub server_name: Option<String>,
 }
 
 impl TlsConfig {
@@ -169,6 +177,7 @@ pub struct TlsSettings {
     authorities: Vec<X509>,
     pub(super) identity: Option<IdentityStore>, // openssl::pkcs12::ParsedPkcs12 doesn't impl Clone yet
     alpn_protocols: Option<Vec<u8>>,
+    server_name: Option<String>,
 }
 
 #[derive(Clone)]
@@ -203,6 +212,7 @@ impl TlsSettings {
             authorities: options.load_authorities()?,
             identity: options.load_identity()?,
             alpn_protocols: options.parse_alpn_protocols()?,
+            server_name: options.server_name.clone(),
         })
     }
 
@@ -333,8 +343,17 @@ impl TlsSettings {
         Ok(())
     }
 
-    pub fn apply_connect_configuration(&self, connection: &mut ConnectConfiguration) {
+    pub fn apply_connect_configuration(
+        &self,
+        connection: &mut ConnectConfiguration,
+    ) -> std::result::Result<(), openssl::error::ErrorStack> {
         connection.set_verify_hostname(self.verify_hostname);
+        if let Some(server_name) = &self.server_name {
+            // Prevent native TLS lib from inferring default SNI using domain name from url.
+            connection.set_use_server_name_indication(false);
+            connection.set_hostname(server_name)?;
+        }
+        Ok(())
     }
 }
 

src/http.rs

@@ -1,11 +1,4 @@
 #![allow(missing_docs)]
-use std::{
-    fmt,
-    net::SocketAddr,
-    task::{Context, Poll},
-    time::Duration,
-};
-
 use futures::future::BoxFuture;
 use headers::{Authorization, HeaderMapExt};
 use http::{
@@ -22,6 +15,12 @@ use hyper_proxy::ProxyConnector;
 use rand::Rng;
 use serde_with::serde_as;
 use snafu::{ResultExt, Snafu};
+use std::{
+    fmt,
+    net::SocketAddr,
+    task::{Context, Poll},
+    time::Duration,
+};
 use tokio::time::Instant;
 use tower::{Layer, Service};
 use tower_http::{
@@ -205,10 +204,10 @@ pub fn build_tls_connector(
     let settings = tls_settings.tls().cloned();
     https.set_callback(move |c, _uri| {
         if let Some(settings) = &settings {
-            settings.apply_connect_configuration(c);
+            settings.apply_connect_configuration(c)
+        } else {
+            Ok(())
         }
-
-        Ok(())
     });
     Ok(https)
 }

website/cue/reference/components/sinks/base/amqp.cue

@@ -387,6 +387,15 @@ base: components: sinks: amqp: configuration: {
 				required: false
 				type: string: examples: ["${KEY_PASS_ENV_VAR}", "PassWord1"]
 			}
+			server_name: {
+				description: """
+					Server name to use when using Server Name Indication (SNI).
+
+					Only relevant for outgoing connections.
+					"""
+				required: false
+				type: string: examples: ["www.example.com"]
+			}
 			verify_certificate: {
 				description: """
 					Enables certificate verification. For components that create a server, this requires that the

website/cue/reference/components/sinks/base/appsignal.cue

@@ -386,6 +386,15 @@ base: components: sinks: appsignal: configuration: {
 				required: false
 				type: string: examples: ["${KEY_PASS_ENV_VAR}", "PassWord1"]
 			}
+			server_name: {
+				description: """
+					Server name to use when using Server Name Indication (SNI).
+
+					Only relevant for outgoing connections.
+					"""
+				required: false
+				type: string: examples: ["www.example.com"]
+			}
 			verify_certificate: {
 				description: """
 					Enables certificate verification. For components that create a server, this requires that the

website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue

@@ -781,6 +781,15 @@ base: components: sinks: aws_cloudwatch_logs: configuration: {
 				required: false
 				type: string: examples: ["${KEY_PASS_ENV_VAR}", "PassWord1"]
 			}
+			server_name: {
+				description: """
+					Server name to use when using Server Name Indication (SNI).
+
+					Only relevant for outgoing connections.
+					"""
+				required: false
+				type: string: examples: ["www.example.com"]
+			}
 			verify_certificate: {
 				description: """
 					Enables certificate verification. For components that create a server, this requires that the

website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue

@@ -460,6 +460,15 @@ base: components: sinks: aws_cloudwatch_metrics: configuration: {
 				required: false
 				type: string: examples: ["${KEY_PASS_ENV_VAR}", "PassWord1"]
 			}
+			server_name: {
+				description: """
+					Server name to use when using Server Name Indication (SNI).
+
+					Only relevant for outgoing connections.
+					"""
+				required: false
+				type: string: examples: ["www.example.com"]
+			}
 			verify_certificate: {
 				description: """
 					Enables certificate verification. For components that create a server, this requires that the

website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue

@@ -730,6 +730,15 @@ base: components: sinks: aws_kinesis_firehose: configuration: {
 				required: false
 				type: string: examples: ["${KEY_PASS_ENV_VAR}", "PassWord1"]
 			}
+			server_name: {
+				description: """
+					Server name to use when using Server Name Indication (SNI).
+
+					Only relevant for outgoing connections.
+					"""
+				required: false
+				type: string: examples: ["www.example.com"]
+			}
 			verify_certificate: {
 				description: """
 					Enables certificate verification. For components that create a server, this requires that the

website/cue/reference/components/sinks/base/aws_kinesis_streams.cue

@@ -730,6 +730,15 @@ base: components: sinks: aws_kinesis_streams: configuration: {
 				required: false
 				type: string: examples: ["${KEY_PASS_ENV_VAR}", "PassWord1"]
 			}
+			server_name: {
+				description: """
+					Server name to use when using Server Name Indication (SNI).
+
+					Only relevant for outgoing connections.
+					"""
+				required: false
+				type: string: examples: ["www.example.com"]
+			}
 			verify_certificate: {
 				description: """
 					Enables certificate verification. For components that create a server, this requires that the

website/cue/reference/components/sinks/base/aws_s3.cue

@@ -1067,6 +1067,15 @@ base: components: sinks: aws_s3: configuration: {
 				required: false
 				type: string: examples: ["${KEY_PASS_ENV_VAR}", "PassWord1"]
 			}
+			server_name: {
+				description: """
+					Server name to use when using Server Name Indication (SNI).
+
+					Only relevant for outgoing connections.
+					"""
+				required: false
+				type: string: examples: ["www.example.com"]
+			}
 			verify_certificate: {
 				description: """
 					Enables certificate verification. For components that create a server, this requires that the

website/cue/reference/components/sinks/base/aws_sns.cue

@@ -659,6 +659,15 @@ base: components: sinks: aws_sns: configuration: {
 				required: false
 				type: string: examples: ["${KEY_PASS_ENV_VAR}", "PassWord1"]
 			}
+			server_name: {
+				description: """
+					Server name to use when using Server Name Indication (SNI).
+
+					Only relevant for outgoing connections.
+					"""
+				required: false
+				type: string: examples: ["www.example.com"]
+			}
 			verify_certificate: {
 				description: """
 					Enables certificate verification. For components that create a server, this requires that the

website/cue/reference/components/sinks/base/aws_sqs.cue

@@ -664,6 +664,15 @@ base: components: sinks: aws_sqs: configuration: {
 				required: false
 				type: string: examples: ["${KEY_PASS_ENV_VAR}", "PassWord1"]
 			}
+			server_name: {
+				description: """
+					Server name to use when using Server Name Indication (SNI).
+
+					Only relevant for outgoing connections.
+					"""
+				required: false
+				type: string: examples: ["www.example.com"]
+			}
 			verify_certificate: {
 				description: """
 					Enables certificate verification. For components that create a server, this requires that the

website/cue/reference/components/sinks/base/axiom.cue

@@ -323,6 +323,15 @@ base: components: sinks: axiom: configuration: {
 				required: false
 				type: string: examples: ["${KEY_PASS_ENV_VAR}", "PassWord1"]
 			}
+			server_name: {
+				description: """
+					Server name to use when using Server Name Indication (SNI).
+
+					Only relevant for outgoing connections.
+					"""
+				required: false
+				type: string: examples: ["www.example.com"]
+			}
 			verify_certificate: {
 				description: """
 					Enables certificate verification. For components that create a server, this requires that the

website/cue/reference/components/sinks/base/azure_monitor_logs.cue

@@ -390,6 +390,15 @@ base: components: sinks: azure_monitor_logs: configuration: {
 				required: false
 				type: string: examples: ["${KEY_PASS_ENV_VAR}", "PassWord1"]
 			}
+			server_name: {
+				description: """
+					Server name to use when using Server Name Indication (SNI).
+
+					Only relevant for outgoing connections.
+					"""
+				required: false
+				type: string: examples: ["www.example.com"]
+			}
 			verify_certificate: {
 				description: """
 					Enables certificate verification. For components that create a server, this requires that the

website/cue/reference/components/sinks/base/clickhouse.cue

@@ -459,6 +459,15 @@ base: components: sinks: clickhouse: configuration: {
 				required: false
 				type: string: examples: ["${KEY_PASS_ENV_VAR}", "PassWord1"]
 			}
+			server_name: {
+				description: """
+					Server name to use when using Server Name Indication (SNI).
+
+					Only relevant for outgoing connections.
+					"""
+				required: false
+				type: string: examples: ["www.example.com"]
+			}
 			verify_certificate: {
 				description: """
 					Enables certificate verification. For components that create a server, this requires that the

website/cue/reference/components/sinks/base/databend.cue

@@ -548,6 +548,15 @@ base: components: sinks: databend: configuration: {
 				required: false
 				type: string: examples: ["${KEY_PASS_ENV_VAR}", "PassWord1"]
 			}
+			server_name: {
+				description: """
+					Server name to use when using Server Name Indication (SNI).
+
+					Only relevant for outgoing connections.
+					"""
+				required: false
+				type: string: examples: ["www.example.com"]
+			}
 			verify_certificate: {
 				description: """
 					Enables certificate verification. For components that create a server, this requires that the

website/cue/reference/components/sinks/base/datadog_events.cue

@@ -320,6 +320,15 @@ base: components: sinks: datadog_events: configuration: {
 				required: false
 				type: string: examples: ["${KEY_PASS_ENV_VAR}", "PassWord1"]
 			}
+			server_name: {
+				description: """
+					Server name to use when using Server Name Indication (SNI).
+
+					Only relevant for outgoing connections.
+					"""
+				required: false
+				type: string: examples: ["www.example.com"]
+			}
 			verify_certificate: {
 				description: """
 					Enables certificate verification. For components that create a server, this requires that the

website/cue/reference/components/sinks/base/datadog_logs.cue

@@ -423,6 +423,15 @@ base: components: sinks: datadog_logs: configuration: {
 				required: false
 				type: string: examples: ["${KEY_PASS_ENV_VAR}", "PassWord1"]
 			}
+			server_name: {
+				description: """
+					Server name to use when using Server Name Indication (SNI).
+
+					Only relevant for outgoing connections.
+					"""
+				required: false
+				type: string: examples: ["www.example.com"]
+			}
 			verify_certificate: {
 				description: """
 					Enables certificate verification. For components that create a server, this requires that the

website/cue/reference/components/sinks/base/datadog_metrics.cue

@@ -362,6 +362,15 @@ base: components: sinks: datadog_metrics: configuration: {
 				required: false
 				type: string: examples: ["${KEY_PASS_ENV_VAR}", "PassWord1"]
 			}
+			server_name: {
+				description: """
+					Server name to use when using Server Name Indication (SNI).
+
+					Only relevant for outgoing connections.
+					"""
+				required: false
+				type: string: examples: ["www.example.com"]
+			}
 			verify_certificate: {
 				description: """
 					Enables certificate verification. For components that create a server, this requires that the