diff --git a/src/utils/structs.rs b/src/utils/structs.rs index 1a64ef9..a6507f7 100644 --- a/src/utils/structs.rs +++ b/src/utils/structs.rs @@ -130,6 +130,9 @@ pub struct AppConfig { pub runuser: Option, pub rungroup: Option, pub log_file: Option, + pub tcp_keepalive_idle: Option, + pub tcp_keepalive_interval: Option, + pub tcp_keepalive_count: Option, } #[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] diff --git a/src/web/proxyhttp.rs b/src/web/proxyhttp.rs index ddea322..93d4c42 100644 --- a/src/web/proxyhttp.rs +++ b/src/web/proxyhttp.rs @@ -177,6 +177,20 @@ impl ProxyHttp for LB { peer.options.verify_cert = false; peer.options.verify_hostname = false; } + /* + Experimental optionsv + The following TCP optimizations were tested but caused performance degrade under heavy load: + peer.options.tcp_keepalive = Some(TcpKeepalive { + idle: Duration::from_secs(60), + interval: Duration::from_secs(10), + count: 5, + user_timeout: Duration::from_secs(30), + }); + + peer.options.idle_timeout = Some(Duration::from_secs(300)); + peer.options.tcp_recv_buf = Some(128 * 1024); + End of experimental options + */ if let Some(_) = ctx.extraparams.sticky_sessions { let mut s = String::with_capacity(64); write!( diff --git a/src/web/start.rs b/src/web/start.rs index e6cdb92..d4cf7ca 100644 --- a/src/web/start.rs +++ b/src/web/start.rs @@ -10,7 +10,9 @@ use dashmap::DashMap; use log::info; use pingora::tls::ssl::{SslAlert, SslRef}; use pingora_core::listeners::tls::TlsSettings; +use pingora_core::listeners::TcpSocketOptions; use pingora_core::prelude::{background_service, Opt}; +use pingora_core::protocols::TcpKeepalive; use pingora_core::server::Server; use privdrop::reexports::libc::SIGQUIT; use sd_notify::NotifyState; @@ -62,12 +64,34 @@ pub fn run() { info!("TLS grade set to: [ {} ]", grade); let bg_srvc = background_service("bgsrvc", lb.clone()); - let mut proxy = pingora_proxy::http_proxy_service(&server.configuration, lb.clone()); let bind_address_http = cfg.proxy_address_http.clone(); let bind_address_tls = cfg.proxy_address_tls.clone(); + let mut proxy = pingora_proxy::http_proxy_service(&server.configuration, lb.clone()); + check_priv(bind_address_http.as_str()); + // let mut tcp_options: Option = Some(TcpSocketOptions::default()); + // let mut tcp_options = TcpSocketOptions::default(); + + let mut tcp_options: Option = None; + if let Some(idle) = cfg.tcp_keepalive_idle { + let mut to = TcpSocketOptions::default(); + to.tcp_keepalive = Some(TcpKeepalive { + idle: Duration::from_secs(idle), + interval: Duration::from_secs(cfg.tcp_keepalive_interval.unwrap_or(60)), + user_timeout: Default::default(), + count: cfg.tcp_keepalive_count.unwrap_or(5usize), + }); + tcp_options = Some(to); + info!( + "Applying kernel tcp_keepalive parameters: idle {}, interval {}, count {}", + idle, + cfg.tcp_keepalive_interval.unwrap_or(60), + cfg.tcp_keepalive_count.unwrap_or(5), + ); + } + if let Some(bind_address_tls) = bind_address_tls { check_priv(bind_address_tls.as_str()); let (tx, rx): (Sender>, Receiver>) = channel(); @@ -95,7 +119,14 @@ pub fn run() { tls_settings.set_servername_callback(move |ssl_ref: &mut SslRef, ssl_alert: &mut SslAlert| certs_for_callback.load().server_name_callback(ssl_ref, ssl_alert)); tls_settings.set_alpn_select_callback(grades::prefer_h2); - proxy.add_tls_with_settings(&bind_address_tls, None, tls_settings); + proxy.add_tls_with_settings(&bind_address_tls, tcp_options.clone(), tls_settings); + + // if let Some(to) = tcp_options.clone() { + // proxy.add_tls_with_settings(&bind_address_tls, Some(to.clone()), tls_settings); + // } else { + // proxy.add_tls_with_settings(&bind_address_tls, None, tls_settings); + // } + // proxy.add_tls_with_settings(&bind_address_tls, None, tls_settings); let certs_for_watcher = certificates.clone(); thread::spawn(move || { @@ -107,8 +138,13 @@ pub fn run() { } }); } - info!("Running HTTP listener on :{}", bind_address_http.as_str()); - proxy.add_tcp(bind_address_http.as_str()); + info!("Running HTTP listener on :{}", bind_address_http); + if let Some(tc) = tcp_options { + proxy.add_tcp_with_settings(&bind_address_http, tc); + } else { + proxy.add_tcp(&bind_address_http) + } + server.add_service(proxy); server.add_service(bg_srvc); thread::spawn(move || server.run_forever());