Skip to content

Commit 52f2ca7

Browse files
relurelu
committed
feat: Allow configuring TCP listen backlog
Signed-off-by: Aurel Canciu <aurel.canciu@nexhealth.com>
1 parent 704cf7a commit 52f2ca7

File tree

10 files changed

+162
-16
lines changed

10 files changed

+162
-16
lines changed

‎linkerd/app/core/src/config.rs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ pub use crate::exp_backoff::ExponentialBackoff;
22
use crate::{
33
proxy::http::{h1, h2},
44
svc::{queue, ExtractParam, Param},
5-
transport::{DualListenAddr, Keepalive, ListenAddr, UserTimeout},
5+
transport::{DualListenAddr, Keepalive, ListenAddr, UserTimeout, Backlog},
66
};
77
use std::time::Duration;
88

@@ -11,6 +11,7 @@ pub struct ServerConfig {
1111
pub addr: DualListenAddr,
1212
pub keepalive: Keepalive,
1313
pub user_timeout: UserTimeout,
14+
pub backlog: Backlog,
1415
pub http2: h2::ServerParams,
1516
}
1617

@@ -84,3 +85,9 @@ impl Param<UserTimeout> for ServerConfig {
8485
self.user_timeout
8586
}
8687
}
88+
89+
impl Param<Backlog> for ServerConfig {
90+
fn param(&self) -> Backlog {
91+
self.backlog
92+
}
93+
}

‎linkerd/app/inbound/src/test_util.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ use linkerd_app_core::{
88
http::{h1, h2},
99
tap,
1010
},
11-
transport::{DualListenAddr, Keepalive, UserTimeout},
11+
transport::{DualListenAddr, Keepalive, UserTimeout, Backlog},
1212
ProxyRuntime,
1313
};
1414
pub use linkerd_app_test as support;
@@ -59,6 +59,7 @@ pub fn default_config() -> Config {
5959
addr: DualListenAddr(([0, 0, 0, 0], 0).into(), None),
6060
keepalive: Keepalive(None),
6161
user_timeout: UserTimeout(None),
62+
backlog: Backlog::default(),
6263
http2: h2::ServerParams::default(),
6364
},
6465
connect: config::ConnectConfig {

‎linkerd/app/integration/src/proxy.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ use super::*;
22
use linkerd_app_core::{
33
svc::Param,
44
transport::{
5-
listen, orig_dst, Keepalive, ListenAddr, Local, OrigDstAddr, ServerAddr, UserTimeout,
5+
listen, orig_dst, Keepalive, ListenAddr, Local, OrigDstAddr, ServerAddr, UserTimeout, Backlog,
66
},
77
Result,
88
};
@@ -70,7 +70,7 @@ struct MockDualOrigDst {
7070

7171
impl<T> listen::Bind<T> for MockOrigDst
7272
where
73-
T: Param<Keepalive> + Param<UserTimeout> + Param<ListenAddr>,
73+
T: Param<Keepalive> + Param<UserTimeout> + Param<ListenAddr> + Param<Backlog>,
7474
{
7575
type Addrs = orig_dst::Addrs;
7676
type BoundAddrs = Local<ServerAddr>;
@@ -120,7 +120,7 @@ impl fmt::Debug for MockOrigDst {
120120

121121
impl<T> listen::Bind<T> for MockDualOrigDst
122122
where
123-
T: Param<Keepalive> + Param<UserTimeout> + Param<ListenAddr>,
123+
T: Param<Keepalive> + Param<UserTimeout> + Param<ListenAddr> + Param<Backlog>,
124124
{
125125
type Addrs = orig_dst::Addrs;
126126
type BoundAddrs = (Local<ServerAddr>, Option<Local<ServerAddr>>);

‎linkerd/app/outbound/src/test_util.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ use linkerd_app_core::{
77
http::{h1, h2},
88
tap,
99
},
10-
transport::{DualListenAddr, Keepalive, UserTimeout},
10+
transport::{DualListenAddr, Keepalive, UserTimeout, Backlog},
1111
IpMatch, IpNet, ProxyRuntime,
1212
};
1313
pub use linkerd_app_test as support;
@@ -27,6 +27,7 @@ pub(crate) fn default_config() -> Config {
2727
addr: DualListenAddr(([0, 0, 0, 0], 0).into(), None),
2828
keepalive: Keepalive(None),
2929
user_timeout: UserTimeout(None),
30+
backlog: Backlog::default(),
3031
http2: h2::ServerParams::default(),
3132
},
3233
connect: config::ConnectConfig {

‎linkerd/app/src/env.rs‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ use linkerd_app_core::{
55
control::{Config as ControlConfig, ControlAddr},
66
proxy::http::{h1, h2},
77
tls,
8-
transport::{DualListenAddr, Keepalive, ListenAddr, UserTimeout},
8+
transport::{DualListenAddr, Keepalive, ListenAddr, UserTimeout, Backlog},
99
AddrMatch, Conditional, IpNet,
1010
};
1111
use std::{collections::HashSet, net::SocketAddr, path::PathBuf, time::Duration};
@@ -133,6 +133,9 @@ const ENV_OUTBOUND_ACCEPT_USER_TIMEOUT: &str = "LINKERD2_PROXY_OUTBOUND_ACCEPT_U
133133
const ENV_INBOUND_CONNECT_USER_TIMEOUT: &str = "LINKERD2_PROXY_INBOUND_CONNECT_USER_TIMEOUT";
134134
const ENV_OUTBOUND_CONNECT_USER_TIMEOUT: &str = "LINKERD2_PROXY_OUTBOUND_CONNECT_USER_TIMEOUT";
135135

136+
const ENV_INBOUND_TCP_LISTEN_BACKLOG: &str = "LINKERD2_PROXY_INBOUND_TCP_LISTEN_BACKLOG";
137+
const ENV_OUTBOUND_TCP_LISTEN_BACKLOG: &str = "LINKERD2_PROXY_OUTBOUND_TCP_LISTEN_BACKLOG";
138+
136139
const ENV_INBOUND_MAX_IDLE_CONNS_PER_ENDPOINT: &str = "LINKERD2_PROXY_MAX_IDLE_CONNS_PER_ENDPOINT";
137140
const ENV_OUTBOUND_MAX_IDLE_CONNS_PER_ENDPOINT: &str =
138141
"LINKERD2_PROXY_OUTBOUND_MAX_IDLE_CONNS_PER_ENDPOINT";
@@ -386,6 +389,9 @@ pub fn parse_config<S: Strings>(strings: &S) -> Result<super::Config, EnvError>
386389
let inbound_accept_keepalive = parse(strings, ENV_INBOUND_ACCEPT_KEEPALIVE, parse_duration);
387390
let outbound_accept_keepalive = parse(strings, ENV_OUTBOUND_ACCEPT_KEEPALIVE, parse_duration);
388391

392+
let inbound_tcp_listen_backlog = parse(strings, ENV_INBOUND_TCP_LISTEN_BACKLOG, parse_number::<u32>);
393+
let outbound_tcp_listen_backlog = parse(strings, ENV_OUTBOUND_TCP_LISTEN_BACKLOG, parse_number::<u32>);
394+
389395
let inbound_connect_keepalive = parse(strings, ENV_INBOUND_CONNECT_KEEPALIVE, parse_duration);
390396
let outbound_connect_keepalive = parse(strings, ENV_OUTBOUND_CONNECT_KEEPALIVE, parse_duration);
391397

@@ -499,10 +505,12 @@ pub fn parse_config<S: Strings>(strings: &S) -> Result<super::Config, EnvError>
499505

500506
let keepalive = Keepalive(outbound_accept_keepalive?);
501507
let user_timeout = UserTimeout(outbound_accept_user_timeout?);
508+
let backlog = Backlog(outbound_tcp_listen_backlog?.unwrap_or(128));
502509
let server = ServerConfig {
503510
addr,
504511
keepalive,
505512
user_timeout,
513+
backlog,
506514
http2: http2::parse_server(strings, "LINKERD2_PROXY_OUTBOUND_SERVER_HTTP2")?,
507515
};
508516
let discovery_idle_timeout =
@@ -591,10 +599,12 @@ pub fn parse_config<S: Strings>(strings: &S) -> Result<super::Config, EnvError>
591599
);
592600
let keepalive = Keepalive(inbound_accept_keepalive?);
593601
let user_timeout = UserTimeout(inbound_accept_user_timeout?);
602+
let backlog = Backlog(inbound_tcp_listen_backlog?.unwrap_or(128));
594603
let server = ServerConfig {
595604
addr,
596605
keepalive,
597606
user_timeout,
607+
backlog,
598608
http2: http2::parse_server(strings, "LINKERD2_PROXY_INBOUND_SERVER_HTTP2")?,
599609
};
600610
let discovery_idle_timeout =
@@ -814,6 +824,7 @@ pub fn parse_config<S: Strings>(strings: &S) -> Result<super::Config, EnvError>
814824
addr: DualListenAddr(admin_listener_addr, None),
815825
keepalive: inbound.proxy.server.keepalive,
816826
user_timeout: inbound.proxy.server.user_timeout,
827+
backlog: inbound.proxy.server.backlog,
817828
http2: inbound.proxy.server.http2.clone(),
818829
},
819830

@@ -868,6 +879,7 @@ pub fn parse_config<S: Strings>(strings: &S) -> Result<super::Config, EnvError>
868879
addr: DualListenAddr(addr, None),
869880
keepalive: inbound.proxy.server.keepalive,
870881
user_timeout: inbound.proxy.server.user_timeout,
882+
backlog: inbound.proxy.server.backlog,
871883
http2: inbound.proxy.server.http2.clone(),
872884
},
873885
})

‎linkerd/app/src/env/types.rs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -357,4 +357,22 @@ mod tests {
357357
assert!(dbg!(parse_port_range_set("69420")).is_err());
358358
assert!(dbg!(parse_port_range_set("1-69420")).is_err());
359359
}
360+
361+
#[test]
362+
fn parse_backlog_valid() {
363+
assert_eq!(parse_number::<u32>("128"), Ok(128));
364+
assert_eq!(parse_number::<u32>("512"), Ok(512));
365+
assert_eq!(parse_number::<u32>("1024"), Ok(1024));
366+
assert_eq!(parse_number::<u32>("65535"), Ok(65535));
367+
assert_eq!(parse_number::<u32>("0"), Ok(0));
368+
}
369+
370+
#[test]
371+
fn parse_backlog_invalid() {
372+
assert!(parse_number::<u32>("").is_err());
373+
assert!(parse_number::<u32>("abc").is_err());
374+
assert!(parse_number::<u32>("-1").is_err());
375+
assert!(parse_number::<u32>("12.5").is_err());
376+
assert!(parse_number::<u32>("999999999999999999999999").is_err());
377+
}
360378
}

‎linkerd/meshtls/tests/util.rs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ use linkerd_meshtls::{self as meshtls, watch};
1111
use linkerd_proxy_transport::{
1212
addrs::*,
1313
listen::{Addrs, Bind, BindTcp},
14-
ConnectTcp, Keepalive, UserTimeout,
14+
ConnectTcp, Keepalive, UserTimeout, Backlog,
1515
};
1616
use linkerd_stack::{
1717
layer::Layer, service_fn, ExtractParam, InsertParam, NewService, Param, ServiceExt,
@@ -408,6 +408,11 @@ impl Param<UserTimeout> for Server {
408408
UserTimeout(None)
409409
}
410410
}
411+
impl Param<Backlog> for Server {
412+
fn param(&self) -> Backlog {
413+
Backlog::default()
414+
}
415+
}
411416

412417
// === impl ServerParams ===
413418

‎linkerd/proxy/transport/src/lib.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,16 @@ impl From<UserTimeout> for Option<Duration> {
4545
}
4646
}
4747

48+
#[derive(Copy, Clone, Debug)]
49+
pub struct Backlog(pub u32);
50+
51+
impl Default for Backlog {
52+
fn default() -> Self {
53+
// Use Rust's default backlog value (128 on most systems)
54+
Backlog(128)
55+
}
56+
}
57+
4858
// Misc.
4959

5060
fn set_nodelay_or_warn(socket: &TcpStream) {

‎linkerd/proxy/transport/src/listen.rs‎

Lines changed: 92 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
mod dual_bind;
22

3-
use crate::{addrs::*, Keepalive, UserTimeout};
3+
use crate::{addrs::*, Keepalive, UserTimeout, Backlog};
44
use dual_bind::DualBind;
55
use futures::prelude::*;
66
use linkerd_error::Result;
@@ -71,7 +71,7 @@ impl BindTcp {
7171

7272
impl<T> Bind<T> for BindTcp
7373
where
74-
T: Param<ListenAddr> + Param<Keepalive> + Param<UserTimeout>,
74+
T: Param<ListenAddr> + Param<Keepalive> + Param<UserTimeout> + Param<Backlog>,
7575
{
7676
type Addrs = Addrs;
7777
type BoundAddrs = Local<ServerAddr>;
@@ -81,10 +81,25 @@ where
8181
fn bind(self, params: &T) -> Result<(Self::BoundAddrs, Self::Incoming)> {
8282
let listen = {
8383
let ListenAddr(addr) = params.param();
84-
let l = std::net::TcpListener::bind(addr)?;
85-
// Ensure that O_NONBLOCK is set on the socket before using it with Tokio.
86-
l.set_nonblocking(true)?;
87-
tokio::net::TcpListener::from_std(l).expect("listener must be valid")
84+
let Backlog(backlog) = params.param();
85+
86+
// Use TcpSocket to create and configure the listener socket before binding.
87+
// This allows us to set socket options and configure the listen backlog.
88+
// TcpSocket::new_v4/v6 automatically sets O_NONBLOCK, which is required
89+
// for Tokio's async I/O operations.
90+
let socket = if addr.is_ipv4() {
91+
tokio::net::TcpSocket::new_v4()?
92+
} else {
93+
tokio::net::TcpSocket::new_v6()?
94+
};
95+
96+
// Enable SO_REUSEADDR to match std::net::TcpListener::bind behavior.
97+
// This allows the server to bind to an address in TIME_WAIT state,
98+
// enabling faster restarts without "address already in use" errors.
99+
socket.set_reuseaddr(true)?;
100+
101+
socket.bind(addr)?;
102+
socket.listen(backlog)?
88103
};
89104
let server = Local(ServerAddr(listen.local_addr()?));
90105
let Keepalive(keepalive) = params.param();
@@ -138,3 +153,74 @@ impl Param<AddrPair> for Addrs {
138153
AddrPair(client, server)
139154
}
140155
}
156+
157+
#[cfg(test)]
158+
mod tests {
159+
use super::*;
160+
161+
#[derive(Clone)]
162+
struct TestParams {
163+
addr: ListenAddr,
164+
keepalive: Keepalive,
165+
user_timeout: UserTimeout,
166+
backlog: crate::Backlog,
167+
}
168+
169+
impl Param<ListenAddr> for TestParams {
170+
fn param(&self) -> ListenAddr {
171+
self.addr
172+
}
173+
}
174+
175+
impl Param<Keepalive> for TestParams {
176+
fn param(&self) -> Keepalive {
177+
self.keepalive
178+
}
179+
}
180+
181+
impl Param<UserTimeout> for TestParams {
182+
fn param(&self) -> UserTimeout {
183+
self.user_timeout
184+
}
185+
}
186+
187+
impl Param<crate::Backlog> for TestParams {
188+
fn param(&self) -> crate::Backlog {
189+
self.backlog
190+
}
191+
}
192+
193+
#[tokio::test]
194+
#[cfg(target_os = "linux")]
195+
async fn listener_is_nonblocking() {
196+
use std::os::unix::io::AsRawFd;
197+
198+
let params = TestParams {
199+
addr: ListenAddr("127.0.0.1:0".parse().unwrap()),
200+
keepalive: Keepalive(None),
201+
user_timeout: UserTimeout(None),
202+
backlog: crate::Backlog(1024),
203+
};
204+
205+
let bind = BindTcp::default();
206+
let (bound_addr, _incoming) = bind.bind(&params).expect("failed to bind");
207+
208+
// Verify the socket is non-blocking by checking the O_NONBLOCK flag.
209+
// Create a new listener to the bound address to check its flags.
210+
let listener = tokio::net::TcpListener::bind(bound_addr.0 .0)
211+
.await
212+
.expect("failed to bind test listener");
213+
214+
let fd = listener.as_raw_fd();
215+
// Safety: We're just reading the file descriptor flags with fcntl(F_GETFL),
216+
// which is safe as long as the fd is valid (guaranteed by the listener being alive).
217+
#[allow(unsafe_code)]
218+
let flags = unsafe { libc::fcntl(fd, libc::F_GETFL) };
219+
assert_ne!(flags, -1, "fcntl failed to get flags");
220+
assert_ne!(
221+
flags & libc::O_NONBLOCK,
222+
0,
223+
"O_NONBLOCK flag is not set on the listener socket"
224+
);
225+
}
226+
}

‎linkerd/proxy/transport/src/listen/dual_bind.rs‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
use crate::{addrs::DualListenAddr, listen::Bind, Keepalive, ListenAddr, UserTimeout};
1+
use crate::{addrs::DualListenAddr, listen::Bind, Keepalive, ListenAddr, UserTimeout, Backlog};
22
use futures::Stream;
33
use linkerd_error::Result;
44
use linkerd_stack::Param;
@@ -26,7 +26,7 @@ impl<B> From<B> for DualBind<B> {
2626

2727
impl<T, B> Bind<T> for DualBind<B>
2828
where
29-
T: Param<DualListenAddr> + Param<Keepalive> + Param<UserTimeout> + Clone,
29+
T: Param<DualListenAddr> + Param<Keepalive> + Param<UserTimeout> + Param<Backlog> + Clone,
3030
B: Bind<Listen<T>, Io = TcpStream> + Clone + 'static,
3131
{
3232
type Addrs = B::Addrs;
@@ -68,6 +68,12 @@ impl<T: Param<UserTimeout>> Param<UserTimeout> for Listen<T> {
6868
}
6969
}
7070

71+
impl<T: Param<Backlog>> Param<Backlog> for Listen<T> {
72+
fn param(&self) -> Backlog {
73+
self.parent.param()
74+
}
75+
}
76+
7177
impl<T> Param<ListenAddr> for Listen<T> {
7278
fn param(&self) -> ListenAddr {
7379
ListenAddr(self.addr)

0 commit comments

Comments
 (0)