From 0f1a1275b07e3c9b98d8c0ad3c34031e9eb4f968 Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Thu, 16 Oct 2025 15:22:56 +0000 Subject: [PATCH 1/2] Introduced ngx_reuseport() and ngx_noreuseport() helpers. --- src/core/ngx_connection.c | 57 +++------------------------------------ src/os/unix/ngx_socket.c | 34 +++++++++++++++++++++++ src/os/unix/ngx_socket.h | 19 +++++++++++++ 3 files changed, 57 insertions(+), 53 deletions(-) diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c index 7cae295eb..12f28e0e9 100644 --- a/src/core/ngx_connection.c +++ b/src/core/ngx_connection.c @@ -444,32 +444,12 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle) * SO_REUSEPORT on the old socket before opening new ones */ - int reuseport = 1; - -#ifdef SO_REUSEPORT_LB - - if (setsockopt(ls[i].fd, SOL_SOCKET, SO_REUSEPORT_LB, - (const void *) &reuseport, sizeof(int)) - == -1) - { + if (ngx_reuseport(ls[i].fd) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, - "setsockopt(SO_REUSEPORT_LB) %V failed, " - "ignored", + ngx_reuseport_n " %V failed, ignored", &ls[i].addr_text); } -#else - - if (setsockopt(ls[i].fd, SOL_SOCKET, SO_REUSEPORT, - (const void *) &reuseport, sizeof(int)) - == -1) - { - ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, - "setsockopt(SO_REUSEPORT) %V failed, ignored", - &ls[i].addr_text); - } -#endif - ls[i].add_reuseport = 0; } #endif @@ -518,37 +498,9 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle) #if (NGX_HAVE_REUSEPORT) if (ls[i].reuseport && !ngx_test_config) { - int reuseport; - - reuseport = 1; - -#ifdef SO_REUSEPORT_LB - - if (setsockopt(s, SOL_SOCKET, SO_REUSEPORT_LB, - (const void *) &reuseport, sizeof(int)) - == -1) - { - ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, - "setsockopt(SO_REUSEPORT_LB) %V failed", - &ls[i].addr_text); - - if (ngx_close_socket(s) == -1) { - ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, - ngx_close_socket_n " %V failed", - &ls[i].addr_text); - } - - return NGX_ERROR; - } - -#else - - if (setsockopt(s, SOL_SOCKET, SO_REUSEPORT, - (const void *) &reuseport, sizeof(int)) - == -1) - { + if (ngx_reuseport(s) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, - "setsockopt(SO_REUSEPORT) %V failed", + ngx_reuseport_n " %V failed", &ls[i].addr_text); if (ngx_close_socket(s) == -1) { @@ -559,7 +511,6 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle) return NGX_ERROR; } -#endif } #endif diff --git a/src/os/unix/ngx_socket.c b/src/os/unix/ngx_socket.c index 3978f655c..7cc242e25 100644 --- a/src/os/unix/ngx_socket.c +++ b/src/os/unix/ngx_socket.c @@ -114,3 +114,37 @@ ngx_tcp_push(ngx_socket_t s) } #endif + + +#if (NGX_HAVE_REUSEPORT) + +int +ngx_reuseport(ngx_socket_t s) +{ + int reuseport = 1; + +#ifdef SO_REUSEPORT_LB + return setsockopt(s, SOL_SOCKET, SO_REUSEPORT_LB, + (const void *) &reuseport, sizeof(int)); +#else + return setsockopt(s, SOL_SOCKET, SO_REUSEPORT, + (const void *) &reuseport, sizeof(int)); +#endif +} + + +int +ngx_noreuseport(ngx_socket_t s) +{ + int reuseport = 0; + +#ifdef SO_REUSEPORT_LB + return setsockopt(s, SOL_SOCKET, SO_REUSEPORT_LB, + (const void *) &reuseport, sizeof(int)); +#else + return setsockopt(s, SOL_SOCKET, SO_REUSEPORT, + (const void *) &reuseport, sizeof(int)); +#endif +} + +#endif diff --git a/src/os/unix/ngx_socket.h b/src/os/unix/ngx_socket.h index 4480adca2..1a1b125e9 100644 --- a/src/os/unix/ngx_socket.h +++ b/src/os/unix/ngx_socket.h @@ -63,6 +63,25 @@ int ngx_tcp_push(ngx_socket_t s); #endif +#if (NGX_HAVE_REUSEPORT) + +int ngx_reuseport(ngx_socket_t s); +int ngx_noreuseport(ngx_socket_t s); + +#ifdef SO_REUSEPORT_LB + +#define ngx_reuseport_n "setsockopt(SO_REUSEPORT_LB)" +#define ngx_noreuseport_n "setsockopt(!SO_REUSEPORT_LB)" + +#else + +#define ngx_reuseport_n "setsockopt(SO_REUSEPORT)" +#define ngx_noreuseport_n "setsockopt(!SO_REUSEPORT)" + +#endif +#endif + + #define ngx_shutdown_socket shutdown #define ngx_shutdown_socket_n "shutdown()" From aa46a5b7fbb3ba33e23f697a93561c4bb6bb527b Mon Sep 17 00:00:00 2001 From: Sergey Kandaurov Date: Wed, 15 Oct 2025 16:20:18 +0000 Subject: [PATCH 2/2] The "multipath" parameter of the "listen" directive. When configured, enables Multipath TCP support on a listen socket. Multipath TCP (MPTCP), standardized in RFC 8684, is enabled with a separate IPPROTO_MPTCP socket(2) protocol. As of now it works on Linux starting with Linux 5.6 and glibc 2.32. To avoid EADDRINUSE errors in bind() and listen() when transitioning between sockets with different protocols, SO_REUSEPORT is temporarily set on both sockets. See f7f1607bf for potential implications. Based on previous work by Maxime Dourov and Anthony Doeraene. --- contrib/vim/syntax/nginx.vim | 2 +- src/core/ngx_connection.c | 54 ++++++++++++++++++++++++++--- src/core/ngx_connection.h | 3 ++ src/core/ngx_cycle.c | 10 ++++-- src/http/ngx_http.c | 1 + src/http/ngx_http_core_module.c | 17 +++++++++ src/http/ngx_http_core_module.h | 1 + src/mail/ngx_mail.c | 1 + src/mail/ngx_mail.h | 1 + src/mail/ngx_mail_core_module.c | 11 ++++++ src/stream/ngx_stream.c | 1 + src/stream/ngx_stream.h | 1 + src/stream/ngx_stream_core_module.c | 17 +++++++++ 13 files changed, 113 insertions(+), 7 deletions(-) diff --git a/contrib/vim/syntax/nginx.vim b/contrib/vim/syntax/nginx.vim index 29eef7a23..ea7c58464 100644 --- a/contrib/vim/syntax/nginx.vim +++ b/contrib/vim/syntax/nginx.vim @@ -65,7 +65,7 @@ syn match ngxListenComment '#.*$' \ contained \ nextgroup=@ngxListenParams skipwhite skipempty syn keyword ngxListenOptions contained - \ default_server ssl quic proxy_protocol + \ default_server ssl quic proxy_protocol multipath \ setfib fastopen backlog rcvbuf sndbuf accept_filter deferred bind \ ipv6only reuseport so_keepalive \ nextgroup=@ngxListenParams skipwhite skipempty diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c index 12f28e0e9..c87901ccd 100644 --- a/src/core/ngx_connection.c +++ b/src/core/ngx_connection.c @@ -315,6 +315,27 @@ ngx_set_inherited_sockets(ngx_cycle_t *cycle) continue; } +#ifdef SO_PROTOCOL + + olen = sizeof(int); + + if (getsockopt(ls[i].fd, SOL_SOCKET, SO_PROTOCOL, + (void *) &ls[i].protocol, &olen) + == -1) + { + ngx_log_error(NGX_LOG_CRIT, cycle->log, ngx_socket_errno, + "getsockopt(SO_PROTOCOL) %V failed", + &ls[i].addr_text); + ls[i].ignore = 1; + continue; + } + + if (ls[i].protocol == IPPROTO_TCP) { + ls[i].protocol = 0; + } + +#endif + #if (NGX_HAVE_TCP_FASTOPEN) olen = sizeof(int); @@ -436,12 +457,15 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle) #if (NGX_HAVE_REUSEPORT) - if (ls[i].add_reuseport) { + if (ls[i].add_reuseport || ls[i].change_protocol) { /* * to allow transition from a socket without SO_REUSEPORT * to multiple sockets with SO_REUSEPORT, we have to set * SO_REUSEPORT on the old socket before opening new ones + * + * to allow socket protocol change (e.g. IPPROTO_MPTCP), + * SO_REUSEPORT is set temporarily on both sockets */ if (ngx_reuseport(ls[i].fd) == -1) { @@ -454,7 +478,7 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle) } #endif - if (ls[i].fd != (ngx_socket_t) -1) { + if (ls[i].fd != (ngx_socket_t) -1 && !ls[i].change_protocol) { continue; } @@ -467,7 +491,8 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle) continue; } - s = ngx_socket(ls[i].sockaddr->sa_family, ls[i].type, 0); + s = ngx_socket(ls[i].sockaddr->sa_family, ls[i].type, + ls[i].protocol); if (s == (ngx_socket_t) -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, @@ -497,7 +522,9 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle) #if (NGX_HAVE_REUSEPORT) - if (ls[i].reuseport && !ngx_test_config) { + if ((ls[i].reuseport || ls[i].change_protocol) + && !ngx_test_config) + { if (ngx_reuseport(s) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, ngx_reuseport_n " %V failed", @@ -637,6 +664,25 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle) continue; } +#if (NGX_HAVE_REUSEPORT) + + if (!ls[i].reuseport && ls[i].change_protocol && !ngx_test_config) { + if (ngx_noreuseport(s) == -1) { + ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, + ngx_noreuseport_n " %V failed", + &ls[i].addr_text); + + if (ngx_close_socket(s) == -1) { + ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, + ngx_close_socket_n " %V failed", + &ls[i].addr_text); + } + + return NGX_ERROR; + } + } +#endif + ls[i].listen = 1; ls[i].fd = s; diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h index 84dd80442..b7d236837 100644 --- a/src/core/ngx_connection.h +++ b/src/core/ngx_connection.h @@ -24,6 +24,7 @@ struct ngx_listening_s { ngx_str_t addr_text; int type; + int protocol; int backlog; int rcvbuf; @@ -75,6 +76,8 @@ struct ngx_listening_s { unsigned keepalive:2; unsigned quic:1; + unsigned change_protocol:1; + unsigned deferred_accept:1; unsigned delete_deferred:1; unsigned add_deferred:1; diff --git a/src/core/ngx_cycle.c b/src/core/ngx_cycle.c index a75bdf878..747fe8ea8 100644 --- a/src/core/ngx_cycle.c +++ b/src/core/ngx_cycle.c @@ -535,9 +535,15 @@ ngx_init_cycle(ngx_cycle_t *old_cycle) == NGX_OK) { nls[n].fd = ls[i].fd; - nls[n].inherited = ls[i].inherited; nls[n].previous = &ls[i]; - ls[i].remain = 1; + + if (ls[i].protocol != nls[n].protocol) { + nls[n].change_protocol = 1; + + } else { + nls[n].inherited = ls[i].inherited; + ls[i].remain = 1; + } if (ls[i].backlog != nls[n].backlog) { nls[n].listen = 1; diff --git a/src/http/ngx_http.c b/src/http/ngx_http.c index 7f2b4225a..a97cc35f1 100644 --- a/src/http/ngx_http.c +++ b/src/http/ngx_http.c @@ -1846,6 +1846,7 @@ ngx_http_add_listening(ngx_conf_t *cf, ngx_http_conf_addr_t *addr) #endif ls->type = addr->opt.type; + ls->protocol = addr->opt.protocol; ls->backlog = addr->opt.backlog; ls->rcvbuf = addr->opt.rcvbuf; ls->sndbuf = addr->opt.sndbuf; diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c index c75ddb849..6091a5713 100644 --- a/src/http/ngx_http_core_module.c +++ b/src/http/ngx_http_core_module.c @@ -4361,6 +4361,17 @@ ngx_http_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) continue; } + if (ngx_strcmp(value[n].data, "multipath") == 0) { +#ifdef IPPROTO_MPTCP + lsopt.protocol = IPPROTO_MPTCP; +#else + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "multipath is not supported " + "on this platform, ignored"); +#endif + continue; + } + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "invalid parameter \"%V\"", &value[n]); return NGX_CONF_ERROR; @@ -4408,6 +4419,12 @@ ngx_http_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) if (lsopt.proxy_protocol) { return "\"proxy_protocol\" parameter is incompatible with \"quic\""; } + +#ifdef IPPROTO_MPTCP + if (lsopt.protocol == IPPROTO_MPTCP) { + return "\"multipath\" parameter is incompatible with \"quic\""; + } +#endif } for (n = 0; n < u.naddrs; n++) { diff --git a/src/http/ngx_http_core_module.h b/src/http/ngx_http_core_module.h index a794144aa..36dbfff9a 100644 --- a/src/http/ngx_http_core_module.h +++ b/src/http/ngx_http_core_module.h @@ -88,6 +88,7 @@ typedef struct { int rcvbuf; int sndbuf; int type; + int protocol; #if (NGX_HAVE_SETFIB) int setfib; #endif diff --git a/src/mail/ngx_mail.c b/src/mail/ngx_mail.c index 890d8153a..cc34cb365 100644 --- a/src/mail/ngx_mail.c +++ b/src/mail/ngx_mail.c @@ -332,6 +332,7 @@ ngx_mail_optimize_servers(ngx_conf_t *cf, ngx_array_t *ports) ls->log.data = &ls->addr_text; ls->log.handler = ngx_accept_log_error; + ls->protocol = addr[i].opt.protocol; ls->backlog = addr[i].opt.backlog; ls->rcvbuf = addr[i].opt.rcvbuf; ls->sndbuf = addr[i].opt.sndbuf; diff --git a/src/mail/ngx_mail.h b/src/mail/ngx_mail.h index e0c62b7ab..221faf67d 100644 --- a/src/mail/ngx_mail.h +++ b/src/mail/ngx_mail.h @@ -47,6 +47,7 @@ typedef struct { int tcp_keepintvl; int tcp_keepcnt; #endif + int protocol; int backlog; int rcvbuf; int sndbuf; diff --git a/src/mail/ngx_mail_core_module.c b/src/mail/ngx_mail_core_module.c index 228a8d0a8..0c31058b6 100644 --- a/src/mail/ngx_mail_core_module.c +++ b/src/mail/ngx_mail_core_module.c @@ -563,6 +563,17 @@ ngx_mail_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) continue; } + if (ngx_strcmp(value[i].data, "multipath") == 0) { +#ifdef IPPROTO_MPTCP + ls->protocol = IPPROTO_MPTCP; +#else + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "multipath is not supported " + "on this platform, ignored"); +#endif + continue; + } + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "invalid parameter \"%V\"", &value[i]); return NGX_CONF_ERROR; diff --git a/src/stream/ngx_stream.c b/src/stream/ngx_stream.c index b6eeb23af..508ce6082 100644 --- a/src/stream/ngx_stream.c +++ b/src/stream/ngx_stream.c @@ -1010,6 +1010,7 @@ ngx_stream_add_listening(ngx_conf_t *cf, ngx_stream_conf_addr_t *addr) ls->log.handler = ngx_accept_log_error; ls->type = addr->opt.type; + ls->protocol = addr->opt.protocol; ls->backlog = addr->opt.backlog; ls->rcvbuf = addr->opt.rcvbuf; ls->sndbuf = addr->opt.sndbuf; diff --git a/src/stream/ngx_stream.h b/src/stream/ngx_stream.h index dc05dc5ba..9bc689e99 100644 --- a/src/stream/ngx_stream.h +++ b/src/stream/ngx_stream.h @@ -62,6 +62,7 @@ typedef struct { int rcvbuf; int sndbuf; int type; + int protocol; #if (NGX_HAVE_SETFIB) int setfib; #endif diff --git a/src/stream/ngx_stream_core_module.c b/src/stream/ngx_stream_core_module.c index 40951c291..077a9afb2 100644 --- a/src/stream/ngx_stream_core_module.c +++ b/src/stream/ngx_stream_core_module.c @@ -1209,6 +1209,17 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) continue; } + if (ngx_strcmp(value[i].data, "multipath") == 0) { +#ifdef IPPROTO_MPTCP + lsopt.protocol = IPPROTO_MPTCP; +#else + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "multipath is not supported " + "on this platform, ignored"); +#endif + continue; + } + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "invalid parameter \"%V\"", &value[i]); return NGX_CONF_ERROR; @@ -1250,6 +1261,12 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) if (lsopt.proxy_protocol) { return "\"proxy_protocol\" parameter is incompatible with \"udp\""; } + +#ifdef IPPROTO_MPTCP + if (lsopt.protocol == IPPROTO_MPTCP) { + return "\"multipath\" parameter is incompatible with \"udp\""; + } +#endif } for (n = 0; n < u.naddrs; n++) {