🔫 macos: set TCP_FASTOPEN_FORCE_ENABLE

This disables the absolutely brutal TFO backoff mechanism, so that TFO can actually work.

src/listener.rs

@@ -6,10 +6,13 @@ use std::{
     task::{Context, Poll},
 };
 
+use cfg_if::cfg_if;
 use futures::{future, ready};
 use tokio::net::{TcpListener as TokioTcpListener, TcpSocket};
 
 use crate::{stream::TfoStream, sys::set_tcp_fastopen};
+#[cfg(target_os = "macos")]
+use crate::sys::set_tcp_fastopen_force_enable;
 
 /// TCP listener with TFO enabled
 pub struct TfoListener {
@@ -36,11 +39,25 @@ impl TfoListener {
         #[cfg(not(windows))]
         socket.set_reuseaddr(true)?;
 
+        // On all other platforms, TCP_FASTOPEN can be set before bind(), between bind() and listen(), or after listen().
+        // We prefer setting it before bind() as this feels most like the natural order of socket initialization sequence.
+        //
+        // On macOS, setting TCP_FASTOPEN_FORCE_ENABLE requires the socket to be in the TCPS_CLOSED state.
+        // TCP_FASTOPEN, on the other hand, can only be set when the socket is in the TCPS_LISTEN state.
+        cfg_if! {
+            if #[cfg(not(target_os = "macos"))] {
+                set_tcp_fastopen(&socket)?;
+            } else {
+                set_tcp_fastopen_force_enable(&socket)?;
+            }
+        }
+
         socket.bind(addr)?;
 
         // mio's default backlog is 1024
         let inner = socket.listen(1024)?;
 
+        #[cfg(target_os = "macos")]
         set_tcp_fastopen(&inner)?;
 
         Ok(TfoListener { inner })

src/sys/unix/bsd/macos.rs

@@ -45,7 +45,7 @@ impl TcpStream {
     }
 
     pub async fn connect_with_socket(socket: TcpSocket, addr: SocketAddr) -> io::Result<TcpStream> {
-        // TFO in macos uses connectx
+        set_tcp_fastopen_force_enable(&socket)?;
 
         unsafe {
             let raddr = SockAddr::from(addr);
@@ -205,6 +205,30 @@ impl AsRawFd for TcpStream {
     }
 }
 
+/// Disables the absolutely brutal TFO backoff mechanism on macOS.
+pub fn set_tcp_fastopen_force_enable<S: AsRawFd>(socket: &S) -> io::Result<()> {
+    const TCP_FASTOPEN_FORCE_ENABLE: libc::c_int = 0x218;
+    let enable: libc::c_int = 1;
+
+    unsafe {
+        let ret = libc::setsockopt(
+            socket.as_raw_fd(),
+            libc::IPPROTO_TCP,
+            TCP_FASTOPEN_FORCE_ENABLE,
+            &enable as *const _ as *const libc::c_void,
+            mem::size_of_val(&enable) as libc::socklen_t,
+        );
+
+        if ret != 0 {
+            let err = io::Error::last_os_error();
+            error!("set TCP_FASTOPEN_FORCE_ENABLE error: {}", err);
+            return Err(err);
+        }
+    }
+
+    Ok(())
+}
+
 /// Enable `TCP_FASTOPEN`
 ///
 /// `TCP_FASTOPEN` was supported since