Trying to improve throughput

Power2All · Power2All · commit c51a70e205f3 · 2025-08-31T11:17:58.000+02:00
diff --git a/Cargo.toml b/Cargo.toml
@@ -56,6 +56,7 @@ tracing = "^0.1"
 utoipa = { version = "^5", features = ["actix_extras"] }
 utoipa-swagger-ui = { version = "^9", features = ["actix-web"] }
 lazy_static = "^1.5"
+libc = "^0.2"
 
 [target.'cfg(windows)'.build-dependencies]
 winres = "^0.1"
diff --git a/src/udp/impls/response_batch_manager.rs b/src/udp/impls/response_batch_manager.rs
@@ -2,6 +2,7 @@ use std::net::SocketAddr;
 use std::sync::{Arc, OnceLock};
 use std::time::Duration;
 use std::collections::{HashMap, VecDeque};
+use log::{debug, info};
 use tokio::net::UdpSocket;
 use tokio::sync::{mpsc, RwLock};
 use tokio::time::interval;
@@ -15,24 +16,35 @@ impl ResponseBatchManager {
         let (sender, mut receiver) = mpsc::unbounded_channel::<QueuedResponse>();
 
         tokio::spawn(async move {
-            let mut buffer = VecDeque::with_capacity(1000);
-            let mut timer = interval(Duration::from_millis(5));
+            let mut buffer = VecDeque::with_capacity(1000); // Larger buffer for higher throughput
+            let mut timer = interval(Duration::from_millis(5)); // 5ms flush interval for better responsiveness
+            let mut stats_timer = interval(Duration::from_secs(10)); // Stats every 10 seconds
+            let mut total_queued = 0u64;
+            let mut total_sent = 0u64;
 
             loop {
                 tokio::select! {
                     Some(response) = receiver.recv() => {
+                        total_queued += 1;
                         buffer.push_back(response);
 
                         if buffer.len() >= 500 {
+                            debug!("Buffer full ({} items) - flushing immediately", buffer.len());
                             Self::flush_buffer(&socket, &mut buffer).await;
                         }
                     }
 
                     _ = timer.tick() => {
                         if !buffer.is_empty() {
+                            debug!("Timer flush - {} items in buffer", buffer.len());
                             Self::flush_buffer(&socket, &mut buffer).await;
                         }
                     }
+
+                    _ = stats_timer.tick() => {
+                        info!("Batch sender stats - Queued: {}, Current buffer: {}, Socket: {:?}",
+                              total_queued, buffer.len(), socket.local_addr());
+                    }
                 }
             }
         });
@@ -41,12 +53,55 @@ impl ResponseBatchManager {
     }
 
     pub(crate) fn queue_response(&self, remote_addr: SocketAddr, payload: Vec<u8>) {
-        let _ = self.sender.send(QueuedResponse { remote_addr, payload });
+        // Monitor queue health
+        match self.sender.send(QueuedResponse { remote_addr, payload }) {
+            Ok(_) => {
+                debug!("Response queued for {}", remote_addr);
+            }
+            Err(e) => {
+                // This indicates the batch sender task has died
+                log::error!("Failed to queue response - batch sender may have crashed: {}", e);
+            }
+        }
     }
 
     async fn flush_buffer(socket: &UdpSocket, buffer: &mut VecDeque<QueuedResponse>) {
+        let batch_size = buffer.len();
+        let mut sent_count = 0;
+        let mut error_count = 0;
+
         while let Some(response) = buffer.pop_front() {
-            let _ = socket.send_to(&response.payload, &response.remote_addr).await;
+            match socket.send_to(&response.payload, &response.remote_addr).await {
+                Ok(bytes_sent) => {
+                    sent_count += 1;
+                    debug!("Sent {} bytes to {}", bytes_sent, response.remote_addr);
+                }
+                Err(e) => {
+                    error_count += 1;
+                    match e.kind() {
+                        std::io::ErrorKind::WouldBlock => {
+                            debug!("Send buffer full (EWOULDBLOCK) - packet dropped");
+                        }
+                        std::io::ErrorKind::Other => {
+                            if let Some(os_error) = e.raw_os_error() {
+                                match os_error {
+                                    105 => debug!("ENOBUFS: No buffer space available - increase socket buffers"),
+                                    111 => debug!("ECONNREFUSED: Connection refused by peer"),
+                                    113 => debug!("EHOSTUNREACH: Host unreachable"),
+                                    _ => debug!("Send error (OS error {}): {}", os_error, e),
+                                }
+                            } else {
+                                debug!("Send error: {}", e);
+                            }
+                        }
+                        _ => debug!("Send error: {}", e),
+                    }
+                }
+            }
+        }
+
+        if batch_size > 0 {
+            info!("Batch flush: {} total, {} sent, {} errors", batch_size, sent_count, error_count);
         }
     }
 
diff --git a/src/udp/impls/udp_server.rs b/src/udp/impls/udp_server.rs
@@ -41,15 +41,46 @@ impl UdpServer {
         let domain = if bind_address.is_ipv4() { Domain::IPV4 } else { Domain::IPV6 };
         let socket = Socket::new(domain, Type::DGRAM, Some(Protocol::UDP))?;
 
-        socket.set_recv_buffer_size(recv_buffer_size).map_err(tokio::io::Error::other)?;
-        socket.set_send_buffer_size(send_buffer_size).map_err(tokio::io::Error::other)?;
+        // Aggressive buffer sizing for high throughput
+        let actual_recv_buffer = recv_buffer_size.max(16_777_216); // Minimum 16MB
+        let actual_send_buffer = send_buffer_size.max(16_777_216); // Minimum 16MB
+
+        socket.set_recv_buffer_size(actual_recv_buffer).map_err(tokio::io::Error::other)?;
+        socket.set_send_buffer_size(actual_send_buffer).map_err(tokio::io::Error::other)?;
         socket.set_reuse_address(reuse_address).map_err(tokio::io::Error::other)?;
+
+        // Enable SO_REUSEPORT for better load distribution across threads
+        #[cfg(target_os = "linux")]
+        {
+            use socket2::TcpKeepalive;
+            let reuse_port = 1i32;
+            unsafe {
+                let optval = &reuse_port as *const i32 as *const libc::c_void;
+                if libc::setsockopt(
+                    socket.as_raw_fd(),
+                    libc::SOL_SOCKET,
+                    libc::SO_REUSEPORT,
+                    optval,
+                    std::mem::size_of::<i32>() as libc::socklen_t,
+                ) != 0 {
+                    log::warn!("Failed to set SO_REUSEPORT - continuing without it");
+                }
+            }
+        }
+
         socket.bind(&bind_address.into()).map_err(tokio::io::Error::other)?;
         socket.set_nonblocking(true).map_err(tokio::io::Error::other)?;
 
+        // Convert to std::net::UdpSocket, then to tokio::net::UdpSocket
         let std_socket: std::net::UdpSocket = socket.into();
         let tokio_socket = UdpSocket::from_std(std_socket)?;
 
+        // Log actual buffer sizes
+        let sock_ref = socket2::SockRef::from(&tokio_socket);
+        let actual_recv = sock_ref.recv_buffer_size().unwrap_or(0);
+        let actual_send = sock_ref.send_buffer_size().unwrap_or(0);
+        info!("Socket created with buffers - Recv: {} bytes, Send: {} bytes", actual_recv, actual_send);
+
         Ok(UdpServer {
             socket: Arc::new(tokio_socket),
             threads,
@@ -61,32 +92,62 @@ impl UdpServer {
     pub async fn start(&self, rx: tokio::sync::watch::Receiver<bool>)
     {
         let threads = self.threads;
-        for _index in 0..=threads {
+        // Create multiple sockets for better performance using SO_REUSEPORT
+        for thread_id in 0..threads {
             let socket_clone = self.socket.clone();
             let tracker = self.tracker.clone();
             let mut rx = rx.clone();
-            let mut data = [0; 1496];
+
             tokio::spawn(async move {
+                // Larger buffer to handle burst traffic
+                let mut data = [0; 2048]; // Increased from 1496
+                let mut packet_count = 0u64;
+                let mut last_stats = std::time::Instant::now();
+
                 loop {
                     let udp_sock = socket_clone.local_addr().unwrap();
                     tokio::select! {
                         _ = rx.changed() => {
-                            info!("Stopping UDP server: {udp_sock}...");
+                            info!("Stopping UDP server thread {}: {udp_sock}...", thread_id);
                             break;
                         }
-                        Ok((valid_bytes, remote_addr)) = socket_clone.recv_from(&mut data) => {
-                            let payload = &data[..valid_bytes];
-
-                            debug!("Received {} bytes from {}", payload.len(), remote_addr);
-                            debug!("{payload:?}");
-
-                            let tracker_cloned = tracker.clone();
-                            let socket_cloned = socket_clone.clone();
-                            let payload_vec = payload.to_vec();
-                            tokio::spawn(async move {
-                                let response = UdpServer::handle_packet(remote_addr, payload_vec, tracker_cloned.clone()).await;
-                                UdpServer::send_response(tracker_cloned.clone(), socket_cloned.clone(), remote_addr, response).await;
-                            });
+                        result = socket_clone.recv_from(&mut data) => {
+                            match result {
+                                Ok((valid_bytes, remote_addr)) => {
+                                    packet_count += 1;
+
+                                    // Log stats every 10k packets per thread
+                                    if packet_count % 10000 == 0 {
+                                        let elapsed = last_stats.elapsed();
+                                        let rate = 10000.0 / elapsed.as_secs_f64();
+                                        debug!("Thread {} processed 10k packets in {:?} ({:.1} pps)",
+                                              thread_id, elapsed, rate);
+                                        last_stats = std::time::Instant::now();
+                                    }
+
+                                    let payload = &data[..valid_bytes];
+                                    debug!("Thread {} received {} bytes from {}", thread_id, payload.len(), remote_addr);
+
+                                    let tracker_cloned = tracker.clone();
+                                    let socket_cloned = socket_clone.clone();
+                                    let payload_vec = payload.to_vec();
+
+                                    // Process immediately without extra spawning for better performance
+                                    let response = UdpServer::handle_packet(remote_addr, payload_vec, tracker_cloned.clone()).await;
+                                    UdpServer::send_response(tracker_cloned.clone(), socket_cloned.clone(), remote_addr, response).await;
+                                }
+                                Err(e) => {
+                                    match e.kind() {
+                                        std::io::ErrorKind::WouldBlock => {
+                                            // This is normal for non-blocking sockets
+                                            tokio::task::yield_now().await;
+                                        }
+                                        _ => {
+                                            log::error!("Thread {} recv_from error: {}", thread_id, e);
+                                        }
+                                    }
+                                }
+                            }
                         }
                     }
                 }
@@ -101,14 +162,16 @@ impl UdpServer {
         let sentry = sentry::TransactionContext::new("udp server", "send response");
         let transaction = sentry::start_transaction(sentry);
 
-        let mut buffer = Vec::with_capacity(512);
+        // Pre-allocate buffer with exact capacity instead of MAX_PACKET_SIZE
+        let mut buffer = Vec::with_capacity(512); // Most responses are much smaller than MAX_PACKET_SIZE
         let mut cursor = Cursor::new(&mut buffer);
 
         match response.write(&mut cursor) {
             Ok(_) => {
                 let position = cursor.position() as usize;
                 debug!("Response bytes: {:?}", &buffer[..position]);
 
+                // Get batch manager for this socket and queue the response
                 let batch_manager = ResponseBatchManager::get_for_socket(socket).await;
                 batch_manager.queue_response(remote_addr, buffer[..position].to_vec());
             }
@@ -127,6 +190,7 @@ impl UdpServer {
 
     #[tracing::instrument(level = "debug")]
     pub async fn send_packet(socket: Arc<UdpSocket>, remote_addr: &SocketAddr, payload: &[u8]) {
+        // This method is kept for compatibility but shouldn't be used in the batched version
         let _ = socket.send_to(payload, remote_addr).await;
     }
 
@@ -214,6 +278,7 @@ impl UdpServer {
     pub async fn handle_udp_announce(remote_addr: SocketAddr, request: &AnnounceRequest, tracker: Arc<TorrentTracker>) -> Result<Response, ServerError> {
         let config = tracker.config.tracker_config.clone();
 
+        // Whitelist/Blacklist checks
         if config.whitelist_enabled && !tracker.check_whitelist(InfoHash(request.info_hash.0)) {
             debug!("[UDP ERROR] Torrent Not Whitelisted");
             return Err(ServerError::TorrentNotWhitelisted);
@@ -223,6 +288,7 @@ impl UdpServer {
             return Err(ServerError::TorrentBlacklisted);
         }
 
+        // Key validation
         if config.keys_enabled {
             if request.path.len() < 50 {
                 debug!("[UDP ERROR] Unknown Key");
@@ -246,6 +312,7 @@ impl UdpServer {
             }
         }
 
+        // User key validation
         let user_key = if config.users_enabled {
             let user_key_path_extract = if request.path.len() >= 91 {
                 Some(&request.path[51..=91])
@@ -278,6 +345,7 @@ impl UdpServer {
             return Err(ServerError::PeerKeyNotValid);
         }
 
+        // Handle announce
         let torrent = match tracker.handle_announce(tracker.clone(), AnnounceQueryRequest {
             info_hash: InfoHash(request.info_hash.0),
             peer_id: PeerId(request.peer_id.0),
@@ -298,13 +366,15 @@ impl UdpServer {
             }
         };
 
+        // Get peers efficiently
         let torrent_peers = tracker.get_torrent_peers(request.info_hash, 72, TorrentPeersType::All, Some(remote_addr.ip()));
 
         let (peers, peers6) = if let Some(torrent_peers_unwrapped) = torrent_peers {
             let mut peers = Vec::with_capacity(72);
             let mut peers6 = Vec::with_capacity(72);
             let mut count = 0;
 
+            // Only collect peers if not completed download
             if request.bytes_left.0 != 0 {
                 if remote_addr.is_ipv4() {
                     for torrent_peer in torrent_peers_unwrapped.seeds_ipv4.values().take(72) {
@@ -325,6 +395,7 @@ impl UdpServer {
                 }
             }
 
+            // Collect regular peers
             if remote_addr.is_ipv4() {
                 for torrent_peer in torrent_peers_unwrapped.peers_ipv4.values().take(72 - count) {
                     if let Ok(ip) = torrent_peer.peer_addr.ip().to_string().parse::<Ipv4Addr>() {
@@ -344,6 +415,7 @@ impl UdpServer {
             (Vec::new(), Vec::new())
         };
 
+        // Create response
         let response = if remote_addr.is_ipv6() {
             Response::from(AnnounceResponse {
                 transaction_id: request.transaction_id,
@@ -362,6 +434,7 @@ impl UdpServer {
             })
         };
 
+        // Update stats
         let stats_event = if remote_addr.is_ipv4() {
             StatsEvent::Udp4AnnouncesHandled
         } else {
