feat: Implement authentication rate limiting (fail2ban-like) (#155)

* feat: Implement authentication rate limiting (fail2ban-like)

Add AuthRateLimiter with ban support to protect against brute-force attacks:

- Track failed authentication attempts per IP address
- Automatically ban IPs that exceed max attempts within time window
- Configurable max attempts, time window, and ban duration
- IP whitelist support for trusted addresses
- Automatic cleanup of expired bans and failure records
- Background cleanup task running every 60 seconds

Configuration options added to SecurityConfig:
- auth_window: Time window for counting attempts (default: 300s)
- whitelist_ips: IPs exempt from rate limiting

Integration with SSH handler:
- Check if IP is banned before authentication
- Record failures and trigger bans on threshold
- Record success to reset failure counter
- Logging for ban events

Closes #140

* fix: Address PR review feedback for auth rate limiting

- Use configuration values instead of hardcoded values for auth_window and ban_time
- Integrate whitelist_ips from configuration with validation and logging
- Fix TOCTOU race condition in record_failure by removing entry atomically
- Add capacity limit (max_tracked_ips) to prevent memory exhaustion DoS
- Use HashSet for whitelist O(1) lookups instead of Vec O(n)
- Add auth rate limit config fields to ServerConfig
- Propagate security config from ServerFileConfig to ServerConfig
- Add test for capacity limit enforcement

* chore: finalize auth rate limiter with docs and formatting fixes

- Fix code formatting (cargo fmt)
- Update ARCHITECTURE.md with Server Security Module documentation
- Update server-configuration.md with auth_window and whitelist_ips options
- All 930 tests passing, clippy clean

Author: inureyes

ARCHITECTURE.md

@@ -189,6 +189,20 @@ Common utilities for code reuse between bssh client and server implementations:
 
 The `security` and `jump::rate_limiter` modules re-export from shared for backward compatibility.
 
+### Server Security Module
+
+Security features for the SSH server (`src/server/security/`):
+
+- **AuthRateLimiter**: Fail2ban-like authentication rate limiting
+  - Tracks failed authentication attempts per IP address
+  - Automatic banning after exceeding configurable threshold
+  - Time-windowed failure counting (failures outside window not counted)
+  - Configurable ban duration with automatic expiration
+  - IP whitelist for exempting trusted addresses from banning
+  - Memory-safe with configurable maximum tracked IPs
+  - Automatic cleanup of expired records via background task
+  - Thread-safe async implementation with `Arc<RwLock<>>`
+
 ### Server CLI Binary
 **Binary**: `bssh-server`
 
@@ -284,7 +298,8 @@ SSH server implementation using the russh library for accepting incoming connect
 
 - **SshHandler**: Per-connection handler for SSH protocol events
   - Public key authentication via AuthProvider trait
-  - Rate limiting for authentication attempts
+  - Rate limiting for authentication attempts (token bucket)
+  - Auth rate limiting with ban support (fail2ban-like)
   - Channel operations (open, close, EOF, data)
   - PTY, exec, shell, and subsystem request handling
   - Command execution with stdout/stderr streaming

docs/architecture/server-configuration.md

@@ -173,9 +173,19 @@ security:
   # Max auth attempts before banning IP
   max_auth_attempts: 5         # Default: 5
 
+  # Time window for counting auth attempts (seconds)
+  # Failed attempts outside this window are not counted
+  auth_window: 300             # Default: 300 (5 minutes)
+
   # Ban duration after exceeding max attempts (seconds)
   ban_time: 300                # Default: 300 (5 minutes)
 
+  # IPs that are never banned (whitelist)
+  # These IPs are exempt from rate limiting and banning
+  whitelist_ips:
+    - "127.0.0.1"
+    - "::1"
+
   # Max concurrent sessions per user
   max_sessions_per_user: 10    # Default: 10
 

src/server/config/mod.rs

@@ -149,6 +149,22 @@ pub struct ServerConfig {
     /// Configuration for command execution.
     #[serde(default)]
     pub exec: ExecConfig,
+
+    /// Time window for counting authentication attempts in seconds.
+    ///
+    /// Default: 300 (5 minutes)
+    #[serde(default = "default_auth_window_secs")]
+    pub auth_window_secs: u64,
+
+    /// Ban duration in seconds after exceeding max auth attempts.
+    ///
+    /// Default: 300 (5 minutes)
+    #[serde(default = "default_ban_time_secs")]
+    pub ban_time_secs: u64,
+
+    /// IP addresses that are never banned (whitelist).
+    #[serde(default)]
+    pub whitelist_ips: Vec<String>,
 }
 
 /// Serializable configuration for public key authentication.
@@ -213,6 +229,14 @@ fn default_idle_timeout_secs() -> u64 {
     0 // 0 means no timeout
 }
 
+fn default_auth_window_secs() -> u64 {
+    300 // 5 minutes
+}
+
+fn default_ban_time_secs() -> u64 {
+    300 // 5 minutes
+}
+
 fn default_true() -> bool {
     true
 }
@@ -233,6 +257,9 @@ impl Default for ServerConfig {
             publickey_auth: PublicKeyAuthConfigSerde::default(),
             password_auth: PasswordAuthConfigSerde::default(),
             exec: ExecConfig::default(),
+            auth_window_secs: default_auth_window_secs(),
+            ban_time_secs: default_ban_time_secs(),
+            whitelist_ips: Vec::new(),
         }
     }
 }
@@ -521,6 +548,9 @@ impl ServerFileConfig {
                 allowed_commands: None,
                 blocked_commands: Vec::new(),
             },
+            auth_window_secs: self.security.auth_window,
+            ban_time_secs: self.security.ban_time,
+            whitelist_ips: self.security.whitelist_ips,
         }
     }
 }

src/server/config/types.rs

@@ -368,12 +368,28 @@ pub struct SecurityConfig {
     #[serde(default = "default_max_auth_attempts")]
     pub max_auth_attempts: u32,
 
+    /// Time window in seconds for counting authentication attempts.
+    ///
+    /// Failed attempts outside this window are not counted toward the ban threshold.
+    ///
+    /// Default: 300 (5 minutes)
+    #[serde(default = "default_auth_window")]
+    pub auth_window: u64,
+
     /// Ban duration in seconds after exceeding max auth attempts.
     ///
     /// Default: 300 (5 minutes)
     #[serde(default = "default_ban_time")]
     pub ban_time: u64,
 
+    /// IP addresses that are never banned (whitelist).
+    ///
+    /// These IPs are exempt from rate limiting and banning.
+    ///
+    /// Example: ["127.0.0.1", "::1"]
+    #[serde(default)]
+    pub whitelist_ips: Vec<String>,
+
     /// Maximum number of concurrent sessions per user.
     ///
     /// Default: 10
@@ -449,6 +465,10 @@ fn default_max_auth_attempts() -> u32 {
     5
 }
 
+fn default_auth_window() -> u64 {
+    300
+}
+
 fn default_ban_time() -> u64 {
     300
 }
@@ -517,7 +537,9 @@ impl Default for SecurityConfig {
     fn default() -> Self {
         Self {
             max_auth_attempts: default_max_auth_attempts(),
+            auth_window: default_auth_window(),
             ban_time: default_ban_time(),
+            whitelist_ips: Vec::new(),
             max_sessions_per_user: default_max_sessions(),
             idle_timeout: default_idle_timeout(),
             allowed_ips: Vec::new(),

src/server/handler.rs

@@ -32,6 +32,7 @@ use super::auth::AuthProvider;
 use super::config::ServerConfig;
 use super::exec::CommandExecutor;
 use super::pty::PtyConfig as PtyMasterConfig;
+use super::security::AuthRateLimiter;
 use super::session::{ChannelState, PtyConfig, SessionId, SessionInfo, SessionManager};
 use super::sftp::SftpHandler;
 use super::shell::ShellSession;
@@ -57,6 +58,9 @@ pub struct SshHandler {
     /// Rate limiter for authentication attempts.
     rate_limiter: RateLimiter<String>,
 
+    /// Auth rate limiter with ban support (fail2ban-like).
+    auth_rate_limiter: Option<AuthRateLimiter>,
+
     /// Session information for this connection.
     session_info: Option<SessionInfo>,
 
@@ -83,6 +87,7 @@ impl SshHandler {
             sessions,
             auth_provider,
             rate_limiter,
+            auth_rate_limiter: None,
             session_info: Some(SessionInfo::new(peer_addr)),
             channels: HashMap::new(),
         }
@@ -106,6 +111,33 @@ impl SshHandler {
             sessions,
             auth_provider,
             rate_limiter,
+            auth_rate_limiter: None,
+            session_info: Some(SessionInfo::new(peer_addr)),
+            channels: HashMap::new(),
+        }
+    }
+
+    /// Create a new SSH handler with shared rate limiters including auth ban support.
+    ///
+    /// This is the preferred constructor for production use as it shares
+    /// both rate limiters across all handlers, providing server-wide rate limiting
+    /// and fail2ban-like functionality.
+    pub fn with_rate_limiters(
+        peer_addr: Option<SocketAddr>,
+        config: Arc<ServerConfig>,
+        sessions: Arc<RwLock<SessionManager>>,
+        rate_limiter: RateLimiter<String>,
+        auth_rate_limiter: AuthRateLimiter,
+    ) -> Self {
+        let auth_provider = config.create_auth_provider();
+
+        Self {
+            peer_addr,
+            config,
+            sessions,
+            auth_provider,
+            rate_limiter,
+            auth_rate_limiter: Some(auth_rate_limiter),
             session_info: Some(SessionInfo::new(peer_addr)),
             channels: HashMap::new(),
         }
@@ -128,6 +160,7 @@ impl SshHandler {
             sessions,
             auth_provider,
             rate_limiter,
+            auth_rate_limiter: None,
             session_info: Some(SessionInfo::new(peer_addr)),
             channels: HashMap::new(),
         }
@@ -284,6 +317,7 @@ impl russh::server::Handler for SshHandler {
         // Clone what we need for the async block
         let auth_provider = Arc::clone(&self.auth_provider);
         let rate_limiter = self.rate_limiter.clone();
+        let auth_rate_limiter = self.auth_rate_limiter.clone();
         let peer_addr = self.peer_addr;
         let user = user.to_string();
         let public_key = public_key.clone();
@@ -292,6 +326,23 @@ impl russh::server::Handler for SshHandler {
         let session_info = &mut self.session_info;
 
         async move {
+            // Check if IP is banned (fail2ban-like check)
+            if let Some(ref limiter) = auth_rate_limiter {
+                if let Some(ip) = peer_addr.map(|a| a.ip()) {
+                    if limiter.is_banned(&ip).await {
+                        tracing::warn!(
+                            user = %user,
+                            peer = ?peer_addr,
+                            "Rejected auth from banned IP"
+                        );
+                        return Ok(Auth::Reject {
+                            proceed_with_methods: None,
+                            partial_success: false,
+                        });
+                    }
+                }
+            }
+
             if exceeded {
                 tracing::warn!(
                     user = %user,
@@ -349,6 +400,13 @@ impl russh::server::Handler for SshHandler {
                         info.authenticate(&user);
                     }
 
+                    // Record success to reset failure counter
+                    if let Some(ref limiter) = auth_rate_limiter {
+                        if let Some(ip) = peer_addr.map(|a| a.ip()) {
+                            limiter.record_success(&ip).await;
+                        }
+                    }
+
                     Ok(Auth::Accept)
                 }
                 Ok(_) => {
@@ -359,6 +417,20 @@ impl russh::server::Handler for SshHandler {
                         "Public key authentication rejected"
                     );
 
+                    // Record failure for ban tracking
+                    if let Some(ref limiter) = auth_rate_limiter {
+                        if let Some(ip) = peer_addr.map(|a| a.ip()) {
+                            let banned = limiter.record_failure(ip).await;
+                            if banned {
+                                tracing::warn!(
+                                    user = %user,
+                                    peer = ?peer_addr,
+                                    "IP banned due to too many failed auth attempts"
+                                );
+                            }
+                        }
+                    }
+
                     let proceed = if methods.is_empty() {
                         None
                     } else {
@@ -378,6 +450,13 @@ impl russh::server::Handler for SshHandler {
                         "Error during public key verification"
                     );
 
+                    // Record failure for ban tracking
+                    if let Some(ref limiter) = auth_rate_limiter {
+                        if let Some(ip) = peer_addr.map(|a| a.ip()) {
+                            limiter.record_failure(ip).await;
+                        }
+                    }
+
                     let proceed = if methods.is_empty() {
                         None
                     } else {
@@ -421,6 +500,7 @@ impl russh::server::Handler for SshHandler {
         // Clone what we need for the async block
         let auth_provider = Arc::clone(&self.auth_provider);
         let rate_limiter = self.rate_limiter.clone();
+        let auth_rate_limiter = self.auth_rate_limiter.clone();
         let peer_addr = self.peer_addr;
         let user = user.to_string();
         // Use Zeroizing to ensure password is securely cleared from memory when dropped
@@ -431,6 +511,23 @@ impl russh::server::Handler for SshHandler {
         let session_info = &mut self.session_info;
 
         async move {
+            // Check if IP is banned (fail2ban-like check)
+            if let Some(ref limiter) = auth_rate_limiter {
+                if let Some(ip) = peer_addr.map(|a| a.ip()) {
+                    if limiter.is_banned(&ip).await {
+                        tracing::warn!(
+                            user = %user,
+                            peer = ?peer_addr,
+                            "Rejected password auth from banned IP"
+                        );
+                        return Ok(Auth::Reject {
+                            proceed_with_methods: None,
+                            partial_success: false,
+                        });
+                    }
+                }
+            }
+
             // Check if password auth is enabled
             if !allow_password {
                 tracing::debug!(
@@ -504,6 +601,13 @@ impl russh::server::Handler for SshHandler {
                         info.authenticate(&user);
                     }
 
+                    // Record success to reset failure counter
+                    if let Some(ref limiter) = auth_rate_limiter {
+                        if let Some(ip) = peer_addr.map(|a| a.ip()) {
+                            limiter.record_success(&ip).await;
+                        }
+                    }
+
                     Ok(Auth::Accept)
                 }
                 Ok(_) => {
@@ -513,6 +617,20 @@ impl russh::server::Handler for SshHandler {
                         "Password authentication rejected"
                     );
 
+                    // Record failure for ban tracking
+                    if let Some(ref limiter) = auth_rate_limiter {
+                        if let Some(ip) = peer_addr.map(|a| a.ip()) {
+                            let banned = limiter.record_failure(ip).await;
+                            if banned {
+                                tracing::warn!(
+                                    user = %user,
+                                    peer = ?peer_addr,
+                                    "IP banned due to too many failed password auth attempts"
+                                );
+                            }
+                        }
+                    }
+
                     let proceed = if methods.is_empty() {
                         None
                     } else {
@@ -532,6 +650,13 @@ impl russh::server::Handler for SshHandler {
                         "Error during password verification"
                     );
 
+                    // Record failure for ban tracking
+                    if let Some(ref limiter) = auth_rate_limiter {
+                        if let Some(ip) = peer_addr.map(|a| a.ip()) {
+                            limiter.record_failure(ip).await;
+                        }
+                    }
+
                     let proceed = if methods.is_empty() {
                         None
                     } else {