Enforce Active Connection limits (#6754)

* Enforce Active Connection limits

1. Throttle connections when there's no room in active conn queue
2. Adjust manage_active_queue() to not fail when the conn is already in active queue
3. Return true for PluginVC (dummy connection) add_to_active_queue
4. Metrics for throttling
5. Allow to disable active connection tracking
6. Doc updates

Author: sudheerv

doc/admin-guide/files/records.config.en.rst

@@ -409,6 +409,22 @@ Network
    handled. This should be tuned according to your memory size, and expected
    work load.  If this is set to 0, the throttling logic is disabled.
 
+.. ts:cv:: CONFIG proxy.config.net.max_connections_in INT 30000
+
+   The total number of client connections that the :program:`traffic_server`
+   can handle simultaneously. This should be tuned according to your memory size,
+   and expected work load (network, cpu etc). This limit includes both keepalive
+   and active client connections that :program:`traffic_server` can handle at
+   any given instant.
+
+.. ts:cv:: CONFIG proxy.config.net.max_active_connections_in INT 10000
+
+   The total number of active client connections that the |TS| can handle
+   simultaneously. This should be tuned according to your memory size,
+   and expected work load (network, cpu etc). If this is set to 0, active
+   connection tracking is disabled and active connections have no separate
+   limit and the total connections follow `proxy.config.net.connections_throttle`
+
 .. ts:cv:: CONFIG proxy.config.net.default_inactivity_timeout INT 86400
    :reloadable:
 

doc/admin-guide/monitoring/statistics/core/network-io.en.rst

@@ -60,8 +60,17 @@ Network I/O
 .. ts:stat:: global proxy.process.net.connections_currently_open integer
    :type: counter
 
+.. ts:stat:: global proxy.process.net.connections_throttled_in integer
+   :type: counter
+
+.. ts:stat:: global proxy.process.net.connections_throttled_out integer
+   :type: counter
+
+.. ts:stat:: global proxy.process.net.max.active.connections_throttled_in integer
+   :type: counter
+
 .. ts:stat:: global proxy.process.net.default_inactivity_timeout_applied integer
-.. ts:stat:: global proxy.process.net.dynamic_keep_alive_timeout_in_count integer
+.. ts:stat:: global proxy.process.net.default_inactivity_timeout_count integer
 .. ts:stat:: global proxy.process.net.dynamic_keep_alive_timeout_in_total integer
 .. ts:stat:: global proxy.process.net.inactivity_cop_lock_acquire_failure integer
 .. ts:stat:: global proxy.process.net.net_handler_run integer

iocore/net/Net.cc

@@ -140,6 +140,8 @@ register_net_stats()
                      (int)net_connections_throttled_in_stat, RecRawStatSyncSum);
   RecRegisterRawStat(net_rsb, RECT_PROCESS, "proxy.process.net.connections_throttled_out", RECD_INT, RECP_PERSISTENT,
                      (int)net_connections_throttled_out_stat, RecRawStatSyncSum);
+  RecRegisterRawStat(net_rsb, RECT_PROCESS, "proxy.process.net.max.active.connections_throttled_in", RECD_INT, RECP_PERSISTENT,
+                     (int)net_connections_max_active_throttled_in_stat, RecRawStatSyncSum);
 }
 
 void

iocore/net/P_Net.h

@@ -57,6 +57,7 @@ enum Net_Stats {
   net_tcp_accept_stat,
   net_connections_throttled_in_stat,
   net_connections_throttled_out_stat,
+  net_connections_max_active_throttled_in_stat,
   Net_Stat_Count
 };
 

iocore/net/UnixNet.cc

@@ -562,6 +562,11 @@ NetHandler::manage_active_queue(bool ignore_queue_size = false)
         max_connections_per_thread_in, max_connections_active_per_thread_in, total_connections_in, active_queue_size,
         keep_alive_queue_size);
 
+  if (!max_connections_active_per_thread_in) {
+    // active queue has no max
+    return true;
+  }
+
   if (ignore_queue_size == false && max_connections_active_per_thread_in > active_queue_size) {
     return true;
   }
@@ -721,16 +726,22 @@ NetHandler::add_to_active_queue(NetEvent *ne)
         max_connections_per_thread_in, active_queue_size, keep_alive_queue_size);
   ink_assert(mutex->thread_holding == this_ethread());
 
+  bool active_queue_full = false;
+
   // if active queue is over size then close inactive connections
   if (manage_active_queue() == false) {
-    // there is no room left in the queue
-    return false;
+    active_queue_full = true;
   }
 
   if (active_queue.in(ne)) {
     // already in the active queue, move the head
     active_queue.remove(ne);
   } else {
+    if (active_queue_full) {
+      // there is no room left in the queue
+      NET_SUM_DYN_STAT(net_connections_max_active_throttled_in_stat, 1);
+      return false;
+    }
     // in the keep-alive queue or no queue, new to this queue
     remove_from_keep_alive_queue(ne);
     ++active_queue_size;

proxy/PluginVC.cc

@@ -926,7 +926,7 @@ bool
 PluginVC::add_to_active_queue()
 {
   // do nothing
-  return false;
+  return true;
 }
 
 SOCKET

proxy/http/Http1ClientSession.cc

@@ -465,6 +465,12 @@ Http1ClientSession::new_transaction()
     return;
   }
 
+  if (!client_vc->add_to_active_queue()) {
+    // no room in the active queue close the connection
+    this->do_io_close();
+    return;
+  }
+
   // Defensive programming, make sure nothing persists across
   // connection re-use
   half_close = false;
@@ -474,7 +480,6 @@ Http1ClientSession::new_transaction()
   trans.set_proxy_ssn(this);
   transact_count++;
 
-  client_vc->add_to_active_queue();
   trans.new_transaction(read_from_early_data > 0 ? true : false);
 }
 

proxy/http2/Http2ConnectionState.cc

@@ -1153,6 +1153,13 @@ Http2ConnectionState::state_closed(int event, void *edata)
 Http2Stream *
 Http2ConnectionState::create_stream(Http2StreamId new_id, Http2Error &error)
 {
+  // first check if we've hit the active connection limit
+  if (!ua_session->get_netvc()->add_to_active_queue()) {
+    error = Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_NO_ERROR,
+                       "refused to create new stream, maxed out active connections");
+    return nullptr;
+  }
+
   // In half_close state, TS doesn't create new stream. Because GOAWAY frame is sent to client
   if (ua_session->get_half_close_local_flag()) {
     error = Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_REFUSED_STREAM,
@@ -1226,7 +1233,6 @@ Http2ConnectionState::create_stream(Http2StreamId new_id, Http2Error &error)
   new_stream->mutex                     = new_ProxyMutex();
   new_stream->is_first_transaction_flag = get_stream_requests() == 0;
   increment_stream_requests();
-  ua_session->get_netvc()->add_to_active_queue();
 
   return new_stream;
 }