Address further review comments

1. Remove proxy.config.io.max_buffer_size
2. Add docs for the new configs

Author: sudheerv

doc/admin-guide/files/records.config.en.rst

@@ -4407,6 +4407,31 @@ Sockets
    platforms.  (Currently only linux).  IO buffers are allocated with the MADV_DONTDUMP
    with madvise() on linux platforms that support MADV_DONTDUMP.  Enabled by default.
 
+.. ts:cv:: CONFIG proxy.config.ssl.misc.io.max_buffer_index INT 8
+
+   Configures the max IOBuffer Block index used for various SSL Operations
+   such as Handshake or Protocol Probe. Default value is 8 which maps to a 32K buffer
+
+.. ts:cv:: CONFIG proxy.config.hostdb.io.max_buffer_index INT 8
+
+   Configures the max IOBuffer Block index used for storing HostDB records.
+   Default value is 8 which maps to a 32K buffer
+
+.. ts:cv:: CONFIG proxy.config.payload.io.max_buffer_index INT 8
+
+   Configures the max IOBuffer Block index used for storing request payload buffer
+   for a POST request. Default value is 8 which maps to a 32K buffer
+
+.. ts:cv:: CONFIG proxy.config.msg.io.max_buffer_index INT 8
+
+   Configures the max IOBuffer Block index used for storing miscellaneous transactional
+   buffers such as error response body. Default value is 8 which maps to a 32K buffer
+
+.. ts:cv:: CONFIG proxy.config.log.io.max_buffer_index INT 8
+
+   Configures the max IOBuffer Block index used for storing an access log entry.
+   Default value is 8 which maps to a 32K buffer
+
 .. ts:cv:: CONFIG proxy.config.http.enabled INT 1
 
    Turn on or off support for HTTP proxying. This is rarely used, the one

iocore/cache/CachePages.cc

@@ -427,7 +427,7 @@ ShowCache::handleCacheEvent(int event, Event *e)
 
   case VC_EVENT_READ_READY:
     if (!cvio) {
-      buffer         = new_empty_MIOBuffer();
+      buffer         = new_empty_MIOBuffer(BUFFER_SIZE_INDEX_32K);
       buffer_reader  = buffer->alloc_reader();
       content_length = cache_vc->get_object_size();
       cvio           = cache_vc->do_io_read(this, content_length, buffer);

iocore/cache/CacheTest.cc

@@ -95,7 +95,7 @@ CacheTestSM::event_handler(int event, void *data)
     cancel_timeout();
     cache_action  = nullptr;
     cache_vc      = static_cast<CacheVConnection *>(data);
-    buffer        = new_empty_MIOBuffer();
+    buffer        = new_empty_MIOBuffer(BUFFER_SIZE_INDEX_32K);
     buffer_reader = buffer->alloc_reader();
     if (open_read_callout() < 0) {
       goto Lclose_error_next;
@@ -129,7 +129,7 @@ CacheTestSM::event_handler(int event, void *data)
     cancel_timeout();
     cache_action  = nullptr;
     cache_vc      = static_cast<CacheVConnection *>(data);
-    buffer        = new_empty_MIOBuffer();
+    buffer        = new_empty_MIOBuffer(BUFFER_SIZE_INDEX_32K);
     buffer_reader = buffer->alloc_reader();
     if (open_write_callout() < 0) {
       goto Lclose_error_next;

iocore/eventsystem/EventSystem.cc

@@ -34,25 +34,14 @@ void
 ink_event_system_init(ts::ModuleVersion v)
 {
   ink_release_assert(v.check(EVENT_SYSTEM_MODULE_INTERNAL_VERSION));
-  int config_max_iobuffer_size = DEFAULT_MAX_BUFFER_SIZE;
-  int iobuffer_advice          = 0;
+  int iobuffer_advice = 0;
 
   // For backwards compatibility make sure to allow thread_freelist_size
   // This needs to change in 6.0
   REC_EstablishStaticConfigInt32(thread_freelist_high_watermark, "proxy.config.allocator.thread_freelist_size");
 
   REC_EstablishStaticConfigInt32(thread_freelist_low_watermark, "proxy.config.allocator.thread_freelist_low_watermark");
 
-  REC_ReadConfigInteger(config_max_iobuffer_size, "proxy.config.io.max_buffer_size");
-
-  max_iobuffer_size = buffer_size_to_index(config_max_iobuffer_size, DEFAULT_BUFFER_SIZES - 1);
-  if (default_small_iobuffer_size > max_iobuffer_size) {
-    default_small_iobuffer_size = max_iobuffer_size;
-  }
-  if (default_large_iobuffer_size > max_iobuffer_size) {
-    default_large_iobuffer_size = max_iobuffer_size;
-  }
-
 #ifdef MADV_DONTDUMP // This should only exist on Linux 3.4 and higher.
   RecBool dont_dump_enabled = true;
   RecGetRecordBool("proxy.config.allocator.dontdump_iobuffers", &dont_dump_enabled, false);

iocore/eventsystem/I_IOBuffer.h

@@ -52,10 +52,6 @@ class MIOBuffer;
 class IOBufferReader;
 class VIO;
 
-inkcoreapi extern int64_t max_iobuffer_size;
-extern int64_t default_small_iobuffer_size;
-extern int64_t default_large_iobuffer_size; // matched to size of OS buffers
-
 enum AllocType {
   NO_ALLOC,
   MEMALIGNED,
@@ -417,7 +413,7 @@ class IOBufferBlock : public RefCountObj
     section in MIOBuffer.
 
   */
-  void alloc(int64_t i = default_large_iobuffer_size);
+  void alloc(int64_t i);
 
   /**
     Clear the IOBufferData this IOBufferBlock handles. Clears this
@@ -1148,7 +1144,7 @@ class MIOBuffer
   void dealloc_all_readers();
 
   void set(void *b, int64_t len);
-  void alloc(int64_t i = default_large_iobuffer_size);
+  void alloc(int64_t i);
   void append_block_internal(IOBufferBlock *b);
   int64_t write(IOBufferBlock const *b, int64_t len, int64_t offset);
 
@@ -1290,7 +1286,7 @@ struct MIOBufferAccessor {
   IOBufferReader *entry = nullptr;
 };
 
-extern MIOBuffer *new_MIOBuffer_internal(const char *loc, int64_t size_index = default_large_iobuffer_size);
+extern MIOBuffer *new_MIOBuffer_internal(const char *loc, int64_t size_index);
 
 class MIOBuffer_tracker
 {
@@ -1299,13 +1295,13 @@ class MIOBuffer_tracker
 public:
   explicit MIOBuffer_tracker(const char *_loc) : loc(_loc) {}
   MIOBuffer *
-  operator()(int64_t size_index = default_large_iobuffer_size)
+  operator()(int64_t size_index)
   {
     return new_MIOBuffer_internal(loc, size_index);
   }
 };
 
-extern MIOBuffer *new_empty_MIOBuffer_internal(const char *loc, int64_t size_index = default_large_iobuffer_size);
+extern MIOBuffer *new_empty_MIOBuffer_internal(const char *loc, int64_t size_index);
 
 class Empty_MIOBuffer_tracker
 {
@@ -1314,7 +1310,7 @@ class Empty_MIOBuffer_tracker
 public:
   explicit Empty_MIOBuffer_tracker(const char *_loc) : loc(_loc) {}
   MIOBuffer *
-  operator()(int64_t size_index = default_large_iobuffer_size)
+  operator()(int64_t size_index)
   {
     return new_empty_MIOBuffer_internal(loc, size_index);
   }
@@ -1352,8 +1348,7 @@ class IOBufferBlock_tracker
 #define new_IOBufferBlock IOBufferBlock_tracker(RES_PATH("memory/IOBuffer/"))
 ////////////////////////////////////////////////////////////
 
-extern IOBufferData *new_IOBufferData_internal(const char *location, int64_t size_index = default_large_iobuffer_size,
-                                               AllocType type = DEFAULT_ALLOC);
+extern IOBufferData *new_IOBufferData_internal(const char *location, int64_t size_index, AllocType type = DEFAULT_ALLOC);
 
 extern IOBufferData *new_xmalloc_IOBufferData_internal(const char *location, void *b, int64_t size);
 
@@ -1364,7 +1359,7 @@ class IOBufferData_tracker
 public:
   explicit IOBufferData_tracker(const char *_loc) : loc(_loc) {}
   IOBufferData *
-  operator()(int64_t size_index = default_large_iobuffer_size, AllocType type = DEFAULT_ALLOC)
+  operator()(int64_t size_index, AllocType type = DEFAULT_ALLOC)
   {
     return new_IOBufferData_internal(loc, size_index, type);
   }

iocore/eventsystem/unit_tests/test_IOBuffer.cc

@@ -35,26 +35,19 @@
 
 TEST_CASE("MIOBuffer", "[iocore]")
 {
-  // These value could be tweaked by `ink_event_system_init()` using `proxy.config.io.max_buffer_size`
-  REQUIRE(default_small_iobuffer_size == DEFAULT_SMALL_BUFFER_SIZE);
-  REQUIRE(default_large_iobuffer_size == DEFAULT_LARGE_BUFFER_SIZE);
-
-  REQUIRE(BUFFER_SIZE_FOR_INDEX(default_small_iobuffer_size) == 512);
-  REQUIRE(BUFFER_SIZE_FOR_INDEX(default_large_iobuffer_size) == 4096);
-
   SECTION("new_MIOBuffer 100 times")
   {
     int64_t read_avail_len1 = 0;
     int64_t read_avail_len2 = 0;
 
     for (unsigned i = 0; i < 100; ++i) {
-      MIOBuffer *b1            = new_MIOBuffer(default_small_iobuffer_size);
+      MIOBuffer *b1            = new_MIOBuffer(BUFFER_SIZE_INDEX_512);
       int64_t len1             = b1->write_avail();
       IOBufferReader *b1reader = b1->alloc_reader();
       b1->fill(len1);
       read_avail_len1 += b1reader->read_avail();
 
-      MIOBuffer *b2            = new_MIOBuffer(default_large_iobuffer_size);
+      MIOBuffer *b2            = new_MIOBuffer(BUFFER_SIZE_INDEX_4K);
       int64_t len2             = b2->write_avail();
       IOBufferReader *b2reader = b2->alloc_reader();
       b2->fill(len2);
@@ -64,19 +57,19 @@ TEST_CASE("MIOBuffer", "[iocore]")
       free_MIOBuffer(b1);
     }
 
-    CHECK(read_avail_len1 == 100 * BUFFER_SIZE_FOR_INDEX(default_small_iobuffer_size));
-    CHECK(read_avail_len2 == 100 * BUFFER_SIZE_FOR_INDEX(default_large_iobuffer_size));
+    CHECK(read_avail_len1 == 100 * BUFFER_SIZE_FOR_INDEX(BUFFER_SIZE_INDEX_512));
+    CHECK(read_avail_len2 == 100 * BUFFER_SIZE_FOR_INDEX(BUFFER_SIZE_INDEX_4K));
   }
 
   SECTION("write")
   {
-    MIOBuffer *miob            = new_MIOBuffer();
+    MIOBuffer *miob            = new_MIOBuffer(BUFFER_SIZE_INDEX_4K);
     IOBufferReader *miob_r     = miob->alloc_reader();
     const IOBufferBlock *block = miob->first_write_block();
 
     SECTION("initial state")
     {
-      CHECK(miob->size_index == default_large_iobuffer_size);
+      CHECK(miob->size_index == BUFFER_SIZE_INDEX_4K);
       CHECK(miob->water_mark == 0);
       CHECK(miob->first_write_block() != nullptr);
       CHECK(miob->block_size() == 4096);
@@ -176,7 +169,7 @@ TEST_CASE("MIOBuffer", "[iocore]")
 
   SECTION("write_avail")
   {
-    MIOBuffer *miob        = new_MIOBuffer();
+    MIOBuffer *miob        = new_MIOBuffer(BUFFER_SIZE_INDEX_4K);
     IOBufferReader *miob_r = miob->alloc_reader();
     uint8_t buf[8192];
     memset(buf, 0xAA, sizeof(buf));
@@ -347,7 +340,6 @@ struct EventProcessorListener : Catch::TestEventListenerBase {
     init_diags("", nullptr);
     RecProcessInit(RECM_STAND_ALONE);
 
-    // Initialize LibRecordsConfig for `proxy.config.io.max_buffer_size` (32K)
     LibRecordsConfigInit();
 
     ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION);

iocore/net/NetVCTest.cc

@@ -138,8 +138,8 @@ NetVCTest::start_test()
   test_vc->set_inactivity_timeout(HRTIME_SECONDS(timeout));
   test_vc->set_active_timeout(HRTIME_SECONDS(timeout + 5));
 
-  read_buffer  = new_MIOBuffer();
-  write_buffer = new_MIOBuffer();
+  read_buffer  = new_MIOBuffer(BUFFER_SIZE_INDEX_32K);
+  write_buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_32K);
 
   reader_for_rbuf = read_buffer->alloc_reader();
   reader_for_wbuf = write_buffer->alloc_reader();

iocore/net/P_SSLConfig.h

@@ -108,6 +108,7 @@ struct SSLConfigParams : public ConfigInfo {
   static bool server_allow_early_data_params;
 
   static int ssl_maxrecord;
+  static int ssl_misc_max_iobuffer_size_index;
   static bool ssl_allow_client_renegotiation;
 
   static bool ssl_ocsp_enabled;

iocore/net/P_SSLNetVConnection.h

@@ -49,6 +49,7 @@
 #include "P_ALPNSupport.h"
 #include "TLSSessionResumptionSupport.h"
 #include "P_SSLUtils.h"
+#include "P_SSLConfig.h"
 
 // These are included here because older OpenSSL libraries don't have them.
 // Don't copy these defines, or use their values directly, they are merely
@@ -186,7 +187,7 @@ class SSLNetVConnection : public UnixNetVConnection, public ALPNSupport, public
   void
   initialize_handshake_buffers()
   {
-    this->handShakeBuffer    = new_MIOBuffer();
+    this->handShakeBuffer    = new_MIOBuffer(SSLConfigParams::ssl_misc_max_iobuffer_size_index);
     this->handShakeReader    = this->handShakeBuffer->alloc_reader();
     this->handShakeHolder    = this->handShakeReader->clone();
     this->handShakeBioStored = 0;

iocore/net/SSLConfig.cc

@@ -52,6 +52,7 @@ int SSLConfig::configid                                     = 0;
 int SSLCertificateConfig::configid                          = 0;
 int SSLTicketKeyConfig::configid                            = 0;
 int SSLConfigParams::ssl_maxrecord                          = 0;
+int SSLConfigParams::ssl_misc_max_iobuffer_size_index       = 8;
 bool SSLConfigParams::ssl_allow_client_renegotiation        = false;
 bool SSLConfigParams::ssl_ocsp_enabled                      = false;
 int SSLConfigParams::ssl_ocsp_cache_timeout                 = 3600;
@@ -454,6 +455,8 @@ SSLConfigParams::initialize()
 
   REC_ReadConfigInt32(ssl_allow_client_renegotiation, "proxy.config.ssl.allow_client_renegotiation");
 
+  REC_ReadConfigInt32(ssl_misc_max_iobuffer_size_index, "proxy.config.ssl.misc.io.max_buffer_index");
+
   // Enable client regardless of config file settings as remap file
   // can cause HTTP layer to connect using SSL. But only if SSL
   // initialization hasn't failed already.