Functions to control the streaming memory/buffer sizes [Fixes #3084]

Client/core/CCore.cpp

@@ -2316,3 +2316,24 @@ SString CCore::GetBlueCopyrightString()
     SString strCopyright = BLUE_COPYRIGHT_STRING;
     return strCopyright.Replace("%BUILD_YEAR%", std::to_string(BUILD_YEAR).c_str());
 }
+
+// Set streaming memory size override [See `engineStreamingSetMemorySize`]
+// Use `0` to turn it off, and thus restore the value to the `cvar` setting
+void CCore::SetCustomStreamingMemory(size_t sizeBytes) {
+    // NOTE: The override is applied to the game in `CClientGame::DoPulsePostFrame`
+    // There's no specific reason we couldn't do it here, but we wont
+    m_CustomStreamingMemoryLimitBytes = sizeBytes;
+}
+
+bool CCore::IsUsingCustomStreamingMemorySize()
+{
+    return m_CustomStreamingMemoryLimitBytes != 0;
+}
+
+// Streaming memory size used [In Bytes]
+size_t CCore::GetStreamingMemory()
+{
+    return IsUsingCustomStreamingMemorySize()
+        ? m_CustomStreamingMemoryLimitBytes
+        : CVARS_GET_VALUE<size_t>("streaming_memory") * 1024 * 1024; // MB to B conversion
+}

Client/core/CCore.h

@@ -279,6 +279,9 @@ class CCore : public CCoreInterface, public CSingleton<CCore>
     SString     GetBlueCopyrightString();
     bool        IsFirstFrame() const noexcept { return m_bFirstFrame; }
 
+    void   SetCustomStreamingMemory(size_t szMB);
+    bool   IsUsingCustomStreamingMemorySize();
+    size_t GetStreamingMemory();
 private:
     void ApplyCoreInitSettings();
 
@@ -360,8 +363,15 @@ class CCore : public CCoreInterface, public CSingleton<CCore>
     bool                 m_bWaitToSetNick;
     uint                 m_uiNewNickWaitFrames;
     EDiagnosticDebugType m_DiagnosticDebug;
-    float                m_fMinStreamingMemory;
-    float                m_fMaxStreamingMemory;
+
+    // Below 2 are used for the UI only
+    float                m_fMinStreamingMemory{};
+    float                m_fMaxStreamingMemory{};
+
+    // Custom streaming memory limit set by `engineStreamingSetMemorySize` - Reset on server connects (= set to 0), or by the scripter
+    // `0` means "not set" [so the value should be ignored]
+    size_t               m_CustomStreamingMemoryLimitBytes{};
+
     bool                 m_bGettingIdleCallsFromMultiplayer;
     bool                 m_bWindowsTimerEnabled;
     bool                 m_bModulesLoaded;

Client/core/CMemStats.cpp

@@ -472,8 +472,8 @@ void CMemStats::SampleState(SMemStatsInfo& memStatsInfo)
         memStatsInfo.iProcessTotalVirtualKB = status.ullTotalVirtual / 1024LL;
     }
 
-    memStatsInfo.iStreamingMemoryUsed = *(int*)0x08E4CB4;
-    memStatsInfo.iStreamingMemoryAvailable = *(int*)0x08A5A80;
+    memStatsInfo.iStreamingMemoryUsed = *(size_t*)0x08E4CB4;
+    memStatsInfo.iStreamingMemoryAvailable = *(size_t*)0x08A5A80;
 
     char*        pFileInfoArray = *(char**)(0x5B8B08 + 6);
     CGame*       pGame = g_pCore->GetGame();

Client/core/CMemStats.h

@@ -26,10 +26,10 @@ struct SMemStatsInfo
     CProxyDirect3DDevice9::SMemoryState d3dMemory;
     CProxyDirect3DDevice9::SFrameStats  frameStats;
     SDxStatus                           dxStatus;
-    int                                 iProcessMemSizeKB;
-    int                                 iProcessTotalVirtualKB;
-    int                                 iStreamingMemoryUsed;
-    int                                 iStreamingMemoryAvailable;
+    size_t                              iProcessMemSizeKB;
+    size_t                              iProcessTotalVirtualKB;
+    size_t                              iStreamingMemoryUsed;
+    size_t                              iStreamingMemoryAvailable;
     SRwResourceStats                    rwResourceStats;
     SClothesCacheStats                  clothesCacheStats;
     SShaderReplacementStats             shaderReplacementStats;

Client/game_sa/CStreamingSA.cpp

@@ -256,39 +256,39 @@ void CStreamingSA::RemoveArchive(unsigned char ucArhiveID)
     m_aStreamingHandlers[uiStreamHandlerID] = NULL;
 }
 
-void CStreamingSA::SetStreamingBufferSize(uint32 uiBlockSize)
+void CStreamingSA::SetStreamingBufferSize(uint32 numBlocks)
 {
-    if (uiBlockSize == ms_streamingHalfOfBufferSize * 2)
+    if (numBlocks == ms_streamingHalfOfBufferSize * 2)
         return;
 
     int pointer = *(int*)0x8E3FFC;
     SGtaStream(&streaming)[5] = *(SGtaStream(*)[5])(pointer);
 
     // Wait while streaming threads ends tasks
-    while (streaming[0].bInUse && streaming[1].bInUse)
-
-        // Suspend streaming handle
-        SuspendThread(*phStreamingThread);
+    while (streaming[0].bInUse && streaming[1].bInUse);
 
+    // Suspend streaming handle
+    SuspendThread(*phStreamingThread);
+
     // Create new buffer
-    if (uiBlockSize & 1)
-        uiBlockSize++;
+    if (numBlocks & 1) // Make it be even
+        numBlocks++;
 
     typedef void*(__cdecl * Function_CMemoryMgr_MallocAlign)(uint32 uiCount, uint32 uiAlign);
-    void* pNewBuffer = ((Function_CMemoryMgr_MallocAlign)(0x72F4C0))(uiBlockSize << 11, 2048);
+    void* pNewBuffer = ((Function_CMemoryMgr_MallocAlign)(0x72F4C0))(numBlocks * 2048, 2048);
 
     // Copy data from old buffer to new buffer
-    uint uiCopySize = std::min(ms_streamingHalfOfBufferSize, uiBlockSize / 2);
+    uint uiCopySize = std::min(ms_streamingHalfOfBufferSize, numBlocks / 2); // TODO: This probably won't work as expected 
     MemCpyFast(pNewBuffer, (void*)ms_pStreamingBuffer[0], uiCopySize);
-    MemCpyFast((void*)(reinterpret_cast<int>(pNewBuffer) + 1024 * uiBlockSize), (void*)ms_pStreamingBuffer[1], uiCopySize);
+    MemCpyFast((void*)(reinterpret_cast<int>(pNewBuffer) + 1024 * numBlocks), (void*)ms_pStreamingBuffer[1], uiCopySize);
 
     typedef void(__cdecl * Function_CMemoryMgr_FreeAlign)(void* pos);
     ((Function_CMemoryMgr_FreeAlign)(0x72F4F0))(ms_pStreamingBuffer[0]);
 
-    ms_streamingHalfOfBufferSize = uiBlockSize / 2;
+    ms_streamingHalfOfBufferSize = numBlocks / 2;
 
     ms_pStreamingBuffer[0] = pNewBuffer;
-    ms_pStreamingBuffer[1] = (void*)(reinterpret_cast<int>(pNewBuffer) + 1024 * uiBlockSize);
+    ms_pStreamingBuffer[1] = (void*)(reinterpret_cast<int>(pNewBuffer) + 1024 * numBlocks);
 
     streaming[0].pBuffer = ms_pStreamingBuffer[0];
     streaming[1].pBuffer = ms_pStreamingBuffer[1];

Client/game_sa/CStreamingSA.h

@@ -66,7 +66,7 @@ class CStreamingSA final : public CStreaming
     unsigned char   AddArchive(const char* szFilePath);
     void            RemoveArchive(unsigned char ucStreamHandler);
     void            SetStreamingBufferSize(uint32 uiSize);
-    uint32          GetStreamingBufferSize() { return ms_streamingHalfOfBufferSize * 2; };
+    uint32          GetStreamingBufferSize() { return ms_streamingHalfOfBufferSize * 2; }; // In bytes
 
     void          MakeSpaceFor(std::uint32_t memoryToCleanInBytes) override;
     std::uint32_t GetMemoryUsed() const override;

Client/mods/deathmatch/logic/CClientGame.cpp

@@ -529,6 +529,9 @@ CClientGame::~CClientGame()
     m_bBeingDeleted = false;
 
     CStaticFunctionDefinitions::ResetAllSurfaceInfo();
+
+    // Reset custom streaming memory size [possibly] set by the server
+    g_pCore->SetCustomStreamingMemory(0);
 }
 
 /*
@@ -880,18 +883,21 @@ void CClientGame::DoPulsePostFrame()
                                                DT_NOCLIP | DT_CENTER);
         }
 
-        // Adjust the streaming memory limit.
-        unsigned int uiStreamingMemoryPrev;
-        g_pCore->GetCVars()->Get("streaming_memory", uiStreamingMemoryPrev);
-        uint uiStreamingMemory = SharedUtil::Clamp(g_pCore->GetMinStreamingMemory(), uiStreamingMemoryPrev, g_pCore->GetMaxStreamingMemory());
-        if (uiStreamingMemory != uiStreamingMemoryPrev)
-            g_pCore->GetCVars()->Set("streaming_memory", uiStreamingMemory);
+        // Adjust the streaming memory size cvar [if needed]
+        if (!g_pCore->IsUsingCustomStreamingMemorySize()) {
+            unsigned int uiStreamingMemoryPrev;
+            g_pCore->GetCVars()->Get("streaming_memory", uiStreamingMemoryPrev);
+            uint uiStreamingMemory = SharedUtil::Clamp(g_pCore->GetMinStreamingMemory(), uiStreamingMemoryPrev, g_pCore->GetMaxStreamingMemory());
+            if (uiStreamingMemory != uiStreamingMemoryPrev)
+                g_pCore->GetCVars()->Set("streaming_memory", uiStreamingMemory);
+        }
 
-        int iStreamingMemoryBytes = static_cast<int>(uiStreamingMemory) * 1024 * 1024;
-        if (g_pMultiplayer->GetLimits()->GetStreamingMemory() != iStreamingMemoryBytes)
-            g_pMultiplayer->GetLimits()->SetStreamingMemory(iStreamingMemoryBytes);
+        const auto streamingMemorySizeBytes = g_pCore->GetStreamingMemory();
+        if (g_pMultiplayer->GetLimits()->GetStreamingMemory() != streamingMemorySizeBytes) {
+            g_pMultiplayer->GetLimits()->SetStreamingMemory(streamingMemorySizeBytes);
+        }
 
-            // If we're in debug mode and are supposed to show task data, do it
+        // If we're in debug mode and are supposed to show task data, do it
         #ifdef MTA_DEBUG
         if (m_pShowPlayerTasks)
         {

Client/mods/deathmatch/logic/CClientGame.h

@@ -804,9 +804,9 @@ class CClientGame
     bool                      m_bDoPaintballs;
     bool                      m_bShowInterpolation;
     #endif
-    bool m_bDevelopmentMode;
-    bool m_bShowCollision;
-    bool m_bShowSound;
+    bool                      m_bDevelopmentMode;
+    bool                      m_bShowCollision;
+    bool                      m_bShowSound;
 
     // Debug class. Empty in release.
 public:

Client/mods/deathmatch/logic/CClientIMG.cpp

@@ -20,7 +20,7 @@ struct tImgHeader
 };
 
 CClientIMG::CClientIMG(class CClientManager* pManager, ElementID ID)
-    : ClassInit(this), CClientEntity(ID), m_pImgManager(pManager->GetIMGManager()), m_ucArchiveID(INVALID_ARCHIVE_ID), m_usRequiredBufferSize(0)
+    : ClassInit(this), CClientEntity(ID), m_pImgManager(pManager->GetIMGManager()), m_ucArchiveID(INVALID_ARCHIVE_ID), m_LargestFileSizeBlocks(0)
 {
     m_pManager = pManager;
     SetTypeName("img");
@@ -106,19 +106,19 @@ bool CClientIMG::GetFile(size_t fileID, std::string& buffer)
     if (!pFileInfo)
         throw std::invalid_argument("Invalid file id");
 
-    const auto ulToReadSize = pFileInfo->usSize * 2048;
+    const auto toReadBytes = (size_t)pFileInfo->usSize * 2048u;
 
     try
     {
-        buffer.resize(ulToReadSize);
+        buffer.resize(toReadBytes);
     }
     catch (const std::bad_alloc&)
     {
         throw std::invalid_argument("Out of memory");
     }
 
     m_ifs.seekg((std::streampos)pFileInfo->uiOffset * 2048);
-    m_ifs.read(buffer.data(), ulToReadSize);
+    m_ifs.read(buffer.data(), toReadBytes);
 
     return !m_ifs.fail() && !m_ifs.eof();
 }
@@ -153,10 +153,10 @@ bool CClientIMG::StreamEnable()
     if (IsStreamed())
         return false;
 
-    if (m_usRequiredBufferSize == 0)
+    if (m_LargestFileSizeBlocks == 0)
     {
         for (const auto& fileInfo : m_fileInfos)
-            m_usRequiredBufferSize = Max(m_usRequiredBufferSize, fileInfo.usSize);
+            m_LargestFileSizeBlocks = std::max(m_LargestFileSizeBlocks, (size_t)fileInfo.usSize);
     }
 
     m_ucArchiveID = g_pGame->GetStreaming()->AddArchive(m_filePath.c_str());

Client/mods/deathmatch/logic/CClientIMG.h

@@ -55,10 +55,10 @@ class CClientIMG : public CClientEntity
     void SetPosition(const CVector& vecPosition){};
 
     eClientEntityType GetType() const { return CCLIENTIMG; }
-    unsigned char     GetArchiveID() { return m_ucArchiveID; }
-    unsigned int      GetFilesCount() { return m_fileInfos.size(); }
+    unsigned char     GetArchiveID() const { return m_ucArchiveID; }
+    unsigned int      GetFilesCount() const { return m_fileInfos.size(); }
     const auto&       GetFileInfos() const noexcept { return m_fileInfos; }
-    unsigned short    GetRequiredBufferSize() { return m_usRequiredBufferSize; }
+    auto              GetLargestFileSizeBlocks() const { return m_LargestFileSizeBlocks; }
 
     bool Load(fs::path filePath);
     void Unload();
@@ -81,7 +81,7 @@ class CClientIMG : public CClientEntity
     std::string               m_filePath;
     unsigned char             m_ucArchiveID;
     std::vector<tImgFileInfo> m_fileInfos;
-    unsigned short            m_usRequiredBufferSize;
+    size_t                    m_LargestFileSizeBlocks; // The size of the largest file [in streaming blocks/sectors]
 
     std::vector<tLinkedModelRestoreInfo> m_restoreInfo;
 };

Client/mods/deathmatch/logic/CClientIMGManager.cpp

@@ -15,6 +15,14 @@ CClientIMGManager::CClientIMGManager(CClientManager* pManager)
 {
     // Init
     m_bRemoveFromList = true;
+
+    // Buffer size as calculated by GTA - This is also the size of the largest file in all of the loaded IMGs
+
+    // g_pGame->GetStreaming()->GetStreamingBufferSize() / 2048;
+    // TODO: In the default gta3.img the biggest file is 1260 sectors, so to be fail safe, we double it
+    // ideally, we'd just take this value from the game, but there's no clean/easy way to do that [without loading the img archives]
+    // so, for now, this is good enough
+    m_LargestFileSizeBlocks = m_GTALargestFileSizeBlocks = 1260 * 2; 
 }
 
 CClientIMGManager::~CClientIMGManager()
@@ -25,7 +33,6 @@ CClientIMGManager::~CClientIMGManager()
 
 void CClientIMGManager::InitDefaultBufferSize()
 {
-    m_uiDefaultStreamerBufferSize = g_pGame->GetStreaming()->GetStreamingBufferSize();
 }
 
 CClientIMG* CClientIMGManager::GetElementFromArchiveID(unsigned char ucArchiveID)
@@ -93,6 +100,21 @@ bool CClientIMGManager::RestoreModel(unsigned int uiModel)
     return false;
 }
 
+size_t CClientIMGManager::CalculateLargestFile() const
+{
+    auto largest = m_GTALargestFileSizeBlocks;
+
+    for (const auto img : m_List)
+    {
+        if (!img->IsStreamed())
+            continue;
+
+        largest = std::max(img->GetLargestFileSizeBlocks(), largest);
+    }
+
+    return largest;
+}
+
 void CClientIMGManager::RemoveFromList(CClientIMG* pIMG)
 {
     // Can we remove anything from the list?
@@ -104,15 +126,10 @@ void CClientIMGManager::RemoveFromList(CClientIMG* pIMG)
 
 void CClientIMGManager::UpdateStreamerBufferSize()
 {
-    ushort usRequestStreamSize = m_uiDefaultStreamerBufferSize;
+    m_LargestFileSizeBlocks = CalculateLargestFile();
 
-    for (CClientIMG* pImg : m_List)
-    {
-        if (!pImg->IsStreamed())
-            continue;
-        ushort usStreamSize = pImg->GetRequiredBufferSize();
-        if (usStreamSize > usRequestStreamSize)
-            usRequestStreamSize = usStreamSize;
+    // Only update if necessary, otherwise leave it be [User might've set it manually - we don't want to touch that]
+    if (const auto s = g_pGame->GetStreaming(); m_LargestFileSizeBlocks > s->GetStreamingBufferSize()) {
+        s->SetStreamingBufferSize(m_LargestFileSizeBlocks);
     }
-    g_pGame->GetStreaming()->SetStreamingBufferSize(usRequestStreamSize);
 }

Client/mods/deathmatch/logic/CClientIMGManager.h

@@ -33,12 +33,15 @@ class CClientIMGManager
     bool        RestoreModel(unsigned int uiModel);
     static bool IsLinkableModel(unsigned int uiModel);
     void        UpdateStreamerBufferSize();
+    auto        GetLargestFileSizeBlocks() const { return m_LargestFileSizeBlocks; }
 
 private:
+    size_t CalculateLargestFile() const;
     void AddToList(CClientIMG* pIMG) { m_List.push_back(pIMG); }
     void RemoveFromList(CClientIMG* pIMG);
 
     std::list<CClientIMG*> m_List;
     bool                   m_bRemoveFromList;
-    uint32                 m_uiDefaultStreamerBufferSize;
+    uint32                 m_GTALargestFileSizeBlocks;
+    uint32                 m_LargestFileSizeBlocks; // Size of the largest file [in streaming blocks/sectors] in any of the loaded imgs
 };