Add per-LLM-node generation_config.json path override (#4233)

docs/llm/reference.md

@@ -109,6 +109,7 @@ The calculator supports the following `node_options` for tuning the pipeline con
 -    `optional string tool_parser` - name of the parser to use for tool calls extraction from model output before creating a response;
 -    `optional bool enable_tool_guided_generation` - enable enforcing tool schema during generation. Requires setting response parser. [default = false];
 -    `optional SparseAttentionConfig sparse_attention_config` - Sparse attention configuration. Disabled if not specified.
+-    `optional string generation_config_path` - path to a `generation_config.json` holding the default generation parameters for this node. Absolute, or relative to `models_path`. When unset, `generation_config.json` from `models_path` is used. Lets several deployments backed by the same model weights use different generation defaults without duplicating the model directory.
 
 ### Caching settings
 The value of `cache_size` might have performance and stability implications. It is used for storing LLM model KV cache data. Adjust it based on your environment capabilities, model size and expected level of concurrency.

src/llm/language_model/continuous_batching/servable_initializer.cpp

@@ -140,9 +140,13 @@ Status ContinuousBatchingServableInitializer::initialize(std::shared_ptr<GenAiSe
     }
     auto properties = std::static_pointer_cast<ContinuousBatchingServableProperties>(servable->getProperties());
     properties->modelsPath = parsedModelsPath;
-    std::filesystem::path modelGenerationConfigPath = std::filesystem::path(parsedModelsPath) / "generation_config.json";
-    if (std::filesystem::exists(modelGenerationConfigPath)) {
-        properties->baseGenerationConfig = ov::genai::GenerationConfig(modelGenerationConfigPath.string());
+    std::string generationConfigPath;
+    status = resolveGenerationConfigPath(generationConfigPath, parsedModelsPath, nodeOptions);
+    if (!status.ok()) {
+        return status;
+    }
+    if (std::filesystem::exists(generationConfigPath)) {
+        properties->baseGenerationConfig = ov::genai::GenerationConfig(generationConfigPath);
     }
     if (nodeOptions.has_tool_parser()) {
         properties->toolParserName = nodeOptions.tool_parser();

src/llm/language_model/legacy/servable_initializer.cpp

@@ -51,9 +51,13 @@ Status LegacyServableInitializer::initialize(std::shared_ptr<GenAiServable>& ser
     auto properties = std::static_pointer_cast<LegacyServableProperties>(servable->getProperties());
 
     properties->modelsPath = parsedModelsPath;
-    std::filesystem::path modelGenerationConfigPath = std::filesystem::path(parsedModelsPath) / "generation_config.json";
-    if (std::filesystem::exists(modelGenerationConfigPath)) {
-        properties->baseGenerationConfig = ov::genai::GenerationConfig(modelGenerationConfigPath.string());
+    std::string generationConfigPath;
+    status = resolveGenerationConfigPath(generationConfigPath, parsedModelsPath, nodeOptions);
+    if (!status.ok()) {
+        return status;
+    }
+    if (std::filesystem::exists(generationConfigPath)) {
+        properties->baseGenerationConfig = ov::genai::GenerationConfig(generationConfigPath);
     }
 
     if (nodeOptions.has_tool_parser()) {

src/llm/llm_calculator.proto

@@ -151,4 +151,11 @@ message LLMCalculatorOptions {
     }
 
     optional ChatTemplateMode chat_template_mode = 26;
+
+    // Optional path to a generation_config.json holding the default generation
+    // parameters for this node. Absolute, or relative to models_path. When unset,
+    // generation_config.json from models_path is used. Allows several deployments
+    // backed by the same model weights to use different generation defaults
+    // without duplicating the model directory.
+    optional string generation_config_path = 27;
 }

src/llm/servable_initializer.cpp

@@ -445,6 +445,27 @@ Status parseModelsPath(std::string& outPath, std::string modelsPath, std::string
     return StatusCode::LLM_NODE_PATH_DOES_NOT_EXIST_AND_NOT_GGUFFILE;
 }
 
+Status resolveGenerationConfigPath(std::string& outPath, const std::string& parsedModelsPath, const mediapipe::LLMCalculatorOptions& nodeOptions) {
+    // Default: generation_config.json inside the model directory.
+    outPath = (std::filesystem::path(parsedModelsPath) / "generation_config.json").string();
+    if (!nodeOptions.has_generation_config_path() || nodeOptions.generation_config_path().empty()) {
+        return StatusCode::OK;
+    }
+    // Explicit per-node override. A relative path is resolved against models_path
+    // (its parent directory when models_path points at a file, e.g. a GGUF).
+    std::filesystem::path base(parsedModelsPath);
+    if (!std::filesystem::is_directory(base)) {
+        base = base.parent_path();
+    }
+    std::filesystem::path overridePath = FileSystem::resolvePathWithBase(nodeOptions.generation_config_path(), base.string());
+    if (!std::filesystem::exists(overridePath) || !std::filesystem::is_regular_file(overridePath)) {
+        SPDLOG_LOGGER_ERROR(modelmanager_logger, "LLM node generation_config_path: {} does not exist or is not a regular file.", overridePath.string());
+        return StatusCode::LLM_NODE_GENERATION_CONFIG_DOES_NOT_EXIST;
+    }
+    outPath = overridePath.string();
+    return StatusCode::OK;
+}
+
 std::optional<uint32_t> parseMaxModelLength(std::string& modelsPath) {
     std::string configPath = FileSystem::appendSlash(modelsPath) + "config.json";
     std::optional<uint32_t> maxModelLength;

src/llm/servable_initializer.hpp

@@ -67,6 +67,7 @@ class GenAiServableInitializer {
     virtual Status initialize(std::shared_ptr<GenAiServable>& servable, const mediapipe::LLMCalculatorOptions& nodeOptions, std::string graphPath) = 0;
 };
 Status parseModelsPath(std::string& outPath, std::string modelsPath, std::string graphPath);
+Status resolveGenerationConfigPath(std::string& outPath, const std::string& parsedModelsPath, const mediapipe::LLMCalculatorOptions& nodeOptions);
 std::optional<uint32_t> parseMaxModelLength(std::string& modelsPath);
 Status determinePipelineType(PipelineType& pipelineType, const mediapipe::LLMCalculatorOptions& nodeOptions, const std::string& graphPath);
 Status initializeGenAiServable(std::shared_ptr<GenAiServable>& servable, const ::mediapipe::CalculatorGraphConfig::Node& graphNodeConfig, std::string graphPath);

src/status.cpp

@@ -225,6 +225,7 @@ const std::unordered_map<StatusCode, std::string> Status::statusMessageMap = {
     {StatusCode::LLM_NODE_NAME_ALREADY_EXISTS, "The LLM Node name is already present in nodes list"},
     {StatusCode::LLM_NODE_DIRECTORY_DOES_NOT_EXIST, "The LLM Node workspace path does not exist"},
     {StatusCode::LLM_NODE_PATH_DOES_NOT_EXIST_AND_NOT_GGUFFILE, "The LLM Node workspace path does not exist and not gguf model file"},
+    {StatusCode::LLM_NODE_GENERATION_CONFIG_DOES_NOT_EXIST, "The LLM Node generation_config_path does not point to an existing file"},
     {StatusCode::LLM_NODE_RESOURCE_STATE_INITIALIZATION_FAILED, "The LLM Node resource initialization failed"},
     {StatusCode::LLM_NODE_MISSING_OPTIONS, "The LLM Node is missing options definition"},
     {StatusCode::LLM_NODE_MISSING_NAME, "The LLM Node is missing name definition"},

src/status.hpp

@@ -271,6 +271,7 @@ enum class StatusCode {
     LLM_NODE_NAME_ALREADY_EXISTS,
     LLM_NODE_DIRECTORY_DOES_NOT_EXIST,
     LLM_NODE_PATH_DOES_NOT_EXIST_AND_NOT_GGUFFILE,
+    LLM_NODE_GENERATION_CONFIG_DOES_NOT_EXIST,
     LLM_NODE_RESOURCE_STATE_INITIALIZATION_FAILED,
     LLM_NODE_MISSING_OPTIONS,
     LLM_NODE_MISSING_NAME,

src/test/llm/llmnode_test.cpp

@@ -51,6 +51,7 @@
 #include "../../mediapipe_internal/mediapipegraphdefinition.hpp"
 #include "../../ov_utils.hpp"
 #include "../../server.hpp"
+#include "src/filesystem/filesystem.hpp"
 #include "src/graph_export/graph_export.hpp"
 #include "rapidjson/document.h"
 #include "rapidjson/stringbuffer.h"
@@ -4504,6 +4505,85 @@ TEST_F(LLMVLMOptionsHttpTest, LLMVLMNodeOptionsCheckPluginConfig) {
     LLMNodeOptionsCheckPluginConfig(modelsPath);
 }
 
+// RAII guard removing a temporary directory tree on scope exit, so a failed ASSERT_*
+// (which returns early from the test body) does not leak the directory.
+struct TempDirGuard {
+    std::filesystem::path path;
+    explicit TempDirGuard(std::filesystem::path p) :
+        path(std::move(p)) {}
+    ~TempDirGuard() {
+        std::error_code ec;
+        std::filesystem::remove_all(path, ec);
+    }
+};
+
+// Unit test for the per-node generation_config.json path override (issue #4233).
+TEST(LLMGenerationConfigPath, ResolveGenerationConfigPath) {
+    // Unique per-run directory so concurrent/parallel test runs cannot collide.
+    std::string tempPath;
+    ASSERT_EQ(ovms::FileSystem::createTempPath(&tempPath), ovms::StatusCode::OK);
+    std::filesystem::path base(tempPath);
+    TempDirGuard cleanup(base);
+    std::filesystem::path modelDir = base / "model";
+    std::filesystem::path overrideDir = base / "overrides";
+    std::filesystem::create_directories(modelDir);
+    std::filesystem::create_directories(overrideDir);
+    auto writeFile = [](const std::filesystem::path& p) {
+        std::ofstream ofs(p);
+        ofs << "{}";
+    };
+    writeFile(modelDir / "generation_config.json");
+    writeFile(overrideDir / "custom_generation_config.json");
+    // A GGUF "models_path" is a file, not a directory - relative overrides resolve against its parent.
+    writeFile(modelDir / "model.gguf");
+
+    // Case 1: no override -> default generation_config.json inside the model dir.
+    {
+        mediapipe::LLMCalculatorOptions nodeOptions;
+        std::string outPath;
+        ASSERT_EQ(ovms::resolveGenerationConfigPath(outPath, modelDir.string(), nodeOptions), ovms::StatusCode::OK);
+        ASSERT_EQ(std::filesystem::path(outPath), modelDir / "generation_config.json");
+    }
+    // Case 2: explicit absolute override path.
+    {
+        mediapipe::LLMCalculatorOptions nodeOptions;
+        nodeOptions.set_generation_config_path((overrideDir / "custom_generation_config.json").string());
+        std::string outPath;
+        ASSERT_EQ(ovms::resolveGenerationConfigPath(outPath, modelDir.string(), nodeOptions), ovms::StatusCode::OK);
+        ASSERT_EQ(std::filesystem::path(outPath), overrideDir / "custom_generation_config.json");
+    }
+    // Case 3: explicit relative override path is resolved against models_path.
+    {
+        mediapipe::LLMCalculatorOptions nodeOptions;
+        nodeOptions.set_generation_config_path("generation_config.json");
+        std::string outPath;
+        ASSERT_EQ(ovms::resolveGenerationConfigPath(outPath, modelDir.string(), nodeOptions), ovms::StatusCode::OK);
+        ASSERT_EQ(std::filesystem::path(outPath), modelDir / "generation_config.json");
+    }
+    // Case 4: explicit override that does not exist -> dedicated error.
+    {
+        mediapipe::LLMCalculatorOptions nodeOptions;
+        nodeOptions.set_generation_config_path((overrideDir / "missing.json").string());
+        std::string outPath;
+        ASSERT_EQ(ovms::resolveGenerationConfigPath(outPath, modelDir.string(), nodeOptions), ovms::StatusCode::LLM_NODE_GENERATION_CONFIG_DOES_NOT_EXIST);
+    }
+    // Case 5: explicit override pointing at a directory -> same error (not a regular file).
+    {
+        mediapipe::LLMCalculatorOptions nodeOptions;
+        nodeOptions.set_generation_config_path(overrideDir.string());
+        std::string outPath;
+        ASSERT_EQ(ovms::resolveGenerationConfigPath(outPath, modelDir.string(), nodeOptions), ovms::StatusCode::LLM_NODE_GENERATION_CONFIG_DOES_NOT_EXIST);
+    }
+    // Case 6: models_path points at a GGUF file - relative override resolves against its parent dir.
+    {
+        mediapipe::LLMCalculatorOptions nodeOptions;
+        nodeOptions.set_generation_config_path("generation_config.json");
+        std::string outPath;
+        ASSERT_EQ(ovms::resolveGenerationConfigPath(outPath, (modelDir / "model.gguf").string(), nodeOptions), ovms::StatusCode::OK);
+        ASSERT_EQ(std::filesystem::path(outPath), modelDir / "generation_config.json");
+    }
+}
+
 // RAII guard that restores the global Config singleton (and optionally removes a temporary
 // cache directory) on scope exit. The cache_dir tests below mutate the process-wide Config
 // singleton; without this guard a failed ASSERT_* mid-test (which returns early) would leak