diff --git a/docs/llm/reference.md b/docs/llm/reference.md index bde4cc792c..5f2831a96b 100644 --- a/docs/llm/reference.md +++ b/docs/llm/reference.md @@ -109,6 +109,7 @@ The calculator supports the following `node_options` for tuning the pipeline con - `optional string tool_parser` - name of the parser to use for tool calls extraction from model output before creating a response; - `optional bool enable_tool_guided_generation` - enable enforcing tool schema during generation. Requires setting response parser. [default = false]; - `optional SparseAttentionConfig sparse_attention_config` - Sparse attention configuration. Disabled if not specified. +- `optional string generation_config_path` - path to a `generation_config.json` holding the default generation parameters for this node. Absolute, or relative to `models_path`. When unset, `generation_config.json` from `models_path` is used. Lets several deployments backed by the same model weights use different generation defaults without duplicating the model directory. ### Caching settings The value of `cache_size` might have performance and stability implications. It is used for storing LLM model KV cache data. Adjust it based on your environment capabilities, model size and expected level of concurrency. diff --git a/src/llm/language_model/continuous_batching/servable_initializer.cpp b/src/llm/language_model/continuous_batching/servable_initializer.cpp index e27f90bd49..3eedfd4923 100644 --- a/src/llm/language_model/continuous_batching/servable_initializer.cpp +++ b/src/llm/language_model/continuous_batching/servable_initializer.cpp @@ -140,9 +140,13 @@ Status ContinuousBatchingServableInitializer::initialize(std::shared_ptr(servable->getProperties()); properties->modelsPath = parsedModelsPath; - std::filesystem::path modelGenerationConfigPath = std::filesystem::path(parsedModelsPath) / "generation_config.json"; - if (std::filesystem::exists(modelGenerationConfigPath)) { - properties->baseGenerationConfig = ov::genai::GenerationConfig(modelGenerationConfigPath.string()); + std::string generationConfigPath; + status = resolveGenerationConfigPath(generationConfigPath, parsedModelsPath, nodeOptions); + if (!status.ok()) { + return status; + } + if (std::filesystem::exists(generationConfigPath)) { + properties->baseGenerationConfig = ov::genai::GenerationConfig(generationConfigPath); } if (nodeOptions.has_tool_parser()) { properties->toolParserName = nodeOptions.tool_parser(); diff --git a/src/llm/language_model/legacy/servable_initializer.cpp b/src/llm/language_model/legacy/servable_initializer.cpp index 8f9d51cee6..b00dbf9759 100644 --- a/src/llm/language_model/legacy/servable_initializer.cpp +++ b/src/llm/language_model/legacy/servable_initializer.cpp @@ -51,9 +51,13 @@ Status LegacyServableInitializer::initialize(std::shared_ptr& ser auto properties = std::static_pointer_cast(servable->getProperties()); properties->modelsPath = parsedModelsPath; - std::filesystem::path modelGenerationConfigPath = std::filesystem::path(parsedModelsPath) / "generation_config.json"; - if (std::filesystem::exists(modelGenerationConfigPath)) { - properties->baseGenerationConfig = ov::genai::GenerationConfig(modelGenerationConfigPath.string()); + std::string generationConfigPath; + status = resolveGenerationConfigPath(generationConfigPath, parsedModelsPath, nodeOptions); + if (!status.ok()) { + return status; + } + if (std::filesystem::exists(generationConfigPath)) { + properties->baseGenerationConfig = ov::genai::GenerationConfig(generationConfigPath); } if (nodeOptions.has_tool_parser()) { diff --git a/src/llm/llm_calculator.proto b/src/llm/llm_calculator.proto index 88e50c128f..c96bdb9778 100644 --- a/src/llm/llm_calculator.proto +++ b/src/llm/llm_calculator.proto @@ -151,4 +151,11 @@ message LLMCalculatorOptions { } optional ChatTemplateMode chat_template_mode = 26; + + // Optional path to a generation_config.json holding the default generation + // parameters for this node. Absolute, or relative to models_path. When unset, + // generation_config.json from models_path is used. Allows several deployments + // backed by the same model weights to use different generation defaults + // without duplicating the model directory. + optional string generation_config_path = 27; } diff --git a/src/llm/servable_initializer.cpp b/src/llm/servable_initializer.cpp index a268abdf9e..414fb1324f 100644 --- a/src/llm/servable_initializer.cpp +++ b/src/llm/servable_initializer.cpp @@ -445,6 +445,27 @@ Status parseModelsPath(std::string& outPath, std::string modelsPath, std::string return StatusCode::LLM_NODE_PATH_DOES_NOT_EXIST_AND_NOT_GGUFFILE; } +Status resolveGenerationConfigPath(std::string& outPath, const std::string& parsedModelsPath, const mediapipe::LLMCalculatorOptions& nodeOptions) { + // Default: generation_config.json inside the model directory. + outPath = (std::filesystem::path(parsedModelsPath) / "generation_config.json").string(); + if (!nodeOptions.has_generation_config_path() || nodeOptions.generation_config_path().empty()) { + return StatusCode::OK; + } + // Explicit per-node override. A relative path is resolved against models_path + // (its parent directory when models_path points at a file, e.g. a GGUF). + std::filesystem::path base(parsedModelsPath); + if (!std::filesystem::is_directory(base)) { + base = base.parent_path(); + } + std::filesystem::path overridePath = FileSystem::resolvePathWithBase(nodeOptions.generation_config_path(), base.string()); + if (!std::filesystem::exists(overridePath) || !std::filesystem::is_regular_file(overridePath)) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "LLM node generation_config_path: {} does not exist or is not a regular file.", overridePath.string()); + return StatusCode::LLM_NODE_GENERATION_CONFIG_DOES_NOT_EXIST; + } + outPath = overridePath.string(); + return StatusCode::OK; +} + std::optional parseMaxModelLength(std::string& modelsPath) { std::string configPath = FileSystem::appendSlash(modelsPath) + "config.json"; std::optional maxModelLength; diff --git a/src/llm/servable_initializer.hpp b/src/llm/servable_initializer.hpp index 7455d1c42b..da9017d9b2 100644 --- a/src/llm/servable_initializer.hpp +++ b/src/llm/servable_initializer.hpp @@ -67,6 +67,7 @@ class GenAiServableInitializer { virtual Status initialize(std::shared_ptr& servable, const mediapipe::LLMCalculatorOptions& nodeOptions, std::string graphPath) = 0; }; Status parseModelsPath(std::string& outPath, std::string modelsPath, std::string graphPath); +Status resolveGenerationConfigPath(std::string& outPath, const std::string& parsedModelsPath, const mediapipe::LLMCalculatorOptions& nodeOptions); std::optional parseMaxModelLength(std::string& modelsPath); Status determinePipelineType(PipelineType& pipelineType, const mediapipe::LLMCalculatorOptions& nodeOptions, const std::string& graphPath); Status initializeGenAiServable(std::shared_ptr& servable, const ::mediapipe::CalculatorGraphConfig::Node& graphNodeConfig, std::string graphPath); diff --git a/src/status.cpp b/src/status.cpp index 0394192e86..f6f9a67257 100644 --- a/src/status.cpp +++ b/src/status.cpp @@ -225,6 +225,7 @@ const std::unordered_map Status::statusMessageMap = { {StatusCode::LLM_NODE_NAME_ALREADY_EXISTS, "The LLM Node name is already present in nodes list"}, {StatusCode::LLM_NODE_DIRECTORY_DOES_NOT_EXIST, "The LLM Node workspace path does not exist"}, {StatusCode::LLM_NODE_PATH_DOES_NOT_EXIST_AND_NOT_GGUFFILE, "The LLM Node workspace path does not exist and not gguf model file"}, + {StatusCode::LLM_NODE_GENERATION_CONFIG_DOES_NOT_EXIST, "The LLM Node generation_config_path does not point to an existing file"}, {StatusCode::LLM_NODE_RESOURCE_STATE_INITIALIZATION_FAILED, "The LLM Node resource initialization failed"}, {StatusCode::LLM_NODE_MISSING_OPTIONS, "The LLM Node is missing options definition"}, {StatusCode::LLM_NODE_MISSING_NAME, "The LLM Node is missing name definition"}, diff --git a/src/status.hpp b/src/status.hpp index 94be7948cb..924c989ea0 100644 --- a/src/status.hpp +++ b/src/status.hpp @@ -271,6 +271,7 @@ enum class StatusCode { LLM_NODE_NAME_ALREADY_EXISTS, LLM_NODE_DIRECTORY_DOES_NOT_EXIST, LLM_NODE_PATH_DOES_NOT_EXIST_AND_NOT_GGUFFILE, + LLM_NODE_GENERATION_CONFIG_DOES_NOT_EXIST, LLM_NODE_RESOURCE_STATE_INITIALIZATION_FAILED, LLM_NODE_MISSING_OPTIONS, LLM_NODE_MISSING_NAME, diff --git a/src/test/llm/llmnode_test.cpp b/src/test/llm/llmnode_test.cpp index 12d77d98d3..3c03e68763 100644 --- a/src/test/llm/llmnode_test.cpp +++ b/src/test/llm/llmnode_test.cpp @@ -51,6 +51,7 @@ #include "../../mediapipe_internal/mediapipegraphdefinition.hpp" #include "../../ov_utils.hpp" #include "../../server.hpp" +#include "src/filesystem/filesystem.hpp" #include "src/graph_export/graph_export.hpp" #include "rapidjson/document.h" #include "rapidjson/stringbuffer.h" @@ -4504,6 +4505,85 @@ TEST_F(LLMVLMOptionsHttpTest, LLMVLMNodeOptionsCheckPluginConfig) { LLMNodeOptionsCheckPluginConfig(modelsPath); } +// RAII guard removing a temporary directory tree on scope exit, so a failed ASSERT_* +// (which returns early from the test body) does not leak the directory. +struct TempDirGuard { + std::filesystem::path path; + explicit TempDirGuard(std::filesystem::path p) : + path(std::move(p)) {} + ~TempDirGuard() { + std::error_code ec; + std::filesystem::remove_all(path, ec); + } +}; + +// Unit test for the per-node generation_config.json path override (issue #4233). +TEST(LLMGenerationConfigPath, ResolveGenerationConfigPath) { + // Unique per-run directory so concurrent/parallel test runs cannot collide. + std::string tempPath; + ASSERT_EQ(ovms::FileSystem::createTempPath(&tempPath), ovms::StatusCode::OK); + std::filesystem::path base(tempPath); + TempDirGuard cleanup(base); + std::filesystem::path modelDir = base / "model"; + std::filesystem::path overrideDir = base / "overrides"; + std::filesystem::create_directories(modelDir); + std::filesystem::create_directories(overrideDir); + auto writeFile = [](const std::filesystem::path& p) { + std::ofstream ofs(p); + ofs << "{}"; + }; + writeFile(modelDir / "generation_config.json"); + writeFile(overrideDir / "custom_generation_config.json"); + // A GGUF "models_path" is a file, not a directory - relative overrides resolve against its parent. + writeFile(modelDir / "model.gguf"); + + // Case 1: no override -> default generation_config.json inside the model dir. + { + mediapipe::LLMCalculatorOptions nodeOptions; + std::string outPath; + ASSERT_EQ(ovms::resolveGenerationConfigPath(outPath, modelDir.string(), nodeOptions), ovms::StatusCode::OK); + ASSERT_EQ(std::filesystem::path(outPath), modelDir / "generation_config.json"); + } + // Case 2: explicit absolute override path. + { + mediapipe::LLMCalculatorOptions nodeOptions; + nodeOptions.set_generation_config_path((overrideDir / "custom_generation_config.json").string()); + std::string outPath; + ASSERT_EQ(ovms::resolveGenerationConfigPath(outPath, modelDir.string(), nodeOptions), ovms::StatusCode::OK); + ASSERT_EQ(std::filesystem::path(outPath), overrideDir / "custom_generation_config.json"); + } + // Case 3: explicit relative override path is resolved against models_path. + { + mediapipe::LLMCalculatorOptions nodeOptions; + nodeOptions.set_generation_config_path("generation_config.json"); + std::string outPath; + ASSERT_EQ(ovms::resolveGenerationConfigPath(outPath, modelDir.string(), nodeOptions), ovms::StatusCode::OK); + ASSERT_EQ(std::filesystem::path(outPath), modelDir / "generation_config.json"); + } + // Case 4: explicit override that does not exist -> dedicated error. + { + mediapipe::LLMCalculatorOptions nodeOptions; + nodeOptions.set_generation_config_path((overrideDir / "missing.json").string()); + std::string outPath; + ASSERT_EQ(ovms::resolveGenerationConfigPath(outPath, modelDir.string(), nodeOptions), ovms::StatusCode::LLM_NODE_GENERATION_CONFIG_DOES_NOT_EXIST); + } + // Case 5: explicit override pointing at a directory -> same error (not a regular file). + { + mediapipe::LLMCalculatorOptions nodeOptions; + nodeOptions.set_generation_config_path(overrideDir.string()); + std::string outPath; + ASSERT_EQ(ovms::resolveGenerationConfigPath(outPath, modelDir.string(), nodeOptions), ovms::StatusCode::LLM_NODE_GENERATION_CONFIG_DOES_NOT_EXIST); + } + // Case 6: models_path points at a GGUF file - relative override resolves against its parent dir. + { + mediapipe::LLMCalculatorOptions nodeOptions; + nodeOptions.set_generation_config_path("generation_config.json"); + std::string outPath; + ASSERT_EQ(ovms::resolveGenerationConfigPath(outPath, (modelDir / "model.gguf").string(), nodeOptions), ovms::StatusCode::OK); + ASSERT_EQ(std::filesystem::path(outPath), modelDir / "generation_config.json"); + } +} + // RAII guard that restores the global Config singleton (and optionally removes a temporary // cache directory) on scope exit. The cache_dir tests below mutate the process-wide Config // singleton; without this guard a failed ASSERT_* mid-test (which returns early) would leak