From 63a1de16e28464f7b56e3f6385dc9531c965a1a1 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sat, 30 May 2026 19:15:52 -0500 Subject: [PATCH 01/11] server : improve message span logic --- common/chat.cpp | 129 +++++++++++++++++++++----------- common/chat.h | 67 +++++++++++++++-- tests/test-chat.cpp | 107 +++++++++++++++++++------- tools/server/server-common.cpp | 10 +-- tools/server/server-context.cpp | 88 +++------------------- tools/server/server-task.cpp | 5 ++ tools/server/server-task.h | 7 +- 7 files changed, 247 insertions(+), 166 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index ded8440e6688..a5559c388928 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -90,41 +90,86 @@ std::string common_chat_msg::render_content(const std::string & delimiter) const return text; } -std::vector common_chat_split_by_role(const std::string & prompt, const std::vector & delims) { - if (delims.empty() || prompt.empty()) { - return {}; +common_chat_role common_chat_role_from_string(const std::string & role) { + if (role == "system") { return COMMON_CHAT_ROLE_SYSTEM; } + if (role == "assistant") { return COMMON_CHAT_ROLE_ASSISTANT; } + if (role == "user") { return COMMON_CHAT_ROLE_USER; } + if (role == "tool") { return COMMON_CHAT_ROLE_TOOL; } + return COMMON_CHAT_ROLE_UNKNOWN; +} + +const char * common_chat_role_to_string(common_chat_role role) { + switch (role) { + case COMMON_CHAT_ROLE_SYSTEM: return "system"; + case COMMON_CHAT_ROLE_ASSISTANT: return "assistant"; + case COMMON_CHAT_ROLE_USER: return "user"; + case COMMON_CHAT_ROLE_TOOL: return "tool"; + case COMMON_CHAT_ROLE_UNKNOWN: return ""; } + return ""; +} + +json common_chat_msg_delimiters::to_json() const { + json result = json::array(); + for (const auto & d : delimiters) { + result.push_back({ + { "role", common_chat_role_to_string(d.role) }, + { "delimiter", d.delimiter }, + }); + } + return result; +} - auto parser = build_peg_parser([&](common_peg_parser_builder & p) { - std::vector all_delims; - std::vector tagged_messages; +common_chat_msg_token_delimiters common_chat_msg_token_delimiters_parse(const llama_vocab * vocab, const json & delimiters) { + common_chat_msg_token_delimiters result; - all_delims.reserve(delims.size()); - tagged_messages.reserve(delims.size()); - for (const auto & d : delims) { - all_delims.push_back(d.delimiter); - } + if (!delimiters.is_array()) { + return result; + } - auto any_delim = p.until_one_of(all_delims); - for (const auto & d : delims) { - tagged_messages.push_back(p.tag(d.role, p.literal(d.delimiter) + any_delim)); + result.delimiters.reserve(delimiters.size()); + for (const auto & d : delimiters) { + if (!d.is_object()) { + continue; } + auto tokens = common_tokenize(vocab, d.value("delimiter", std::string()), false, true); + if (tokens.empty()) { + continue; + } + result.delimiters.push_back({ + common_chat_role_from_string(d.value("role", std::string())), + std::move(tokens), + }); + } - return any_delim + p.zero_or_more(p.choice(tagged_messages)) + p.end(); - }); + return result; +} - common_peg_parse_context ctx(prompt); - const auto result = parser.parse(ctx); - if (!result.success()) { - return {}; +common_chat_msg_spans common_chat_msg_token_delimiters::split(const llama_tokens & tokens) const { + std::vector> matches; + for (size_t i = 0; i < tokens.size(); i++) { + if (tokens[i] == LLAMA_TOKEN_NULL) { + continue; + } + for (const auto & d : delimiters) { + if (i + d.tokens.size() > tokens.size()) { + continue; + } + if (std::equal(d.tokens.begin(), d.tokens.end(), tokens.begin() + i)) { + matches.emplace_back(d.role, i); + break; + } + } } - std::vector spans; - ctx.ast.visit(result, [&](const common_peg_ast_node & node) { - if (!node.tag.empty()) { - spans.push_back({ node.tag, node.start, node.end - node.start }); - } - }); + matches.emplace_back(COMMON_CHAT_ROLE_UNKNOWN, tokens.size()); + + common_chat_msg_spans spans; + for (size_t i = 0; i + 1 < matches.size(); i++) { + const auto & curr = matches[i]; + const auto & next = matches[i + 1]; + spans.add(curr.first, curr.second, next.second - curr.second); + } return spans; } @@ -1081,13 +1126,13 @@ static common_chat_params common_chat_params_init_gpt_oss(const common_chat_temp data.prompt = prompt; data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override= */ adjusted_messages); - data.message_spans = common_chat_split_by_role(prompt, { - { "assistant", "<|start|>assistant" }, - { "user", "<|start|>user" }, - { "system", "<|start|>developer" }, - { "system", "<|start|>system" }, - { "tool", "<|start|>functions" }, - }); + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" }, + { COMMON_CHAT_ROLE_USER, "<|start|>user" }, + { COMMON_CHAT_ROLE_SYSTEM, "<|start|>developer" }, + { COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" }, + { COMMON_CHAT_ROLE_TOOL, "<|start|>functions" }, + }; data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; data.supports_thinking = true; @@ -1228,10 +1273,10 @@ static common_chat_params common_chat_params_init_gemma4(const common_chat_templ data.prompt += data.generation_prompt; } - data.message_spans = common_chat_split_by_role(data.prompt, { - { "user", "<|turn>user\n" }, - { "assistant", "<|turn>model\n" }, - }); + data.message_delimiters = { + { COMMON_CHAT_ROLE_USER, "<|turn>user\n" }, + { COMMON_CHAT_ROLE_ASSISTANT, "<|turn>model\n" }, + }; data.format = COMMON_CHAT_FORMAT_PEG_GEMMA4; data.supports_thinking = true; @@ -2526,17 +2571,15 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_ autoparser.analyze_template(tmpl); auto auto_params = autoparser::peg_generator::generate_parser(tmpl, params, autoparser); - std::vector delimiters; + common_chat_msg_delimiters delimiters; if (!autoparser.assistant_start.empty()) { - delimiters.push_back({ "assistant", autoparser.assistant_start }); + delimiters.add(COMMON_CHAT_ROLE_ASSISTANT, autoparser.assistant_start); } if (!autoparser.user_start.empty()) { - delimiters.push_back({ "user", autoparser.user_start }); + delimiters.add(COMMON_CHAT_ROLE_USER, autoparser.user_start); } - if (!delimiters.empty()) { - auto_params.message_spans = common_chat_split_by_role(auto_params.prompt, delimiters); - } + auto_params.message_delimiters = std::move(delimiters); auto_params.supports_thinking = autoparser.reasoning.mode != autoparser::reasoning_mode::NONE; if (auto_params.supports_thinking) { diff --git a/common/chat.h b/common/chat.h index 5659cd42a07c..e844dfb5ea8e 100644 --- a/common/chat.h +++ b/common/chat.h @@ -143,15 +143,71 @@ struct common_chat_msg_diff { } }; +enum common_chat_role { + COMMON_CHAT_ROLE_UNKNOWN, + COMMON_CHAT_ROLE_SYSTEM, + COMMON_CHAT_ROLE_ASSISTANT, + COMMON_CHAT_ROLE_USER, + COMMON_CHAT_ROLE_TOOL +}; + +common_chat_role common_chat_role_from_string(const std::string & role); +const char * common_chat_role_to_string(common_chat_role role); + struct common_chat_msg_span { - std::string role; + common_chat_role role = COMMON_CHAT_ROLE_UNKNOWN; std::size_t pos = 0; std::size_t len = 0; + + bool valid() const { + return role != COMMON_CHAT_ROLE_UNKNOWN; + } +}; + +struct common_chat_msg_spans { + std::vector spans; + + void add(common_chat_role role, size_t pos, size_t len) { + spans.push_back({ role, pos, len }); + } + + common_chat_msg_span last_user() const { + for (auto it = spans.rbegin(); it != spans.rend(); ++it) { + if (it->role == COMMON_CHAT_ROLE_USER) { + return *it; + } + } + return {}; + } }; struct common_chat_msg_delimiter { - std::string role; - std::string delimiter; + common_chat_role role = COMMON_CHAT_ROLE_UNKNOWN; + std::string delimiter; +}; + +struct common_chat_msg_token_delimiter { + common_chat_role role = COMMON_CHAT_ROLE_UNKNOWN; + llama_tokens tokens; +}; + +struct common_chat_msg_token_delimiters { + std::vector delimiters; + + common_chat_msg_spans split(const llama_tokens & tokens) const; +}; + +struct common_chat_msg_delimiters { + std::vector delimiters; + + common_chat_msg_delimiters() = default; + common_chat_msg_delimiters(std::initializer_list delims) : delimiters(delims) {} + + void add(common_chat_role role, const std::string & delimiter) { + delimiters.push_back({ role, delimiter }); + } + + nlohmann::ordered_json to_json() const; }; struct common_chat_tool { @@ -219,7 +275,7 @@ struct common_chat_params { std::vector preserved_tokens; std::vector additional_stops; std::string parser; - std::vector message_spans; + common_chat_msg_delimiters message_delimiters; }; // per-message parsing syntax @@ -325,5 +381,4 @@ struct common_chat_prompt_preset { common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates); -std::vector common_chat_split_by_role(const std::string & prompt, const std::vector & delims); - +common_chat_msg_token_delimiters common_chat_msg_token_delimiters_parse(const llama_vocab * vocab, const nlohmann::ordered_json & delimiters); diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 902a4c135abe..ecc76f64b45c 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -1562,37 +1562,90 @@ static void test_msgs_oaicompat_json_conversion() { } } -static void test_split_by_role() { +static void test_msg_token_delimiters_split() { LOG_DBG("%s\n", __func__); + // Delimiters that share a leading token, distinguished by the second token, + // to exercise the per-position token matching. + const common_chat_msg_token_delimiters delims = { + { { COMMON_CHAT_ROLE_USER, { 10, 11 } }, + { COMMON_CHAT_ROLE_ASSISTANT, { 10, 12 } } } + }; + // Empty inputs - assert_equals(0, common_chat_split_by_role("", {}).size()); - assert_equals(0, common_chat_split_by_role("hello", {}).size()); - assert_equals(0, common_chat_split_by_role("", { { "user", "<|user|>" } }).size()); + assert_equals(0, common_chat_msg_token_delimiters{}.split({}).spans.size()); + assert_equals(0, common_chat_msg_token_delimiters{}.split({ 10, 11 }).spans.size()); + assert_equals(0, delims.split({}).spans.size()); + + // No delimiters match -> no spans + assert_equals(0, delims.split({ 100, 101, 102 }).spans.size()); - // Multi-role conversation, no leading/trailing content + // Multi-role conversation: HiHelloBye { - const std::string prompt = "<|user|>Hi<|assistant|>Hello<|user|>Bye"; - const auto splits = common_chat_split_by_role(prompt, { - { "user", "<|user|>" }, - { "assistant", "<|assistant|>" }, - }); - assert_equals(3, splits.size()); - - assert_equals("user", splits[0].role); - assert_equals(0, splits[0].pos); - assert_equals(10, splits[0].len); - assert_equals("<|user|>Hi", prompt.substr(splits[0].pos, splits[0].len)); - - assert_equals("assistant", splits[1].role); - assert_equals(10, splits[1].pos); - assert_equals(18, splits[1].len); - assert_equals("<|assistant|>Hello", prompt.substr(splits[1].pos, splits[1].len)); - - assert_equals("user", splits[2].role); - assert_equals(28, splits[2].pos); - assert_equals(11, splits[2].len); - assert_equals("<|user|>Bye", prompt.substr(splits[2].pos, splits[2].len)); + const llama_tokens tokens = { + 10, 11, // + 100, 101, // Hi + 10, 12, // + 200, 201, 202, // Hello + 10, 11, // + 300, 301, // Bye + }; + + const auto result = delims.split(tokens); + const auto & spans = result.spans; + assert_equals(3, spans.size()); + + assert_equals(COMMON_CHAT_ROLE_USER, spans[0].role); + assert_equals(0, spans[0].pos); + assert_equals(4, spans[0].len); + + assert_equals(COMMON_CHAT_ROLE_ASSISTANT, spans[1].role); + assert_equals(4, spans[1].pos); + assert_equals(5, spans[1].len); + + assert_equals(COMMON_CHAT_ROLE_USER, spans[2].role); + assert_equals(9, spans[2].pos); + assert_equals(4, spans[2].len); + + // last_user() returns the most recent user span + assert_equals(COMMON_CHAT_ROLE_USER, result.last_user().role); + assert_equals(9, result.last_user().pos); + } + + // Content before the first delimiter is not captured as a span + { + const llama_tokens tokens = { + 500, 501, // leading content (dropped) + 10, 11, // + 100, // Hi + }; + + const auto spans = delims.split(tokens).spans; + assert_equals(1, spans.size()); + assert_equals(COMMON_CHAT_ROLE_USER, spans[0].role); + assert_equals(2, spans[0].pos); + assert_equals(3, spans[0].len); + } + + // LLAMA_TOKEN_NULL tokens cannot start a delimiter match but count as content + { + const llama_tokens tokens = { + 10, 11, // + LLAMA_TOKEN_NULL, // media placeholder + 100, // Hi + 10, 12, // + }; + + const auto spans = delims.split(tokens).spans; + assert_equals(2, spans.size()); + + assert_equals(COMMON_CHAT_ROLE_USER, spans[0].role); + assert_equals(0, spans[0].pos); + assert_equals(4, spans[0].len); + + assert_equals(COMMON_CHAT_ROLE_ASSISTANT, spans[1].role); + assert_equals(4, spans[1].pos); + assert_equals(2, spans[1].len); } } @@ -5857,7 +5910,7 @@ int main(int argc, char ** argv) { { test_msg_diffs_compute(); test_msgs_oaicompat_json_conversion(); - test_split_by_role(); + test_msg_token_delimiters_split(); test_tools_oaicompat_json_conversion(); test_convert_responses_to_chatcmpl(); test_developer_role_to_system_workaround(); diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index 75729e62dd68..0891e843fb98 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -1091,15 +1091,7 @@ json oaicompat_chat_params_parse( llama_params["chat_parser"] = chat_params.parser; } - llama_params["message_spans"] = json::array(); - - for (const auto & span : chat_params.message_spans) { - llama_params["message_spans"].push_back({ - { "role", span.role }, - { "pos", span.pos }, - { "len", span.len }, - }); - } + llama_params["message_delimiters"] = chat_params.message_delimiters.to_json(); // Reasoning budget: pass parameters through to sampling layer { diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index bcae39a10966..5ff135227c9a 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3098,8 +3098,9 @@ struct server_context_impl { has_mtmd = true; } - const int32_t n_before_user = slot.task->params.n_before_user; - const bool n_before_user_known = n_before_user > 0; + // find span for last user message + const auto & spans = slot.task->params.message_spans; + const auto span_last_user = spans.last_user(); // add prompt tokens for processing in the current batch while (slot.prompt.n_tokens() < slot.task->n_tokens() && batch.n_tokens < n_batch) { @@ -3131,8 +3132,7 @@ struct server_context_impl { // stop the prompt batch exactly before the latest user input, so a checkpoint // can be created after the previous messages - if (n_before_user_known && - slot.prompt.n_tokens() == n_before_user) { + if (span_last_user.valid() && slot.prompt.n_tokens() == span_last_user.pos) { break; } @@ -3178,7 +3178,7 @@ struct server_context_impl { slot.init_sampler(); } else { // skip ordinary mid-prompt checkpoints - if (!n_before_user_known && !near_prompt_end) { + if (!span_last_user.valid() && !near_prompt_end) { do_checkpoint = false; } } @@ -3190,20 +3190,10 @@ struct server_context_impl { // their token position is the batch start rather than the prompt end const int32_t n_tokens_start = slot.prompt.n_tokens() - n_tokens_cur; - { - const bool is_on_user = - n_before_user_known && - n_tokens_start == n_before_user; - - const bool is_after_user = - n_before_user_known && - n_tokens_start > n_before_user; - - const bool is_allowed = - !n_before_user_known || - is_on_user || - (is_after_user && near_prompt_end); - + if (span_last_user.valid()) { + const bool is_on_user = n_tokens_start == span_last_user.pos; + const bool is_after_user = n_tokens_start > span_last_user.pos; + const bool is_allowed = is_on_user || (is_after_user && near_prompt_end); if (do_checkpoint && !is_allowed) { do_checkpoint = false; } @@ -3695,54 +3685,6 @@ void server_context::on_sleeping_changed(std::function callback) { impl->queue_tasks.on_sleeping_state(std::move(callback)); } -// compute the number of tokens before the last user message in the prompt -static int32_t prompt_get_n_before_user( - const json & message_spans, - const std::string & prompt, - const std::vector & files, - const llama_vocab * vocab, - mtmd_context * mctx) { - int32_t result = -1; - int32_t byte_pos = -1; - - for (const auto & span : message_spans) { - const std::string role = json_value(span, "role", std::string()); - - if (role == "user") { - byte_pos = json_value(span, "pos", -1); - } - } - - if (byte_pos >= 0) { - GGML_ASSERT((size_t) byte_pos <= prompt.size()); - - const std::string prefix = prompt.substr(0, (size_t) byte_pos); - - const std::string marker = get_media_marker(); - size_t n_prefix_media = 0; - for (size_t pos = 0; (pos = prefix.find(marker, pos)) != std::string::npos; pos += marker.size()) { - n_prefix_media++; - } - - GGML_ASSERT(n_prefix_media <= files.size()); - - if (mctx != nullptr && n_prefix_media > 0) { - // TODO: this makes a copy - avoid it - std::vector prefix_files(files.begin(), files.begin() + n_prefix_media); - - result = (int32_t) process_mtmd_prompt(mctx, prefix, prefix_files).size(); - } else { - result = (int32_t) tokenize_input_prompts(vocab, nullptr, prefix, true, true)[0].size(); - } - - SRV_TRC("message_spans: last user message: byte_pos=%d, media=%zu, n_before_user=%d\n", - byte_pos, n_prefix_media, result); - } - - return result; -} - - // // server_routes // @@ -3797,23 +3739,13 @@ std::unique_ptr server_routes::handle_completions_impl( task.tokens = std::move(inputs[i]); task.params = server_task::params_from_json_cmpl( + task.tokens.get_tokens(), ctx_server.vocab, params, meta->slot_n_ctx, meta->logit_bias_eog, data); - const auto message_spans = json_value(data, "message_spans", json::array()); - if (prompt.is_string() && message_spans.is_array()) { - task.params.n_before_user = - prompt_get_n_before_user( - message_spans, - prompt.get(), - files, - ctx_server.vocab, - ctx_server.mctx); - } - task.id_slot = json_value(data, "id_slot", -1); // OAI-compat diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index 72a4bd076ad7..dbc5683cd4ab 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -237,6 +237,7 @@ common_chat_msg task_result_state::update_chat_msg( // task_params server_task::params_from_json_cmpl( + const llama_tokens & tokens, const llama_vocab * vocab, const common_params & params_base, const int n_ctx_slot, @@ -614,6 +615,10 @@ task_params server_task::params_from_json_cmpl( } } + // message spans for checkpointing + auto delimiters = common_chat_msg_token_delimiters_parse(vocab, json_value(data, "message_delimiters", json::array())); + params.message_spans = delimiters.split(tokens); + if (params.n_cmpl > params_base.n_parallel) { throw std::runtime_error("n_cmpl cannot be greater than the number of slots, please increase -np"); } diff --git a/tools/server/server-task.h b/tools/server/server-task.h index bdadcff76527..b0d9028d534a 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -62,9 +62,6 @@ struct task_params { int32_t n_cache_reuse = 0; // min chunk size to attempt reusing from the cache via KV shifting (0 = disabled) - // number of prompt tokens before the latest user message - int32_t n_before_user = -1; - int64_t t_max_prompt_ms = -1; // TODO: implement int64_t t_max_predict_ms = -1; // if positive, limit the generation phase to this time limit @@ -92,6 +89,9 @@ struct task_params { // per-request parameters for chat parsing common_chat_parser_params chat_parser_params; + // spans for message-aware checkpointing + common_chat_msg_spans message_spans; + // Embeddings int32_t embd_normalize = 2; // (-1=none, 0=max absolute int16, 1=taxicab, 2=Euclidean/L2, >2=p-norm) @@ -211,6 +211,7 @@ struct server_task { } static task_params params_from_json_cmpl( + const llama_tokens & tokens, const llama_vocab * vocab, const common_params & params_base, const int n_ctx_slot, From d26756c3001121aa739e7c5c260a483cbd95bfac Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Sun, 31 May 2026 16:12:56 -0500 Subject: [PATCH 02/11] cont : cast size_t to int32_t in comparisons --- tools/server/server-context.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 5ff135227c9a..4d047642c0d5 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3132,7 +3132,7 @@ struct server_context_impl { // stop the prompt batch exactly before the latest user input, so a checkpoint // can be created after the previous messages - if (span_last_user.valid() && slot.prompt.n_tokens() == span_last_user.pos) { + if (span_last_user.valid() && slot.prompt.n_tokens() == (int32_t) span_last_user.pos) { break; } @@ -3191,8 +3191,8 @@ struct server_context_impl { const int32_t n_tokens_start = slot.prompt.n_tokens() - n_tokens_cur; if (span_last_user.valid()) { - const bool is_on_user = n_tokens_start == span_last_user.pos; - const bool is_after_user = n_tokens_start > span_last_user.pos; + const bool is_on_user = n_tokens_start == (int32_t) span_last_user.pos; + const bool is_after_user = n_tokens_start > (int32_t) span_last_user.pos; const bool is_allowed = is_on_user || (is_after_user && near_prompt_end); if (do_checkpoint && !is_allowed) { do_checkpoint = false; From 096942264cb55d51b2f2951c9f42985139415290 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 5 Jun 2026 05:57:35 -0500 Subject: [PATCH 03/11] server : create checkpoints before every user msg --- common/chat.h | 10 +++++----- tools/server/server-context.cpp | 29 +++++++++-------------------- 2 files changed, 14 insertions(+), 25 deletions(-) diff --git a/common/chat.h b/common/chat.h index e844dfb5ea8e..721a18fc969d 100644 --- a/common/chat.h +++ b/common/chat.h @@ -171,13 +171,13 @@ struct common_chat_msg_spans { spans.push_back({ role, pos, len }); } - common_chat_msg_span last_user() const { - for (auto it = spans.rbegin(); it != spans.rend(); ++it) { - if (it->role == COMMON_CHAT_ROLE_USER) { - return *it; + bool is_user_start(int32_t pos) const { + for (auto it = spans.begin(); it != spans.end(); ++it) { + if (it->role == COMMON_CHAT_ROLE_USER && pos == (int32_t) it->pos) { + return true; } } - return {}; + return false; } }; diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 4d047642c0d5..ed323e493c8c 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3098,9 +3098,7 @@ struct server_context_impl { has_mtmd = true; } - // find span for last user message const auto & spans = slot.task->params.message_spans; - const auto span_last_user = spans.last_user(); // add prompt tokens for processing in the current batch while (slot.prompt.n_tokens() < slot.task->n_tokens() && batch.n_tokens < n_batch) { @@ -3130,9 +3128,8 @@ struct server_context_impl { slot.n_prompt_tokens_processed++; - // stop the prompt batch exactly before the latest user input, so a checkpoint - // can be created after the previous messages - if (span_last_user.valid() && slot.prompt.n_tokens() == (int32_t) span_last_user.pos) { + // stop the prompt batch exactly before a user message + if (spans.is_user_start(slot.prompt.n_tokens())) { break; } @@ -3161,8 +3158,12 @@ struct server_context_impl { // the number of tokens added to the batch for the current slot const auto n_tokens_cur = batch.n_tokens - n_tokens_prev; + const auto n_tokens_start = slot.prompt.n_tokens() - n_tokens_cur; + const bool near_prompt_end = slot.task->n_tokens() < slot.prompt.n_tokens() + n_ubatch; + const bool is_user_start = spans.is_user_start(n_tokens_start); + // entire prompt has been processed if (slot.prompt.n_tokens() == slot.task->n_tokens()) { slot.state = SLOT_STATE_DONE_PROMPT; @@ -3177,8 +3178,9 @@ struct server_context_impl { slot.init_sampler(); } else { - // skip ordinary mid-prompt checkpoints - if (!span_last_user.valid() && !near_prompt_end) { + // skip ordinary mid-prompt checkpoints, unless the batch starts a user + // message or we are near the end of the prompt + if (!is_user_start && !near_prompt_end) { do_checkpoint = false; } } @@ -3186,19 +3188,6 @@ struct server_context_impl { const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot.id); const auto pos_max = llama_memory_seq_pos_max(llama_get_memory(ctx_tgt), slot.id); - // checkpoints are created before the current batch is decoded, so - // their token position is the batch start rather than the prompt end - const int32_t n_tokens_start = slot.prompt.n_tokens() - n_tokens_cur; - - if (span_last_user.valid()) { - const bool is_on_user = n_tokens_start == (int32_t) span_last_user.pos; - const bool is_after_user = n_tokens_start > (int32_t) span_last_user.pos; - const bool is_allowed = is_on_user || (is_after_user && near_prompt_end); - if (do_checkpoint && !is_allowed) { - do_checkpoint = false; - } - } - // nothing to checkpoint yet // TODO: is this check needed? if (do_checkpoint && pos_min < 0) { From 30a1c4c11e91e7046a7fa83ef219c7179f321371 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 5 Jun 2026 05:58:24 -0500 Subject: [PATCH 04/11] chat : remove \n in gemma4 delimiters --- common/chat.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index a5559c388928..a843e87b6059 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1274,8 +1274,8 @@ static common_chat_params common_chat_params_init_gemma4(const common_chat_templ } data.message_delimiters = { - { COMMON_CHAT_ROLE_USER, "<|turn>user\n" }, - { COMMON_CHAT_ROLE_ASSISTANT, "<|turn>model\n" }, + { COMMON_CHAT_ROLE_USER, "<|turn>user" }, + { COMMON_CHAT_ROLE_ASSISTANT, "<|turn>model" }, }; data.format = COMMON_CHAT_FORMAT_PEG_GEMMA4; From d9e4f9b5673bb7053fe3e3352a330921898f6460 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 5 Jun 2026 06:12:10 -0500 Subject: [PATCH 05/11] chat : merge msg delimiter structs into one --- common/chat.cpp | 19 +++++++++++-------- common/chat.h | 18 ++++++------------ tests/test-chat.cpp | 17 +++++++++-------- tools/server/server-task.cpp | 3 ++- 4 files changed, 28 insertions(+), 29 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index a843e87b6059..b98e276a8868 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -120,8 +120,8 @@ json common_chat_msg_delimiters::to_json() const { return result; } -common_chat_msg_token_delimiters common_chat_msg_token_delimiters_parse(const llama_vocab * vocab, const json & delimiters) { - common_chat_msg_token_delimiters result; +common_chat_msg_delimiters common_chat_msg_delimiters_parse(const json & delimiters) { + common_chat_msg_delimiters result; if (!delimiters.is_array()) { return result; @@ -132,20 +132,23 @@ common_chat_msg_token_delimiters common_chat_msg_token_delimiters_parse(const ll if (!d.is_object()) { continue; } - auto tokens = common_tokenize(vocab, d.value("delimiter", std::string()), false, true); - if (tokens.empty()) { - continue; - } result.delimiters.push_back({ common_chat_role_from_string(d.value("role", std::string())), - std::move(tokens), + d.value("delimiter", std::string()), + {}, }); } return result; } -common_chat_msg_spans common_chat_msg_token_delimiters::split(const llama_tokens & tokens) const { +void common_chat_msg_delimiters::tokenize(const llama_vocab * vocab) { + for (auto & d : delimiters) { + d.tokens = common_tokenize(vocab, d.delimiter, false, true); + } +} + +common_chat_msg_spans common_chat_msg_delimiters::split(const llama_tokens & tokens) const { std::vector> matches; for (size_t i = 0; i < tokens.size(); i++) { if (tokens[i] == LLAMA_TOKEN_NULL) { diff --git a/common/chat.h b/common/chat.h index 721a18fc969d..04aa83f21601 100644 --- a/common/chat.h +++ b/common/chat.h @@ -184,19 +184,9 @@ struct common_chat_msg_spans { struct common_chat_msg_delimiter { common_chat_role role = COMMON_CHAT_ROLE_UNKNOWN; std::string delimiter; -}; - -struct common_chat_msg_token_delimiter { - common_chat_role role = COMMON_CHAT_ROLE_UNKNOWN; llama_tokens tokens; }; -struct common_chat_msg_token_delimiters { - std::vector delimiters; - - common_chat_msg_spans split(const llama_tokens & tokens) const; -}; - struct common_chat_msg_delimiters { std::vector delimiters; @@ -204,9 +194,13 @@ struct common_chat_msg_delimiters { common_chat_msg_delimiters(std::initializer_list delims) : delimiters(delims) {} void add(common_chat_role role, const std::string & delimiter) { - delimiters.push_back({ role, delimiter }); + delimiters.push_back({ role, delimiter, {} }); } + void tokenize(const llama_vocab * vocab); + + common_chat_msg_spans split(const llama_tokens & tokens) const; + nlohmann::ordered_json to_json() const; }; @@ -381,4 +375,4 @@ struct common_chat_prompt_preset { common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates); -common_chat_msg_token_delimiters common_chat_msg_token_delimiters_parse(const llama_vocab * vocab, const nlohmann::ordered_json & delimiters); +common_chat_msg_delimiters common_chat_msg_delimiters_parse(const nlohmann::ordered_json & delimiters); diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index ecc76f64b45c..ed638011f5a5 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -1567,14 +1567,14 @@ static void test_msg_token_delimiters_split() { // Delimiters that share a leading token, distinguished by the second token, // to exercise the per-position token matching. - const common_chat_msg_token_delimiters delims = { - { { COMMON_CHAT_ROLE_USER, { 10, 11 } }, - { COMMON_CHAT_ROLE_ASSISTANT, { 10, 12 } } } + const common_chat_msg_delimiters delims = { + { { COMMON_CHAT_ROLE_USER, "", { 10, 11 } }, + { COMMON_CHAT_ROLE_ASSISTANT, "", { 10, 12 } } } }; // Empty inputs - assert_equals(0, common_chat_msg_token_delimiters{}.split({}).spans.size()); - assert_equals(0, common_chat_msg_token_delimiters{}.split({ 10, 11 }).spans.size()); + assert_equals(0, common_chat_msg_delimiters{}.split({}).spans.size()); + assert_equals(0, common_chat_msg_delimiters{}.split({ 10, 11 }).spans.size()); assert_equals(0, delims.split({}).spans.size()); // No delimiters match -> no spans @@ -1607,9 +1607,10 @@ static void test_msg_token_delimiters_split() { assert_equals(9, spans[2].pos); assert_equals(4, spans[2].len); - // last_user() returns the most recent user span - assert_equals(COMMON_CHAT_ROLE_USER, result.last_user().role); - assert_equals(9, result.last_user().pos); + // is_user_start() is true at the token position where a user span begins + assert_equals(true, result.is_user_start(0)); + assert_equals(false, result.is_user_start(4)); // assistant span + assert_equals(true, result.is_user_start(9)); } // Content before the first delimiter is not captured as a span diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index dbc5683cd4ab..fede6735a5c3 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -616,7 +616,8 @@ task_params server_task::params_from_json_cmpl( } // message spans for checkpointing - auto delimiters = common_chat_msg_token_delimiters_parse(vocab, json_value(data, "message_delimiters", json::array())); + auto delimiters = common_chat_msg_delimiters_parse(json_value(data, "message_delimiters", json::array())); + delimiters.tokenize(vocab); params.message_spans = delimiters.split(tokens); if (params.n_cmpl > params_base.n_parallel) { From 9367e2e5dcb8669ddec9d39a7d365b1101d0a56b Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 5 Jun 2026 06:16:05 -0500 Subject: [PATCH 06/11] cont : reword comment --- tools/server/server-task.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/server/server-task.h b/tools/server/server-task.h index b0d9028d534a..2c261efabd10 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -89,7 +89,7 @@ struct task_params { // per-request parameters for chat parsing common_chat_parser_params chat_parser_params; - // spans for message-aware checkpointing + // message spans for checkpointing common_chat_msg_spans message_spans; // Embeddings From f72e7f4fc65f472a4a1fca78fa16d4b3cfdb137c Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 5 Jun 2026 06:49:19 -0500 Subject: [PATCH 07/11] cont : initialize tokens in delimiter --- common/chat.cpp | 1 - common/chat.h | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index b98e276a8868..8fe4af4ab499 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -135,7 +135,6 @@ common_chat_msg_delimiters common_chat_msg_delimiters_parse(const json & delimit result.delimiters.push_back({ common_chat_role_from_string(d.value("role", std::string())), d.value("delimiter", std::string()), - {}, }); } diff --git a/common/chat.h b/common/chat.h index 04aa83f21601..82a3635fb4d2 100644 --- a/common/chat.h +++ b/common/chat.h @@ -184,7 +184,7 @@ struct common_chat_msg_spans { struct common_chat_msg_delimiter { common_chat_role role = COMMON_CHAT_ROLE_UNKNOWN; std::string delimiter; - llama_tokens tokens; + llama_tokens tokens = {}; }; struct common_chat_msg_delimiters { @@ -194,7 +194,7 @@ struct common_chat_msg_delimiters { common_chat_msg_delimiters(std::initializer_list delims) : delimiters(delims) {} void add(common_chat_role role, const std::string & delimiter) { - delimiters.push_back({ role, delimiter, {} }); + delimiters.push_back({ role, delimiter }); } void tokenize(const llama_vocab * vocab); From 5e946f3322258bdc0d6ff4e3780f0652d2a187bc Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Wed, 10 Jun 2026 04:50:30 -0500 Subject: [PATCH 08/11] cont : add server_tokens::get_raw_tokens() for mtmd --- tools/server/server-common.cpp | 4 ++++ tools/server/server-common.h | 3 +++ tools/server/server-context.cpp | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index 0891e843fb98..fe5920e91da5 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -407,6 +407,10 @@ const llama_tokens & server_tokens::get_tokens() const { return tokens; } +const llama_tokens & server_tokens::get_raw_tokens() const { + return tokens; +} + llama_tokens server_tokens::get_text_tokens() const { llama_tokens res; res.reserve(tokens.size()); diff --git a/tools/server/server-common.h b/tools/server/server-common.h index f286b3d15688..95168d6e68ca 100644 --- a/tools/server/server-common.h +++ b/tools/server/server-common.h @@ -198,6 +198,9 @@ struct server_tokens { // for compatibility with speculative decoding, ctx shift, slot save/load const llama_tokens & get_tokens() const; + // tokens including LLAMA_TOKEN_NULL placeholders for media chunks + const llama_tokens & get_raw_tokens() const; + llama_tokens get_text_tokens() const; // for compatibility with speculative decoding diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index ed323e493c8c..1556d25a449c 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3728,7 +3728,7 @@ std::unique_ptr server_routes::handle_completions_impl( task.tokens = std::move(inputs[i]); task.params = server_task::params_from_json_cmpl( - task.tokens.get_tokens(), + task.tokens.get_raw_tokens(), ctx_server.vocab, params, meta->slot_n_ctx, From 504d8a2fa04b8651ada00ce25a24790084ef0010 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Mon, 15 Jun 2026 22:08:00 -0500 Subject: [PATCH 09/11] cont : move message finding to server_tokens and skip mtmd tokens --- common/chat.cpp | 11 ++++++++--- common/chat.h | 3 ++- tests/test-chat.cpp | 31 ++++++++++++++++++++++++++----- tools/server/server-common.cpp | 12 ++++++++---- tools/server/server-common.h | 6 +++--- tools/server/server-context.cpp | 7 ++++++- tools/server/server-task.cpp | 6 ------ tools/server/server-task.h | 1 - 8 files changed, 53 insertions(+), 24 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 8fe4af4ab499..0a58ea5a9f01 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -147,10 +147,14 @@ void common_chat_msg_delimiters::tokenize(const llama_vocab * vocab) { } } -common_chat_msg_spans common_chat_msg_delimiters::split(const llama_tokens & tokens) const { +common_chat_msg_spans common_chat_msg_delimiters::split(const llama_tokens & tokens, const std::map & skips) const { std::vector> matches; - for (size_t i = 0; i < tokens.size(); i++) { - if (tokens[i] == LLAMA_TOKEN_NULL) { + + auto skip = skips.begin(); + for (size_t i = 0; i < tokens.size();) { + if (skip != skips.end() && i == skip->first) { + i += skip->second; + ++skip; continue; } for (const auto & d : delimiters) { @@ -162,6 +166,7 @@ common_chat_msg_spans common_chat_msg_delimiters::split(const llama_tokens & tok break; } } + i++; } matches.emplace_back(COMMON_CHAT_ROLE_UNKNOWN, tokens.size()); diff --git a/common/chat.h b/common/chat.h index 82a3635fb4d2..ec47e6bf4c43 100644 --- a/common/chat.h +++ b/common/chat.h @@ -199,7 +199,8 @@ struct common_chat_msg_delimiters { void tokenize(const llama_vocab * vocab); - common_chat_msg_spans split(const llama_tokens & tokens) const; + // split tokens into message spans. skips maps a start index to a length of a region to jump over without matching + common_chat_msg_spans split(const llama_tokens & tokens, const std::map & skips = {}) const; nlohmann::ordered_json to_json() const; }; diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index ed638011f5a5..fa790cb4b049 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -1628,26 +1628,47 @@ static void test_msg_token_delimiters_split() { assert_equals(3, spans[0].len); } - // LLAMA_TOKEN_NULL tokens cannot start a delimiter match but count as content + // Skipped regions (media chunks) are jumped over but still count as span content { const llama_tokens tokens = { 10, 11, // - LLAMA_TOKEN_NULL, // media placeholder + LLAMA_TOKEN_NULL, // media chunk (3 tokens) + LLAMA_TOKEN_NULL, + LLAMA_TOKEN_NULL, 100, // Hi 10, 12, // }; - const auto spans = delims.split(tokens).spans; + const std::map skips = { { 2, 3 } }; + + const auto spans = delims.split(tokens, skips).spans; assert_equals(2, spans.size()); assert_equals(COMMON_CHAT_ROLE_USER, spans[0].role); assert_equals(0, spans[0].pos); - assert_equals(4, spans[0].len); + assert_equals(6, spans[0].len); assert_equals(COMMON_CHAT_ROLE_ASSISTANT, spans[1].role); - assert_equals(4, spans[1].pos); + assert_equals(6, spans[1].pos); assert_equals(2, spans[1].len); } + + // A delimiter sequence inside a skipped region is not matched + { + const llama_tokens tokens = { + 10, 11, // + 10, 12, // skipped region that happens to contain delimiter tokens + 100, // Hi + }; + + const std::map skips = { { 2, 2 } }; + + const auto spans = delims.split(tokens, skips).spans; + assert_equals(1, spans.size()); + assert_equals(COMMON_CHAT_ROLE_USER, spans[0].role); + assert_equals(0, spans[0].pos); + assert_equals(5, spans[0].len); + } } static void test_tools_oaicompat_json_conversion() { diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index fe5920e91da5..a60b9b3c1371 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -407,10 +407,6 @@ const llama_tokens & server_tokens::get_tokens() const { return tokens; } -const llama_tokens & server_tokens::get_raw_tokens() const { - return tokens; -} - llama_tokens server_tokens::get_text_tokens() const { llama_tokens res; res.reserve(tokens.size()); @@ -521,6 +517,14 @@ size_t server_tokens::get_common_prefix(const server_tokens & b) const { return max_idx; // all tokens are equal } +common_chat_msg_spans server_tokens::find_message_spans(const common_chat_msg_delimiters & delims) const { + std::map skips; + for (const auto & it : map_idx_to_media) { + skips[it.first] = mtmd_input_chunk_get_n_tokens(it.second.get()); + } + return delims.split(tokens, skips); +} + bool server_tokens::validate(const struct llama_context * ctx) const { const llama_model * model = llama_get_model(ctx); const llama_vocab * vocab = llama_model_get_vocab(model); diff --git a/tools/server/server-common.h b/tools/server/server-common.h index 95168d6e68ca..400c048d6503 100644 --- a/tools/server/server-common.h +++ b/tools/server/server-common.h @@ -198,9 +198,6 @@ struct server_tokens { // for compatibility with speculative decoding, ctx shift, slot save/load const llama_tokens & get_tokens() const; - // tokens including LLAMA_TOKEN_NULL placeholders for media chunks - const llama_tokens & get_raw_tokens() const; - llama_tokens get_text_tokens() const; // for compatibility with speculative decoding @@ -221,6 +218,9 @@ struct server_tokens { size_t get_common_prefix(const server_tokens & b) const; + // split the tokens into message spans, skipping over media chunks + common_chat_msg_spans find_message_spans(const common_chat_msg_delimiters & delims) const; + // make sure all text tokens are within the vocab range bool validate(const struct llama_context * ctx) const; diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 1556d25a449c..189a17cc22a3 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3721,6 +3721,10 @@ std::unique_ptr server_routes::handle_completions_impl( // tasks.reserve(inputs.size()); // TODO: this is inaccurate due to child tasks + // message delimiters for checkpointing + auto delimiters = common_chat_msg_delimiters_parse(json_value(data, "message_delimiters", json::array())); + delimiters.tokenize(ctx_server.vocab); + for (size_t i = 0; i < inputs.size(); i++) { server_task task = server_task(type); @@ -3728,13 +3732,14 @@ std::unique_ptr server_routes::handle_completions_impl( task.tokens = std::move(inputs[i]); task.params = server_task::params_from_json_cmpl( - task.tokens.get_raw_tokens(), ctx_server.vocab, params, meta->slot_n_ctx, meta->logit_bias_eog, data); + task.params.message_spans = task.tokens.find_message_spans(delimiters); + task.id_slot = json_value(data, "id_slot", -1); // OAI-compat diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index fede6735a5c3..72a4bd076ad7 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -237,7 +237,6 @@ common_chat_msg task_result_state::update_chat_msg( // task_params server_task::params_from_json_cmpl( - const llama_tokens & tokens, const llama_vocab * vocab, const common_params & params_base, const int n_ctx_slot, @@ -615,11 +614,6 @@ task_params server_task::params_from_json_cmpl( } } - // message spans for checkpointing - auto delimiters = common_chat_msg_delimiters_parse(json_value(data, "message_delimiters", json::array())); - delimiters.tokenize(vocab); - params.message_spans = delimiters.split(tokens); - if (params.n_cmpl > params_base.n_parallel) { throw std::runtime_error("n_cmpl cannot be greater than the number of slots, please increase -np"); } diff --git a/tools/server/server-task.h b/tools/server/server-task.h index 2c261efabd10..a96349a5c776 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -211,7 +211,6 @@ struct server_task { } static task_params params_from_json_cmpl( - const llama_tokens & tokens, const llama_vocab * vocab, const common_params & params_base, const int n_ctx_slot, From fd324b1e037034f601dc3b6e0773a01748b7335b Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Mon, 15 Jun 2026 22:19:42 -0500 Subject: [PATCH 10/11] cont : update cohere2moe parser --- common/chat.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index 0a58ea5a9f01..cee6ad650ac0 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -2082,15 +2082,15 @@ static common_chat_params common_chat_params_init_cohere2moe(const common_chat_t RESULT_START, RESULT_END, }; - // Split the rendered prompt into per-role message spans. Tool results are rendered with the + // Declare per-role message delimiters. Tool results are rendered with the // system token followed by <|START_TOOL_RESULT|>, so the "tool" delimiter must be listed before // the plain "system" one (it is a strict superset, and the role split tries delimiters in order). - data.message_spans = common_chat_split_by_role(data.prompt, { - { "assistant", GEN_PREFIX }, - { "user", TURN_START + USER }, - { "tool", TURN_START + SYSTEM + RESULT_START }, - { "system", TURN_START + SYSTEM }, - }); + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, GEN_PREFIX }, + { COMMON_CHAT_ROLE_USER, TURN_START + USER }, + { COMMON_CHAT_ROLE_TOOL, TURN_START + SYSTEM + RESULT_START }, + { COMMON_CHAT_ROLE_SYSTEM, TURN_START + SYSTEM }, + }; auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; From 93f97007612674406e76d6eafbb4a5c2ba1e89a3 Mon Sep 17 00:00:00 2001 From: Alde Rojas Date: Fri, 19 Jun 2026 20:44:04 -0500 Subject: [PATCH 11/11] cont : increase min-step to 8192 and always produce a chkpt for last user message --- common/chat.h | 9 +++++++++ common/common.h | 2 +- tools/server/server-context.cpp | 6 ++++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/common/chat.h b/common/chat.h index ec47e6bf4c43..7898f1623f54 100644 --- a/common/chat.h +++ b/common/chat.h @@ -179,6 +179,15 @@ struct common_chat_msg_spans { } return false; } + + int32_t last_user_message_pos() const { + for (auto it = spans.rbegin(); it != spans.rend(); ++it) { + if (it->role == COMMON_CHAT_ROLE_USER) { + return (int32_t) it->pos; + } + } + return -1; + } }; struct common_chat_msg_delimiter { diff --git a/common/common.h b/common/common.h index 0b284cbb36c7..0c0a75279e34 100644 --- a/common/common.h +++ b/common/common.h @@ -600,7 +600,7 @@ struct common_params { bool cache_prompt = true; // whether to enable prompt caching bool cache_idle_slots = true; // save and clear idle slots upon starting a new task int32_t n_ctx_checkpoints = 32; // max number of context checkpoints per slot - int32_t checkpoint_min_step = 256; // minimum spacing between context checkpoints + int32_t checkpoint_min_step = 8192; // minimum spacing between context checkpoints int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc. std::string hostname = "127.0.0.1"; diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 189a17cc22a3..9fcbbc910227 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3099,6 +3099,7 @@ struct server_context_impl { } const auto & spans = slot.task->params.message_spans; + const auto last_user_pos = spans.last_user_message_pos(); // add prompt tokens for processing in the current batch while (slot.prompt.n_tokens() < slot.task->n_tokens() && batch.n_tokens < n_batch) { @@ -3163,6 +3164,7 @@ struct server_context_impl { const bool near_prompt_end = slot.task->n_tokens() < slot.prompt.n_tokens() + n_ubatch; const bool is_user_start = spans.is_user_start(n_tokens_start); + const bool is_last_user_message = n_tokens_start == last_user_pos; // entire prompt has been processed if (slot.prompt.n_tokens() == slot.task->n_tokens()) { @@ -3197,8 +3199,8 @@ struct server_context_impl { // do not checkpoint after mtmd chunks do_checkpoint = do_checkpoint && !has_mtmd; - // no need to create checkpoints that are too close together - do_checkpoint = do_checkpoint && (slot.prompt.checkpoints.empty() || n_tokens_start > slot.prompt.checkpoints.back().n_tokens + params_base.checkpoint_min_step); + // no need to create checkpoints that are too close together, unless it's the last user message + do_checkpoint = do_checkpoint && (slot.prompt.checkpoints.empty() || is_last_user_message || n_tokens_start > slot.prompt.checkpoints.back().n_tokens + params_base.checkpoint_min_step); SLT_DBG(slot, "main/do_checkpoint = %s, pos_min = %d, pos_max = %d\n", do_checkpoint ? "yes" : "no", pos_min, pos_max); // note: we create the checkpoint before calling llama_decode(), so the current batch is not