diff --git a/codex-rs/core/src/context/token_budget_context.rs b/codex-rs/core/src/context/token_budget_context.rs index 55a94b2b11b0..3b48aeaa722a 100644 --- a/codex-rs/core/src/context/token_budget_context.rs +++ b/codex-rs/core/src/context/token_budget_context.rs @@ -1,5 +1,7 @@ use super::ContextualUserFragment; use codex_protocol::ThreadId; +use codex_protocol::protocol::CONTEXT_WINDOW_CLOSE_TAG; +use codex_protocol::protocol::CONTEXT_WINDOW_OPEN_TAG; use uuid::Uuid; #[derive(Debug, Clone, PartialEq, Eq)] @@ -39,7 +41,7 @@ impl ContextualUserFragment for TokenBudgetContext { } fn type_markers() -> (&'static str, &'static str) { - ("", "") + (CONTEXT_WINDOW_OPEN_TAG, CONTEXT_WINDOW_CLOSE_TAG) } fn body(&self) -> String { @@ -47,7 +49,7 @@ impl ContextualUserFragment for TokenBudgetContext { let first_window_id = self.first_window_id; let window_id = self.window_id; let mut lines = vec![ - format!("Thread id {thread_id}."), + format!("Thread id: {thread_id}"), format!("First context window id: {first_window_id}"), format!("Current context window id: {window_id}"), ]; @@ -57,7 +59,7 @@ impl ContextualUserFragment for TokenBudgetContext { if let Some(mcp_result) = &self.mcp_result { lines.push(mcp_result.clone()); } - lines.join("\n") + format!("\n{}\n", lines.join("\n")) } } diff --git a/codex-rs/core/src/event_mapping.rs b/codex-rs/core/src/event_mapping.rs index 4c57c5cadc9f..07f06c44b627 100644 --- a/codex-rs/core/src/event_mapping.rs +++ b/codex-rs/core/src/event_mapping.rs @@ -15,6 +15,7 @@ use codex_protocol::models::is_image_open_tag_text; use codex_protocol::models::is_local_image_close_tag_text; use codex_protocol::models::is_local_image_open_tag_text; use codex_protocol::protocol::COLLABORATION_MODE_OPEN_TAG; +use codex_protocol::protocol::CONTEXT_WINDOW_OPEN_TAG; use codex_protocol::protocol::MULTI_AGENT_MODE_OPEN_TAG; use codex_protocol::protocol::REALTIME_CONVERSATION_OPEN_TAG; use codex_protocol::protocol::SKILLS_INSTRUCTIONS_OPEN_TAG; @@ -36,6 +37,7 @@ const CONTEXTUAL_DEVELOPER_PREFIXES: &[&str] = &[ "", // Keep recognizing token-budget wrappers persisted by older versions. "", + CONTEXT_WINDOW_OPEN_TAG, "", ]; diff --git a/codex-rs/core/src/event_mapping_tests.rs b/codex-rs/core/src/event_mapping_tests.rs index f4c3ddd3a644..d7e999afb97e 100644 --- a/codex-rs/core/src/event_mapping_tests.rs +++ b/codex-rs/core/src/event_mapping_tests.rs @@ -15,6 +15,8 @@ use codex_protocol::models::ReasoningItemContent; use codex_protocol::models::ReasoningItemReasoningSummary; use codex_protocol::models::ResponseItem; use codex_protocol::models::WebSearchAction; +use codex_protocol::protocol::CONTEXT_WINDOW_CLOSE_TAG; +use codex_protocol::protocol::CONTEXT_WINDOW_OPEN_TAG; use codex_protocol::protocol::SKILLS_INSTRUCTIONS_OPEN_TAG; use codex_protocol::user_input::UserInput; use pretty_assertions::assert_eq; @@ -39,6 +41,20 @@ fn recognizes_legacy_token_budget_as_contextual_developer_content() { assert!(!has_non_contextual_dev_message_content(&content)); } +#[test] +fn recognizes_context_window_as_contextual_developer_content() { + let content = vec![ContentItem::InputText { + text: format!( + r#"{CONTEXT_WINDOW_OPEN_TAG} +Thread id: 00000000-0000-0000-0000-000000000000 +{CONTEXT_WINDOW_CLOSE_TAG}"# + ), + }]; + + assert!(is_contextual_dev_message_content(&content)); + assert!(!has_non_contextual_dev_message_content(&content)); +} + #[test] fn parses_user_message_with_text_and_two_images() { let img1 = "https://example.com/one.png".to_string(); diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__token_budget__token_budget_new_context_window_tool_full_context.snap b/codex-rs/core/tests/suite/snapshots/all__suite__token_budget__token_budget_new_context_window_tool_full_context.snap index c0c905625fa4..49687d347b34 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__token_budget__token_budget_new_context_window_tool_full_context.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__token_budget__token_budget_new_context_window_tool_full_context.snap @@ -1,6 +1,6 @@ --- source: core/tests/suite/token_budget.rs -assertion_line: 588 +assertion_line: 545 expression: snapshot --- Scenario: New context window tool installs fresh full context before the next follow-up request. @@ -9,7 +9,7 @@ Scenario: New context window tool installs fresh full context before the next fo 00:message/developer[3]: [01] [02] - [03] Thread id .\nFirst context window id: \nCurrent context window id: \nPrevious context window id: + [03] \nThread id: \nFirst context window id: \nCurrent context window id: \nPrevious context window id: \n 01:message/user:> 02:function_call/update_plan 03:function_call_output:Plan updated diff --git a/codex-rs/core/tests/suite/token_budget.rs b/codex-rs/core/tests/suite/token_budget.rs index ec86ce5946ce..09399e511e54 100644 --- a/codex-rs/core/tests/suite/token_budget.rs +++ b/codex-rs/core/tests/suite/token_budget.rs @@ -4,6 +4,8 @@ use codex_config::types::McpServerTransportConfig; use codex_core::config::TokenBudgetConfig; use codex_features::Feature; use codex_model_provider_info::built_in_model_providers; +use codex_protocol::protocol::CONTEXT_WINDOW_CLOSE_TAG; +use codex_protocol::protocol::CONTEXT_WINDOW_OPEN_TAG; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; use core_test_support::PathBufExt; @@ -34,10 +36,11 @@ use std::time::Duration; const CONFIGURED_CONTEXT_WINDOW: i64 = 128_000; fn token_budget_contexts(request: &ResponsesRequest) -> Vec { + let context_window_prefix = format!("{CONTEXT_WINDOW_OPEN_TAG}\nThread id: "); request .message_input_texts("developer") .into_iter() - .filter(|text| text.starts_with("Thread id ")) + .filter(|text| text.starts_with(&context_window_prefix)) .collect() } @@ -47,7 +50,7 @@ fn token_budget_window_ids( ) -> (String, Option, String) { let captures = assert_regex_match( &format!( - r"^Thread id {thread_id}\.\nFirst context window id: ([0-9a-f-]{{36}})\nCurrent context window id: ([0-9a-f-]{{36}})(?:\nPrevious context window id: ([0-9a-f-]{{36}}))?$" + r"^{CONTEXT_WINDOW_OPEN_TAG}\nThread id: {thread_id}\nFirst context window id: ([0-9a-f-]{{36}})\nCurrent context window id: ([0-9a-f-]{{36}})(?:\nPrevious context window id: ([0-9a-f-]{{36}}))?\n{CONTEXT_WINDOW_CLOSE_TAG}$" ), text, ); @@ -191,7 +194,7 @@ async fn token_budget_context_injects_plain_thread_hint_text() -> Result<()> { assert_eq!(token_budgets.len(), 1); let captures = assert_regex_match( &format!( - r"^Thread id {thread_id}\.\nFirst context window id: ([0-9a-f-]{{36}})\nCurrent context window id: ([0-9a-f-]{{36}})\nmanual history hint for thread {thread_id}\nunstructured notes/thread_hint fixture result$" + r"^{CONTEXT_WINDOW_OPEN_TAG}\nThread id: {thread_id}\nFirst context window id: ([0-9a-f-]{{36}})\nCurrent context window id: ([0-9a-f-]{{36}})\nmanual history hint for thread {thread_id}\nunstructured notes/thread_hint fixture result\n{CONTEXT_WINDOW_CLOSE_TAG}$" ), &token_budgets[0], ); diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index dfcc6cb7a561..600aa475bdfd 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -92,8 +92,8 @@ use crate::permissions::default_read_only_subpaths_for_writable_root; pub use crate::request_permissions::RequestPermissionsArgs; pub use crate::request_user_input::RequestUserInputEvent; -/// Open/close tags for special user-input blocks. Used across crates to avoid -/// duplicated hardcoded strings. +/// Open/close tags for special context blocks. Used across crates to avoid duplicated hardcoded +/// strings. pub const USER_INSTRUCTIONS_OPEN_TAG: &str = ""; pub const USER_INSTRUCTIONS_CLOSE_TAG: &str = ""; pub const ENVIRONMENT_CONTEXT_OPEN_TAG: &str = ""; @@ -110,6 +110,8 @@ pub const MULTI_AGENT_MODE_OPEN_TAG: &str = ""; pub const MULTI_AGENT_MODE_CLOSE_TAG: &str = ""; pub const REALTIME_CONVERSATION_OPEN_TAG: &str = ""; pub const REALTIME_CONVERSATION_CLOSE_TAG: &str = ""; +pub const CONTEXT_WINDOW_OPEN_TAG: &str = ""; +pub const CONTEXT_WINDOW_CLOSE_TAG: &str = ""; pub const USER_MESSAGE_BEGIN: &str = "## My request for Codex:"; // TODO(anp): Replace `TurnEnvironmentSelection` with `PathUri` once path URIs carry environment