Skip to content

[codex] Test code-mode variable truncation #28471

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
aibrahim-oai merged 6 commits into from
Jun 17, 2026
Merged

[codex] Test code-mode variable truncation #28471

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
3c682e7
Preserve code mode output budgets in history
aibrahim-oai Jun 16, 2026
a1a28e7
Test code mode variable truncation
aibrahim-oai Jun 16, 2026
29c1e73
Clarify code mode truncation tests
aibrahim-oai Jun 16, 2026
0270bc4
codex: address PR review feedback (#28471)
aibrahim-oai Jun 16, 2026
29dcd3f
codex: address configured cap review (#28471)
aibrahim-oai Jun 16, 2026
289bbe6
codex: clarify configured truncation assertions (#28471)
aibrahim-oai Jun 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
143 changes: 91 additions & 52 deletions codex-rs/core/tests/suite/code_mode.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -940,9 +940,15 @@ text(JSON.stringify(results));
Ok(())
}

// This model uses token-based tool-output truncation, giving the downstream
// history assertions a stable `…N tokens truncated…` marker.
const TOKEN_POLICY_TEST_MODEL: &str = "gpt-5.4";

// A nested `exec_command` limit applies to `result.output` inside JavaScript.
// The outer code-mode and history budgets apply after the script calls `text`.
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn code_mode_exec_command_explicit_max_output_tokens_truncates() -> Result<()> {
async fn code_mode_exec_nested_limit_formats_truncated_result_with_warning() -> Result<()> {
skip_if_no_network!(Ok(()));

let server = responses::start_mock_server().await;
Expand All @@ -964,15 +970,16 @@ text(result.output);
&custom_tool_output_items(&second_mock.single_request(), "call-1"),
/*index*/ 1
),
"Total output lines: 1\n\n0123456789…5 tokens truncated…0123456789"
"Warning: truncated output (original token count: 10)\nTotal output lines: 1\n\n0123456789…5 tokens truncated…0123456789"
);

Ok(())
}

#[cfg_attr(windows, ignore = "no exec_command on Windows")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn code_mode_exec_explicit_max_above_default_preserves_output() -> Result<()> {
async fn code_mode_exec_nested_limit_preserves_result_variable_before_default_history_truncation()
-> Result<()> {
// TODO(anp): Remove after Wine exec returns complete nested-tool output to code mode.
skip_if_wine_exec!(
Ok(()),
Expand All @@ -981,33 +988,35 @@ async fn code_mode_exec_explicit_max_above_default_preserves_output() -> Result<
skip_if_no_network!(Ok(()));

let server = responses::start_mock_server().await;
let (_test, second_mock) = run_code_mode_turn(
let (_test, second_mock) = run_code_mode_turn_with_model_and_config(
&server,
"use exec_command from code mode",
r#"// @exec: {"max_output_tokens": 20000}
const result = await tools.exec_command({
cmd: "python3 -c \"import sys; sys.stdout.write('x' * 50000)\"",
max_output_tokens: 20000
});
text(result.output);
const resultVariableWasTruncated = result.output.length !== 50000;
text(`Variable truncated: ${resultVariableWasTruncated ? "True" : "False"}. Variable: ${result.output}`);
"#,
TOKEN_POLICY_TEST_MODEL,
|_| {},
)
.await?;

assert_eq!(
text_item(
&custom_tool_output_items(&second_mock.single_request(), "call-1"),
/*index*/ 1
),
"x".repeat(50_000)
let items = custom_tool_output_items(&second_mock.single_request(), "call-1");
let output = text_item(&items, /*index*/ 1);
assert_regex_match(
r"^Variable truncated: False\. Variable: x+…\d+ tokens truncated…x+$",
output,
);

Ok(())
}

#[cfg_attr(windows, ignore = "no exec_command on Windows")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn code_mode_exec_explicit_max_above_default_truncates_larger_output() -> Result<()> {
async fn code_mode_exec_nested_limit_truncates_result_variable_when_exceeded() -> Result<()> {
// TODO(anp): Remove after Wine exec returns complete nested-tool output to code mode.
skip_if_wine_exec!(
Ok(()),
Expand All @@ -1016,37 +1025,45 @@ async fn code_mode_exec_explicit_max_above_default_truncates_larger_output() ->
skip_if_no_network!(Ok(()));

let server = responses::start_mock_server().await;
let (_test, second_mock) = run_code_mode_turn(
let (_test, second_mock) = run_code_mode_turn_with_model_and_config(
&server,
"use exec_command from code mode",
r#"// @exec: {"max_output_tokens": 25000}
const result = await tools.exec_command({
cmd: "python3 -c \"import sys; sys.stdout.write('A' * 90000)\"",
max_output_tokens: 20000
});
text(result.output);
const resultVariableWasTruncated = result.output.includes("…2500 tokens truncated…");
text(`Variable truncated: ${resultVariableWasTruncated ? "True" : "False"}. Variable: ${result.output}`);
"#,
TOKEN_POLICY_TEST_MODEL,
|_| {},
)
.await?;

assert_eq!(
text_item(
&custom_tool_output_items(&second_mock.single_request(), "call-1"),
/*index*/ 1
),
format!(
"Warning: truncated output (original token count: 22500)\nTotal output lines: 1\n\n{}…2500 tokens truncated…{}",
"A".repeat(40_000),
"A".repeat(40_000)
)
let items = custom_tool_output_items(&second_mock.single_request(), "call-1");
let output = text_item(&items, /*index*/ 1);
// The nested 20,000-token budget leaves about 80,000 characters. This
// ceiling independently proves that history applied its smaller cap.
assert!(
output.len() < 60_000,
"expected history to truncate the emitted value, got {} bytes",
output.len()
);
// The boolean describes the nested result; the marker below comes from
// history truncating the value emitted with `text` afterward.
assert_regex_match(
r"(?s)^Variable truncated: True\. Variable: .*…\d+ tokens truncated…A+$",
Comment thread
aibrahim-oai marked this conversation as resolved.
output,
);

Ok(())
}

#[cfg_attr(windows, ignore = "no exec_command on Windows")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn code_mode_exec_explicit_max_above_truncation_policy_preserves_output() -> Result<()> {
async fn code_mode_exec_nested_limit_preserves_result_variable_before_configured_history_truncation()
-> Result<()> {
// TODO(anp): Remove after Wine exec returns complete nested-tool output to code mode.
skip_if_wine_exec!(
Ok(()),
Expand All @@ -1055,36 +1072,45 @@ async fn code_mode_exec_explicit_max_above_truncation_policy_preserves_output()
skip_if_no_network!(Ok(()));

let server = responses::start_mock_server().await;
let (_test, second_mock) = run_code_mode_turn_with_config(
let (_test, second_mock) = run_code_mode_turn_with_model_and_config(
&server,
"use exec_command from code mode",
r#"// @exec: {"max_output_tokens": 20000}
const result = await tools.exec_command({
cmd: "python3 -c \"import sys; sys.stdout.write('x' * 50000)\"",
max_output_tokens: 20000
});
text(result.output);
const resultVariableWasTruncated = result.output.length !== 50000;
text(`Variable truncated: ${resultVariableWasTruncated ? "True" : "False"}. Variable: ${result.output}`);
"#,
TOKEN_POLICY_TEST_MODEL,
|config| {
config.tool_output_token_limit = Some(50);
},
)
.await?;

assert_eq!(
text_item(
&custom_tool_output_items(&second_mock.single_request(), "call-1"),
/*index*/ 1
),
"x".repeat(50_000)
let items = custom_tool_output_items(&second_mock.single_request(), "call-1");
let output = text_item(&items, /*index*/ 1);
// The 50-token override must shrink this 50,000-character value far below
// what the default 10,000-token history cap would retain.
assert!(
output.len() < 1_000,
"expected configured history cap to truncate the emitted value, got {} bytes",
output.len()
);
assert_regex_match(
r"^Variable truncated: False\. Variable: x+…\d+ tokens truncated…x+$",
output,
);

Ok(())
}

#[cfg_attr(windows, ignore = "no exec_command on Windows")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn code_mode_exec_without_max_preserves_output_beyond_default() -> Result<()> {
async fn code_mode_exec_without_nested_limit_preserves_result_variable_before_default_history_truncation()
-> Result<()> {
// TODO(anp): Remove after Wine exec returns complete nested-tool output to code mode.
skip_if_wine_exec!(
Ok(()),
Expand All @@ -1093,32 +1119,35 @@ async fn code_mode_exec_without_max_preserves_output_beyond_default() -> Result<
skip_if_no_network!(Ok(()));

let server = responses::start_mock_server().await;
let (_test, second_mock) = run_code_mode_turn(
let (_test, second_mock) = run_code_mode_turn_with_model_and_config(
&server,
"use exec_command from code mode",
r#"// @exec: {"max_output_tokens": 20000}
const result = await tools.exec_command({
cmd: "python3 -c \"import sys; sys.stdout.write('x' * 50000)\""
});
text(result.output);
const resultVariableWasTruncated = result.output.length !== 50000;
text(`Variable truncated: ${resultVariableWasTruncated ? "True" : "False"}. Variable: ${result.output}`);
"#,
TOKEN_POLICY_TEST_MODEL,
|_| {},
)
.await?;

assert_eq!(
text_item(
&custom_tool_output_items(&second_mock.single_request(), "call-1"),
/*index*/ 1
),
"x".repeat(50_000)
let items = custom_tool_output_items(&second_mock.single_request(), "call-1");
let output = text_item(&items, /*index*/ 1);
assert_regex_match(
r"^Variable truncated: False\. Variable: x+…\d+ tokens truncated…x+$",
output,
);

Ok(())
}

#[cfg_attr(windows, ignore = "no exec_command on Windows")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn code_mode_exec_without_max_preserves_output_beyond_truncation_policy() -> Result<()> {
async fn code_mode_exec_without_nested_limit_preserves_result_variable_before_configured_history_truncation()
-> Result<()> {
// TODO(anp): Remove after Wine exec returns complete nested-tool output to code mode.
skip_if_wine_exec!(
Ok(()),
Expand All @@ -1127,35 +1156,45 @@ async fn code_mode_exec_without_max_preserves_output_beyond_truncation_policy()
skip_if_no_network!(Ok(()));

let server = responses::start_mock_server().await;
let (_test, second_mock) = run_code_mode_turn_with_config(
let (_test, second_mock) = run_code_mode_turn_with_model_and_config(
&server,
"use exec_command from code mode",
r#"// @exec: {"max_output_tokens": 20000}
const result = await tools.exec_command({
cmd: "python3 -c \"import sys; sys.stdout.write('x' * 50000)\""
});
text(result.output);
const resultVariableWasTruncated = result.output.length !== 50000;
text(`Variable truncated: ${resultVariableWasTruncated ? "True" : "False"}. Variable: ${result.output}`);
"#,
TOKEN_POLICY_TEST_MODEL,
|config| {
config.tool_output_token_limit = Some(50);
},
)
.await?;

assert_eq!(
text_item(
&custom_tool_output_items(&second_mock.single_request(), "call-1"),
/*index*/ 1
),
"x".repeat(50_000)
let items = custom_tool_output_items(&second_mock.single_request(), "call-1");
let output = text_item(&items, /*index*/ 1);
// The 50-token override must shrink this 50,000-character value far below
// what the default 10,000-token history cap would retain.
assert!(
output.len() < 1_000,
"expected configured history cap to truncate the emitted value, got {} bytes",
output.len()
);
assert_regex_match(
r"^Variable truncated: False\. Variable: x+…\d+ tokens truncated…x+$",
output,
);

Ok(())
}

// The outer directive limits output after JavaScript emits it; it does not
// limit `result.output` returned by the nested command.
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn code_mode_exec_explicit_max_output_tokens_truncates() -> Result<()> {
async fn code_mode_exec_outer_limit_truncates_emitted_output() -> Result<()> {
skip_if_no_network!(Ok(()));

let server = responses::start_mock_server().await;
Expand Down
Loading