Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions codex-rs/app-server/tests/suite/v2/dynamic_tools.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ use tempfile::TempDir;
use tokio::time::timeout;
use wiremock::MockServer;

const TINY_PNG_DATA_URL: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==";

// macOS and Windows Bazel CI can spend tens of seconds starting app-server
// subprocesses or processing test RPCs under load.
#[cfg(any(target_os = "macos", windows))]
Expand Down Expand Up @@ -650,7 +652,7 @@ async fn dynamic_tool_call_round_trip_sends_content_items_to_model() -> Result<(
text: "dynamic-ok".to_string(),
},
DynamicToolCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
image_url: TINY_PNG_DATA_URL.to_string(),
},
];
let content_items = response_content_items
Expand Down Expand Up @@ -695,7 +697,7 @@ async fn dynamic_tool_call_round_trip_sends_content_items_to_model() -> Result<(
text: "dynamic-ok".to_string(),
},
DynamicToolCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
image_url: TINY_PNG_DATA_URL.to_string(),
},
])
);
Expand All @@ -721,7 +723,7 @@ async fn dynamic_tool_call_round_trip_sends_content_items_to_model() -> Result<(
},
{
"type": "input_image",
"image_url": "data:image/png;base64,AAA",
"image_url": TINY_PNG_DATA_URL,
"detail": "high"
}
])
Expand Down
10 changes: 5 additions & 5 deletions codex-rs/app-server/tests/suite/v2/imagegen_extension.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,13 @@ use wiremock::ResponseTemplate;
use wiremock::matchers::method;
use wiremock::matchers::path;

const RESULT: &str = "cG5n";
const RESULT: &str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==";
const TINY_PNG_BYTES: &[u8] = &[
137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 1, 0, 0, 0, 1, 8, 6, 0,
0, 0, 31, 21, 196, 137, 0, 0, 0, 11, 73, 68, 65, 84, 120, 156, 99, 96, 0, 2, 0, 0, 5, 0, 1,
122, 94, 171, 63, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130,
0, 0, 31, 21, 196, 137, 0, 0, 0, 13, 73, 68, 65, 84, 120, 156, 99, 248, 207, 192, 240, 31, 0,
5, 0, 1, 255, 137, 153, 61, 29, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130,
];
const TINY_PNG_DATA_URL: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR4nGNgAAIAAAUAAXpeqz8AAAAASUVORK5CYII=";
const TINY_PNG_DATA_URL: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==";

#[derive(Clone, Copy)]
enum ImagegenTestMode {
Expand Down Expand Up @@ -116,7 +116,7 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model() -> Resul
assert_eq!(status, "completed");
assert_eq!(revised_prompt.as_deref(), Some("paint a blue whale"));
assert_eq!(result, RESULT);
assert_eq!(std::fs::read(&saved_path)?, b"png");
assert_eq!(std::fs::read(&saved_path)?, TINY_PNG_BYTES);

let requests = response_mock.requests();
assert_eq!(requests.len(), 2);
Expand Down
2 changes: 1 addition & 1 deletion codex-rs/core/src/prompt_debug.rs
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ pub(crate) async fn build_prompt_input_from_session(
.await;

if !input.is_empty() {
let response_item = sess.response_item_from_user_input(turn_context.as_ref(), input);
let response_item = sess.response_item_from_user_input(input);
sess.record_conversation_items(turn_context.as_ref(), std::slice::from_ref(&response_item))
.await;
}
Expand Down
43 changes: 9 additions & 34 deletions codex-rs/core/src/session/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1371,17 +1371,11 @@ impl Session {
} = self
.reconstruct_history_from_rollout(turn_context, rollout_items)
.await;
if turn_context
.config
.features
.enabled(Feature::ResizeAllImages)
{
// Keep the recorded rollout unchanged. Prepare its reconstructed history before
// installing it, so legacy images are processed once for this resume or fork and
// will be processed again if the rollout is reconstructed in a future session.
// This meets image resizing requirements without modifying persisted rollouts.
prepare_response_items(&mut history);
}
// Keep the recorded rollout unchanged. Prepare its reconstructed history before
// installing it, so legacy images are processed once for this resume or fork and
// will be processed again if the rollout is reconstructed in a future session.
// This meets image resizing requirements without modifying persisted rollouts.
prepare_response_items(&mut history);
{
let mut state = self.state.lock().await;
state.replace_history(history, reference_context_item);
Expand Down Expand Up @@ -2688,13 +2682,7 @@ impl Session {
items: &'a [ResponseItem],
) -> Cow<'a, [ResponseItem]> {
let mut items = Cow::Borrowed(items);
if turn_context
.config
.features
.enabled(Feature::ResizeAllImages)
{
prepare_response_items(items.to_mut());
}
prepare_response_items(items.to_mut());

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Preserve low-detail code-mode images

When code mode emits a data-URL image with detail: "low" (which the code-mode parser still accepts and maps to ImageDetail::Low), this unconditional preparation path now runs before the tool output reaches the next model request. prepare_image treats ImageDetail::Low as unsupported and replaces the image with a text placeholder, so previously valid low-detail code-mode image outputs silently stop being images. Please either stop accepting low in code mode or normalize it to a supported detail before calling prepare_response_items.

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

expected

if turn_context.config.features.enabled(Feature::ItemIds) {
Self::assign_missing_response_item_ids(items)
} else {
Expand Down Expand Up @@ -2733,23 +2721,10 @@ impl Session {
items
}

pub(crate) fn response_item_from_user_input(
&self,
turn_context: &TurnContext,
input: Vec<UserInput>,
) -> ResponseItem {
let local_image_preparation = if turn_context
.config
.features
.enabled(Feature::ResizeAllImages)
{
LocalImagePreparation::Defer
} else {
LocalImagePreparation::Process
};
pub(crate) fn response_item_from_user_input(&self, input: Vec<UserInput>) -> ResponseItem {
ResponseItem::from(ResponseInputItem::from_user_input(
input,
local_image_preparation,
LocalImagePreparation::Defer,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Cap local images before deferring bytes

When a user accidentally attaches a large non-image or oversized local file, this now always takes the Defer path, whose from_user_input branch wraps the entire file in a base64 data URL before prepare_response_items can validate or reject it. The previous default Process path could fail during image loading without creating that extra 4/3-sized string and then decoding it again, so a bad local attachment can now cause large transient memory spikes before it becomes a placeholder. Please reject/cap the byte buffer before converting it to a data URL, or prepare directly from bytes.

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not an issue

))
}

Expand Down Expand Up @@ -3615,7 +3590,7 @@ impl Session {
// Persist the user message to history, but emit the turn item from `UserInput` so
// UI-only `text_elements` are preserved. `ResponseItem::Message` does not carry
// those spans, and `record_response_item_and_emit_turn_item` would drop them.
let response_item = self.response_item_from_user_input(turn_context, input.to_vec());
let response_item = self.response_item_from_user_input(input.to_vec());
self.record_conversation_items(turn_context, std::slice::from_ref(&response_item))
.await;
let mut user_message_item = UserMessageItem::new(input);
Expand Down
14 changes: 3 additions & 11 deletions codex-rs/core/src/session/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1694,12 +1694,11 @@ async fn record_initial_history_reconstructs_resumed_transcript() {
}

#[tokio::test]
async fn resize_all_images_prepares_failures_before_history_insertion() {
async fn prepares_image_failures_before_history_insertion() {
let (session, turn_context, _rx) = make_session_and_context_with_auth_and_config_and_rx(
CodexAuth::from_api_key("Test API Key"),
Vec::new(),
|config| {
let _ = config.features.enable(Feature::ResizeAllImages);
let _ = config.features.enable(Feature::ItemIds);
},
)
Expand Down Expand Up @@ -1764,15 +1763,8 @@ async fn resize_all_images_prepares_failures_before_history_insertion() {
}

#[tokio::test]
async fn resize_all_images_prepares_resumed_history_before_installing_it() {
let (session, _turn_context, _rx) = make_session_and_context_with_auth_and_config_and_rx(
CodexAuth::from_api_key("Test API Key"),
Vec::new(),
|config| {
let _ = config.features.enable(Feature::ResizeAllImages);
},
)
.await;
async fn prepares_resumed_history_before_installing_it() {
let (session, _turn_context) = make_session_and_context().await;
let resumed_item = ResponseItem::Message {
id: None,
role: "user".to_string(),
Expand Down
28 changes: 3 additions & 25 deletions codex-rs/core/src/tools/handlers/view_image.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
use codex_features::Feature;
use codex_protocol::items::ImageViewItem;
use codex_protocol::items::TurnItem;
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
Expand All @@ -8,9 +7,7 @@ use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::openai_models::InputModality;
use codex_utils_image::PromptImageMode;
use codex_utils_image::data_url_from_bytes;
use codex_utils_image::load_for_prompt_bytes;
use serde::Deserialize;

use crate::function_tool::FunctionCallError;
Expand Down Expand Up @@ -195,24 +192,8 @@ impl ViewImageHandler {
DEFAULT_IMAGE_DETAIL
};

let image_url = if turn.config.features.enabled(Feature::ResizeAllImages) {
// The history insertion path owns image decoding and resizing when this is enabled.
data_url_from_bytes("application/octet-stream", &file_bytes)
} else {
let image_mode = if use_original_detail {
PromptImageMode::Original
} else {
PromptImageMode::ResizeToFit
};
load_for_prompt_bytes(abs_path.as_path(), file_bytes, image_mode)
.map_err(|error| {
FunctionCallError::RespondToModel(format!(
"unable to process image at `{}`: {error}",
abs_path.display()
))
})?
.into_data_url()
};
// The history insertion path owns image decoding and resizing.
let image_url = data_url_from_bytes("application/octet-stream", &file_bytes);

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Avoid exposing unprepared view_image bytes outside history

When a PostToolUse hook matches view_image or rollout tracing is enabled, those payloads are built from ViewImageOutput::to_response_item() before record_conversation_items() calls prepare_response_items, so this deferred URL is sent to those external surfaces as full unvalidated data:application/octet-stream bytes. That means a malformed/non-image file now appears as a successful tool output to hooks/traces, and large images bypass the bounded prepared/placeholder value that the model later sees. Please prepare before constructing non-history payloads or override those surfaces to emit a bounded placeholder/processed value.

Useful? React with 👍 / 👎.

@rka-oai rka-oai Jun 22, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not a serious issue


let item = TurnItem::ImageView(ImageViewItem {
id: call_id,
Expand Down Expand Up @@ -420,9 +401,6 @@ mod tests {
})
.await;

let Err(FunctionCallError::RespondToModel(message)) = result else {
panic!("expected image processing error");
};
assert!(message.contains("unable to process image"), "{message}");
result.expect("explicit high detail should be accepted");
}
}
4 changes: 2 additions & 2 deletions codex-rs/core/tests/suite/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -860,7 +860,7 @@ async fn resume_replays_legacy_js_repl_image_rollout_shapes() {
async fn resume_replays_image_tool_outputs_with_detail() {
skip_if_no_network!();

let image_url = "data:image/webp;base64,UklGRiIAAABXRUJQVlA4IBYAAAAwAQCdASoBAAEAAUAmJaACdLoB+AADsAD+8ut//NgVzXPv9//S4P0uD9Lg/9KQAAA=";
let image_url = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==";
let function_call_id = "view-image-call";
let custom_call_id = "js-repl-call";
let thread_id = ThreadId::default();
Expand Down Expand Up @@ -888,7 +888,7 @@ async fn resume_replays_image_tool_outputs_with_detail() {
id: None,
name: "view_image".to_string(),
namespace: None,
arguments: "{\"path\":\"/tmp/example.webp\"}".to_string(),
arguments: "{\"path\":\"/tmp/example.png\"}".to_string(),
call_id: function_call_id.to_string(),
metadata: None,
}),
Expand Down
20 changes: 6 additions & 14 deletions codex-rs/core/tests/suite/code_mode.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2681,7 +2681,7 @@ async fn code_mode_can_output_images_via_global_helper() -> Result<()> {
&server,
"use exec to return images",
r#"
image("data:image/png;base64,AAA");
image("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==");
"#,
)
.await?;
Expand All @@ -2706,7 +2706,7 @@ image("data:image/png;base64,AAA");
items[1],
serde_json::json!({
"type": "input_image",
"image_url": "data:image/png;base64,AAA",
"image_url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==",
"detail": "high"
}),
);
Expand All @@ -2715,17 +2715,14 @@ image("data:image/png;base64,AAA");
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn resize_all_images_replaces_malformed_code_mode_image() -> Result<()> {
async fn code_mode_replaces_malformed_image() -> Result<()> {
skip_if_no_network!(Ok(()));

let server = responses::start_mock_server().await;
let (_test, second_mock) = run_code_mode_turn_with_config(
let (_test, second_mock) = run_code_mode_turn(
&server,
"use exec to return an image",
r#"image("data:image/png;base64,AAA");"#,
|config| {
let _ = config.features.enable(Feature::ResizeAllImages);
},
)
.await?;

Expand All @@ -2746,7 +2743,7 @@ async fn resize_all_images_replaces_malformed_code_mode_image() -> Result<()> {
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn resize_all_images_resizes_explicit_original_code_mode_image() -> Result<()> {
async fn code_mode_resizes_explicit_original_image() -> Result<()> {
skip_if_no_network!(Ok(()));

let original_dimensions = (6401, 100);
Expand All @@ -2772,12 +2769,7 @@ async fn resize_all_images_resizes_explicit_original_code_mode_image() -> Result
"use exec to return a large original-detail image",
&code,
"gpt-5.3-codex",
|config| {
config
.features
.enable(Feature::ResizeAllImages)
.expect("resize_all_images should be enabled");
},
|_| {},
)
.await?;

Expand Down
2 changes: 1 addition & 1 deletion codex-rs/core/tests/suite/compact.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4403,7 +4403,7 @@ async fn snapshot_request_shape_pre_turn_compaction_including_incoming_user_mess
)
.await
.expect("override thread settings");
let image_url = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGNgYAAAAAMAASsJTYQAAAAASUVORK5CYII="
let image_url = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg=="
.to_string();
codex
.submit(Op::UserInput {
Expand Down
10 changes: 6 additions & 4 deletions codex-rs/core/tests/suite/extension_sandbox.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,11 @@ use wiremock::matchers::path;

const TINY_PNG_BYTES: &[u8] = &[
137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 1, 0, 0, 0, 1, 8, 6, 0,
0, 0, 31, 21, 196, 137, 0, 0, 0, 11, 73, 68, 65, 84, 120, 156, 99, 96, 0, 2, 0, 0, 5, 0, 1,
122, 94, 171, 63, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130,
0, 0, 31, 21, 196, 137, 0, 0, 0, 13, 73, 68, 65, 84, 120, 156, 99, 248, 207, 192, 240, 31, 0,
5, 0, 1, 255, 137, 153, 61, 29, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130,
];
const TINY_PNG_BASE64: &str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==";
const TINY_PNG_DATA_URL: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==";

fn image_generation_extensions(auth: &CodexAuth) -> Arc<ExtensionRegistry<Config>> {
let auth_manager = codex_core::test_support::auth_manager_from_auth(auth.clone());
Expand Down Expand Up @@ -148,7 +150,7 @@ async fn extension_tool_uses_granted_turn_permissions() -> Result<()> {
.and(path("/v1/images/edits"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"created": 1,
"data": [{"b64_json": "cG5n"}],
"data": [{"b64_json": TINY_PNG_BASE64}],
})))
.expect(1)
.mount(&server)
Expand Down Expand Up @@ -299,7 +301,7 @@ async fn extension_tool_uses_granted_turn_permissions() -> Result<()> {
let output = request.function_call_output(image_call_id);
let image = &output["output"][0];
assert_eq!(image["type"], "input_image");
assert_eq!(image["image_url"], "data:image/png;base64,cG5n");
assert_eq!(image["image_url"], TINY_PNG_DATA_URL);

Ok(())
}
2 changes: 1 addition & 1 deletion codex-rs/core/tests/suite/image_rollout.rs
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ async fn drag_drop_image_persists_rollout_request_shape() -> anyhow::Result<()>
..
} = test_codex().build(&server).await?;

let image_url = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGNgYAAAAAMAASsJTYQAAAAASUVORK5CYII=".to_string();
let image_url = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==".to_string();

let response = sse(vec![
ev_response_created("resp-1"),
Expand Down
2 changes: 1 addition & 1 deletion codex-rs/core/tests/suite/model_switching.rs
Original file line number Diff line number Diff line change
Expand Up @@ -506,7 +506,7 @@ async fn model_change_from_image_to_text_strips_prior_image_content() -> Result<
let _ = models_manager
.list_models(RefreshStrategy::OnlineIfUncached)
.await;
let image_url = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4nGNgYAAAAAMAASsJTYQAAAAASUVORK5CYII="
let image_url = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg=="
.to_string();

test.codex
Expand Down
2 changes: 1 addition & 1 deletion codex-rs/core/tests/suite/responses_lite.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ fn has_hosted_tool(tools: &[Value], tool_type: &str) -> bool {
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn responses_lite_strips_data_image_detail_without_resize_all_images() -> Result<()> {
async fn responses_lite_strips_data_image_detail() -> Result<()> {
skip_if_no_network!(Ok(()));

let server = responses::start_mock_server().await;
Expand Down
5 changes: 0 additions & 5 deletions codex-rs/core/tests/suite/rmcp_client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ use codex_core::config::Config;
use codex_exec_server::CreateDirectoryOptions;
use codex_exec_server::Environment;
use codex_exec_server::HttpRequestParams;
use codex_features::Feature;
use codex_login::CodexAuth;
use codex_mcp::MCP_SANDBOX_STATE_META_CAPABILITY;
use codex_models_manager::manager::RefreshStrategy;
Expand Down Expand Up @@ -1455,10 +1454,6 @@ async fn stdio_image_responses_resize_large_image() -> anyhow::Result<()> {
let rmcp_test_server_bin = remote_aware_stdio_server_bin()?;
let fixture = test_codex()
.with_config(move |config| {
config
.features
.enable(Feature::ResizeAllImages)
.expect("resize_all_images should be enabled");
insert_mcp_server(
config,
server_name,
Expand Down
Loading
Loading