Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions codex-rs/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

19 changes: 15 additions & 4 deletions codex-rs/app-server/tests/suite/v2/imagegen_extension.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(60);
const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(10);

#[tokio::test]
async fn standalone_image_generation_persists_image_and_returns_it_to_model() -> Result<()> {
async fn standalone_image_generation_returns_saved_path_hint_to_model() -> Result<()> {
let call_id = "image-run-1";
let server = responses::start_mock_server().await;
mount_image_response(&server).await;
Expand Down Expand Up @@ -124,7 +124,13 @@ async fn standalone_image_generation_persists_image_and_returns_it_to_model() ->
"detail": "high",
})
);
assert_eq!(output["output"].as_array().map(Vec::len), Some(1));
let output_hint = output["output"][1]["text"]
.as_str()
.context("image output should include model-visible path hint")?;
assert!(
output_hint.contains(&saved_path.display().to_string()),
"output hint should identify the path core saved"
);
assert!(
!requests[1]
.message_input_texts("developer")
Expand Down Expand Up @@ -156,7 +162,7 @@ const result = await tools.image_gen__imagegen({
action: "generate",
prompt: "paint a blue whale",
});
image(result);
generatedImage(result);
"#,
),
responses::ev_completed("resp-1"),
Expand Down Expand Up @@ -203,7 +209,12 @@ image(result);
"detail": "high",
})
);
assert_eq!(output["output"].as_array().map(Vec::len), Some(2));
assert!(
output["output"][2]["text"]
.as_str()
.is_some_and(|text| text.contains("Generated images are saved"))
);
assert_eq!(output["output"].as_array().map(Vec::len), Some(3));

Ok(())
}
Expand Down
1 change: 1 addition & 0 deletions codex-rs/code-mode/src/description.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ const EXEC_DESCRIPTION_TEMPLATE: &str = r#"Run JavaScript code to orchestrate/co
- `exit()`: Immediately ends the current script successfully (like an early return from the top level).
- `text(value: string | number | boolean | undefined | null)`: Appends a text item. Non-string values are stringified with `JSON.stringify(...)` when possible.
- `image(imageUrlOrItem: string | { image_url: string; detail?: "auto" | "low" | "high" | "original" | null } | ImageContent, detail?: "auto" | "low" | "high" | "original" | null)`: Appends an image item. `image_url` can be an HTTPS URL or a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`. MCP image blocks may request detail with `_meta: { "codex/imageDetail": "original" }`. When provided, the second `detail` argument overrides any detail embedded in the first argument.
- `generatedImage(result: { image_url: string; output_hint?: string })`: Appends an image-generation result and its optional output hint.
- `store(key: string, value: any)`: stores a serializable value under a string key for later `exec` calls in the same session.
- `load(key: string)`: returns the stored value for a string key, or `undefined` if it is missing.
- `notify(value: string | number | boolean | undefined | null)`: immediately injects an extra `custom_tool_call_output` for the current `exec` call. Values are stringified like `text(...)`.
Expand Down
52 changes: 52 additions & 0 deletions codex-rs/code-mode/src/runtime/callbacks.rs
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,58 @@ pub(super) fn image_callback(
retval.set(v8::undefined(scope).into());
}

pub(super) fn generated_image_callback(
scope: &mut v8::PinScope<'_, '_>,
args: v8::FunctionCallbackArguments,
mut retval: v8::ReturnValue<v8::Value>,
) {
let value = if args.length() == 0 {
v8::undefined(scope).into()
} else {
args.get(0)
};
let output_hint = match generated_image_output_hint(scope, value) {
Ok(output_hint) => output_hint,
Err(error_text) => {
throw_type_error(scope, &error_text);
return;
}
};
let image_item = match normalize_output_image(scope, value, /*detail_override*/ None) {
Ok(image_item) => image_item,
Err(()) => return,
};
if let Some(state) = scope.get_slot::<RuntimeState>() {
let _ = state.event_tx.send(RuntimeEvent::ContentItem(image_item));
if let Some(text) = output_hint {
let _ = state.event_tx.send(RuntimeEvent::ContentItem(
FunctionCallOutputContentItem::InputText { text },
));
}
}
retval.set(v8::undefined(scope).into());
}

fn generated_image_output_hint(
scope: &mut v8::PinScope<'_, '_>,
value: v8::Local<'_, v8::Value>,
) -> Result<Option<String>, String> {
let object = v8::Local::<v8::Object>::try_from(value)
.map_err(|_| "generatedImage expects an image generation result object".to_string())?;
let key = v8::String::new(scope, "output_hint")
.ok_or_else(|| "failed to allocate generatedImage helper keys".to_string())?;
let output_hint = object
.get(scope, key.into())
.ok_or_else(|| "failed to read generatedImage output_hint".to_string())?;
if output_hint.is_undefined() {
return Ok(None);
}
if !output_hint.is_string() {
return Err("generatedImage output_hint must be a string when provided".to_string());
}
Ok(Some(output_hint.to_rust_string_lossy(scope)))
}

pub(super) fn store_callback(
scope: &mut v8::PinScope<'_, '_>,
args: v8::FunctionCallbackArguments,
Expand Down
3 changes: 3 additions & 0 deletions codex-rs/code-mode/src/runtime/globals.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use super::RuntimeState;
use super::callbacks::clear_timeout_callback;
use super::callbacks::exit_callback;
use super::callbacks::generated_image_callback;
use super::callbacks::image_callback;
use super::callbacks::load_callback;
use super::callbacks::notify_callback;
Expand All @@ -23,6 +24,7 @@ pub(super) fn install_globals(scope: &mut v8::PinScope<'_, '_>) -> Result<(), St
let set_timeout = helper_function(scope, "setTimeout", set_timeout_callback)?;
let text = helper_function(scope, "text", text_callback)?;
let image = helper_function(scope, "image", image_callback)?;
let generated_image = helper_function(scope, "generatedImage", generated_image_callback)?;
let store = helper_function(scope, "store", store_callback)?;
let load = helper_function(scope, "load", load_callback)?;
let notify = helper_function(scope, "notify", notify_callback)?;
Expand All @@ -35,6 +37,7 @@ pub(super) fn install_globals(scope: &mut v8::PinScope<'_, '_>) -> Result<(), St
set_global(scope, global, "setTimeout", set_timeout.into())?;
set_global(scope, global, "text", text.into())?;
set_global(scope, global, "image", image.into())?;
set_global(scope, global, "generatedImage", generated_image.into())?;
set_global(scope, global, "store", store.into())?;
set_global(scope, global, "load", load.into())?;
set_global(scope, global, "notify", notify.into())?;
Expand Down
38 changes: 38 additions & 0 deletions codex-rs/code-mode/src/service.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1563,6 +1563,44 @@ image({
);
}

#[tokio::test]
async fn generated_image_helper_appends_image_and_output_hint() {
let service = CodeModeService::new();

let response = execute(
&service,
ExecuteRequest {
source: r#"
generatedImage({
image_url: "https://example.com/image.jpg",
output_hint: "generated image save hint",
});
"#
.to_string(),
yield_time_ms: None,
..execute_request("")
},
)
.await;

assert_eq!(
response,
RuntimeResponse::Result {
cell_id: cell_id("1"),
content_items: vec![
FunctionCallOutputContentItem::InputImage {
image_url: "https://example.com/image.jpg".to_string(),
detail: Some(crate::DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputText {
text: "generated image save hint".to_string(),
},
],
error_text: None,
}
);
}

#[tokio::test]
async fn image_helper_second_arg_overrides_explicit_object_detail() {
let service = CodeModeService::new();
Expand Down
26 changes: 22 additions & 4 deletions codex-rs/core/src/context/image_generation_instructions.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,27 @@
use super::ContextualUserFragment;
use std::fmt::Display;

/// Maximum size of the extension's model-facing generated-image path hint.
const MAX_IMAGE_GENERATION_OUTPUT_HINT_BYTES: usize = 1024;

/// Returns the extension's model-facing hint, or omits it if the path makes it too large.
pub fn extension_image_generation_output_hint(
image_output_dir: impl Display,
image_output_path: impl Display,
) -> Option<String> {
let hint = image_generation_hint(image_output_dir, image_output_path);
(hint.len() <= MAX_IMAGE_GENERATION_OUTPUT_HINT_BYTES).then_some(hint)
}

fn image_generation_hint(
image_output_dir: impl Display,
image_output_path: impl Display,
) -> String {
format!(
"Generated images are saved to {image_output_dir} as {image_output_path} by default.\nIf you need to use a generated image at another path, copy it and leave the original in place unless the user explicitly asks you to delete it."
)
}

#[derive(Debug, Clone, PartialEq)]
pub(crate) struct ImageGenerationInstructions {
image_output_dir: String,
Expand Down Expand Up @@ -30,9 +51,6 @@ impl ContextualUserFragment for ImageGenerationInstructions {
}

fn body(&self) -> String {
format!(
"Generated images are saved to {} as {} by default.\nIf you need to use a generated image at another path, copy it and leave the original in place unless the user explicitly asks you to delete it.",
self.image_output_dir, self.image_output_path
)
image_generation_hint(&self.image_output_dir, &self.image_output_path)
}
}
1 change: 1 addition & 0 deletions codex-rs/core/src/context/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ pub(crate) use fragments::AdditionalContextUserFragment;
pub(crate) use guardian_followup_review_reminder::GuardianFollowupReviewReminder;
pub(crate) use hook_additional_context::HookAdditionalContext;
pub(crate) use image_generation_instructions::ImageGenerationInstructions;
pub use image_generation_instructions::extension_image_generation_output_hint;
pub use internal_model_context::InternalContextSource;
pub use internal_model_context::InternalModelContextFragment;
pub use internal_model_context::InvalidInternalContextSource;
Expand Down
1 change: 1 addition & 0 deletions codex-rs/core/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ pub(crate) use skills::manager;
pub(crate) use skills::maybe_emit_implicit_skill_invocation;
pub(crate) use skills::skills_load_input_from_config;
mod stream_events_utils;
pub use stream_events_utils::image_generation_artifact_path;
pub mod test_support;
mod unified_exec;
pub mod windows_sandbox;
Expand Down
3 changes: 2 additions & 1 deletion codex-rs/core/src/stream_events_utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ use tracing::warn;

const GENERATED_IMAGE_ARTIFACTS_DIR: &str = "generated_images";

pub(crate) fn image_generation_artifact_path(
/// Returns the host-owned default artifact path for a generated image.
pub fn image_generation_artifact_path(
codex_home: &AbsolutePathBuf,
session_id: &str,
call_id: &str,
Expand Down
1 change: 1 addition & 0 deletions codex-rs/ext/image-generation/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ codex-model-provider = { workspace = true }
codex-model-provider-info = { workspace = true }
codex-protocol = { workspace = true }
codex-tools = { workspace = true }
codex-utils-absolute-path = { workspace = true }

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Refresh Bazel lockfile for the new dependency

Adding this workspace dependency changes the Rust dependency graph, but the commit does not include the corresponding MODULE.bazel.lock update required by the root AGENTS.md dependency-change rule. I checked this commit locally with just bazel-lock-check; it exits with MODULE.bazel.lock is out of date. Run 'just bazel-lock-update' and commit the updated lockfile., so Bazel lock validation will fail until the lockfile update is committed.

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

already ran this and no change

http = { workspace = true }
schemars = { workspace = true }
serde = { workspace = true, features = ["derive"] }
Expand Down
1 change: 1 addition & 0 deletions codex-rs/ext/image-generation/imagegen_description.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ The `image_gen.imagegen` tool enables image generation from descriptions and edi
- The user wants to modify an attached or previously generated image with specific changes, including adding or removing elements, altering colors, improving quality/resolution, or transforming the style (e.g., cartoon, oil painting).

Guidelines:
- In code mode, pass the result to `generatedImage(result)`.
- Set `action` to `generate` when the user asks for a brand new image.
- Set `action` to `edit` when the user asks to modify an existing image from the conversation history.
- Directly generate the image without reconfirmation or clarification.
Expand Down
14 changes: 11 additions & 3 deletions codex-rs/ext/image-generation/src/extension.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ use codex_features::Feature;
use codex_login::AuthManager;
use codex_model_provider::create_model_provider;
use codex_model_provider_info::ModelProviderInfo;
use codex_utils_absolute_path::AbsolutePathBuf;

use crate::backend::CodexImagesBackend;
use crate::tool::ImageGenerationTool;
Expand All @@ -26,6 +27,7 @@ struct ImageGenerationExtension {
struct ImageGenerationExtensionConfig {
enabled: bool,
provider: ModelProviderInfo,
codex_home: AbsolutePathBuf,
}

impl From<&Config> for ImageGenerationExtensionConfig {
Expand All @@ -35,6 +37,7 @@ impl From<&Config> for ImageGenerationExtensionConfig {
enabled: config.features.enabled(Feature::ImageGenExt)
&& config.model_provider.is_openai(),
provider: config.model_provider.clone(),
codex_home: config.codex_home.clone(),
}
}
}
Expand Down Expand Up @@ -76,9 +79,14 @@ impl ToolContributor for ImageGenerationExtension {
return Vec::new();
}

vec![Arc::new(ImageGenerationTool::new(CodexImagesBackend::new(
create_model_provider(config.provider.clone(), Some(self.auth_manager.clone())),
)))]
vec![Arc::new(ImageGenerationTool::new(
CodexImagesBackend::new(create_model_provider(
config.provider.clone(),
Some(self.auth_manager.clone()),
)),
config.codex_home.clone(),
thread_store.level_id().to_string(),
))]
}
}

Expand Down
52 changes: 51 additions & 1 deletion codex-rs/ext/image-generation/src/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ use codex_api::ImageEditRequest;
use codex_api::ImageGenerationRequest;
use codex_api::ImageQuality;
use codex_api::ImageUrl;
use codex_core::context::extension_image_generation_output_hint;
use codex_extension_api::ToolOutput;
use codex_extension_api::ToolPayload;
use codex_extension_api::ToolSpec;
Expand Down Expand Up @@ -54,9 +55,58 @@ fn generate_uses_fixed_request_defaults() {
}

#[test]
fn generated_output_returns_image_input() {
fn generated_output_returns_image_input_and_output_hint() {
let output_hint =
extension_image_generation_output_hint("/tmp", "/tmp/call-1.png").expect("hint should fit");
let output = GeneratedImageOutput {
result: RESULT.to_string(),
output_hint: Some(output_hint.clone()),
};

let ResponseInputItem::FunctionCallOutput {
output: response_output,
..
} = output.to_response_item("call-1", &function_payload())
else {
panic!("imagegen should return function tool output");
};
let FunctionCallOutputBody::ContentItems(content_items) = response_output.body else {
panic!("imagegen output should contain generated image bytes");
};
assert_eq!(
content_items,
vec![
FunctionCallOutputContentItem::InputImage {
image_url: format!("data:image/png;base64,{RESULT}"),
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputText { text: output_hint },
]
);
}

#[test]
fn generated_output_returns_generated_image_helper_input_in_code_mode() {
let output = GeneratedImageOutput {
result: RESULT.to_string(),
output_hint: Some("generated image save hint".to_string()),
};

assert_eq!(
output.code_mode_result(&function_payload()),
serde_json::json!({
"image_url": format!("data:image/png;base64,{RESULT}"),
"output_hint": "generated image save hint",
})
);
}

#[test]
fn generated_output_omits_oversized_output_hint() {
let long_path = "x".repeat(1024);
let output = GeneratedImageOutput {
result: RESULT.to_string(),
output_hint: extension_image_generation_output_hint("/tmp", long_path),
};

let ResponseInputItem::FunctionCallOutput {
Expand Down
Loading
Loading