From 23899f7cb63a1510e53fddd68740dfc325853e3b Mon Sep 17 00:00:00 2001 From: nhamidi-oai Date: Fri, 17 Jul 2026 23:55:04 +0000 Subject: [PATCH] Forward audio inputs to the Responses API (#33932) ## Why Audio variants were available in the user-input protocol but were replaced with unsupported-input placeholders instead of being sent to the model. ## What changed - Serialize audio data URLs as `input_audio` content and convert local `wav`, `mp3`, `m4a`, `webm`, and `ogg` files to labeled data URLs. - Validate and canonicalize base64 audio before requests, enforce the 50 MiB input limit, and replace invalid, unsupported, or oversized audio with explanatory text. - Preserve audio attachments when mapping response items back to user-message events and document the app-server input variants. ## Testing - Add unit coverage for local-file conversion, data URL validation, size and format errors, event mapping, and attachment extraction. - Add client tests that verify data URL and local audio payloads sent to the Responses API. GitOrigin-RevId: f72cd6b6e43ab12757eeb47621b1d670594ac7d9 --- codex-rs/app-server/README.md | 7 +- codex-rs/core/src/audio_preparation.rs | 179 ++++++++++++ codex-rs/core/src/audio_preparation_tests.rs | 159 +++++++++++ codex-rs/core/src/event_mapping.rs | 19 +- codex-rs/core/src/event_mapping_tests.rs | 44 +++ codex-rs/core/src/lib.rs | 1 + codex-rs/core/src/session/mod.rs | 13 +- .../src/tools/handlers/multi_agents_spec.rs | 5 +- codex-rs/core/tests/suite/client.rs | 100 +++++++ codex-rs/protocol/src/items.rs | 44 +++ codex-rs/protocol/src/models.rs | 259 ++++++++++++++++-- codex-rs/protocol/src/user_input.rs | 4 +- codex-rs/tui/src/chatwidget/user_messages.rs | 8 +- 13 files changed, 802 insertions(+), 40 deletions(-) create mode 100644 codex-rs/core/src/audio_preparation.rs create mode 100644 codex-rs/core/src/audio_preparation_tests.rs diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index afa340e8ec02..13a19c3be4e6 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -757,13 +757,16 @@ If the thread does not already have an active turn, the server starts a standalo ### Example: Start a turn (send user input) -Turns attach user input (text or images) to a thread and trigger Codex generation. The `input` field is a list of discriminated unions: +Turns attach user input (text, images, or audio) to a thread and trigger Codex generation. The `input` field is a list of discriminated unions: - `{"type":"text","text":"Explain this diff"}` - `{"type":"image","url":"data:image/png;base64,…"}` - `{"type":"localImage","path":"/tmp/screenshot.png"}` +- `{"type":"audio","url":"data:audio/wav;base64,…"}` +- `{"type":"localAudio","path":"/tmp/recording.mp3"}` The `image` variant accepts inline data URLs. Remote HTTP(S) image URLs are rejected; use a data URL or `localImage` instead. +The `audio` variant accepts data URLs. Other URL schemes are rejected. `localAudio` reads local wav, mp3, m4a, webm, and ogg files and converts them to data URLs before the Responses API request. You can optionally specify config overrides on the new turn. If specified, these settings become the default for subsequent turns on the same thread. `outputSchema` applies only to the current turn. Experimental `environments` is turn-scoped: omit it to inherit the thread's sticky environments, pass `[]` to run the turn with no environments, or pass explicit environment ids to override the sticky selection for this turn only. @@ -1420,7 +1423,7 @@ Today both notifications carry an empty `items` array even when item events were `ThreadItem` is the tagged union carried in turn responses and `item/*` notifications. Currently we support events for the following items: -- `userMessage` — `{id, clientId, content}` where `clientId` is the optional `clientUserMessageId` supplied to `turn/start` or `turn/steer`, and `content` is a list of user inputs (`text`, `image`, or `localImage`). +- `userMessage` — `{id, clientId, content}` where `clientId` is the optional `clientUserMessageId` supplied to `turn/start` or `turn/steer`, and `content` is a list of user inputs (`text`, `image`, `localImage`, `audio`, or `localAudio`). - `agentMessage` — `{id, text}` containing the accumulated agent reply. - `plan` — `{id, text}` emitted for plan-mode turns; plan text can stream via `item/plan/delta` (experimental). - `reasoning` — `{id, summary, content}` where `summary` holds streamed reasoning summaries (applicable for most OpenAI models) and `content` holds raw reasoning blocks (applicable for e.g. open source models). diff --git a/codex-rs/core/src/audio_preparation.rs b/codex-rs/core/src/audio_preparation.rs new file mode 100644 index 000000000000..e0ed11fb938f --- /dev/null +++ b/codex-rs/core/src/audio_preparation.rs @@ -0,0 +1,179 @@ +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; +use codex_protocol::models::ContentItem; +use codex_protocol::models::FunctionCallOutputContentItem; +use codex_protocol::models::ResponseItem; +use tracing::warn; + +const AUDIO_PROCESSING_ERROR_PLACEHOLDER: &str = + "audio content omitted because it could not be processed"; +const AUDIO_TOO_LARGE_PLACEHOLDER: &str = + "audio content omitted because it exceeded the supported size limit; use a smaller audio file"; +const UNSUPPORTED_AUDIO_FORMAT_PLACEHOLDER: &str = + "audio content omitted because its format is not supported; use wav, mp3, m4a, webm, or ogg"; + +/// Maximum accepted decoded byte length for prompt audio inputs. +/// +/// This matches the Responses API audio input limit. +const MAX_PROMPT_AUDIO_INPUT_BYTES: usize = 50 * 1024 * 1024; +const MAX_PROMPT_AUDIO_BASE64_BYTES: usize = MAX_PROMPT_AUDIO_INPUT_BYTES.div_ceil(3) * 4; + +#[derive(Debug, thiserror::Error)] +enum AudioPreparationError { + #[error("invalid audio data URL: {reason}")] + InvalidDataUrl { reason: &'static str }, + #[error("unsupported audio format")] + UnsupportedFormat, + #[error("audio input is too large ({size} bytes; max {MAX_PROMPT_AUDIO_INPUT_BYTES} bytes)")] + AudioTooLarge { size: usize }, +} + +impl AudioPreparationError { + fn placeholder(&self) -> &'static str { + match self { + AudioPreparationError::InvalidDataUrl { .. } => AUDIO_PROCESSING_ERROR_PLACEHOLDER, + AudioPreparationError::UnsupportedFormat => UNSUPPORTED_AUDIO_FORMAT_PLACEHOLDER, + AudioPreparationError::AudioTooLarge { .. } => AUDIO_TOO_LARGE_PLACEHOLDER, + } + } +} + +pub(crate) fn prepare_response_items(items: &mut [ResponseItem]) { + for item in items { + match item { + ResponseItem::Message { content, .. } => prepare_message_content(content), + ResponseItem::FunctionCallOutput { output, .. } + | ResponseItem::CustomToolCallOutput { output, .. } => { + if let Some(content) = output.content_items_mut() { + prepare_tool_output_content(content); + } + } + ResponseItem::AdditionalTools { .. } + | ResponseItem::Reasoning { .. } + | ResponseItem::AgentMessage { .. } + | ResponseItem::LocalShellCall { .. } + | ResponseItem::FunctionCall { .. } + | ResponseItem::ToolSearchCall { .. } + | ResponseItem::CustomToolCall { .. } + | ResponseItem::ToolSearchOutput { .. } + | ResponseItem::WebSearchCall { .. } + | ResponseItem::ImageGenerationCall { .. } + | ResponseItem::Compaction { .. } + | ResponseItem::CompactionTrigger { .. } + | ResponseItem::ContextCompaction { .. } + | ResponseItem::Other => {} + } + } +} + +fn prepare_message_content(items: &mut [ContentItem]) { + for item in items { + if let ContentItem::InputAudio { audio_url } = item + && let Err(error) = prepare_audio(audio_url) + { + warn!(%error, "failed to prepare message audio"); + *item = ContentItem::InputText { + text: error.placeholder().to_string(), + }; + } + } +} + +fn prepare_tool_output_content(items: &mut [FunctionCallOutputContentItem]) { + for item in items { + if let FunctionCallOutputContentItem::InputAudio { audio_url } = item + && let Err(error) = prepare_audio(audio_url) + { + warn!(%error, "failed to prepare tool output audio"); + *item = FunctionCallOutputContentItem::InputText { + text: error.placeholder().to_string(), + }; + } + } +} + +fn is_data_url(audio_url: &str) -> bool { + audio_url + .get(.."data:".len()) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:")) +} + +fn canonical_audio_mime(mime: &str) -> Option<&'static str> { + if mime.eq_ignore_ascii_case("audio/wav") + || mime.eq_ignore_ascii_case("audio/x-wav") + || mime.eq_ignore_ascii_case("audio/wave") + || mime.eq_ignore_ascii_case("audio/vnd.wave") + { + Some("audio/wav") + } else if mime.eq_ignore_ascii_case("audio/mpeg") || mime.eq_ignore_ascii_case("audio/mp3") { + Some("audio/mpeg") + } else if mime.eq_ignore_ascii_case("audio/mp4") + || mime.eq_ignore_ascii_case("audio/m4a") + || mime.eq_ignore_ascii_case("audio/x-m4a") + { + Some("audio/mp4") + } else if mime.eq_ignore_ascii_case("audio/webm") { + Some("audio/webm") + } else if mime.eq_ignore_ascii_case("audio/ogg") { + Some("audio/ogg") + } else { + None + } +} + +fn prepare_audio(audio_url: &mut String) -> Result<(), AudioPreparationError> { + if !is_data_url(audio_url) { + return Err(AudioPreparationError::InvalidDataUrl { + reason: "audio input must be a data URL", + }); + } + + let (metadata, payload) = + audio_url + .split_once(',') + .ok_or(AudioPreparationError::InvalidDataUrl { + reason: "missing payload separator", + })?; + let metadata = metadata + .get("data:".len()..) + .ok_or(AudioPreparationError::InvalidDataUrl { + reason: "missing data URL prefix", + })?; + let mut metadata_parts = metadata.split(';'); + let mime = metadata_parts + .next() + .filter(|mime| !mime.is_empty()) + .ok_or(AudioPreparationError::InvalidDataUrl { + reason: "missing media type", + })?; + let canonical_mime = + canonical_audio_mime(mime).ok_or(AudioPreparationError::UnsupportedFormat)?; + if !metadata_parts.any(|part| part.eq_ignore_ascii_case("base64")) { + return Err(AudioPreparationError::InvalidDataUrl { + reason: "audio payload is not base64 encoded", + }); + } + if payload.len() > MAX_PROMPT_AUDIO_BASE64_BYTES { + return Err(AudioPreparationError::AudioTooLarge { + size: payload.len(), + }); + } + + let bytes = + BASE64_STANDARD + .decode(payload) + .map_err(|_| AudioPreparationError::InvalidDataUrl { + reason: "invalid base64 payload", + })?; + if bytes.len() > MAX_PROMPT_AUDIO_INPUT_BYTES { + return Err(AudioPreparationError::AudioTooLarge { size: bytes.len() }); + } + + let encoded = BASE64_STANDARD.encode(bytes); + *audio_url = format!("data:{canonical_mime};base64,{encoded}"); + Ok(()) +} + +#[cfg(test)] +#[path = "audio_preparation_tests.rs"] +mod tests; diff --git a/codex-rs/core/src/audio_preparation_tests.rs b/codex-rs/core/src/audio_preparation_tests.rs new file mode 100644 index 000000000000..9307f8a84e81 --- /dev/null +++ b/codex-rs/core/src/audio_preparation_tests.rs @@ -0,0 +1,159 @@ +use codex_protocol::models::FunctionCallOutputBody; +use codex_protocol::models::FunctionCallOutputPayload; +use pretty_assertions::assert_eq; + +use super::*; + +#[test] +fn preparation_canonicalizes_data_urls_and_rejects_remote_urls() { + let mut items = vec![ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ + ContentItem::InputAudio { + audio_url: "data:audio/x-wav;base64,YXVkaW8=".to_string(), + }, + ContentItem::InputAudio { + audio_url: "data:audio/ogg;base64,YXVkaW8=".to_string(), + }, + ContentItem::InputAudio { + audio_url: "https://example.com/audio.mp3".to_string(), + }, + ], + phase: None, + internal_chat_message_metadata_passthrough: None, + }]; + + prepare_response_items(&mut items); + + assert_eq!( + items, + vec![ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ + ContentItem::InputAudio { + audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(), + }, + ContentItem::InputAudio { + audio_url: "data:audio/ogg;base64,YXVkaW8=".to_string(), + }, + ContentItem::InputText { + text: "audio content omitted because it could not be processed".to_string(), + }, + ], + phase: None, + internal_chat_message_metadata_passthrough: None, + }] + ); +} + +#[test] +fn preparation_replaces_invalid_message_audio_with_placeholders() { + let mut items = vec![ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ + ContentItem::InputAudio { + audio_url: "data:audio/wav;base64,%%%".to_string(), + }, + ContentItem::InputAudio { + audio_url: "data:audio/flac;base64,YXVkaW8=".to_string(), + }, + ], + phase: None, + internal_chat_message_metadata_passthrough: None, + }]; + + prepare_response_items(&mut items); + + assert_eq!( + items, + vec![ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ + ContentItem::InputText { + text: "audio content omitted because it could not be processed".to_string(), + }, + ContentItem::InputText { + text: "audio content omitted because its format is not supported; use wav, mp3, m4a, webm, or ogg".to_string(), + }, + ], + phase: None, + internal_chat_message_metadata_passthrough: None, + }] + ); +} + +#[test] +fn preparation_replaces_only_failed_tool_audio_and_preserves_metadata() { + let mut items = vec![ResponseItem::FunctionCallOutput { + id: None, + call_id: "call-1".to_string(), + output: FunctionCallOutputPayload { + body: FunctionCallOutputBody::ContentItems(vec![ + FunctionCallOutputContentItem::InputText { + text: "before".to_string(), + }, + FunctionCallOutputContentItem::InputAudio { + audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(), + }, + FunctionCallOutputContentItem::InputAudio { + audio_url: "data:audio/wav,not-base64".to_string(), + }, + ]), + success: Some(true), + }, + internal_chat_message_metadata_passthrough: None, + }]; + + prepare_response_items(&mut items); + + assert_eq!( + items, + vec![ResponseItem::FunctionCallOutput { + id: None, + call_id: "call-1".to_string(), + output: FunctionCallOutputPayload { + body: FunctionCallOutputBody::ContentItems(vec![ + FunctionCallOutputContentItem::InputText { + text: "before".to_string(), + }, + FunctionCallOutputContentItem::InputAudio { + audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(), + }, + FunctionCallOutputContentItem::InputText { + text: "audio content omitted because it could not be processed".to_string(), + }, + ]), + success: Some(true), + }, + internal_chat_message_metadata_passthrough: None, + }] + ); +} + +#[test] +fn preparation_errors_map_to_expected_placeholders() { + let cases = [ + ( + AudioPreparationError::InvalidDataUrl { + reason: "details remain in logs", + }, + "audio content omitted because it could not be processed", + ), + ( + AudioPreparationError::UnsupportedFormat, + "audio content omitted because its format is not supported; use wav, mp3, m4a, webm, or ogg", + ), + ( + AudioPreparationError::AudioTooLarge { size: usize::MAX }, + "audio content omitted because it exceeded the supported size limit; use a smaller audio file", + ), + ]; + + for (error, expected) in cases { + assert_eq!(error.placeholder(), expected); + } +} diff --git a/codex-rs/core/src/event_mapping.rs b/codex-rs/core/src/event_mapping.rs index f3b714c8ef3c..c92b79a7d921 100644 --- a/codex-rs/core/src/event_mapping.rs +++ b/codex-rs/core/src/event_mapping.rs @@ -10,8 +10,12 @@ use codex_protocol::models::ReasoningItemContent; use codex_protocol::models::ReasoningItemReasoningSummary; use codex_protocol::models::ResponseItem; use codex_protocol::models::WebSearchAction; +use codex_protocol::models::is_audio_close_tag_text; +use codex_protocol::models::is_audio_open_tag_text; use codex_protocol::models::is_image_close_tag_text; use codex_protocol::models::is_image_open_tag_text; +use codex_protocol::models::is_local_audio_close_tag_text; +use codex_protocol::models::is_local_audio_open_tag_text; use codex_protocol::models::is_local_image_close_tag_text; use codex_protocol::models::is_local_image_open_tag_text; use codex_protocol::protocol::APPS_INSTRUCTIONS_OPEN_TAG; @@ -93,12 +97,19 @@ fn parse_user_message(message: &[ContentItem]) -> Option { for (idx, content_item) in message.iter().enumerate() { match content_item { ContentItem::InputText { text } => { - if (is_local_image_open_tag_text(text) || is_image_open_tag_text(text)) - && (matches!(message.get(idx + 1), Some(ContentItem::InputImage { .. }))) + let is_image_label = ((is_local_image_open_tag_text(text) + || is_image_open_tag_text(text)) + && matches!(message.get(idx + 1), Some(ContentItem::InputImage { .. }))) || (idx > 0 && (is_local_image_close_tag_text(text) || is_image_close_tag_text(text)) - && matches!(message.get(idx - 1), Some(ContentItem::InputImage { .. }))) - { + && matches!(message.get(idx - 1), Some(ContentItem::InputImage { .. }))); + let is_audio_label = ((is_local_audio_open_tag_text(text) + || is_audio_open_tag_text(text)) + && matches!(message.get(idx + 1), Some(ContentItem::InputAudio { .. }))) + || (idx > 0 + && (is_local_audio_close_tag_text(text) || is_audio_close_tag_text(text)) + && matches!(message.get(idx - 1), Some(ContentItem::InputAudio { .. }))); + if is_image_label || is_audio_label { continue; } content.push(UserInput::Text { diff --git a/codex-rs/core/src/event_mapping_tests.rs b/codex-rs/core/src/event_mapping_tests.rs index 36f840c0c019..ac8ed66c5e87 100644 --- a/codex-rs/core/src/event_mapping_tests.rs +++ b/codex-rs/core/src/event_mapping_tests.rs @@ -165,6 +165,50 @@ fn skips_local_image_label_text() { } } +#[test] +fn skips_local_audio_label_text() { + let audio_url = "data:audio/wav;base64,abc".to_string(); + let label = r#"".to_string(), + }, + ContentItem::InputText { + text: user_text.clone(), + }, + ], + phase: None, + internal_chat_message_metadata_passthrough: None, + }; + + let turn_item = parse_turn_item(&item).expect("expected user message turn item"); + + match turn_item { + TurnItem::UserMessage(user) => { + assert_eq!( + user.content, + vec![ + UserInput::Audio { audio_url }, + UserInput::Text { + text: user_text, + text_elements: Vec::new(), + }, + ] + ); + } + other => panic!("expected TurnItem::UserMessage, got {other:?}"), + } +} + #[test] fn parses_assistant_message_input_text_for_backward_compatibility() { let item = ResponseItem::Message { diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 0341090b20be..b7572a66cd46 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -7,6 +7,7 @@ mod apply_patch; mod apps; +mod audio_preparation; mod client; mod client_common; mod realtime_context; diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 0d5e5b714459..f08cb036dc7d 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -17,6 +17,7 @@ use crate::agent::status::is_final; use crate::agent_communication::AgentCommunicationContext; use crate::agent_communication::AgentCommunicationKind; use crate::attestation::AttestationProvider; +use crate::audio_preparation::prepare_response_items as prepare_audio_response_items; use crate::build_available_skills; use crate::compact; use crate::config::ManagedFeatures; @@ -35,7 +36,7 @@ use crate::current_time::TimeProvider; use crate::default_skill_metadata_budget; use crate::environment_selection::TurnEnvironmentSnapshot; use crate::exec_policy::ExecPolicyManager; -use crate::image_preparation::prepare_response_items; +use crate::image_preparation::prepare_response_items as prepare_image_response_items; use crate::parse_turn_item; use crate::realtime_conversation::RealtimeConversationManager; use crate::session::step_context::StepContext; @@ -1361,10 +1362,11 @@ impl Session { .reconstruct_history_from_rollout(turn_context, rollout_items) .await; // Keep the recorded rollout unchanged. Prepare its reconstructed history before - // installing it, so legacy images are processed once for this resume or fork and + // installing it, so legacy media is processed once for this resume or fork and // will be processed again if the rollout is reconstructed in a future session. - // This meets image resizing requirements without modifying persisted rollouts. - prepare_response_items(&mut history); + // This meets media preparation requirements without modifying persisted rollouts. + prepare_image_response_items(&mut history); + prepare_audio_response_items(&mut history); { let mut state = self.state.lock().await; state.replace_history(history, reference_context_item); @@ -2766,7 +2768,8 @@ impl Session { items: &'a [ResponseItem], ) -> Cow<'a, [ResponseItem]> { let mut items = Cow::Borrowed(items); - prepare_response_items(items.to_mut()); + prepare_image_response_items(items.to_mut()); + prepare_audio_response_items(items.to_mut()); // Most response items get their passthrough turn ID at the durable history boundary. for item in items.to_mut() { item.set_turn_id_if_missing(&turn_context.sub_id); diff --git a/codex-rs/core/src/tools/handlers/multi_agents_spec.rs b/codex-rs/core/src/tools/handlers/multi_agents_spec.rs index 9e74b01a5077..2eb82e175b45 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_spec.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_spec.rs @@ -548,7 +548,8 @@ fn create_collab_input_items_schema() -> JsonSchema { ( "type".to_string(), JsonSchema::string(Some( - "Input item type: text, image, local_image, skill, or mention.".to_string(), + "Input item type: text, image, local_image, audio, local_audio, skill, or mention." + .to_string(), )), ), ( @@ -561,7 +562,7 @@ fn create_collab_input_items_schema() -> JsonSchema { ), ( "audio_url".to_string(), - JsonSchema::string(Some("Audio URL when type is audio.".to_string())), + JsonSchema::string(Some("Audio data URL when type is audio.".to_string())), ), ( "path".to_string(), diff --git a/codex-rs/core/tests/suite/client.rs b/codex-rs/core/tests/suite/client.rs index 827708516d4a..2789cb6d111b 100644 --- a/codex-rs/core/tests/suite/client.rs +++ b/codex-rs/core/tests/suite/client.rs @@ -304,6 +304,106 @@ async fn non_openai_responses_requests_omit_item_passthrough_metadata() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn sends_audio_urls_to_responses() { + skip_if_no_network!(); + + let server = MockServer::start().await; + let response_mock = mount_sse_once( + &server, + sse(vec![ev_response_created("resp1"), ev_completed("resp1")]), + ) + .await; + let codex = test_codex().build(&server).await.unwrap().codex; + let audio_url = "data:audio/wav;base64,AAAA"; + + codex + .submit(Op::UserInput { + items: vec![UserInput::Audio { + audio_url: audio_url.to_string(), + }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: Default::default(), + thread_settings: Default::default(), + }) + .await + .unwrap(); + wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + + let user_message = response_mock + .single_request() + .input() + .into_iter() + .rev() + .find(|item| item.get("role").and_then(serde_json::Value::as_str) == Some("user")) + .expect("request should include a user message"); + assert_eq!( + user_message["content"], + json!([{ + "type": "input_audio", + "audio_url": audio_url, + }]) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn sends_local_audio_to_responses() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = MockServer::start().await; + let response_mock = mount_sse_once( + &server, + sse(vec![ev_response_created("resp1"), ev_completed("resp1")]), + ) + .await; + let codex = test_codex().build(&server).await?.codex; + let temp_dir = tempfile::tempdir()?; + let audio_path = temp_dir.path().join("recording.wav"); + std::fs::write(&audio_path, b"audio")?; + + codex + .submit(Op::UserInput { + items: vec![UserInput::LocalAudio { + path: audio_path.clone(), + }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: Default::default(), + thread_settings: Default::default(), + }) + .await?; + wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + + let user_message = response_mock + .single_request() + .input() + .into_iter() + .rev() + .find(|item| item.get("role").and_then(serde_json::Value::as_str) == Some("user")) + .expect("request should include a user message"); + let audio_path = audio_path.display(); + assert_eq!( + user_message["content"], + json!([ + { + "type": "input_text", + "text": format!(r#"", + }, + ]) + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn response_item_ids_persist_across_resume_and_preserve_server_ids() -> anyhow::Result<()> { let server = MockServer::start().await; diff --git a/codex-rs/protocol/src/items.rs b/codex-rs/protocol/src/items.rs index a418e63d57de..5c1b85bb675c 100644 --- a/codex-rs/protocol/src/items.rs +++ b/codex-rs/protocol/src/items.rs @@ -516,6 +516,26 @@ impl UserMessageItem { .collect(), ) } + + pub fn audio_urls(&self) -> Vec { + self.content + .iter() + .filter_map(|c| match c { + UserInput::Audio { audio_url } => Some(audio_url.clone()), + _ => None, + }) + .collect() + } + + pub fn local_audio_paths(&self) -> Vec { + self.content + .iter() + .filter_map(|c| match c { + UserInput::LocalAudio { path } => Some(path.clone()), + _ => None, + }) + .collect() + } } fn trim_trailing_default_image_details( @@ -671,6 +691,30 @@ mod tests { ); } + #[test] + fn user_message_item_extracts_audio_attachments() { + let item = UserMessageItem::new(&[ + UserInput::Text { + text: "transcribe these".to_string(), + text_elements: Vec::new(), + }, + UserInput::Audio { + audio_url: "https://example.com/remote.mp3".to_string(), + }, + UserInput::LocalAudio { + path: std::path::PathBuf::from("local.wav"), + }, + ]); + + assert_eq!( + (item.audio_urls(), item.local_audio_paths()), + ( + vec!["https://example.com/remote.mp3".to_string()], + vec![std::path::PathBuf::from("local.wav")], + ) + ); + } + #[test] fn hook_prompt_roundtrips_multiple_fragments() { let original = vec![ diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index 1774921288c6..222c1ad8d69d 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -1364,13 +1364,6 @@ fn local_media_error_placeholder( } } -fn local_media_kind_unsupported(media_kind: LocalMediaKind) -> ContentItem { - let media_name = media_kind.name(); - ContentItem::InputText { - text: format!("Codex does not support local {media_name} input yet."), - } -} - pub const VIEW_IMAGE_TOOL_NAME: &str = "view_image"; const IMAGE_OPEN_TAG: &str = ""; @@ -1378,6 +1371,11 @@ const IMAGE_CLOSE_TAG: &str = ""; const LOCAL_IMAGE_OPEN_TAG_PREFIX: &str = ""; const LOCAL_IMAGE_CLOSE_TAG: &str = IMAGE_CLOSE_TAG; +const AUDIO_OPEN_TAG: &str = ""; +const LOCAL_AUDIO_OPEN_TAG_PREFIX: &str = "