diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 4cd4636b9af1..6ffe55efafcc 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -11550,6 +11550,13 @@ "image" ], "type": "string" + }, + { + "description": "Audio attachments included in user turns.", + "enum": [ + "audio" + ], + "type": "string" } ] }, diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 2ba1134dc6a3..0076353c8f38 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -7907,6 +7907,13 @@ "image" ], "type": "string" + }, + { + "description": "Audio attachments included in user turns.", + "enum": [ + "audio" + ], + "type": "string" } ] }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ModelListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ModelListResponse.json index ce6c976d340c..c50a6bdfa029 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ModelListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ModelListResponse.json @@ -17,6 +17,13 @@ "image" ], "type": "string" + }, + { + "description": "Audio attachments included in user turns.", + "enum": [ + "audio" + ], + "type": "string" } ] }, diff --git a/codex-rs/app-server-protocol/schema/typescript/InputModality.ts b/codex-rs/app-server-protocol/schema/typescript/InputModality.ts index 73661938b38a..40d598df3db5 100644 --- a/codex-rs/app-server-protocol/schema/typescript/InputModality.ts +++ b/codex-rs/app-server-protocol/schema/typescript/InputModality.ts @@ -5,4 +5,4 @@ /** * Canonical user-input modality tags advertised by a model. */ -export type InputModality = "text" | "image"; +export type InputModality = "text" | "image" | "audio"; diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs index 318c5d4d2fb3..464a94cbedcc 100644 --- a/codex-rs/core/src/context_manager/history.rs +++ b/codex-rs/core/src/context_manager/history.rs @@ -135,9 +135,8 @@ impl ContextManager { } /// Returns the history prepared for sending to the model. This applies a proper - /// normalization and drops un-suited items. When `input_modalities` does not - /// include `InputModality::Image`, images are stripped from messages and tool - /// outputs. + /// normalization and drops un-suited items. Unsupported image and audio content + /// is stripped from messages and tool outputs according to `input_modalities`. pub(crate) fn for_prompt(mut self, input_modalities: &[InputModality]) -> Vec { self.normalize_history(input_modalities); self.items @@ -355,7 +354,7 @@ impl ContextManager { /// This function enforces a couple of invariants on the in-memory history: /// 1. every call (function/custom) has a corresponding output entry /// 2. every output has a corresponding call entry - /// 3. when images are unsupported, image content is stripped from messages and tool outputs + /// 3. unsupported image and audio content is stripped from messages and tool outputs fn normalize_history(&mut self, input_modalities: &[InputModality]) { // all function/tool calls must have a corresponding output normalize::ensure_call_outputs_present(&mut self.items); @@ -365,6 +364,9 @@ impl ContextManager { // strip images when model does not support them normalize::strip_images_when_unsupported(input_modalities, &mut self.items); + + // strip audio when model does not support it + normalize::strip_audio_when_unsupported(input_modalities, &mut self.items); } fn process_item(&self, item: &ResponseItem, policy: TruncationPolicy) -> ResponseItem { diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs index 8a21b6b77587..e983655df0a8 100644 --- a/codex-rs/core/src/context_manager/history_tests.rs +++ b/codex-rs/core/src/context_manager/history_tests.rs @@ -471,7 +471,7 @@ fn total_token_usage_includes_all_items_after_last_model_generated_item() { } #[test] -fn for_prompt_strips_images_when_model_does_not_support_images() { +fn for_prompt_strips_media_when_model_does_not_support_it() { let items = vec![ ResponseItem::Message { id: None, @@ -484,6 +484,9 @@ fn for_prompt_strips_images_when_model_does_not_support_images() { image_url: "https://example.com/img.png".to_string(), detail: Some(DEFAULT_IMAGE_DETAIL), }, + ContentItem::InputAudio { + audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(), + }, ContentItem::InputText { text: "caption".to_string(), }, @@ -554,6 +557,10 @@ fn for_prompt_strips_images_when_model_does_not_support_images() { text: "image content omitted because you do not support image input" .to_string(), }, + ContentItem::InputText { + text: "audio content omitted because you do not support audio input" + .to_string(), + }, ContentItem::InputText { text: "caption".to_string(), }, @@ -635,6 +642,21 @@ fn for_prompt_strips_images_when_model_does_not_support_images() { } else { panic!("expected Message"); } + + let audio_message = ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputAudio { + audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(), + }], + phase: None, + internal_chat_message_metadata_passthrough: None, + }; + let with_audio = create_history_with_items(vec![audio_message.clone()]); + assert_eq!( + with_audio.for_prompt(&[InputModality::Text, InputModality::Audio]), + vec![audio_message] + ); } #[test] diff --git a/codex-rs/core/src/context_manager/normalize.rs b/codex-rs/core/src/context_manager/normalize.rs index 35230301961f..a98c47efd8d3 100644 --- a/codex-rs/core/src/context_manager/normalize.rs +++ b/codex-rs/core/src/context_manager/normalize.rs @@ -12,6 +12,8 @@ use tracing::info; const IMAGE_CONTENT_OMITTED_PLACEHOLDER: &str = "image content omitted because you do not support image input"; +const AUDIO_CONTENT_OMITTED_PLACEHOLDER: &str = + "audio content omitted because you do not support audio input"; // Changing this value would change model-visible IDs and invalidate prompt caches. const SYNTHETIC_OUTPUT_ID_NAMESPACE: Uuid = Uuid::from_u128(0x90d38d3e_6a5b_4d52_bfe2_2f1e634bfac4); @@ -366,3 +368,26 @@ pub(crate) fn strip_images_when_unsupported( } } } + +/// Strip audio content from messages when the model does not support audio. +/// When `input_modalities` contains `InputModality::Audio`, no stripping is performed. +pub(crate) fn strip_audio_when_unsupported( + input_modalities: &[InputModality], + items: &mut [ResponseItem], +) { + if input_modalities.contains(&InputModality::Audio) { + return; + } + + for item in items.iter_mut() { + if let ResponseItem::Message { content, .. } = item { + for content_item in content.iter_mut() { + if matches!(content_item, ContentItem::InputAudio { .. }) { + *content_item = ContentItem::InputText { + text: AUDIO_CONTENT_OMITTED_PLACEHOLDER.to_string(), + }; + } + } + } + } +} diff --git a/codex-rs/core/tests/common/responses.rs b/codex-rs/core/tests/common/responses.rs index 1ab3a70cbb9e..d2f335747399 100644 --- a/codex-rs/core/tests/common/responses.rs +++ b/codex-rs/core/tests/common/responses.rs @@ -216,6 +216,22 @@ impl ResponsesRequest { .collect() } + /// Returns all `input_audio` `audio_url` spans from `message` inputs for the provided role. + pub fn message_input_audio_urls(&self, role: &str) -> Vec { + self.inputs_of_type("message") + .into_iter() + .filter(|item| item.get("role").and_then(Value::as_str) == Some(role)) + .filter_map(|item| item.get("content").and_then(Value::as_array).cloned()) + .flatten() + .filter(|span| span.get("type").and_then(Value::as_str) == Some("input_audio")) + .filter_map(|span| { + span.get("audio_url") + .and_then(Value::as_str) + .map(str::to_owned) + }) + .collect() + } + pub fn input(&self) -> Vec { self.body_json()["input"] .as_array() diff --git a/codex-rs/core/tests/suite/client.rs b/codex-rs/core/tests/suite/client.rs index 13667b74ba35..d78855ffa2cc 100644 --- a/codex-rs/core/tests/suite/client.rs +++ b/codex-rs/core/tests/suite/client.rs @@ -43,6 +43,7 @@ use codex_protocol::models::ReasoningItemContent; use codex_protocol::models::ReasoningItemReasoningSummary; use codex_protocol::models::ResponseItem; use codex_protocol::models::WebSearchAction; +use codex_protocol::openai_models::InputModality; use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; @@ -314,7 +315,14 @@ async fn sends_audio_urls_to_responses() { sse(vec![ev_response_created("resp1"), ev_completed("resp1")]), ) .await; - let codex = test_codex().build(&server).await.unwrap().codex; + let codex = test_codex() + .with_model_info_override("gpt-5.5", |model_info| { + model_info.input_modalities.push(InputModality::Audio); + }) + .build(&server) + .await + .unwrap() + .codex; let audio_url = "data:audio/wav;base64,AAAA"; codex @@ -357,7 +365,13 @@ async fn sends_local_audio_to_responses() -> anyhow::Result<()> { sse(vec![ev_response_created("resp1"), ev_completed("resp1")]), ) .await; - let codex = test_codex().build(&server).await?.codex; + let codex = test_codex() + .with_model_info_override("gpt-5.5", |model_info| { + model_info.input_modalities.push(InputModality::Audio); + }) + .build(&server) + .await? + .codex; let temp_dir = tempfile::tempdir()?; let audio_path = temp_dir.path().join("recording.wav"); std::fs::write(&audio_path, b"audio")?; diff --git a/codex-rs/core/tests/suite/model_switching.rs b/codex-rs/core/tests/suite/model_switching.rs index 58275c07d32d..b33555814e1e 100644 --- a/codex-rs/core/tests/suite/model_switching.rs +++ b/codex-rs/core/tests/suite/model_switching.rs @@ -477,17 +477,21 @@ async fn null_service_tier_override_is_omitted_from_http_turn_with_catalog_defau } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn model_change_from_image_to_text_strips_prior_image_content() -> Result<()> { +async fn model_change_from_multimodal_to_text_strips_prior_media_content() -> Result<()> { skip_if_no_network!(Ok(())); let server = MockServer::start().await; - let image_model_slug = "test-image-model"; + let multimodal_model_slug = "test-multimodal-model"; let text_model_slug = "test-text-only-model"; - let image_model = test_model_info( - image_model_slug, - "Test Image Model", - "supports image input", - default_input_modalities(), + let multimodal_model = test_model_info( + multimodal_model_slug, + "Test Multimodal Model", + "supports image and audio input", + vec![ + InputModality::Text, + InputModality::Image, + InputModality::Audio, + ], ); let text_model = test_model_info( text_model_slug, @@ -498,7 +502,7 @@ async fn model_change_from_image_to_text_strips_prior_image_content() -> Result< mount_models_once( &server, ModelsResponse { - models: vec![image_model, text_model], + models: vec![multimodal_model, text_model], }, ) .await; @@ -512,7 +516,7 @@ async fn model_change_from_image_to_text_strips_prior_image_content() -> Result< let mut builder = test_codex() .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) .with_config(move |config| { - config.model = Some(image_model_slug.to_string()); + config.model = Some(multimodal_model_slug.to_string()); }); let test = builder.build(&server).await?; let models_manager = test.thread_manager.get_models_manager(); @@ -533,12 +537,15 @@ async fn model_change_from_image_to_text_strips_prior_image_content() -> Result< image_url: image_url.clone(), detail: None, }, + UserInput::Audio { + audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(), + }, UserInput::Text { text: "first turn".to_string(), text_elements: Vec::new(), }, ], - image_model_slug.to_string(), + multimodal_model_slug.to_string(), )) .await?; wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; @@ -563,12 +570,20 @@ async fn model_change_from_image_to_text_strips_prior_image_content() -> Result< !first_request.message_input_image_urls("user").is_empty(), "first request should include the uploaded image" ); + assert_eq!( + first_request.message_input_audio_urls("user"), + vec!["data:audio/wav;base64,YXVkaW8=".to_string()] + ); let second_request = requests.last().expect("expected second request"); assert!( second_request.message_input_image_urls("user").is_empty(), "second request should strip unsupported image content" ); + assert!( + second_request.message_input_audio_urls("user").is_empty(), + "second request should strip unsupported audio content" + ); let second_user_texts = second_request.message_input_texts("user"); assert!( second_user_texts @@ -576,6 +591,12 @@ async fn model_change_from_image_to_text_strips_prior_image_content() -> Result< .any(|text| text == "image content omitted because you do not support image input"), "second request should include the image-omitted placeholder text" ); + assert!( + second_user_texts + .iter() + .any(|text| text == "audio content omitted because you do not support audio input"), + "second request should include the audio-omitted placeholder text" + ); Ok(()) } diff --git a/codex-rs/protocol/src/openai_models.rs b/codex-rs/protocol/src/openai_models.rs index 9734c77d2233..b3523be33ee8 100644 --- a/codex-rs/protocol/src/openai_models.rs +++ b/codex-rs/protocol/src/openai_models.rs @@ -157,6 +157,8 @@ pub enum InputModality { Text, /// Image attachments included in user turns. Image, + /// Audio attachments included in user turns. + Audio, } /// Backward-compatible default when `input_modalities` is omitted on the wire. @@ -1101,12 +1103,15 @@ mod tests { "context_window": null, "auto_compact_token_limit": null, "effective_context_window_percent": 95, - "experimental_supported_tools": [], - "input_modalities": ["text", "image"] + "experimental_supported_tools": [] })) .expect("deserialize model info"); assert_eq!(model.availability_nux, None); + assert_eq!( + model.input_modalities, + vec![InputModality::Text, InputModality::Image] + ); assert!(!model.include_skills_usage_instructions); assert!(model.supports_reasoning_summary_parameter); assert!(!model.supports_image_detail_original);