Forward audio inputs to the Responses API

codex-rs/app-server/README.md

@@ -757,13 +757,16 @@ If the thread does not already have an active turn, the server starts a standalo
 
 ### Example: Start a turn (send user input)
 
-Turns attach user input (text or images) to a thread and trigger Codex generation. The `input` field is a list of discriminated unions:
+Turns attach user input (text, images, or audio) to a thread and trigger Codex generation. The `input` field is a list of discriminated unions:
 
 - `{"type":"text","text":"Explain this diff"}`
 - `{"type":"image","url":"data:image/png;base64,…"}`
 - `{"type":"localImage","path":"/tmp/screenshot.png"}`
+- `{"type":"audio","url":"data:audio/wav;base64,…"}`
+- `{"type":"localAudio","path":"/tmp/recording.mp3"}`
 
 The `image` variant accepts inline data URLs. Remote HTTP(S) image URLs are rejected; use a data URL or `localImage` instead.
+The `audio` variant accepts data URLs. Other URL schemes are rejected. `localAudio` reads local wav, mp3, m4a, webm, and ogg files and converts them to data URLs before the Responses API request.
 
 You can optionally specify config overrides on the new turn. If specified, these settings become the default for subsequent turns on the same thread. `outputSchema` applies only to the current turn. Experimental `environments` is turn-scoped: omit it to inherit the thread's sticky environments, pass `[]` to run the turn with no environments, or pass explicit environment ids to override the sticky selection for this turn only.
 
@@ -1420,7 +1423,7 @@ Today both notifications carry an empty `items` array even when item events were
 
 `ThreadItem` is the tagged union carried in turn responses and `item/*` notifications. Currently we support events for the following items:
 
-- `userMessage` — `{id, clientId, content}` where `clientId` is the optional `clientUserMessageId` supplied to `turn/start` or `turn/steer`, and `content` is a list of user inputs (`text`, `image`, or `localImage`).
+- `userMessage` — `{id, clientId, content}` where `clientId` is the optional `clientUserMessageId` supplied to `turn/start` or `turn/steer`, and `content` is a list of user inputs (`text`, `image`, `localImage`, `audio`, or `localAudio`).
 - `agentMessage` — `{id, text}` containing the accumulated agent reply.
 - `plan` — `{id, text}` emitted for plan-mode turns; plan text can stream via `item/plan/delta` (experimental).
 - `reasoning` — `{id, summary, content}` where `summary` holds streamed reasoning summaries (applicable for most OpenAI models) and `content` holds raw reasoning blocks (applicable for e.g. open source models).

codex-rs/core/src/audio_preparation.rs

@@ -0,0 +1,179 @@
+use base64::Engine;
+use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
+use codex_protocol::models::ContentItem;
+use codex_protocol::models::FunctionCallOutputContentItem;
+use codex_protocol::models::ResponseItem;
+use tracing::warn;
+
+const AUDIO_PROCESSING_ERROR_PLACEHOLDER: &str =
+    "audio content omitted because it could not be processed";
+const AUDIO_TOO_LARGE_PLACEHOLDER: &str =
+    "audio content omitted because it exceeded the supported size limit; use a smaller audio file";
+const UNSUPPORTED_AUDIO_FORMAT_PLACEHOLDER: &str =
+    "audio content omitted because its format is not supported; use wav, mp3, m4a, webm, or ogg";
+
+/// Maximum accepted decoded byte length for prompt audio inputs.
+///
+/// This matches the Responses API audio input limit.
+const MAX_PROMPT_AUDIO_INPUT_BYTES: usize = 50 * 1024 * 1024;
+const MAX_PROMPT_AUDIO_BASE64_BYTES: usize = MAX_PROMPT_AUDIO_INPUT_BYTES.div_ceil(3) * 4;
+
+#[derive(Debug, thiserror::Error)]
+enum AudioPreparationError {
+    #[error("invalid audio data URL: {reason}")]
+    InvalidDataUrl { reason: &'static str },
+    #[error("unsupported audio format")]
+    UnsupportedFormat,
+    #[error("audio input is too large ({size} bytes; max {MAX_PROMPT_AUDIO_INPUT_BYTES} bytes)")]
+    AudioTooLarge { size: usize },
+}
+
+impl AudioPreparationError {
+    fn placeholder(&self) -> &'static str {
+        match self {
+            AudioPreparationError::InvalidDataUrl { .. } => AUDIO_PROCESSING_ERROR_PLACEHOLDER,
+            AudioPreparationError::UnsupportedFormat => UNSUPPORTED_AUDIO_FORMAT_PLACEHOLDER,
+            AudioPreparationError::AudioTooLarge { .. } => AUDIO_TOO_LARGE_PLACEHOLDER,
+        }
+    }
+}
+
+pub(crate) fn prepare_response_items(items: &mut [ResponseItem]) {
+    for item in items {
+        match item {
+            ResponseItem::Message { content, .. } => prepare_message_content(content),
+            ResponseItem::FunctionCallOutput { output, .. }
+            | ResponseItem::CustomToolCallOutput { output, .. } => {
+                if let Some(content) = output.content_items_mut() {
+                    prepare_tool_output_content(content);
+                }
+            }
+            ResponseItem::AdditionalTools { .. }
+            | ResponseItem::Reasoning { .. }
+            | ResponseItem::AgentMessage { .. }
+            | ResponseItem::LocalShellCall { .. }
+            | ResponseItem::FunctionCall { .. }
+            | ResponseItem::ToolSearchCall { .. }
+            | ResponseItem::CustomToolCall { .. }
+            | ResponseItem::ToolSearchOutput { .. }
+            | ResponseItem::WebSearchCall { .. }
+            | ResponseItem::ImageGenerationCall { .. }
+            | ResponseItem::Compaction { .. }
+            | ResponseItem::CompactionTrigger { .. }
+            | ResponseItem::ContextCompaction { .. }
+            | ResponseItem::Other => {}
+        }
+    }
+}
+
+fn prepare_message_content(items: &mut [ContentItem]) {
+    for item in items {
+        if let ContentItem::InputAudio { audio_url } = item
+            && let Err(error) = prepare_audio(audio_url)
+        {
+            warn!(%error, "failed to prepare message audio");
+            *item = ContentItem::InputText {
+                text: error.placeholder().to_string(),
+            };
+        }
+    }
+}
+
+fn prepare_tool_output_content(items: &mut [FunctionCallOutputContentItem]) {
+    for item in items {
+        if let FunctionCallOutputContentItem::InputAudio { audio_url } = item
+            && let Err(error) = prepare_audio(audio_url)
+        {
+            warn!(%error, "failed to prepare tool output audio");
+            *item = FunctionCallOutputContentItem::InputText {
+                text: error.placeholder().to_string(),
+            };
+        }
+    }
+}
+
+fn is_data_url(audio_url: &str) -> bool {
+    audio_url
+        .get(.."data:".len())
+        .is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:"))
+}
+
+fn canonical_audio_mime(mime: &str) -> Option<&'static str> {
+    if mime.eq_ignore_ascii_case("audio/wav")
+        || mime.eq_ignore_ascii_case("audio/x-wav")
+        || mime.eq_ignore_ascii_case("audio/wave")
+        || mime.eq_ignore_ascii_case("audio/vnd.wave")
+    {
+        Some("audio/wav")
+    } else if mime.eq_ignore_ascii_case("audio/mpeg") || mime.eq_ignore_ascii_case("audio/mp3") {
+        Some("audio/mpeg")
+    } else if mime.eq_ignore_ascii_case("audio/mp4")
+        || mime.eq_ignore_ascii_case("audio/m4a")
+        || mime.eq_ignore_ascii_case("audio/x-m4a")
+    {
+        Some("audio/mp4")
+    } else if mime.eq_ignore_ascii_case("audio/webm") {
+        Some("audio/webm")
+    } else if mime.eq_ignore_ascii_case("audio/ogg") {
+        Some("audio/ogg")
+    } else {
+        None
+    }
+}
+
+fn prepare_audio(audio_url: &mut String) -> Result<(), AudioPreparationError> {
+    if !is_data_url(audio_url) {
+        return Err(AudioPreparationError::InvalidDataUrl {
+            reason: "audio input must be a data URL",
+        });
+    }
+
+    let (metadata, payload) =
+        audio_url
+            .split_once(',')
+            .ok_or(AudioPreparationError::InvalidDataUrl {
+                reason: "missing payload separator",
+            })?;
+    let metadata = metadata
+        .get("data:".len()..)
+        .ok_or(AudioPreparationError::InvalidDataUrl {
+            reason: "missing data URL prefix",
+        })?;
+    let mut metadata_parts = metadata.split(';');
+    let mime = metadata_parts
+        .next()
+        .filter(|mime| !mime.is_empty())
+        .ok_or(AudioPreparationError::InvalidDataUrl {
+            reason: "missing media type",
+        })?;
+    let canonical_mime =
+        canonical_audio_mime(mime).ok_or(AudioPreparationError::UnsupportedFormat)?;
+    if !metadata_parts.any(|part| part.eq_ignore_ascii_case("base64")) {
+        return Err(AudioPreparationError::InvalidDataUrl {
+            reason: "audio payload is not base64 encoded",
+        });
+    }
+    if payload.len() > MAX_PROMPT_AUDIO_BASE64_BYTES {
+        return Err(AudioPreparationError::AudioTooLarge {
+            size: payload.len(),
+        });
+    }
+
+    let bytes =
+        BASE64_STANDARD
+            .decode(payload)
+            .map_err(|_| AudioPreparationError::InvalidDataUrl {
+                reason: "invalid base64 payload",
+            })?;
+    if bytes.len() > MAX_PROMPT_AUDIO_INPUT_BYTES {
+        return Err(AudioPreparationError::AudioTooLarge { size: bytes.len() });
+    }
+
+    let encoded = BASE64_STANDARD.encode(bytes);
+    *audio_url = format!("data:{canonical_mime};base64,{encoded}");
+    Ok(())
+}
+
+#[cfg(test)]
+#[path = "audio_preparation_tests.rs"]
+mod tests;

codex-rs/core/src/audio_preparation_tests.rs

@@ -0,0 +1,159 @@
+use codex_protocol::models::FunctionCallOutputBody;
+use codex_protocol::models::FunctionCallOutputPayload;
+use pretty_assertions::assert_eq;
+
+use super::*;
+
+#[test]
+fn preparation_canonicalizes_data_urls_and_rejects_remote_urls() {
+    let mut items = vec![ResponseItem::Message {
+        id: None,
+        role: "user".to_string(),
+        content: vec![
+            ContentItem::InputAudio {
+                audio_url: "data:audio/x-wav;base64,YXVkaW8=".to_string(),
+            },
+            ContentItem::InputAudio {
+                audio_url: "data:audio/ogg;base64,YXVkaW8=".to_string(),
+            },
+            ContentItem::InputAudio {
+                audio_url: "https://example.com/audio.mp3".to_string(),
+            },
+        ],
+        phase: None,
+        internal_chat_message_metadata_passthrough: None,
+    }];
+
+    prepare_response_items(&mut items);
+
+    assert_eq!(
+        items,
+        vec![ResponseItem::Message {
+            id: None,
+            role: "user".to_string(),
+            content: vec![
+                ContentItem::InputAudio {
+                    audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
+                },
+                ContentItem::InputAudio {
+                    audio_url: "data:audio/ogg;base64,YXVkaW8=".to_string(),
+                },
+                ContentItem::InputText {
+                    text: "audio content omitted because it could not be processed".to_string(),
+                },
+            ],
+            phase: None,
+            internal_chat_message_metadata_passthrough: None,
+        }]
+    );
+}
+
+#[test]
+fn preparation_replaces_invalid_message_audio_with_placeholders() {
+    let mut items = vec![ResponseItem::Message {
+        id: None,
+        role: "user".to_string(),
+        content: vec![
+            ContentItem::InputAudio {
+                audio_url: "data:audio/wav;base64,%%%".to_string(),
+            },
+            ContentItem::InputAudio {
+                audio_url: "data:audio/flac;base64,YXVkaW8=".to_string(),
+            },
+        ],
+        phase: None,
+        internal_chat_message_metadata_passthrough: None,
+    }];
+
+    prepare_response_items(&mut items);
+
+    assert_eq!(
+        items,
+        vec![ResponseItem::Message {
+            id: None,
+            role: "user".to_string(),
+            content: vec![
+                ContentItem::InputText {
+                    text: "audio content omitted because it could not be processed".to_string(),
+                },
+                ContentItem::InputText {
+                    text: "audio content omitted because its format is not supported; use wav, mp3, m4a, webm, or ogg".to_string(),
+                },
+            ],
+            phase: None,
+            internal_chat_message_metadata_passthrough: None,
+        }]
+    );
+}
+
+#[test]
+fn preparation_replaces_only_failed_tool_audio_and_preserves_metadata() {
+    let mut items = vec![ResponseItem::FunctionCallOutput {
+        id: None,
+        call_id: "call-1".to_string(),
+        output: FunctionCallOutputPayload {
+            body: FunctionCallOutputBody::ContentItems(vec![
+                FunctionCallOutputContentItem::InputText {
+                    text: "before".to_string(),
+                },
+                FunctionCallOutputContentItem::InputAudio {
+                    audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
+                },
+                FunctionCallOutputContentItem::InputAudio {
+                    audio_url: "data:audio/wav,not-base64".to_string(),
+                },
+            ]),
+            success: Some(true),
+        },
+        internal_chat_message_metadata_passthrough: None,
+    }];
+
+    prepare_response_items(&mut items);
+
+    assert_eq!(
+        items,
+        vec![ResponseItem::FunctionCallOutput {
+            id: None,
+            call_id: "call-1".to_string(),
+            output: FunctionCallOutputPayload {
+                body: FunctionCallOutputBody::ContentItems(vec![
+                    FunctionCallOutputContentItem::InputText {
+                        text: "before".to_string(),
+                    },
+                    FunctionCallOutputContentItem::InputAudio {
+                        audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
+                    },
+                    FunctionCallOutputContentItem::InputText {
+                        text: "audio content omitted because it could not be processed".to_string(),
+                    },
+                ]),
+                success: Some(true),
+            },
+            internal_chat_message_metadata_passthrough: None,
+        }]
+    );
+}
+
+#[test]
+fn preparation_errors_map_to_expected_placeholders() {
+    let cases = [
+        (
+            AudioPreparationError::InvalidDataUrl {
+                reason: "details remain in logs",
+            },
+            "audio content omitted because it could not be processed",
+        ),
+        (
+            AudioPreparationError::UnsupportedFormat,
+            "audio content omitted because its format is not supported; use wav, mp3, m4a, webm, or ogg",
+        ),
+        (
+            AudioPreparationError::AudioTooLarge { size: usize::MAX },
+            "audio content omitted because it exceeded the supported size limit; use a smaller audio file",
+        ),
+    ];
+
+    for (error, expected) in cases {
+        assert_eq!(error.placeholder(), expected);
+    }
+}