From 58cec52819abcec54bce14133986a1e4bd4d3a3e Mon Sep 17 00:00:00 2001 From: Guinness Chen Date: Fri, 12 Jun 2026 10:44:28 -0700 Subject: [PATCH 01/11] Add realtime handoff append controls --- .../src/protocol/common.rs | 36 +++++++ .../src/protocol/v2/realtime.rs | 19 ++++ codex-rs/app-server/README.md | 6 +- codex-rs/app-server/src/message_processor.rs | 5 + codex-rs/app-server/src/request_processors.rs | 3 + .../src/request_processors/turn_processor.rs | 38 +++++++ .../tests/common/test_app_server.rs | 11 ++ .../tests/suite/v2/experimental_api.rs | 2 + .../tests/suite/v2/realtime_conversation.rs | 102 ++++++++++++++++++ codex-rs/core/src/realtime_conversation.rs | 89 ++++++++++++--- .../core/src/realtime_conversation_tests.rs | 2 +- codex-rs/core/src/session/handlers.rs | 5 + codex-rs/core/tests/suite/compact_remote.rs | 1 + .../core/tests/suite/realtime_conversation.rs | 36 +++++++ codex-rs/protocol/src/protocol.rs | 11 ++ 15 files changed, 350 insertions(+), 16 deletions(-) diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index 5526f5c54836..425b9ad9c101 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -814,6 +814,12 @@ client_request_definitions! { serialization: thread_id(params.thread_id), response: v2::ThreadRealtimeAppendTextResponse, }, + #[experimental("thread/realtime/appendHandoff")] + ThreadRealtimeAppendHandoff => "thread/realtime/appendHandoff" { + params: v2::ThreadRealtimeAppendHandoffParams, + serialization: thread_id(params.thread_id), + response: v2::ThreadRealtimeAppendHandoffResponse, + }, #[experimental("thread/realtime/stop")] ThreadRealtimeStop => "thread/realtime/stop" { params: v2::ThreadRealtimeStopParams, @@ -3010,6 +3016,7 @@ mod tests { thread_id: "thr_123".to_string(), model: Some("realtime-treatment-model".to_string()), output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: Some(false), prompt: Some(Some("You are on a call".to_string())), realtime_session_id: Some("sess_456".to_string()), transport: None, @@ -3025,6 +3032,7 @@ mod tests { "threadId": "thr_123", "model": "realtime-treatment-model", "outputModality": "audio", + "autoHandoffAppends": false, "prompt": "You are on a call", "realtimeSessionId": "sess_456", "transport": null, @@ -3045,6 +3053,7 @@ mod tests { thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: None, prompt: None, realtime_session_id: None, transport: None, @@ -3060,6 +3069,7 @@ mod tests { "threadId": "thr_123", "model": null, "outputModality": "audio", + "autoHandoffAppends": null, "realtimeSessionId": null, "transport": null, "version": null, @@ -3075,6 +3085,7 @@ mod tests { thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: None, prompt: Some(None), realtime_session_id: None, transport: None, @@ -3090,6 +3101,7 @@ mod tests { "threadId": "thr_123", "model": null, "outputModality": "audio", + "autoHandoffAppends": null, "prompt": null, "realtimeSessionId": null, "transport": null, @@ -3136,6 +3148,29 @@ mod tests { Ok(()) } + #[test] + fn serialize_thread_realtime_append_handoff() -> Result<()> { + let request = ClientRequest::ThreadRealtimeAppendHandoff { + request_id: RequestId::Integer(10), + params: v2::ThreadRealtimeAppendHandoffParams { + thread_id: "thr_123".to_string(), + output_text: "Short voice update".to_string(), + }, + }; + assert_eq!( + json!({ + "method": "thread/realtime/appendHandoff", + "id": 10, + "params": { + "threadId": "thr_123", + "outputText": "Short voice update" + } + }), + serde_json::to_value(&request)?, + ); + Ok(()) + } + #[test] fn serialize_thread_status_changed_notification() -> Result<()> { let notification = @@ -3248,6 +3283,7 @@ mod tests { thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: None, prompt: Some(Some("You are on a call".to_string())), realtime_session_id: None, transport: None, diff --git a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs index a86cd8067dc2..9d2c8db62523 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs @@ -71,6 +71,10 @@ pub struct ThreadRealtimeStartParams { /// Selects text or audio output for the realtime session. Transport and voice stay /// independent so clients can choose how they connect separately from what the model emits. pub output_modality: RealtimeOutputModality, + /// Controls whether Codex automatically appends standalone assistant output to realtime. + /// Omit to keep the default behavior enabled. + #[ts(optional = nullable)] + pub auto_handoff_appends: Option, #[serde( default, deserialize_with = "crate::protocol::serde_helpers::deserialize_double_option", @@ -139,6 +143,21 @@ pub struct ThreadRealtimeAppendTextParams { #[ts(export_to = "v2/")] pub struct ThreadRealtimeAppendTextResponse {} +/// EXPERIMENTAL - append assistant output to thread realtime. +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadRealtimeAppendHandoffParams { + pub thread_id: String, + pub output_text: String, +} + +/// EXPERIMENTAL - response for appending realtime assistant output. +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadRealtimeAppendHandoffResponse {} + /// EXPERIMENTAL - stop thread realtime. #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 0532ffa706e8..ebfe38dd074f 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -165,9 +165,10 @@ Example with notification opt-out: - `thread/inject_items` — append raw Responses API items to a loaded thread’s model-visible history without starting a user turn; returns `{}` on success. - `turn/steer` — add user input to an already in-flight regular turn without starting a new turn; returns the active `turnId` that accepted the input. `clientUserMessageId` is optional; when supplied, the corresponding `userMessage` item echoes it as `clientId`. Review and manual compaction turns reject `turn/steer`. - `turn/interrupt` — request cancellation of an in-flight turn by `(thread_id, turn_id)`; success is an empty `{}` response and the turn finishes with `status: "interrupted"`. -- `thread/realtime/start` — start a thread-scoped realtime session (experimental); pass `outputModality: "text"` or `outputModality: "audio"` to choose model output, and optionally pass `model` and `version` to override configured realtime selection for this session only. Returns `{}` and streams `thread/realtime/*` notifications. Omit `transport` for the websocket transport, or pass `{ "type": "webrtc", "sdp": "..." }` to create a WebRTC session from a browser-generated SDP offer; the remote answer SDP is emitted as `thread/realtime/sdp`. +- `thread/realtime/start` — start a thread-scoped realtime session (experimental); pass `outputModality: "text"` or `outputModality: "audio"` to choose model output, and optionally pass `model` and `version` to override configured realtime selection for this session only. Pass `autoHandoffAppends: false` to stop Codex from automatically appending standalone assistant output back into realtime. Returns `{}` and streams `thread/realtime/*` notifications. Omit `transport` for the websocket transport, or pass `{ "type": "webrtc", "sdp": "..." }` to create a WebRTC session from a browser-generated SDP offer; the remote answer SDP is emitted as `thread/realtime/sdp`. - `thread/realtime/appendAudio` — append an input audio chunk to the active realtime session (experimental); returns `{}`. - `thread/realtime/appendText` — append text input to the active realtime session (experimental); returns `{}`. +- `thread/realtime/appendHandoff` — append assistant output to the active realtime session (experimental); returns `{}`. For v1 sessions this sends `conversation.handoff.append`. - `thread/realtime/stop` — stop the active realtime session for the thread (experimental); returns `{}`. - `review/start` — kick off Codex’s automated reviewer for a thread; responds like `turn/start` and emits `item/started`/`item/completed` notifications with `enteredReviewMode` and `exitedReviewMode` items, plus a final assistant `agentMessage` containing the review. - `command/exec` — run a single command under the server sandbox without starting a thread/turn (handy for utilities and validation). @@ -852,6 +853,9 @@ Omit `prompt` to use Codex's default realtime backend prompt. Send `prompt: null `prompt: ""` when the session should start without that default backend prompt. Clients may also pass `model` and `version` on `thread/realtime/start` to select a different realtime session configuration without changing thread or user config. +Pass `autoHandoffAppends: false` to suppress automatic standalone assistant output +from the backend Codex model, then call `thread/realtime/appendHandoff` when the +app decides a realtime update should be spoken. ```javascript await pc.setRemoteDescription({ diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index 6ba59e733fe8..d5b11e7157ee 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -1293,6 +1293,11 @@ impl MessageProcessor { .thread_realtime_append_text(&request_id, params) .await } + ClientRequest::ThreadRealtimeAppendHandoff { params, .. } => { + self.turn_processor + .thread_realtime_append_handoff(&request_id, params) + .await + } ClientRequest::ThreadRealtimeStop { params, .. } => { self.turn_processor .thread_realtime_stop(&request_id, params) diff --git a/codex-rs/app-server/src/request_processors.rs b/codex-rs/app-server/src/request_processors.rs index 83e704cdfdcb..4899249003d6 100644 --- a/codex-rs/app-server/src/request_processors.rs +++ b/codex-rs/app-server/src/request_processors.rs @@ -219,6 +219,8 @@ use codex_app_server_protocol::ThreadReadParams; use codex_app_server_protocol::ThreadReadResponse; use codex_app_server_protocol::ThreadRealtimeAppendAudioParams; use codex_app_server_protocol::ThreadRealtimeAppendAudioResponse; +use codex_app_server_protocol::ThreadRealtimeAppendHandoffParams; +use codex_app_server_protocol::ThreadRealtimeAppendHandoffResponse; use codex_app_server_protocol::ThreadRealtimeAppendTextParams; use codex_app_server_protocol::ThreadRealtimeAppendTextResponse; use codex_app_server_protocol::ThreadRealtimeListVoicesResponse; @@ -390,6 +392,7 @@ use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::permissions::FileSystemSandboxPolicy; use codex_protocol::protocol::AgentStatus; use codex_protocol::protocol::ConversationAudioParams; +use codex_protocol::protocol::ConversationHandoffParams; use codex_protocol::protocol::ConversationStartParams; use codex_protocol::protocol::ConversationStartTransport; use codex_protocol::protocol::ConversationTextParams; diff --git a/codex-rs/app-server/src/request_processors/turn_processor.rs b/codex-rs/app-server/src/request_processors/turn_processor.rs index b2e6ddc0b916..c9dfdec3b5a9 100644 --- a/codex-rs/app-server/src/request_processors/turn_processor.rs +++ b/codex-rs/app-server/src/request_processors/turn_processor.rs @@ -181,6 +181,16 @@ impl TurnRequestProcessor { .map(|response| response.map(Into::into)) } + pub(crate) async fn thread_realtime_append_handoff( + &self, + request_id: &ConnectionRequestId, + params: ThreadRealtimeAppendHandoffParams, + ) -> Result, JSONRPCErrorError> { + self.thread_realtime_append_handoff_inner(request_id, params) + .await + .map(|response| response.map(Into::into)) + } + pub(crate) async fn thread_realtime_stop( &self, request_id: &ConnectionRequestId, @@ -936,6 +946,7 @@ impl TurnRequestProcessor { Op::RealtimeConversationStart(ConversationStartParams { model: params.model, output_modality: params.output_modality, + auto_handoff_appends: params.auto_handoff_appends.unwrap_or(true), prompt: params.prompt, realtime_session_id: params.realtime_session_id, transport: params.transport.map(|transport| match transport { @@ -1007,6 +1018,33 @@ impl TurnRequestProcessor { Ok(Some(ThreadRealtimeAppendTextResponse::default())) } + async fn thread_realtime_append_handoff_inner( + &self, + request_id: &ConnectionRequestId, + params: ThreadRealtimeAppendHandoffParams, + ) -> Result, JSONRPCErrorError> { + let Some((_, thread)) = self + .prepare_realtime_conversation_thread(request_id, ¶ms.thread_id) + .await? + else { + return Ok(None); + }; + self.submit_core_op( + request_id, + thread.as_ref(), + Op::RealtimeConversationHandoff(ConversationHandoffParams { + output_text: params.output_text, + }), + ) + .await + .map_err(|err| { + internal_error(format!( + "failed to append realtime conversation handoff output: {err}" + )) + })?; + Ok(Some(ThreadRealtimeAppendHandoffResponse::default())) + } + async fn thread_realtime_stop_inner( &self, request_id: &ConnectionRequestId, diff --git a/codex-rs/app-server/tests/common/test_app_server.rs b/codex-rs/app-server/tests/common/test_app_server.rs index 91c42e2fe437..b7d22f151273 100644 --- a/codex-rs/app-server/tests/common/test_app_server.rs +++ b/codex-rs/app-server/tests/common/test_app_server.rs @@ -88,6 +88,7 @@ use codex_app_server_protocol::ThreadMemoryModeSetParams; use codex_app_server_protocol::ThreadMetadataUpdateParams; use codex_app_server_protocol::ThreadReadParams; use codex_app_server_protocol::ThreadRealtimeAppendAudioParams; +use codex_app_server_protocol::ThreadRealtimeAppendHandoffParams; use codex_app_server_protocol::ThreadRealtimeAppendTextParams; use codex_app_server_protocol::ThreadRealtimeListVoicesParams; use codex_app_server_protocol::ThreadRealtimeStartParams; @@ -1000,6 +1001,16 @@ impl TestAppServer { .await } + /// Send a `thread/realtime/appendHandoff` JSON-RPC request (v2). + pub async fn send_thread_realtime_append_handoff_request( + &mut self, + params: ThreadRealtimeAppendHandoffParams, + ) -> anyhow::Result { + let params = Some(serde_json::to_value(params)?); + self.send_request("thread/realtime/appendHandoff", params) + .await + } + /// Send a `thread/realtime/stop` JSON-RPC request (v2). pub async fn send_thread_realtime_stop_request( &mut self, diff --git a/codex-rs/app-server/tests/suite/v2/experimental_api.rs b/codex-rs/app-server/tests/suite/v2/experimental_api.rs index 03c9d455f05d..357ce4c8e1e5 100644 --- a/codex-rs/app-server/tests/suite/v2/experimental_api.rs +++ b/codex-rs/app-server/tests/suite/v2/experimental_api.rs @@ -82,6 +82,7 @@ async fn realtime_conversation_start_requires_experimental_api_capability() -> R thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: None, prompt: Some(Some("hello".to_string())), realtime_session_id: None, transport: None, @@ -190,6 +191,7 @@ async fn realtime_webrtc_start_requires_experimental_api_capability() -> Result< thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: None, prompt: Some(Some("hello".to_string())), realtime_session_id: None, transport: Some(ThreadRealtimeStartTransport::Webrtc { diff --git a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs index ddc0eae404b9..f40c01facff8 100644 --- a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs +++ b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs @@ -15,6 +15,8 @@ use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::ThreadRealtimeAppendAudioParams; use codex_app_server_protocol::ThreadRealtimeAppendAudioResponse; +use codex_app_server_protocol::ThreadRealtimeAppendHandoffParams; +use codex_app_server_protocol::ThreadRealtimeAppendHandoffResponse; use codex_app_server_protocol::ThreadRealtimeAppendTextParams; use codex_app_server_protocol::ThreadRealtimeAppendTextResponse; use codex_app_server_protocol::ThreadRealtimeAudioChunk; @@ -308,6 +310,17 @@ impl RealtimeE2eHarness { } async fn start_webrtc_realtime(&mut self, offer_sdp: &str) -> Result { + self.start_webrtc_realtime_with_auto_handoff_appends( + offer_sdp, /*auto_handoff_appends*/ None, + ) + .await + } + + async fn start_webrtc_realtime_with_auto_handoff_appends( + &mut self, + offer_sdp: &str, + auto_handoff_appends: Option, + ) -> Result { // Starts realtime through the public JSON-RPC method, then waits for the same client-visible // notifications a desktop app needs: started first, SDP answer second. let start_request_id = self @@ -316,6 +329,7 @@ impl RealtimeE2eHarness { thread_id: self.thread_id.clone(), model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: Some(ThreadRealtimeStartTransport::Webrtc { @@ -404,6 +418,24 @@ impl RealtimeE2eHarness { Ok(()) } + async fn append_handoff(&mut self, thread_id: String, output_text: &str) -> Result<()> { + let request_id = self + .mcp + .send_thread_realtime_append_handoff_request(ThreadRealtimeAppendHandoffParams { + thread_id, + output_text: output_text.to_string(), + }) + .await?; + let response: JSONRPCResponse = timeout( + DEFAULT_TIMEOUT, + self.mcp + .read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let _: ThreadRealtimeAppendHandoffResponse = to_response(response)?; + Ok(()) + } + async fn main_loop_responses_requests(&self) -> Result> { responses_requests(&self.main_loop_responses_server).await } @@ -563,6 +595,7 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> { thread_id: thread_start.thread.id.clone(), model: Some("realtime-treatment-model".to_string()), output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: None, prompt: None, realtime_session_id: None, transport: None, @@ -818,6 +851,7 @@ async fn realtime_text_output_modality_requests_text_output_and_final_transcript thread_id: thread_start.thread.id.clone(), model: None, output_modality: RealtimeOutputModality::Text, + auto_handoff_appends: None, prompt: None, realtime_session_id: None, transport: None, @@ -994,6 +1028,7 @@ async fn realtime_conversation_stop_emits_closed_notification() -> Result<()> { thread_id: thread_start.thread.id.clone(), model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: None, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1093,6 +1128,7 @@ async fn realtime_webrtc_start_emits_sdp_notification() -> Result<()> { thread_id: thread_id.clone(), model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: None, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: Some(ThreadRealtimeStartTransport::Webrtc { @@ -1345,6 +1381,70 @@ async fn webrtc_v1_handoff_request_delegates_and_appends_result() -> Result<()> Ok(()) } +#[tokio::test] +async fn realtime_start_can_disable_auto_handoff_appends_and_append_manually() -> Result<()> { + skip_if_no_network!(Ok(())); + + let mut harness = RealtimeE2eHarness::new( + RealtimeTestVersion::V1, + main_loop_responses(vec![create_final_assistant_message_sse_response( + "automatic output", + )?]), + realtime_sideband(vec![realtime_sideband_connection(vec![ + vec![session_updated("sess_manual_handoff")], + vec![], + ])]), + ) + .await?; + + let started = harness + .start_webrtc_realtime_with_auto_handoff_appends( + "v=offer\r\n", + /*auto_handoff_appends*/ Some(false), + ) + .await?; + assert_eq!(started.started.version, RealtimeConversationVersion::V1); + assert_v1_session_update(&harness.sideband_outbound_request(/*request_index*/ 0).await)?; + + let turn_request_id = harness + .mcp + .send_turn_start_request(TurnStartParams { + thread_id: harness.thread_id.clone(), + input: vec![V2UserInput::Text { + text: "do something quietly".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let turn_response: JSONRPCResponse = timeout( + DEFAULT_TIMEOUT, + harness + .mcp + .read_stream_until_response_message(RequestId::Integer(turn_request_id)), + ) + .await??; + let _: TurnStartResponse = to_response(turn_response)?; + let _ = harness + .read_notification::("turn/completed") + .await?; + + harness + .append_handoff(harness.thread_id.clone(), "manual voice update") + .await?; + assert_eq!( + harness.sideband_outbound_request(/*request_index*/ 1).await, + json!({ + "type": "conversation.handoff.append", + "handoff_id": "codex", + "output_text": "manual voice update", + }) + ); + + harness.shutdown().await; + Ok(()) +} + #[tokio::test] async fn webrtc_assistant_output_without_handoff_reaches_realtime() -> Result<()> { skip_if_no_network!(Ok(())); @@ -2140,6 +2240,7 @@ async fn realtime_webrtc_start_surfaces_backend_error() -> Result<()> { thread_id: thread_start.thread.id, model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: None, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: Some(ThreadRealtimeStartTransport::Webrtc { @@ -2201,6 +2302,7 @@ async fn realtime_conversation_requires_feature_flag() -> Result<()> { thread_id: thread_start.thread.id.clone(), model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: None, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, diff --git a/codex-rs/core/src/realtime_conversation.rs b/codex-rs/core/src/realtime_conversation.rs index 33568f6f38ff..50fd86dfaa33 100644 --- a/codex-rs/core/src/realtime_conversation.rs +++ b/codex-rs/core/src/realtime_conversation.rs @@ -32,6 +32,7 @@ use codex_protocol::error::CodexErr; use codex_protocol::error::Result as CodexResult; use codex_protocol::protocol::CodexErrorInfo; use codex_protocol::protocol::ConversationAudioParams; +use codex_protocol::protocol::ConversationHandoffParams; use codex_protocol::protocol::ConversationStartParams; use codex_protocol::protocol::ConversationStartTransport; use codex_protocol::protocol::ConversationTextParams; @@ -105,6 +106,7 @@ struct RealtimeHandoffState { active_handoff: Arc>>, last_output_text: Arc>>, session_kind: RealtimeSessionKind, + auto_handoff_appends: bool, } #[derive(Debug, PartialEq, Eq)] @@ -209,12 +211,17 @@ struct RealtimeInputChannels { } impl RealtimeHandoffState { - fn new(output_tx: Sender, session_kind: RealtimeSessionKind) -> Self { + fn new( + output_tx: Sender, + session_kind: RealtimeSessionKind, + auto_handoff_appends: bool, + ) -> Self { Self { output_tx, active_handoff: Arc::new(Mutex::new(None)), last_output_text: Arc::new(Mutex::new(None)), session_kind, + auto_handoff_appends, } } } @@ -234,6 +241,7 @@ struct RealtimeStart { api_provider: ApiProvider, extra_headers: Option, session_config: RealtimeSessionConfig, + auto_handoff_appends: bool, model_client: ModelClient, sdp: Option, } @@ -286,6 +294,7 @@ impl RealtimeConversationManager { api_provider, extra_headers, session_config, + auto_handoff_appends, model_client, sdp, } = start; @@ -305,7 +314,8 @@ impl RealtimeConversationManager { async_channel::bounded::(OUTPUT_EVENTS_QUEUE_CAPACITY); let realtime_active = Arc::new(AtomicBool::new(true)); - let handoff = RealtimeHandoffState::new(handoff_output_tx, session_kind); + let handoff = + RealtimeHandoffState::new(handoff_output_tx, session_kind, auto_handoff_appends); let input_channels = RealtimeInputChannels { user_text_rx, handoff_output_rx, @@ -481,19 +491,10 @@ impl RealtimeConversationManager { } } None if output_text.trim().is_empty() => return Ok(()), - None => { - let output_text = prefix_realtime_text( - output_text, - REALTIME_BACKEND_TEXT_PREFIX, - handoff.session_kind, - ); - HandoffOutput::StandaloneAssistantOutput { - output_text: truncate_realtime_text_to_token_budget( - &output_text, - REALTIME_ASSISTANT_OUTPUT_TOKEN_BUDGET, - ), - } + None if handoff.auto_handoff_appends => { + standalone_assistant_output(output_text, handoff.session_kind) } + None => return Ok(()), }; handoff .output_tx @@ -503,6 +504,32 @@ impl RealtimeConversationManager { Ok(()) } + pub(crate) async fn handoff_append(&self, output_text: String) -> CodexResult<()> { + if output_text.trim().is_empty() { + return Ok(()); + } + + let handoff = { + let guard = self.state.lock().await; + let Some(state) = guard.as_ref() else { + return Err(CodexErr::InvalidRequest( + "conversation is not running".to_string(), + )); + }; + state.handoff.clone() + }; + + handoff + .output_tx + .send(standalone_assistant_output( + output_text, + handoff.session_kind, + )) + .await + .map_err(|_| CodexErr::InvalidRequest("conversation is not running".to_string()))?; + Ok(()) + } + pub(crate) async fn handoff_complete(&self) -> CodexResult<()> { let handoff = { let guard = self.state.lock().await; @@ -615,6 +642,7 @@ struct PreparedRealtimeConversationStart { api_provider: ApiProvider, extra_headers: Option, requested_realtime_session_id: Option, + auto_handoff_appends: bool, version: RealtimeWsVersion, session_config: RealtimeSessionConfig, transport: ConversationStartTransport, @@ -672,6 +700,7 @@ async fn prepare_realtime_start( api_provider, extra_headers, requested_realtime_session_id, + auto_handoff_appends: params.auto_handoff_appends, version, session_config, transport, @@ -754,6 +783,19 @@ fn prefix_realtime_text(text: String, prefix: &str, session_kind: RealtimeSessio format!("{prefix}{text}") } +fn standalone_assistant_output( + output_text: String, + session_kind: RealtimeSessionKind, +) -> HandoffOutput { + let output_text = prefix_realtime_text(output_text, REALTIME_BACKEND_TEXT_PREFIX, session_kind); + HandoffOutput::StandaloneAssistantOutput { + output_text: truncate_realtime_text_to_token_budget( + &output_text, + REALTIME_ASSISTANT_OUTPUT_TOKEN_BUDGET, + ), + } +} + fn validate_realtime_voice(version: RealtimeWsVersion, voice: RealtimeVoice) -> CodexResult<()> { let voices = RealtimeVoicesList::builtin(); let allowed = match version { @@ -788,6 +830,7 @@ async fn handle_start_inner( api_provider, extra_headers, requested_realtime_session_id, + auto_handoff_appends, version, session_config, transport, @@ -801,6 +844,7 @@ async fn handle_start_inner( api_provider, extra_headers, session_config, + auto_handoff_appends, model_client: sess.services.model_client.clone(), sdp, }; @@ -1027,6 +1071,23 @@ pub(crate) async fn handle_text( } } +pub(crate) async fn handle_handoff( + sess: &Arc, + sub_id: String, + params: ConversationHandoffParams, +) { + debug!(text = %params.output_text, "[realtime-text] appending realtime handoff output"); + if let Err(err) = sess.conversation.handoff_append(params.output_text).await { + error!("failed to append realtime handoff output: {err}"); + if sess.conversation.running_state().await.is_some() { + warn!("realtime handoff append failed while the session was already ending"); + } else { + send_conversation_error(sess, sub_id, err.to_string(), CodexErrorInfo::BadRequest) + .await; + } + } +} + pub(crate) async fn handle_close(sess: &Arc, sub_id: String) { end_realtime_conversation(sess, sub_id, RealtimeConversationEnd::Requested).await; } diff --git a/codex-rs/core/src/realtime_conversation_tests.rs b/codex-rs/core/src/realtime_conversation_tests.rs index b67205ef8fc6..745d9169285d 100644 --- a/codex-rs/core/src/realtime_conversation_tests.rs +++ b/codex-rs/core/src/realtime_conversation_tests.rs @@ -128,7 +128,7 @@ fn wraps_realtime_delegation_input_with_xml_escaping_without_transcript() { #[tokio::test] async fn clears_active_handoff_explicitly() { let (tx, _rx) = bounded(1); - let state = RealtimeHandoffState::new(tx, RealtimeSessionKind::V1); + let state = RealtimeHandoffState::new(tx, RealtimeSessionKind::V1, true); *state.active_handoff.lock().await = Some("handoff_1".to_string()); assert_eq!( diff --git a/codex-rs/core/src/session/handlers.rs b/codex-rs/core/src/session/handlers.rs index bfb87a0ff3ea..2e7cd1ffbb6e 100644 --- a/codex-rs/core/src/session/handlers.rs +++ b/codex-rs/core/src/session/handlers.rs @@ -1,5 +1,6 @@ use crate::realtime_conversation::handle_audio as handle_realtime_conversation_audio; use crate::realtime_conversation::handle_close as handle_realtime_conversation_close; +use crate::realtime_conversation::handle_handoff as handle_realtime_conversation_handoff; use crate::realtime_conversation::handle_start as handle_realtime_conversation_start; use crate::realtime_conversation::handle_text as handle_realtime_conversation_text; use async_channel::Receiver; @@ -737,6 +738,10 @@ pub(super) async fn submission_loop( handle_realtime_conversation_text(&sess, sub.id.clone(), params).await; false } + Op::RealtimeConversationHandoff(params) => { + handle_realtime_conversation_handoff(&sess, sub.id.clone(), params).await; + false + } Op::RealtimeConversationClose => { handle_realtime_conversation_close(&sess, sub.id.clone()).await; false diff --git a/codex-rs/core/tests/suite/compact_remote.rs b/codex-rs/core/tests/suite/compact_remote.rs index eee445ea9fa1..5afab90d6b65 100644 --- a/codex-rs/core/tests/suite/compact_remote.rs +++ b/codex-rs/core/tests/suite/compact_remote.rs @@ -196,6 +196,7 @@ async fn start_realtime_conversation(codex: &codex_core::CodexThread) -> Result< .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, diff --git a/codex-rs/core/tests/suite/realtime_conversation.rs b/codex-rs/core/tests/suite/realtime_conversation.rs index 25f0516b0d91..9e992903e4e3 100644 --- a/codex-rs/core/tests/suite/realtime_conversation.rs +++ b/codex-rs/core/tests/suite/realtime_conversation.rs @@ -284,6 +284,7 @@ async fn conversation_start_audio_text_close_round_trip() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -424,6 +425,7 @@ async fn conversation_start_defaults_to_v2_and_gpt_realtime_1_5() -> Result<()> .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -514,6 +516,7 @@ async fn conversation_webrtc_start_posts_generated_session() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: Some("session-override-model".to_string()), output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: Some(ConversationStartTransport::Webrtc { @@ -686,6 +689,7 @@ async fn conversation_webrtc_close_while_sideband_connecting_drops_pending_join( .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: Some(ConversationStartTransport::Webrtc { @@ -772,6 +776,7 @@ async fn conversation_webrtc_sideband_connect_failure_closes_with_error() -> Res .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: Some(ConversationStartTransport::Webrtc { @@ -859,6 +864,7 @@ async fn conversation_start_uses_openai_env_key_fallback_with_chatgpt_auth() -> .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -927,6 +933,7 @@ async fn conversation_transport_close_emits_closed_event() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1019,6 +1026,7 @@ async fn conversation_start_preflight_failure_emits_realtime_error_only() -> Res .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1065,6 +1073,7 @@ async fn conversation_start_connect_failure_emits_realtime_error_only() -> Resul .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1158,6 +1167,7 @@ async fn conversation_second_start_replaces_runtime() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("old".to_string())), realtime_session_id: Some("conv_old".to_string()), transport: None, @@ -1183,6 +1193,7 @@ async fn conversation_second_start_replaces_runtime() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("new".to_string())), realtime_session_id: Some("conv_new".to_string()), transport: None, @@ -1279,6 +1290,7 @@ async fn conversation_uses_experimental_realtime_ws_base_url_override() -> Resul .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1343,6 +1355,7 @@ async fn conversation_uses_default_realtime_backend_prompt() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: None, realtime_session_id: None, transport: None, @@ -1415,6 +1428,7 @@ async fn conversation_uses_empty_instructions_for_null_or_empty_prompt() -> Resu .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt, realtime_session_id: None, transport: None, @@ -1480,6 +1494,7 @@ async fn conversation_uses_explicit_start_voice() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1537,6 +1552,7 @@ async fn conversation_uses_configured_realtime_voice() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1582,6 +1598,7 @@ async fn conversation_rejects_voice_for_wrong_realtime_version() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1628,6 +1645,7 @@ async fn conversation_uses_experimental_realtime_ws_backend_prompt_override() -> .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("prompt from op".to_string())), realtime_session_id: None, transport: None, @@ -1700,6 +1718,7 @@ async fn conversation_uses_experimental_realtime_ws_startup_context_override() - .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("prompt from op".to_string())), realtime_session_id: None, transport: None, @@ -1766,6 +1785,7 @@ async fn conversation_disables_realtime_startup_context_with_empty_override() -> .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("prompt from op".to_string())), realtime_session_id: None, transport: None, @@ -1825,6 +1845,7 @@ async fn conversation_start_injects_startup_context_from_thread_history() -> Res .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1936,6 +1957,7 @@ async fn conversation_startup_context_current_thread_selects_many_turns_by_budge .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2043,6 +2065,7 @@ async fn conversation_startup_context_falls_back_to_workspace_map() -> Result<() .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2102,6 +2125,7 @@ async fn conversation_startup_context_is_truncated_and_sent_once_per_start() -> .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2181,6 +2205,7 @@ async fn conversation_user_text_turn_is_not_sent_to_realtime() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2277,6 +2302,7 @@ async fn realtime_v2_noop_tool_call_returns_empty_function_output_without_respon .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2375,6 +2401,7 @@ async fn conversation_mirrors_assistant_message_text_to_realtime_handoff() -> Re .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2511,6 +2538,7 @@ async fn conversation_handoff_persists_across_item_done_until_turn_complete() -> .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2662,6 +2690,7 @@ async fn inbound_handoff_request_starts_turn() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2763,6 +2792,7 @@ async fn inbound_handoff_request_uses_active_transcript() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2865,6 +2895,7 @@ async fn inbound_handoff_request_sends_transcript_delta_after_each_handoff() -> .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2965,6 +2996,7 @@ async fn inbound_conversation_item_does_not_start_turn_and_still_forwards_audio( .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -3087,6 +3119,7 @@ async fn delegated_turn_user_role_echo_does_not_redelegate_and_still_forwards_au .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -3239,6 +3272,7 @@ async fn inbound_handoff_request_does_not_block_realtime_event_forwarding() -> R .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -3375,6 +3409,7 @@ async fn inbound_handoff_request_steers_active_turn() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -3527,6 +3562,7 @@ async fn inbound_handoff_request_starts_turn_and_does_not_block_realtime_audio() .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, + auto_handoff_appends: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 01e664c73499..92160741a05d 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -187,6 +187,8 @@ pub struct ConversationStartParams { pub model: Option, /// Selects whether the realtime session should produce text or audio output. pub output_modality: RealtimeOutputModality, + /// Whether Codex should automatically append standalone assistant output back to realtime. + pub auto_handoff_appends: bool, pub prompt: Option>, pub realtime_session_id: Option, pub transport: Option, @@ -398,6 +400,11 @@ pub struct ConversationTextParams { pub text: String, } +#[derive(Debug, Clone, PartialEq)] +pub struct ConversationHandoffParams { + pub output_text: String, +} + /// Persistent thread-settings overrides that can be applied before user input or /// on their own. #[derive(Debug, Clone, Default, PartialEq)] @@ -494,6 +501,9 @@ pub enum Op { /// Send text input to the running realtime conversation stream. RealtimeConversationText(ConversationTextParams), + /// Append assistant output to the running realtime conversation stream. + RealtimeConversationHandoff(ConversationHandoffParams), + /// Close the running realtime conversation stream. RealtimeConversationClose, @@ -744,6 +754,7 @@ impl Op { Self::RealtimeConversationStart(_) => "realtime_conversation_start", Self::RealtimeConversationAudio(_) => "realtime_conversation_audio", Self::RealtimeConversationText(_) => "realtime_conversation_text", + Self::RealtimeConversationHandoff(_) => "realtime_conversation_handoff", Self::RealtimeConversationClose => "realtime_conversation_close", Self::RealtimeConversationListVoices => "realtime_conversation_list_voices", Self::UserInput { .. } => "user_input", From 4c2a85316907f0ee7f6093a1368877cb119e176b Mon Sep 17 00:00:00 2001 From: Guinness Chen Date: Fri, 12 Jun 2026 11:20:47 -0700 Subject: [PATCH 02/11] Add realtime handoff update opt-out --- .../src/protocol/common.rs | 7 ++ .../src/protocol/v2/realtime.rs | 4 + codex-rs/app-server/README.md | 8 +- .../src/request_processors/turn_processor.rs | 1 + .../tests/suite/v2/experimental_api.rs | 2 + .../tests/suite/v2/realtime_conversation.rs | 98 ++++++++++++++++++- codex-rs/core/src/realtime_conversation.rs | 54 +++++++++- .../core/src/realtime_conversation_tests.rs | 7 +- codex-rs/core/tests/suite/compact_remote.rs | 1 + .../core/tests/suite/realtime_conversation.rs | 36 +++++++ codex-rs/protocol/src/protocol.rs | 2 + 11 files changed, 209 insertions(+), 11 deletions(-) diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index 425b9ad9c101..e8e6553f1a02 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -3017,6 +3017,7 @@ mod tests { model: Some("realtime-treatment-model".to_string()), output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: Some(false), + auto_handoff_updates: Some(false), prompt: Some(Some("You are on a call".to_string())), realtime_session_id: Some("sess_456".to_string()), transport: None, @@ -3033,6 +3034,7 @@ mod tests { "model": "realtime-treatment-model", "outputModality": "audio", "autoHandoffAppends": false, + "autoHandoffUpdates": false, "prompt": "You are on a call", "realtimeSessionId": "sess_456", "transport": null, @@ -3054,6 +3056,7 @@ mod tests { model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: None, + auto_handoff_updates: None, prompt: None, realtime_session_id: None, transport: None, @@ -3070,6 +3073,7 @@ mod tests { "model": null, "outputModality": "audio", "autoHandoffAppends": null, + "autoHandoffUpdates": null, "realtimeSessionId": null, "transport": null, "version": null, @@ -3086,6 +3090,7 @@ mod tests { model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: None, + auto_handoff_updates: None, prompt: Some(None), realtime_session_id: None, transport: None, @@ -3102,6 +3107,7 @@ mod tests { "model": null, "outputModality": "audio", "autoHandoffAppends": null, + "autoHandoffUpdates": null, "prompt": null, "realtimeSessionId": null, "transport": null, @@ -3284,6 +3290,7 @@ mod tests { model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: None, + auto_handoff_updates: None, prompt: Some(Some("You are on a call".to_string())), realtime_session_id: None, transport: None, diff --git a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs index 9d2c8db62523..fb821137dd89 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs @@ -75,6 +75,10 @@ pub struct ThreadRealtimeStartParams { /// Omit to keep the default behavior enabled. #[ts(optional = nullable)] pub auto_handoff_appends: Option, + /// Controls whether Codex automatically forwards active handoff output to realtime. + /// Omit to keep the default behavior enabled. + #[ts(optional = nullable)] + pub auto_handoff_updates: Option, #[serde( default, deserialize_with = "crate::protocol::serde_helpers::deserialize_double_option", diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index ebfe38dd074f..3a16547e7b1e 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -165,7 +165,7 @@ Example with notification opt-out: - `thread/inject_items` — append raw Responses API items to a loaded thread’s model-visible history without starting a user turn; returns `{}` on success. - `turn/steer` — add user input to an already in-flight regular turn without starting a new turn; returns the active `turnId` that accepted the input. `clientUserMessageId` is optional; when supplied, the corresponding `userMessage` item echoes it as `clientId`. Review and manual compaction turns reject `turn/steer`. - `turn/interrupt` — request cancellation of an in-flight turn by `(thread_id, turn_id)`; success is an empty `{}` response and the turn finishes with `status: "interrupted"`. -- `thread/realtime/start` — start a thread-scoped realtime session (experimental); pass `outputModality: "text"` or `outputModality: "audio"` to choose model output, and optionally pass `model` and `version` to override configured realtime selection for this session only. Pass `autoHandoffAppends: false` to stop Codex from automatically appending standalone assistant output back into realtime. Returns `{}` and streams `thread/realtime/*` notifications. Omit `transport` for the websocket transport, or pass `{ "type": "webrtc", "sdp": "..." }` to create a WebRTC session from a browser-generated SDP offer; the remote answer SDP is emitted as `thread/realtime/sdp`. +- `thread/realtime/start` — start a thread-scoped realtime session (experimental); pass `outputModality: "text"` or `outputModality: "audio"` to choose model output, and optionally pass `model` and `version` to override configured realtime selection for this session only. Pass `autoHandoffAppends: false` to stop Codex from automatically appending standalone assistant output back into realtime, and `autoHandoffUpdates: false` to stop Codex from automatically forwarding active handoff output back into realtime. Returns `{}` and streams `thread/realtime/*` notifications. Omit `transport` for the websocket transport, or pass `{ "type": "webrtc", "sdp": "..." }` to create a WebRTC session from a browser-generated SDP offer; the remote answer SDP is emitted as `thread/realtime/sdp`. - `thread/realtime/appendAudio` — append an input audio chunk to the active realtime session (experimental); returns `{}`. - `thread/realtime/appendText` — append text input to the active realtime session (experimental); returns `{}`. - `thread/realtime/appendHandoff` — append assistant output to the active realtime session (experimental); returns `{}`. For v1 sessions this sends `conversation.handoff.append`. @@ -854,8 +854,10 @@ Omit `prompt` to use Codex's default realtime backend prompt. Send `prompt: null Clients may also pass `model` and `version` on `thread/realtime/start` to select a different realtime session configuration without changing thread or user config. Pass `autoHandoffAppends: false` to suppress automatic standalone assistant output -from the backend Codex model, then call `thread/realtime/appendHandoff` when the -app decides a realtime update should be spoken. +from the backend Codex model. Pass `autoHandoffUpdates: false` to suppress +automatic assistant output while Codex is handling an active realtime handoff. +Then call `thread/realtime/appendHandoff` when the app decides a realtime update +should be spoken. ```javascript await pc.setRemoteDescription({ diff --git a/codex-rs/app-server/src/request_processors/turn_processor.rs b/codex-rs/app-server/src/request_processors/turn_processor.rs index c9dfdec3b5a9..b4f3fd00d10f 100644 --- a/codex-rs/app-server/src/request_processors/turn_processor.rs +++ b/codex-rs/app-server/src/request_processors/turn_processor.rs @@ -947,6 +947,7 @@ impl TurnRequestProcessor { model: params.model, output_modality: params.output_modality, auto_handoff_appends: params.auto_handoff_appends.unwrap_or(true), + auto_handoff_updates: params.auto_handoff_updates.unwrap_or(true), prompt: params.prompt, realtime_session_id: params.realtime_session_id, transport: params.transport.map(|transport| match transport { diff --git a/codex-rs/app-server/tests/suite/v2/experimental_api.rs b/codex-rs/app-server/tests/suite/v2/experimental_api.rs index 357ce4c8e1e5..b5baeb1f8cfa 100644 --- a/codex-rs/app-server/tests/suite/v2/experimental_api.rs +++ b/codex-rs/app-server/tests/suite/v2/experimental_api.rs @@ -83,6 +83,7 @@ async fn realtime_conversation_start_requires_experimental_api_capability() -> R model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: None, + auto_handoff_updates: None, prompt: Some(Some("hello".to_string())), realtime_session_id: None, transport: None, @@ -192,6 +193,7 @@ async fn realtime_webrtc_start_requires_experimental_api_capability() -> Result< model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: None, + auto_handoff_updates: None, prompt: Some(Some("hello".to_string())), realtime_session_id: None, transport: Some(ThreadRealtimeStartTransport::Webrtc { diff --git a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs index f40c01facff8..11f78541fdf9 100644 --- a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs +++ b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs @@ -310,8 +310,8 @@ impl RealtimeE2eHarness { } async fn start_webrtc_realtime(&mut self, offer_sdp: &str) -> Result { - self.start_webrtc_realtime_with_auto_handoff_appends( - offer_sdp, /*auto_handoff_appends*/ None, + self.start_webrtc_realtime_with_handoff_options( + offer_sdp, /*auto_handoff_appends*/ None, /*auto_handoff_updates*/ None, ) .await } @@ -320,6 +320,20 @@ impl RealtimeE2eHarness { &mut self, offer_sdp: &str, auto_handoff_appends: Option, + ) -> Result { + self.start_webrtc_realtime_with_handoff_options( + offer_sdp, + auto_handoff_appends, + /*auto_handoff_updates*/ None, + ) + .await + } + + async fn start_webrtc_realtime_with_handoff_options( + &mut self, + offer_sdp: &str, + auto_handoff_appends: Option, + auto_handoff_updates: Option, ) -> Result { // Starts realtime through the public JSON-RPC method, then waits for the same client-visible // notifications a desktop app needs: started first, SDP answer second. @@ -330,6 +344,7 @@ impl RealtimeE2eHarness { model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends, + auto_handoff_updates, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: Some(ThreadRealtimeStartTransport::Webrtc { @@ -596,6 +611,7 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> { model: Some("realtime-treatment-model".to_string()), output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: None, + auto_handoff_updates: None, prompt: None, realtime_session_id: None, transport: None, @@ -852,6 +868,7 @@ async fn realtime_text_output_modality_requests_text_output_and_final_transcript model: None, output_modality: RealtimeOutputModality::Text, auto_handoff_appends: None, + auto_handoff_updates: None, prompt: None, realtime_session_id: None, transport: None, @@ -1029,6 +1046,7 @@ async fn realtime_conversation_stop_emits_closed_notification() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: None, + auto_handoff_updates: None, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1129,6 +1147,7 @@ async fn realtime_webrtc_start_emits_sdp_notification() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: None, + auto_handoff_updates: None, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: Some(ThreadRealtimeStartTransport::Webrtc { @@ -1445,6 +1464,79 @@ async fn realtime_start_can_disable_auto_handoff_appends_and_append_manually() - Ok(()) } +#[tokio::test] +async fn realtime_start_can_disable_auto_handoff_updates_and_append_manually() -> Result<()> { + skip_if_no_network!(Ok(())); + + let mut harness = RealtimeE2eHarness::new( + RealtimeTestVersion::V2, + main_loop_responses(vec![create_final_assistant_message_sse_response( + "automatic final response", + )?]), + realtime_sideband(vec![realtime_sideband_connection(vec![ + vec![ + session_updated("sess_manual_update"), + v2_background_agent_tool_call("call_quiet", "delegate quietly"), + ], + vec![], + vec![], + vec![], + ])]), + ) + .await?; + + let started = harness + .start_webrtc_realtime_with_handoff_options( + "v=offer\r\n", + /*auto_handoff_appends*/ Some(false), + /*auto_handoff_updates*/ Some(false), + ) + .await?; + assert_eq!(started.started.version, RealtimeConversationVersion::V2); + assert_eq!( + harness.sideband_outbound_request(/*request_index*/ 0).await["type"].as_str(), + Some("session.update") + ); + + let turn_started = harness + .read_notification::("turn/started") + .await?; + assert_eq!(turn_started.thread_id, harness.thread_id); + let turn_completed = harness + .read_notification::("turn/completed") + .await?; + assert_eq!(turn_completed.thread_id, harness.thread_id); + + assert_v2_function_call_output( + &harness.sideband_outbound_request(/*request_index*/ 1).await, + "call_quiet", + "", + ); + let automatic_response_create = timeout( + Duration::from_millis(200), + harness + .realtime_server + .wait_for_request(/*connection_index*/ 0, /*request_index*/ 2), + ) + .await; + assert!( + automatic_response_create.is_err(), + "disabled handoff updates should not request an automatic realtime response" + ); + + harness + .append_handoff(harness.thread_id.clone(), "manual spoken update") + .await?; + assert_v2_progress_update( + &harness.sideband_outbound_request(/*request_index*/ 2).await, + "manual spoken update", + ); + assert_v2_response_create(&harness.sideband_outbound_request(/*request_index*/ 3).await); + + harness.shutdown().await; + Ok(()) +} + #[tokio::test] async fn webrtc_assistant_output_without_handoff_reaches_realtime() -> Result<()> { skip_if_no_network!(Ok(())); @@ -2241,6 +2333,7 @@ async fn realtime_webrtc_start_surfaces_backend_error() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: None, + auto_handoff_updates: None, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: Some(ThreadRealtimeStartTransport::Webrtc { @@ -2303,6 +2396,7 @@ async fn realtime_conversation_requires_feature_flag() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: None, + auto_handoff_updates: None, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, diff --git a/codex-rs/core/src/realtime_conversation.rs b/codex-rs/core/src/realtime_conversation.rs index 50fd86dfaa33..8f77220dea45 100644 --- a/codex-rs/core/src/realtime_conversation.rs +++ b/codex-rs/core/src/realtime_conversation.rs @@ -107,6 +107,7 @@ struct RealtimeHandoffState { last_output_text: Arc>>, session_kind: RealtimeSessionKind, auto_handoff_appends: bool, + auto_handoff_updates: bool, } #[derive(Debug, PartialEq, Eq)] @@ -121,6 +122,7 @@ enum HandoffOutput { FinalUpdate { handoff_id: String, output_text: String, + create_response: bool, }, } @@ -215,6 +217,7 @@ impl RealtimeHandoffState { output_tx: Sender, session_kind: RealtimeSessionKind, auto_handoff_appends: bool, + auto_handoff_updates: bool, ) -> Self { Self { output_tx, @@ -222,6 +225,7 @@ impl RealtimeHandoffState { last_output_text: Arc::new(Mutex::new(None)), session_kind, auto_handoff_appends, + auto_handoff_updates, } } } @@ -242,6 +246,7 @@ struct RealtimeStart { extra_headers: Option, session_config: RealtimeSessionConfig, auto_handoff_appends: bool, + auto_handoff_updates: bool, model_client: ModelClient, sdp: Option, } @@ -295,6 +300,7 @@ impl RealtimeConversationManager { extra_headers, session_config, auto_handoff_appends, + auto_handoff_updates, model_client, sdp, } = start; @@ -314,8 +320,12 @@ impl RealtimeConversationManager { async_channel::bounded::(OUTPUT_EVENTS_QUEUE_CAPACITY); let realtime_active = Arc::new(AtomicBool::new(true)); - let handoff = - RealtimeHandoffState::new(handoff_output_tx, session_kind, auto_handoff_appends); + let handoff = RealtimeHandoffState::new( + handoff_output_tx, + session_kind, + auto_handoff_appends, + auto_handoff_updates, + ); let input_channels = RealtimeInputChannels { user_text_rx, handoff_output_rx, @@ -478,7 +488,7 @@ impl RealtimeConversationManager { let active_handoff = handoff.active_handoff.lock().await.clone(); let output = match active_handoff { - Some(handoff_id) => { + Some(handoff_id) if handoff.auto_handoff_updates => { let output_text = prefix_realtime_text( output_text, REALTIME_BACKEND_TEXT_PREFIX, @@ -490,6 +500,7 @@ impl RealtimeConversationManager { output_text, } } + Some(_) => return Ok(()), None if output_text.trim().is_empty() => return Ok(()), None if handoff.auto_handoff_appends => { standalone_assistant_output(output_text, handoff.session_kind) @@ -546,6 +557,26 @@ impl RealtimeConversationManager { let Some(handoff_id) = handoff.active_handoff.lock().await.clone() else { return Ok(()); }; + + if !handoff.auto_handoff_updates { + match handoff.session_kind { + RealtimeSessionKind::V1 => return Ok(()), + RealtimeSessionKind::V2 => { + return handoff + .output_tx + .send(HandoffOutput::FinalUpdate { + handoff_id, + output_text: String::new(), + create_response: false, + }) + .await + .map_err(|_| { + CodexErr::InvalidRequest("conversation is not running".to_string()) + }); + } + } + } + let Some(output_text) = handoff.last_output_text.lock().await.clone() else { return Ok(()); }; @@ -555,6 +586,7 @@ impl RealtimeConversationManager { .send(HandoffOutput::FinalUpdate { handoff_id, output_text, + create_response: true, }) .await .map_err(|_| CodexErr::InvalidRequest("conversation is not running".to_string())) @@ -643,6 +675,7 @@ struct PreparedRealtimeConversationStart { extra_headers: Option, requested_realtime_session_id: Option, auto_handoff_appends: bool, + auto_handoff_updates: bool, version: RealtimeWsVersion, session_config: RealtimeSessionConfig, transport: ConversationStartTransport, @@ -701,6 +734,7 @@ async fn prepare_realtime_start( extra_headers, requested_realtime_session_id, auto_handoff_appends: params.auto_handoff_appends, + auto_handoff_updates: params.auto_handoff_updates, version, session_config, transport, @@ -831,6 +865,7 @@ async fn handle_start_inner( extra_headers, requested_realtime_session_id, auto_handoff_appends, + auto_handoff_updates, version, session_config, transport, @@ -845,6 +880,7 @@ async fn handle_start_inner( extra_headers, session_config, auto_handoff_appends, + auto_handoff_updates, model_client: sess.services.model_client.clone(), sdp, }; @@ -1279,6 +1315,7 @@ async fn handle_handoff_output( | HandoffOutput::FinalUpdate { handoff_id, output_text, + create_response: _, } => { writer .send_conversation_function_call_output(handoff_id, output_text) @@ -1312,19 +1349,26 @@ async fn handle_handoff_output( HandoffOutput::FinalUpdate { handoff_id, output_text: _, + create_response, } => { if let Err(err) = writer .send_conversation_function_call_output( handoff_id, - REALTIME_V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT.to_string(), + if create_response { + REALTIME_V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT.to_string() + } else { + String::new() + }, ) .await { Err(err) - } else { + } else if create_response { return response_create_queue .request_create(writer, events_tx, "handoff") .await; + } else { + Ok(()) } } }, diff --git a/codex-rs/core/src/realtime_conversation_tests.rs b/codex-rs/core/src/realtime_conversation_tests.rs index 745d9169285d..f441770c344e 100644 --- a/codex-rs/core/src/realtime_conversation_tests.rs +++ b/codex-rs/core/src/realtime_conversation_tests.rs @@ -128,7 +128,12 @@ fn wraps_realtime_delegation_input_with_xml_escaping_without_transcript() { #[tokio::test] async fn clears_active_handoff_explicitly() { let (tx, _rx) = bounded(1); - let state = RealtimeHandoffState::new(tx, RealtimeSessionKind::V1, true); + let state = RealtimeHandoffState::new( + tx, + RealtimeSessionKind::V1, + /*auto_handoff_appends*/ true, + /*auto_handoff_updates*/ true, + ); *state.active_handoff.lock().await = Some("handoff_1".to_string()); assert_eq!( diff --git a/codex-rs/core/tests/suite/compact_remote.rs b/codex-rs/core/tests/suite/compact_remote.rs index 5afab90d6b65..d09ece95490a 100644 --- a/codex-rs/core/tests/suite/compact_remote.rs +++ b/codex-rs/core/tests/suite/compact_remote.rs @@ -197,6 +197,7 @@ async fn start_realtime_conversation(codex: &codex_core::CodexThread) -> Result< model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, diff --git a/codex-rs/core/tests/suite/realtime_conversation.rs b/codex-rs/core/tests/suite/realtime_conversation.rs index 9e992903e4e3..19f2e8f18dee 100644 --- a/codex-rs/core/tests/suite/realtime_conversation.rs +++ b/codex-rs/core/tests/suite/realtime_conversation.rs @@ -285,6 +285,7 @@ async fn conversation_start_audio_text_close_round_trip() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -426,6 +427,7 @@ async fn conversation_start_defaults_to_v2_and_gpt_realtime_1_5() -> Result<()> model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -517,6 +519,7 @@ async fn conversation_webrtc_start_posts_generated_session() -> Result<()> { model: Some("session-override-model".to_string()), output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: Some(ConversationStartTransport::Webrtc { @@ -690,6 +693,7 @@ async fn conversation_webrtc_close_while_sideband_connecting_drops_pending_join( model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: Some(ConversationStartTransport::Webrtc { @@ -777,6 +781,7 @@ async fn conversation_webrtc_sideband_connect_failure_closes_with_error() -> Res model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: Some(ConversationStartTransport::Webrtc { @@ -865,6 +870,7 @@ async fn conversation_start_uses_openai_env_key_fallback_with_chatgpt_auth() -> model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -934,6 +940,7 @@ async fn conversation_transport_close_emits_closed_event() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1027,6 +1034,7 @@ async fn conversation_start_preflight_failure_emits_realtime_error_only() -> Res model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1074,6 +1082,7 @@ async fn conversation_start_connect_failure_emits_realtime_error_only() -> Resul model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1168,6 +1177,7 @@ async fn conversation_second_start_replaces_runtime() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("old".to_string())), realtime_session_id: Some("conv_old".to_string()), transport: None, @@ -1194,6 +1204,7 @@ async fn conversation_second_start_replaces_runtime() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("new".to_string())), realtime_session_id: Some("conv_new".to_string()), transport: None, @@ -1291,6 +1302,7 @@ async fn conversation_uses_experimental_realtime_ws_base_url_override() -> Resul model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1356,6 +1368,7 @@ async fn conversation_uses_default_realtime_backend_prompt() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: None, realtime_session_id: None, transport: None, @@ -1429,6 +1442,7 @@ async fn conversation_uses_empty_instructions_for_null_or_empty_prompt() -> Resu model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt, realtime_session_id: None, transport: None, @@ -1495,6 +1509,7 @@ async fn conversation_uses_explicit_start_voice() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1553,6 +1568,7 @@ async fn conversation_uses_configured_realtime_voice() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1599,6 +1615,7 @@ async fn conversation_rejects_voice_for_wrong_realtime_version() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1646,6 +1663,7 @@ async fn conversation_uses_experimental_realtime_ws_backend_prompt_override() -> model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("prompt from op".to_string())), realtime_session_id: None, transport: None, @@ -1719,6 +1737,7 @@ async fn conversation_uses_experimental_realtime_ws_startup_context_override() - model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("prompt from op".to_string())), realtime_session_id: None, transport: None, @@ -1786,6 +1805,7 @@ async fn conversation_disables_realtime_startup_context_with_empty_override() -> model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("prompt from op".to_string())), realtime_session_id: None, transport: None, @@ -1846,6 +1866,7 @@ async fn conversation_start_injects_startup_context_from_thread_history() -> Res model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1958,6 +1979,7 @@ async fn conversation_startup_context_current_thread_selects_many_turns_by_budge model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2066,6 +2088,7 @@ async fn conversation_startup_context_falls_back_to_workspace_map() -> Result<() model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2126,6 +2149,7 @@ async fn conversation_startup_context_is_truncated_and_sent_once_per_start() -> model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2206,6 +2230,7 @@ async fn conversation_user_text_turn_is_not_sent_to_realtime() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2303,6 +2328,7 @@ async fn realtime_v2_noop_tool_call_returns_empty_function_output_without_respon model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2402,6 +2428,7 @@ async fn conversation_mirrors_assistant_message_text_to_realtime_handoff() -> Re model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2539,6 +2566,7 @@ async fn conversation_handoff_persists_across_item_done_until_turn_complete() -> model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2691,6 +2719,7 @@ async fn inbound_handoff_request_starts_turn() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2793,6 +2822,7 @@ async fn inbound_handoff_request_uses_active_transcript() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2896,6 +2926,7 @@ async fn inbound_handoff_request_sends_transcript_delta_after_each_handoff() -> model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2997,6 +3028,7 @@ async fn inbound_conversation_item_does_not_start_turn_and_still_forwards_audio( model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -3120,6 +3152,7 @@ async fn delegated_turn_user_role_echo_does_not_redelegate_and_still_forwards_au model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -3273,6 +3306,7 @@ async fn inbound_handoff_request_does_not_block_realtime_event_forwarding() -> R model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -3410,6 +3444,7 @@ async fn inbound_handoff_request_steers_active_turn() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -3563,6 +3598,7 @@ async fn inbound_handoff_request_starts_turn_and_does_not_block_realtime_audio() model: None, output_modality: RealtimeOutputModality::Audio, auto_handoff_appends: true, + auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 92160741a05d..e22c8a72965b 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -189,6 +189,8 @@ pub struct ConversationStartParams { pub output_modality: RealtimeOutputModality, /// Whether Codex should automatically append standalone assistant output back to realtime. pub auto_handoff_appends: bool, + /// Whether Codex should automatically forward active handoff output back to realtime. + pub auto_handoff_updates: bool, pub prompt: Option>, pub realtime_session_id: Option, pub transport: Option, From aa1b854043fcd6fa152b0d2dee3f7edc2268af66 Mon Sep 17 00:00:00 2001 From: Guinness Chen Date: Fri, 12 Jun 2026 13:59:19 -0700 Subject: [PATCH 03/11] Use explicit realtime handoff speech --- .../src/protocol/common.rs | 14 - .../src/protocol/v2/realtime.rs | 8 - codex-rs/app-server/README.md | 12 +- .../src/request_processors/turn_processor.rs | 2 - .../tests/suite/v2/experimental_api.rs | 4 - .../tests/suite/v2/realtime_conversation.rs | 310 +++++++----------- .../endpoint/realtime_websocket/methods.rs | 12 + .../realtime_websocket/methods_common.rs | 11 + .../endpoint/realtime_websocket/methods_v2.rs | 13 +- .../endpoint/realtime_websocket/protocol.rs | 1 + codex-rs/core/src/realtime_conversation.rs | 210 +++--------- .../core/src/realtime_conversation_tests.rs | 7 +- codex-rs/core/tests/suite/compact_remote.rs | 2 - .../core/tests/suite/realtime_conversation.rs | 72 ---- codex-rs/protocol/src/protocol.rs | 4 - 15 files changed, 218 insertions(+), 464 deletions(-) diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index e8e6553f1a02..334a7a537eb9 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -3016,8 +3016,6 @@ mod tests { thread_id: "thr_123".to_string(), model: Some("realtime-treatment-model".to_string()), output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: Some(false), - auto_handoff_updates: Some(false), prompt: Some(Some("You are on a call".to_string())), realtime_session_id: Some("sess_456".to_string()), transport: None, @@ -3033,8 +3031,6 @@ mod tests { "threadId": "thr_123", "model": "realtime-treatment-model", "outputModality": "audio", - "autoHandoffAppends": false, - "autoHandoffUpdates": false, "prompt": "You are on a call", "realtimeSessionId": "sess_456", "transport": null, @@ -3055,8 +3051,6 @@ mod tests { thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: None, - auto_handoff_updates: None, prompt: None, realtime_session_id: None, transport: None, @@ -3072,8 +3066,6 @@ mod tests { "threadId": "thr_123", "model": null, "outputModality": "audio", - "autoHandoffAppends": null, - "autoHandoffUpdates": null, "realtimeSessionId": null, "transport": null, "version": null, @@ -3089,8 +3081,6 @@ mod tests { thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: None, - auto_handoff_updates: None, prompt: Some(None), realtime_session_id: None, transport: None, @@ -3106,8 +3096,6 @@ mod tests { "threadId": "thr_123", "model": null, "outputModality": "audio", - "autoHandoffAppends": null, - "autoHandoffUpdates": null, "prompt": null, "realtimeSessionId": null, "transport": null, @@ -3289,8 +3277,6 @@ mod tests { thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: None, - auto_handoff_updates: None, prompt: Some(Some("You are on a call".to_string())), realtime_session_id: None, transport: None, diff --git a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs index fb821137dd89..2925b4f1f2f0 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs @@ -71,14 +71,6 @@ pub struct ThreadRealtimeStartParams { /// Selects text or audio output for the realtime session. Transport and voice stay /// independent so clients can choose how they connect separately from what the model emits. pub output_modality: RealtimeOutputModality, - /// Controls whether Codex automatically appends standalone assistant output to realtime. - /// Omit to keep the default behavior enabled. - #[ts(optional = nullable)] - pub auto_handoff_appends: Option, - /// Controls whether Codex automatically forwards active handoff output to realtime. - /// Omit to keep the default behavior enabled. - #[ts(optional = nullable)] - pub auto_handoff_updates: Option, #[serde( default, deserialize_with = "crate::protocol::serde_helpers::deserialize_double_option", diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 3a16547e7b1e..939235c6ffd9 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -165,7 +165,7 @@ Example with notification opt-out: - `thread/inject_items` — append raw Responses API items to a loaded thread’s model-visible history without starting a user turn; returns `{}` on success. - `turn/steer` — add user input to an already in-flight regular turn without starting a new turn; returns the active `turnId` that accepted the input. `clientUserMessageId` is optional; when supplied, the corresponding `userMessage` item echoes it as `clientId`. Review and manual compaction turns reject `turn/steer`. - `turn/interrupt` — request cancellation of an in-flight turn by `(thread_id, turn_id)`; success is an empty `{}` response and the turn finishes with `status: "interrupted"`. -- `thread/realtime/start` — start a thread-scoped realtime session (experimental); pass `outputModality: "text"` or `outputModality: "audio"` to choose model output, and optionally pass `model` and `version` to override configured realtime selection for this session only. Pass `autoHandoffAppends: false` to stop Codex from automatically appending standalone assistant output back into realtime, and `autoHandoffUpdates: false` to stop Codex from automatically forwarding active handoff output back into realtime. Returns `{}` and streams `thread/realtime/*` notifications. Omit `transport` for the websocket transport, or pass `{ "type": "webrtc", "sdp": "..." }` to create a WebRTC session from a browser-generated SDP offer; the remote answer SDP is emitted as `thread/realtime/sdp`. +- `thread/realtime/start` — start a thread-scoped realtime session (experimental); pass `outputModality: "text"` or `outputModality: "audio"` to choose model output, and optionally pass `model` and `version` to override configured realtime selection for this session only. Backend Codex text is sent to v2 realtime sessions as silent developer context; use `thread/realtime/appendHandoff` when the app wants realtime to speak a backend update. Returns `{}` and streams `thread/realtime/*` notifications. Omit `transport` for the websocket transport, or pass `{ "type": "webrtc", "sdp": "..." }` to create a WebRTC session from a browser-generated SDP offer; the remote answer SDP is emitted as `thread/realtime/sdp`. - `thread/realtime/appendAudio` — append an input audio chunk to the active realtime session (experimental); returns `{}`. - `thread/realtime/appendText` — append text input to the active realtime session (experimental); returns `{}`. - `thread/realtime/appendHandoff` — append assistant output to the active realtime session (experimental); returns `{}`. For v1 sessions this sends `conversation.handoff.append`. @@ -853,11 +853,11 @@ Omit `prompt` to use Codex's default realtime backend prompt. Send `prompt: null `prompt: ""` when the session should start without that default backend prompt. Clients may also pass `model` and `version` on `thread/realtime/start` to select a different realtime session configuration without changing thread or user config. -Pass `autoHandoffAppends: false` to suppress automatic standalone assistant output -from the backend Codex model. Pass `autoHandoffUpdates: false` to suppress -automatic assistant output while Codex is handling an active realtime handoff. -Then call `thread/realtime/appendHandoff` when the app decides a realtime update -should be spoken. +Backend Codex output is injected into v2 realtime sessions as silent developer +context so the realtime model stays coherent without automatically speaking +backend preambles, progress, or final assistant text. Call +`thread/realtime/appendHandoff` when the app decides a realtime update should be +spoken. ```javascript await pc.setRemoteDescription({ diff --git a/codex-rs/app-server/src/request_processors/turn_processor.rs b/codex-rs/app-server/src/request_processors/turn_processor.rs index b4f3fd00d10f..713f535e963d 100644 --- a/codex-rs/app-server/src/request_processors/turn_processor.rs +++ b/codex-rs/app-server/src/request_processors/turn_processor.rs @@ -946,8 +946,6 @@ impl TurnRequestProcessor { Op::RealtimeConversationStart(ConversationStartParams { model: params.model, output_modality: params.output_modality, - auto_handoff_appends: params.auto_handoff_appends.unwrap_or(true), - auto_handoff_updates: params.auto_handoff_updates.unwrap_or(true), prompt: params.prompt, realtime_session_id: params.realtime_session_id, transport: params.transport.map(|transport| match transport { diff --git a/codex-rs/app-server/tests/suite/v2/experimental_api.rs b/codex-rs/app-server/tests/suite/v2/experimental_api.rs index b5baeb1f8cfa..03c9d455f05d 100644 --- a/codex-rs/app-server/tests/suite/v2/experimental_api.rs +++ b/codex-rs/app-server/tests/suite/v2/experimental_api.rs @@ -82,8 +82,6 @@ async fn realtime_conversation_start_requires_experimental_api_capability() -> R thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: None, - auto_handoff_updates: None, prompt: Some(Some("hello".to_string())), realtime_session_id: None, transport: None, @@ -192,8 +190,6 @@ async fn realtime_webrtc_start_requires_experimental_api_capability() -> Result< thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: None, - auto_handoff_updates: None, prompt: Some(Some("hello".to_string())), realtime_session_id: None, transport: Some(ThreadRealtimeStartTransport::Webrtc { diff --git a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs index 11f78541fdf9..a088dff3bbec 100644 --- a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs +++ b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs @@ -310,31 +310,6 @@ impl RealtimeE2eHarness { } async fn start_webrtc_realtime(&mut self, offer_sdp: &str) -> Result { - self.start_webrtc_realtime_with_handoff_options( - offer_sdp, /*auto_handoff_appends*/ None, /*auto_handoff_updates*/ None, - ) - .await - } - - async fn start_webrtc_realtime_with_auto_handoff_appends( - &mut self, - offer_sdp: &str, - auto_handoff_appends: Option, - ) -> Result { - self.start_webrtc_realtime_with_handoff_options( - offer_sdp, - auto_handoff_appends, - /*auto_handoff_updates*/ None, - ) - .await - } - - async fn start_webrtc_realtime_with_handoff_options( - &mut self, - offer_sdp: &str, - auto_handoff_appends: Option, - auto_handoff_updates: Option, - ) -> Result { // Starts realtime through the public JSON-RPC method, then waits for the same client-visible // notifications a desktop app needs: started first, SDP answer second. let start_request_id = self @@ -343,8 +318,6 @@ impl RealtimeE2eHarness { thread_id: self.thread_id.clone(), model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends, - auto_handoff_updates, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: Some(ThreadRealtimeStartTransport::Webrtc { @@ -610,8 +583,6 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> { thread_id: thread_start.thread.id.clone(), model: Some("realtime-treatment-model".to_string()), output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: None, - auto_handoff_updates: None, prompt: None, realtime_session_id: None, transport: None, @@ -867,8 +838,6 @@ async fn realtime_text_output_modality_requests_text_output_and_final_transcript thread_id: thread_start.thread.id.clone(), model: None, output_modality: RealtimeOutputModality::Text, - auto_handoff_appends: None, - auto_handoff_updates: None, prompt: None, realtime_session_id: None, transport: None, @@ -1045,8 +1014,6 @@ async fn realtime_conversation_stop_emits_closed_notification() -> Result<()> { thread_id: thread_start.thread.id.clone(), model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: None, - auto_handoff_updates: None, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1146,8 +1113,6 @@ async fn realtime_webrtc_start_emits_sdp_notification() -> Result<()> { thread_id: thread_id.clone(), model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: None, - auto_handoff_updates: None, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: Some(ThreadRealtimeStartTransport::Webrtc { @@ -1401,29 +1366,29 @@ async fn webrtc_v1_handoff_request_delegates_and_appends_result() -> Result<()> } #[tokio::test] -async fn realtime_start_can_disable_auto_handoff_appends_and_append_manually() -> Result<()> { +async fn realtime_automatic_standalone_output_is_context_and_append_speaks() -> Result<()> { skip_if_no_network!(Ok(())); let mut harness = RealtimeE2eHarness::new( - RealtimeTestVersion::V1, + RealtimeTestVersion::V2, main_loop_responses(vec![create_final_assistant_message_sse_response( "automatic output", )?]), realtime_sideband(vec![realtime_sideband_connection(vec![ vec![session_updated("sess_manual_handoff")], vec![], + vec![], + vec![], ])]), ) .await?; - let started = harness - .start_webrtc_realtime_with_auto_handoff_appends( - "v=offer\r\n", - /*auto_handoff_appends*/ Some(false), - ) - .await?; - assert_eq!(started.started.version, RealtimeConversationVersion::V1); - assert_v1_session_update(&harness.sideband_outbound_request(/*request_index*/ 0).await)?; + let started = harness.start_webrtc_realtime("v=offer\r\n").await?; + assert_eq!(started.started.version, RealtimeConversationVersion::V2); + assert_eq!( + harness.sideband_outbound_request(/*request_index*/ 0).await["type"].as_str(), + Some("session.update") + ); let turn_request_id = harness .mcp @@ -1448,24 +1413,37 @@ async fn realtime_start_can_disable_auto_handoff_appends_and_append_manually() - .read_notification::("turn/completed") .await?; + assert_v2_context_update( + &harness.sideband_outbound_request(/*request_index*/ 1).await, + "automatic output", + ); + let automatic_response_create = timeout( + Duration::from_millis(200), + harness + .realtime_server + .wait_for_request(/*connection_index*/ 0, /*request_index*/ 2), + ) + .await; + assert!( + automatic_response_create.is_err(), + "automatic context should not request a realtime response" + ); + harness .append_handoff(harness.thread_id.clone(), "manual voice update") .await?; - assert_eq!( - harness.sideband_outbound_request(/*request_index*/ 1).await, - json!({ - "type": "conversation.handoff.append", - "handoff_id": "codex", - "output_text": "manual voice update", - }) + assert_v2_progress_update( + &harness.sideband_outbound_request(/*request_index*/ 2).await, + "manual voice update", ); + assert_v2_response_create(&harness.sideband_outbound_request(/*request_index*/ 3).await); harness.shutdown().await; Ok(()) } #[tokio::test] -async fn realtime_start_can_disable_auto_handoff_updates_and_append_manually() -> Result<()> { +async fn realtime_automatic_handoff_output_is_context_and_append_speaks() -> Result<()> { skip_if_no_network!(Ok(())); let mut harness = RealtimeE2eHarness::new( @@ -1481,17 +1459,12 @@ async fn realtime_start_can_disable_auto_handoff_updates_and_append_manually() - vec![], vec![], vec![], + vec![], ])]), ) .await?; - let started = harness - .start_webrtc_realtime_with_handoff_options( - "v=offer\r\n", - /*auto_handoff_appends*/ Some(false), - /*auto_handoff_updates*/ Some(false), - ) - .await?; + let started = harness.start_webrtc_realtime("v=offer\r\n").await?; assert_eq!(started.started.version, RealtimeConversationVersion::V2); assert_eq!( harness.sideband_outbound_request(/*request_index*/ 0).await["type"].as_str(), @@ -1507,8 +1480,12 @@ async fn realtime_start_can_disable_auto_handoff_updates_and_append_manually() - .await?; assert_eq!(turn_completed.thread_id, harness.thread_id); - assert_v2_function_call_output( + assert_v2_context_update( &harness.sideband_outbound_request(/*request_index*/ 1).await, + "automatic final response", + ); + assert_v2_function_call_output( + &harness.sideband_outbound_request(/*request_index*/ 2).await, "call_quiet", "", ); @@ -1516,153 +1493,101 @@ async fn realtime_start_can_disable_auto_handoff_updates_and_append_manually() - Duration::from_millis(200), harness .realtime_server - .wait_for_request(/*connection_index*/ 0, /*request_index*/ 2), + .wait_for_request(/*connection_index*/ 0, /*request_index*/ 3), ) .await; assert!( automatic_response_create.is_err(), - "disabled handoff updates should not request an automatic realtime response" + "automatic handoff context should not request a realtime response" ); harness .append_handoff(harness.thread_id.clone(), "manual spoken update") .await?; assert_v2_progress_update( - &harness.sideband_outbound_request(/*request_index*/ 2).await, + &harness.sideband_outbound_request(/*request_index*/ 3).await, "manual spoken update", ); - assert_v2_response_create(&harness.sideband_outbound_request(/*request_index*/ 3).await); + assert_v2_response_create(&harness.sideband_outbound_request(/*request_index*/ 4).await); harness.shutdown().await; Ok(()) } #[tokio::test] -async fn webrtc_assistant_output_without_handoff_reaches_realtime() -> Result<()> { +async fn webrtc_v2_assistant_output_without_handoff_reaches_realtime_context() -> Result<()> { skip_if_no_network!(Ok(())); let final_answer = "long output ".repeat(1_000); - for (version, expected_version, preamble) in [ - ( - RealtimeTestVersion::V1, - RealtimeConversationVersion::V1, - "direct preamble from v1", - ), - ( - RealtimeTestVersion::V2, - RealtimeConversationVersion::V2, - "direct preamble from v2", - ), - ] { - let mut harness = RealtimeE2eHarness::new( - version, - main_loop_responses(vec![responses::sse(vec![ - responses::ev_response_created("resp-1"), - json!({ - "type": "response.output_item.done", - "item": { - "type": "message", - "role": "assistant", - "id": "msg-preamble", - "phase": "commentary", - "content": [{"type": "output_text", "text": preamble}] - } - }), - responses::ev_assistant_message("msg-final", &final_answer), - responses::ev_completed("resp-1"), - ])]), - realtime_sideband(vec![realtime_sideband_connection(vec![ - vec![session_updated("sess_standalone_output")], - vec![], - match version { - RealtimeTestVersion::V1 => vec![], - RealtimeTestVersion::V2 => vec![ - json!({ - "type": "response.created", - "response": { "id": "resp_preamble" } - }), - json!({ - "type": "response.done", - "response": { "id": "resp_preamble" } - }), - ], - }, - vec![], - vec![], - ])]), - ) - .await?; + let preamble = "direct preamble from v2"; + let mut harness = RealtimeE2eHarness::new( + RealtimeTestVersion::V2, + main_loop_responses(vec![responses::sse(vec![ + responses::ev_response_created("resp-1"), + json!({ + "type": "response.output_item.done", + "item": { + "type": "message", + "role": "assistant", + "id": "msg-preamble", + "phase": "commentary", + "content": [{"type": "output_text", "text": preamble}] + } + }), + responses::ev_assistant_message("msg-final", &final_answer), + responses::ev_completed("resp-1"), + ])]), + realtime_sideband(vec![realtime_sideband_connection(vec![ + vec![session_updated("sess_standalone_output")], + vec![], + vec![], + ])]), + ) + .await?; - let started = harness.start_webrtc_realtime("v=offer\r\n").await?; - assert_eq!(started.started.version, expected_version); + let started = harness.start_webrtc_realtime("v=offer\r\n").await?; + assert_eq!(started.started.version, RealtimeConversationVersion::V2); - let request_id = harness + let request_id = harness + .mcp + .send_turn_start_request(TurnStartParams { + thread_id: harness.thread_id.clone(), + input: vec![V2UserInput::Text { + text: "direct text turn".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let response: JSONRPCResponse = timeout( + DEFAULT_TIMEOUT, + harness .mcp - .send_turn_start_request(TurnStartParams { - thread_id: harness.thread_id.clone(), - input: vec![V2UserInput::Text { - text: "direct text turn".to_string(), - text_elements: Vec::new(), - }], - ..Default::default() - }) - .await?; - let response: JSONRPCResponse = timeout( - DEFAULT_TIMEOUT, - harness - .mcp - .read_stream_until_response_message(RequestId::Integer(request_id)), - ) - .await??; - let _: TurnStartResponse = to_response(response)?; - let _ = harness - .read_notification::("turn/completed") - .await?; + .read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let _: TurnStartResponse = to_response(response)?; + let _ = harness + .read_notification::("turn/completed") + .await?; - let preamble_request = harness.sideband_outbound_request(/*request_index*/ 1).await; - let output_text = match version { - RealtimeTestVersion::V1 => { - let final_request = harness.sideband_outbound_request(/*request_index*/ 2).await; - assert_eq!( - preamble_request, - json!({ - "type": "conversation.handoff.append", - "handoff_id": "codex", - "output_text": preamble, - }) - ); - assert_eq!(final_request["type"], "conversation.handoff.append"); - assert_eq!(final_request["handoff_id"], "codex"); - final_request["output_text"] - .as_str() - .expect("output text") - .to_string() - } - RealtimeTestVersion::V2 => { - assert_v2_progress_update(&preamble_request, preamble); - assert_v2_response_create( - &harness.sideband_outbound_request(/*request_index*/ 2).await, - ); - let final_request = harness.sideband_outbound_request(/*request_index*/ 3).await; - assert_eq!(final_request["type"], "conversation.item.create"); - assert_eq!(final_request["item"]["type"], "message"); - assert_eq!(final_request["item"]["role"], "user"); - assert_eq!(final_request["item"]["content"][0]["type"], "input_text"); - let output_text = final_request["item"]["content"][0]["text"] - .as_str() - .expect("output text"); - assert!(output_text.starts_with("[BACKEND] ")); - assert_v2_response_create( - &harness.sideband_outbound_request(/*request_index*/ 4).await, - ); - output_text.to_string() - } - }; - assert!(output_text.contains("tokens truncated")); - assert!(output_text.len() <= 4_000); + assert_v2_context_update( + &harness.sideband_outbound_request(/*request_index*/ 1).await, + preamble, + ); + let final_request = harness.sideband_outbound_request(/*request_index*/ 2).await; + assert_eq!(final_request["type"], "conversation.item.create"); + assert_eq!(final_request["item"]["type"], "message"); + assert_eq!(final_request["item"]["role"], "developer"); + assert_eq!(final_request["item"]["content"][0]["type"], "input_text"); + let output_text = final_request["item"]["content"][0]["text"] + .as_str() + .expect("output text"); + assert!(output_text.starts_with("[BACKEND] ")); + assert!(output_text.contains("tokens truncated")); + assert!(output_text.len() <= 4_000); - harness.shutdown().await; - } + harness.shutdown().await; Ok(()) } @@ -2332,8 +2257,6 @@ async fn realtime_webrtc_start_surfaces_backend_error() -> Result<()> { thread_id: thread_start.thread.id, model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: None, - auto_handoff_updates: None, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: Some(ThreadRealtimeStartTransport::Webrtc { @@ -2395,8 +2318,6 @@ async fn realtime_conversation_requires_feature_flag() -> Result<()> { thread_id: thread_start.thread.id.clone(), model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: None, - auto_handoff_updates: None, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2560,6 +2481,23 @@ fn assert_v2_progress_update(request: &Value, expected_text: &str) { ); } +fn assert_v2_context_update(request: &Value, expected_text: &str) { + assert_eq!( + request, + &json!({ + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "developer", + "content": [{ + "type": "input_text", + "text": format!("[BACKEND] {expected_text}") + }] + } + }) + ); +} + fn assert_v2_user_text_item(request: &Value, expected_text: &str) { assert_eq!( request, diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs index d9a67c7b97c0..3b9bcbab5ecc 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs @@ -1,3 +1,4 @@ +use crate::endpoint::realtime_websocket::methods_common::conversation_context_item_create_message; use crate::endpoint::realtime_websocket::methods_common::conversation_function_call_output_message; use crate::endpoint::realtime_websocket::methods_common::conversation_item_create_message; use crate::endpoint::realtime_websocket::methods_common::normalized_session_mode; @@ -291,6 +292,17 @@ impl RealtimeWebsocketWriter { .await } + pub async fn send_conversation_context_item_create( + &self, + text: String, + ) -> Result<(), ApiError> { + let Some(message) = conversation_context_item_create_message(self.event_parser, text) + else { + return Ok(()); + }; + self.send_json(&message).await + } + pub async fn send_conversation_handoff_append( &self, handoff_id: String, diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_common.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_common.rs index 1e47fb6fbf4f..8e98df6f437d 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_common.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_common.rs @@ -2,6 +2,7 @@ use crate::endpoint::realtime_websocket::methods_v1::conversation_handoff_append use crate::endpoint::realtime_websocket::methods_v1::conversation_item_create_message as v1_conversation_item_create_message; use crate::endpoint::realtime_websocket::methods_v1::session_update_session as v1_session_update_session; use crate::endpoint::realtime_websocket::methods_v1::websocket_intent as v1_websocket_intent; +use crate::endpoint::realtime_websocket::methods_v2::conversation_context_item_create_message as v2_conversation_context_item_create_message; use crate::endpoint::realtime_websocket::methods_v2::conversation_function_call_output_message as v2_conversation_function_call_output_message; use crate::endpoint::realtime_websocket::methods_v2::conversation_item_create_message as v2_conversation_item_create_message; use crate::endpoint::realtime_websocket::methods_v2::session_update_session as v2_session_update_session; @@ -40,6 +41,16 @@ pub(super) fn conversation_item_create_message( } } +pub(super) fn conversation_context_item_create_message( + event_parser: RealtimeEventParser, + text: String, +) -> Option { + match event_parser { + RealtimeEventParser::V1 => None, + RealtimeEventParser::RealtimeV2 => Some(v2_conversation_context_item_create_message(text)), + } +} + pub(super) fn conversation_function_call_output_message( event_parser: RealtimeEventParser, call_id: String, diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v2.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v2.rs index 292067748390..b1978aae8aa3 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v2.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v2.rs @@ -37,10 +37,21 @@ const REALTIME_V2_SILENCE_TOOL_DESCRIPTION: &str = "Call this when the best resp const REALTIME_V2_INPUT_TRANSCRIPTION_MODEL: &str = "gpt-4o-mini-transcribe"; pub(super) fn conversation_item_create_message(text: String) -> RealtimeOutboundMessage { + conversation_message_item_create_message(ConversationRole::User, text) +} + +pub(super) fn conversation_context_item_create_message(text: String) -> RealtimeOutboundMessage { + conversation_message_item_create_message(ConversationRole::Developer, text) +} + +fn conversation_message_item_create_message( + role: ConversationRole, + text: String, +) -> RealtimeOutboundMessage { RealtimeOutboundMessage::ConversationItemCreate { item: ConversationItemPayload::Message(ConversationMessageItem { r#type: ConversationItemType::Message, - role: ConversationRole::User, + role, content: vec![ConversationItemContent { r#type: ConversationContentType::InputText, text, diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/protocol.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/protocol.rs index 5df4c0c50349..2e66bc5b72cf 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/protocol.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/protocol.rs @@ -171,6 +171,7 @@ pub(super) enum ConversationItemType { #[derive(Debug, Clone, Copy, Serialize)] #[serde(rename_all = "snake_case")] pub(super) enum ConversationRole { + Developer, User, } diff --git a/codex-rs/core/src/realtime_conversation.rs b/codex-rs/core/src/realtime_conversation.rs index 8f77220dea45..8ab1c547bde3 100644 --- a/codex-rs/core/src/realtime_conversation.rs +++ b/codex-rs/core/src/realtime_conversation.rs @@ -71,8 +71,6 @@ const STANDALONE_HANDOFF_ID: &str = "codex"; const DEFAULT_REALTIME_MODEL: &str = "gpt-realtime-1.5"; pub(crate) const REALTIME_USER_TEXT_PREFIX: &str = "[USER] "; pub(crate) const REALTIME_BACKEND_TEXT_PREFIX: &str = "[BACKEND] "; -const REALTIME_V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT: &str = - "Background agent finished. Use the preceding [BACKEND] messages as the result."; const REALTIME_V2_STEER_ACKNOWLEDGEMENT: &str = "This was sent to steer the previous background agent task."; const REALTIME_ACTIVE_RESPONSE_ERROR_PREFIX: &str = @@ -104,25 +102,20 @@ enum RealtimeSessionKind { struct RealtimeHandoffState { output_tx: Sender, active_handoff: Arc>>, - last_output_text: Arc>>, session_kind: RealtimeSessionKind, - auto_handoff_appends: bool, - auto_handoff_updates: bool, } #[derive(Debug, PartialEq, Eq)] enum HandoffOutput { - StandaloneAssistantOutput { + ContextUpdate { + handoff_id: Option, output_text: String, }, - ProgressUpdate { + HandoffComplete { handoff_id: String, - output_text: String, }, - FinalUpdate { - handoff_id: String, + SpokenAppend { output_text: String, - create_response: bool, }, } @@ -213,19 +206,11 @@ struct RealtimeInputChannels { } impl RealtimeHandoffState { - fn new( - output_tx: Sender, - session_kind: RealtimeSessionKind, - auto_handoff_appends: bool, - auto_handoff_updates: bool, - ) -> Self { + fn new(output_tx: Sender, session_kind: RealtimeSessionKind) -> Self { Self { output_tx, active_handoff: Arc::new(Mutex::new(None)), - last_output_text: Arc::new(Mutex::new(None)), session_kind, - auto_handoff_appends, - auto_handoff_updates, } } } @@ -245,8 +230,6 @@ struct RealtimeStart { api_provider: ApiProvider, extra_headers: Option, session_config: RealtimeSessionConfig, - auto_handoff_appends: bool, - auto_handoff_updates: bool, model_client: ModelClient, sdp: Option, } @@ -299,8 +282,6 @@ impl RealtimeConversationManager { api_provider, extra_headers, session_config, - auto_handoff_appends, - auto_handoff_updates, model_client, sdp, } = start; @@ -320,12 +301,7 @@ impl RealtimeConversationManager { async_channel::bounded::(OUTPUT_EVENTS_QUEUE_CAPACITY); let realtime_active = Arc::new(AtomicBool::new(true)); - let handoff = RealtimeHandoffState::new( - handoff_output_tx, - session_kind, - auto_handoff_appends, - auto_handoff_updates, - ); + let handoff = RealtimeHandoffState::new(handoff_output_tx, session_kind); let input_channels = RealtimeInputChannels { user_text_rx, handoff_output_rx, @@ -486,30 +462,16 @@ impl RealtimeConversationManager { state.handoff.clone() }; - let active_handoff = handoff.active_handoff.lock().await.clone(); - let output = match active_handoff { - Some(handoff_id) if handoff.auto_handoff_updates => { - let output_text = prefix_realtime_text( - output_text, - REALTIME_BACKEND_TEXT_PREFIX, - handoff.session_kind, - ); - *handoff.last_output_text.lock().await = Some(output_text.clone()); - HandoffOutput::ProgressUpdate { - handoff_id, - output_text, - } - } - Some(_) => return Ok(()), - None if output_text.trim().is_empty() => return Ok(()), - None if handoff.auto_handoff_appends => { - standalone_assistant_output(output_text, handoff.session_kind) - } - None => return Ok(()), - }; + if output_text.trim().is_empty() { + return Ok(()); + } + let handoff_id = handoff.active_handoff.lock().await.clone(); handoff .output_tx - .send(output) + .send(HandoffOutput::ContextUpdate { + handoff_id, + output_text: realtime_backend_output(output_text, handoff.session_kind), + }) .await .map_err(|_| CodexErr::InvalidRequest("conversation is not running".to_string()))?; Ok(()) @@ -532,10 +494,9 @@ impl RealtimeConversationManager { handoff .output_tx - .send(standalone_assistant_output( - output_text, - handoff.session_kind, - )) + .send(HandoffOutput::SpokenAppend { + output_text: realtime_backend_output(output_text, handoff.session_kind), + }) .await .map_err(|_| CodexErr::InvalidRequest("conversation is not running".to_string()))?; Ok(()) @@ -558,36 +519,9 @@ impl RealtimeConversationManager { return Ok(()); }; - if !handoff.auto_handoff_updates { - match handoff.session_kind { - RealtimeSessionKind::V1 => return Ok(()), - RealtimeSessionKind::V2 => { - return handoff - .output_tx - .send(HandoffOutput::FinalUpdate { - handoff_id, - output_text: String::new(), - create_response: false, - }) - .await - .map_err(|_| { - CodexErr::InvalidRequest("conversation is not running".to_string()) - }); - } - } - } - - let Some(output_text) = handoff.last_output_text.lock().await.clone() else { - return Ok(()); - }; - handoff .output_tx - .send(HandoffOutput::FinalUpdate { - handoff_id, - output_text, - create_response: true, - }) + .send(HandoffOutput::HandoffComplete { handoff_id }) .await .map_err(|_| CodexErr::InvalidRequest("conversation is not running".to_string())) } @@ -599,7 +533,6 @@ impl RealtimeConversationManager { }; if let Some(handoff) = handoff { *handoff.active_handoff.lock().await = None; - *handoff.last_output_text.lock().await = None; } } @@ -674,8 +607,6 @@ struct PreparedRealtimeConversationStart { api_provider: ApiProvider, extra_headers: Option, requested_realtime_session_id: Option, - auto_handoff_appends: bool, - auto_handoff_updates: bool, version: RealtimeWsVersion, session_config: RealtimeSessionConfig, transport: ConversationStartTransport, @@ -733,8 +664,6 @@ async fn prepare_realtime_start( api_provider, extra_headers, requested_realtime_session_id, - auto_handoff_appends: params.auto_handoff_appends, - auto_handoff_updates: params.auto_handoff_updates, version, session_config, transport, @@ -817,17 +746,9 @@ fn prefix_realtime_text(text: String, prefix: &str, session_kind: RealtimeSessio format!("{prefix}{text}") } -fn standalone_assistant_output( - output_text: String, - session_kind: RealtimeSessionKind, -) -> HandoffOutput { +fn realtime_backend_output(output_text: String, session_kind: RealtimeSessionKind) -> String { let output_text = prefix_realtime_text(output_text, REALTIME_BACKEND_TEXT_PREFIX, session_kind); - HandoffOutput::StandaloneAssistantOutput { - output_text: truncate_realtime_text_to_token_budget( - &output_text, - REALTIME_ASSISTANT_OUTPUT_TOKEN_BUDGET, - ), - } + truncate_realtime_text_to_token_budget(&output_text, REALTIME_ASSISTANT_OUTPUT_TOKEN_BUDGET) } fn validate_realtime_voice(version: RealtimeWsVersion, voice: RealtimeVoice) -> CodexResult<()> { @@ -864,8 +785,6 @@ async fn handle_start_inner( api_provider, extra_headers, requested_realtime_session_id, - auto_handoff_appends, - auto_handoff_updates, version, session_config, transport, @@ -879,8 +798,6 @@ async fn handle_start_inner( api_provider, extra_headers, session_config, - auto_handoff_appends, - auto_handoff_updates, model_client: sess.services.model_client.clone(), sdp, }; @@ -1238,7 +1155,6 @@ async fn run_realtime_input_task(input: RealtimeInputTask) { background_agent_output, &writer, &events_tx, - &handoff_state, event_parser, &mut response_create_queue, ) @@ -1291,7 +1207,6 @@ async fn handle_handoff_output( handoff_output: Result, writer: &RealtimeWebsocketWriter, events_tx: &Sender, - handoff_state: &RealtimeHandoffState, event_parser: RealtimeEventParser, response_create_queue: &mut RealtimeResponseCreateQueue, ) -> anyhow::Result<()> { @@ -1299,7 +1214,25 @@ async fn handle_handoff_output( let result = match event_parser { RealtimeEventParser::V1 => match handoff_output { - HandoffOutput::StandaloneAssistantOutput { output_text } => { + HandoffOutput::ContextUpdate { + handoff_id, + output_text, + } => { + if let Some(handoff_id) = handoff_id { + writer + .send_conversation_function_call_output(handoff_id, output_text) + .await + } else { + writer + .send_conversation_handoff_append( + STANDALONE_HANDOFF_ID.to_string(), + output_text, + ) + .await + } + } + HandoffOutput::HandoffComplete { .. } => Ok(()), + HandoffOutput::SpokenAppend { output_text } => { // TODO(guinness): Use the new client event for standalone handoffs once the API changes are complete. writer .send_conversation_handoff_append( @@ -1308,69 +1241,30 @@ async fn handle_handoff_output( ) .await } - HandoffOutput::ProgressUpdate { - handoff_id, - output_text, - } - | HandoffOutput::FinalUpdate { - handoff_id, + }, + RealtimeEventParser::RealtimeV2 => match handoff_output { + HandoffOutput::ContextUpdate { + handoff_id: _, output_text, - create_response: _, } => { writer - .send_conversation_function_call_output(handoff_id, output_text) + .send_conversation_context_item_create(output_text) .await } - }, - RealtimeEventParser::RealtimeV2 => match handoff_output { - HandoffOutput::StandaloneAssistantOutput { output_text } => { + HandoffOutput::HandoffComplete { handoff_id } => { + writer + .send_conversation_function_call_output(handoff_id, String::new()) + .await + } + HandoffOutput::SpokenAppend { output_text } => { if let Err(err) = writer.send_conversation_item_create(output_text).await { Err(err) } else { return response_create_queue - .request_create(writer, events_tx, "standalone assistant output") + .request_create(writer, events_tx, "handoff append") .await; } } - HandoffOutput::ProgressUpdate { - handoff_id, - output_text, - } => { - let active_handoff = handoff_state.active_handoff.lock().await.clone(); - match active_handoff { - Some(active_handoff) if active_handoff == handoff_id => {} - Some(_) | None => { - debug!("dropping stale realtime handoff progress update"); - return Ok(()); - } - } - writer.send_conversation_item_create(output_text).await - } - HandoffOutput::FinalUpdate { - handoff_id, - output_text: _, - create_response, - } => { - if let Err(err) = writer - .send_conversation_function_call_output( - handoff_id, - if create_response { - REALTIME_V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT.to_string() - } else { - String::new() - }, - ) - .await - { - Err(err) - } else if create_response { - return response_create_queue - .request_create(writer, events_tx, "handoff") - .await; - } else { - Ok(()) - } - } }, }; if let Err(err) = result { @@ -1484,7 +1378,6 @@ async fn handle_realtime_server_event( match session_kind { RealtimeSessionKind::V1 => { - *handoff_state.last_output_text.lock().await = None; *handoff_state.active_handoff.lock().await = Some(handoff.handoff_id.clone()); } RealtimeSessionKind::V2 => { @@ -1512,7 +1405,6 @@ async fn handle_realtime_server_event( .await?; } None => { - *handoff_state.last_output_text.lock().await = None; *handoff_state.active_handoff.lock().await = Some(handoff.handoff_id.clone()); } diff --git a/codex-rs/core/src/realtime_conversation_tests.rs b/codex-rs/core/src/realtime_conversation_tests.rs index f441770c344e..b67205ef8fc6 100644 --- a/codex-rs/core/src/realtime_conversation_tests.rs +++ b/codex-rs/core/src/realtime_conversation_tests.rs @@ -128,12 +128,7 @@ fn wraps_realtime_delegation_input_with_xml_escaping_without_transcript() { #[tokio::test] async fn clears_active_handoff_explicitly() { let (tx, _rx) = bounded(1); - let state = RealtimeHandoffState::new( - tx, - RealtimeSessionKind::V1, - /*auto_handoff_appends*/ true, - /*auto_handoff_updates*/ true, - ); + let state = RealtimeHandoffState::new(tx, RealtimeSessionKind::V1); *state.active_handoff.lock().await = Some("handoff_1".to_string()); assert_eq!( diff --git a/codex-rs/core/tests/suite/compact_remote.rs b/codex-rs/core/tests/suite/compact_remote.rs index d09ece95490a..eee445ea9fa1 100644 --- a/codex-rs/core/tests/suite/compact_remote.rs +++ b/codex-rs/core/tests/suite/compact_remote.rs @@ -196,8 +196,6 @@ async fn start_realtime_conversation(codex: &codex_core::CodexThread) -> Result< .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, diff --git a/codex-rs/core/tests/suite/realtime_conversation.rs b/codex-rs/core/tests/suite/realtime_conversation.rs index 19f2e8f18dee..25f0516b0d91 100644 --- a/codex-rs/core/tests/suite/realtime_conversation.rs +++ b/codex-rs/core/tests/suite/realtime_conversation.rs @@ -284,8 +284,6 @@ async fn conversation_start_audio_text_close_round_trip() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -426,8 +424,6 @@ async fn conversation_start_defaults_to_v2_and_gpt_realtime_1_5() -> Result<()> .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -518,8 +514,6 @@ async fn conversation_webrtc_start_posts_generated_session() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: Some("session-override-model".to_string()), output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: Some(ConversationStartTransport::Webrtc { @@ -692,8 +686,6 @@ async fn conversation_webrtc_close_while_sideband_connecting_drops_pending_join( .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: Some(ConversationStartTransport::Webrtc { @@ -780,8 +772,6 @@ async fn conversation_webrtc_sideband_connect_failure_closes_with_error() -> Res .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: Some(ConversationStartTransport::Webrtc { @@ -869,8 +859,6 @@ async fn conversation_start_uses_openai_env_key_fallback_with_chatgpt_auth() -> .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -939,8 +927,6 @@ async fn conversation_transport_close_emits_closed_event() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1033,8 +1019,6 @@ async fn conversation_start_preflight_failure_emits_realtime_error_only() -> Res .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1081,8 +1065,6 @@ async fn conversation_start_connect_failure_emits_realtime_error_only() -> Resul .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1176,8 +1158,6 @@ async fn conversation_second_start_replaces_runtime() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("old".to_string())), realtime_session_id: Some("conv_old".to_string()), transport: None, @@ -1203,8 +1183,6 @@ async fn conversation_second_start_replaces_runtime() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("new".to_string())), realtime_session_id: Some("conv_new".to_string()), transport: None, @@ -1301,8 +1279,6 @@ async fn conversation_uses_experimental_realtime_ws_base_url_override() -> Resul .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1367,8 +1343,6 @@ async fn conversation_uses_default_realtime_backend_prompt() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: None, realtime_session_id: None, transport: None, @@ -1441,8 +1415,6 @@ async fn conversation_uses_empty_instructions_for_null_or_empty_prompt() -> Resu .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt, realtime_session_id: None, transport: None, @@ -1508,8 +1480,6 @@ async fn conversation_uses_explicit_start_voice() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1567,8 +1537,6 @@ async fn conversation_uses_configured_realtime_voice() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1614,8 +1582,6 @@ async fn conversation_rejects_voice_for_wrong_realtime_version() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1662,8 +1628,6 @@ async fn conversation_uses_experimental_realtime_ws_backend_prompt_override() -> .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("prompt from op".to_string())), realtime_session_id: None, transport: None, @@ -1736,8 +1700,6 @@ async fn conversation_uses_experimental_realtime_ws_startup_context_override() - .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("prompt from op".to_string())), realtime_session_id: None, transport: None, @@ -1804,8 +1766,6 @@ async fn conversation_disables_realtime_startup_context_with_empty_override() -> .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("prompt from op".to_string())), realtime_session_id: None, transport: None, @@ -1865,8 +1825,6 @@ async fn conversation_start_injects_startup_context_from_thread_history() -> Res .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -1978,8 +1936,6 @@ async fn conversation_startup_context_current_thread_selects_many_turns_by_budge .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2087,8 +2043,6 @@ async fn conversation_startup_context_falls_back_to_workspace_map() -> Result<() .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2148,8 +2102,6 @@ async fn conversation_startup_context_is_truncated_and_sent_once_per_start() -> .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2229,8 +2181,6 @@ async fn conversation_user_text_turn_is_not_sent_to_realtime() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2327,8 +2277,6 @@ async fn realtime_v2_noop_tool_call_returns_empty_function_output_without_respon .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2427,8 +2375,6 @@ async fn conversation_mirrors_assistant_message_text_to_realtime_handoff() -> Re .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2565,8 +2511,6 @@ async fn conversation_handoff_persists_across_item_done_until_turn_complete() -> .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2718,8 +2662,6 @@ async fn inbound_handoff_request_starts_turn() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2821,8 +2763,6 @@ async fn inbound_handoff_request_uses_active_transcript() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -2925,8 +2865,6 @@ async fn inbound_handoff_request_sends_transcript_delta_after_each_handoff() -> .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -3027,8 +2965,6 @@ async fn inbound_conversation_item_does_not_start_turn_and_still_forwards_audio( .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -3151,8 +3087,6 @@ async fn delegated_turn_user_role_echo_does_not_redelegate_and_still_forwards_au .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -3305,8 +3239,6 @@ async fn inbound_handoff_request_does_not_block_realtime_event_forwarding() -> R .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -3443,8 +3375,6 @@ async fn inbound_handoff_request_steers_active_turn() -> Result<()> { .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, @@ -3597,8 +3527,6 @@ async fn inbound_handoff_request_starts_turn_and_does_not_block_realtime_audio() .submit(Op::RealtimeConversationStart(ConversationStartParams { model: None, output_modality: RealtimeOutputModality::Audio, - auto_handoff_appends: true, - auto_handoff_updates: true, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index e22c8a72965b..192dbdd2c4a7 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -187,10 +187,6 @@ pub struct ConversationStartParams { pub model: Option, /// Selects whether the realtime session should produce text or audio output. pub output_modality: RealtimeOutputModality, - /// Whether Codex should automatically append standalone assistant output back to realtime. - pub auto_handoff_appends: bool, - /// Whether Codex should automatically forward active handoff output back to realtime. - pub auto_handoff_updates: bool, pub prompt: Option>, pub realtime_session_id: Option, pub transport: Option, From 38a8d2412d2ff035532caba22b7c1a24f6a4bd94 Mon Sep 17 00:00:00 2001 From: Guinness Chen Date: Fri, 12 Jun 2026 16:04:30 -0700 Subject: [PATCH 04/11] Add realtime handoff context opt-in --- .../src/protocol/common.rs | 7 + .../src/protocol/v2/realtime.rs | 4 + codex-rs/app-server/README.md | 13 +- .../src/request_processors/turn_processor.rs | 3 + .../tests/suite/v2/experimental_api.rs | 2 + .../tests/suite/v2/realtime_conversation.rs | 136 ++++++++++++++++-- .../endpoint/realtime_websocket/methods.rs | 5 +- .../realtime_websocket/methods_common.rs | 7 +- .../endpoint/realtime_websocket/methods_v1.rs | 13 +- codex-rs/core/src/realtime_conversation.rs | 44 +++++- .../core/src/realtime_conversation_tests.rs | 6 +- codex-rs/core/tests/suite/compact_remote.rs | 1 + .../core/tests/suite/realtime_conversation.rs | 36 +++++ codex-rs/protocol/src/protocol.rs | 3 + 14 files changed, 248 insertions(+), 32 deletions(-) diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index 334a7a537eb9..543a453e5a43 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -3013,6 +3013,7 @@ mod tests { let request = ClientRequest::ThreadRealtimeStart { request_id: RequestId::Integer(9), params: v2::ThreadRealtimeStartParams { + auto_handoff_output_as_context: None, thread_id: "thr_123".to_string(), model: Some("realtime-treatment-model".to_string()), output_modality: RealtimeOutputModality::Audio, @@ -3029,6 +3030,7 @@ mod tests { "id": 9, "params": { "threadId": "thr_123", + "autoHandoffOutputAsContext": null, "model": "realtime-treatment-model", "outputModality": "audio", "prompt": "You are on a call", @@ -3048,6 +3050,7 @@ mod tests { let default_prompt_request = ClientRequest::ThreadRealtimeStart { request_id: RequestId::Integer(9), params: v2::ThreadRealtimeStartParams { + auto_handoff_output_as_context: None, thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, @@ -3064,6 +3067,7 @@ mod tests { "id": 9, "params": { "threadId": "thr_123", + "autoHandoffOutputAsContext": null, "model": null, "outputModality": "audio", "realtimeSessionId": null, @@ -3078,6 +3082,7 @@ mod tests { let null_prompt_request = ClientRequest::ThreadRealtimeStart { request_id: RequestId::Integer(9), params: v2::ThreadRealtimeStartParams { + auto_handoff_output_as_context: None, thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, @@ -3094,6 +3099,7 @@ mod tests { "id": 9, "params": { "threadId": "thr_123", + "autoHandoffOutputAsContext": null, "model": null, "outputModality": "audio", "prompt": null, @@ -3274,6 +3280,7 @@ mod tests { let request = ClientRequest::ThreadRealtimeStart { request_id: RequestId::Integer(1), params: v2::ThreadRealtimeStartParams { + auto_handoff_output_as_context: None, thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, diff --git a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs index 2925b4f1f2f0..b2d2d8be476b 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs @@ -65,6 +65,10 @@ impl From for CoreRealtimeAudioFrame { #[ts(export_to = "v2/")] pub struct ThreadRealtimeStartParams { pub thread_id: String, + /// Sends automatic backend Codex output as silent realtime context instead of speakable + /// handoff output. + #[ts(optional = nullable)] + pub auto_handoff_output_as_context: Option, /// Overrides the configured realtime model for this session only. #[ts(optional = nullable)] pub model: Option, diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 939235c6ffd9..b305e40634ff 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -165,7 +165,7 @@ Example with notification opt-out: - `thread/inject_items` — append raw Responses API items to a loaded thread’s model-visible history without starting a user turn; returns `{}` on success. - `turn/steer` — add user input to an already in-flight regular turn without starting a new turn; returns the active `turnId` that accepted the input. `clientUserMessageId` is optional; when supplied, the corresponding `userMessage` item echoes it as `clientId`. Review and manual compaction turns reject `turn/steer`. - `turn/interrupt` — request cancellation of an in-flight turn by `(thread_id, turn_id)`; success is an empty `{}` response and the turn finishes with `status: "interrupted"`. -- `thread/realtime/start` — start a thread-scoped realtime session (experimental); pass `outputModality: "text"` or `outputModality: "audio"` to choose model output, and optionally pass `model` and `version` to override configured realtime selection for this session only. Backend Codex text is sent to v2 realtime sessions as silent developer context; use `thread/realtime/appendHandoff` when the app wants realtime to speak a backend update. Returns `{}` and streams `thread/realtime/*` notifications. Omit `transport` for the websocket transport, or pass `{ "type": "webrtc", "sdp": "..." }` to create a WebRTC session from a browser-generated SDP offer; the remote answer SDP is emitted as `thread/realtime/sdp`. +- `thread/realtime/start` — start a thread-scoped realtime session (experimental); pass `outputModality: "text"` or `outputModality: "audio"` to choose model output, and optionally pass `model` and `version` to override configured realtime selection for this session only. By default, automatic backend Codex text follows the legacy speakable handoff path. Pass `autoHandoffOutputAsContext: true` to send automatic backend Codex text as silent developer context, then use `thread/realtime/appendHandoff` when the app wants realtime to speak a backend update. Returns `{}` and streams `thread/realtime/*` notifications. Omit `transport` for the websocket transport, or pass `{ "type": "webrtc", "sdp": "..." }` to create a WebRTC session from a browser-generated SDP offer; the remote answer SDP is emitted as `thread/realtime/sdp`. - `thread/realtime/appendAudio` — append an input audio chunk to the active realtime session (experimental); returns `{}`. - `thread/realtime/appendText` — append text input to the active realtime session (experimental); returns `{}`. - `thread/realtime/appendHandoff` — append assistant output to the active realtime session (experimental); returns `{}`. For v1 sessions this sends `conversation.handoff.append`. @@ -853,11 +853,12 @@ Omit `prompt` to use Codex's default realtime backend prompt. Send `prompt: null `prompt: ""` when the session should start without that default backend prompt. Clients may also pass `model` and `version` on `thread/realtime/start` to select a different realtime session configuration without changing thread or user config. -Backend Codex output is injected into v2 realtime sessions as silent developer -context so the realtime model stays coherent without automatically speaking -backend preambles, progress, or final assistant text. Call -`thread/realtime/appendHandoff` when the app decides a realtime update should be -spoken. +Pass `autoHandoffOutputAsContext: true` to inject automatic backend Codex output +as silent developer context so the realtime model stays coherent without +automatically speaking backend preambles, progress, or final assistant text. Omit +the field, or pass `false`, to preserve the legacy speakable handoff behavior. +Call `thread/realtime/appendHandoff` when the app decides a realtime update +should be spoken. ```javascript await pc.setRemoteDescription({ diff --git a/codex-rs/app-server/src/request_processors/turn_processor.rs b/codex-rs/app-server/src/request_processors/turn_processor.rs index 713f535e963d..dd353fa751fb 100644 --- a/codex-rs/app-server/src/request_processors/turn_processor.rs +++ b/codex-rs/app-server/src/request_processors/turn_processor.rs @@ -944,6 +944,9 @@ impl TurnRequestProcessor { request_id, thread.as_ref(), Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: params + .auto_handoff_output_as_context + .unwrap_or(false), model: params.model, output_modality: params.output_modality, prompt: params.prompt, diff --git a/codex-rs/app-server/tests/suite/v2/experimental_api.rs b/codex-rs/app-server/tests/suite/v2/experimental_api.rs index 03c9d455f05d..83ee660db86f 100644 --- a/codex-rs/app-server/tests/suite/v2/experimental_api.rs +++ b/codex-rs/app-server/tests/suite/v2/experimental_api.rs @@ -79,6 +79,7 @@ async fn realtime_conversation_start_requires_experimental_api_capability() -> R let request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { + auto_handoff_output_as_context: None, thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, @@ -187,6 +188,7 @@ async fn realtime_webrtc_start_requires_experimental_api_capability() -> Result< let request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { + auto_handoff_output_as_context: None, thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, diff --git a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs index a088dff3bbec..afad74ada0f3 100644 --- a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs +++ b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs @@ -310,12 +310,35 @@ impl RealtimeE2eHarness { } async fn start_webrtc_realtime(&mut self, offer_sdp: &str) -> Result { + self.start_webrtc_realtime_with_auto_handoff_output_as_context( + offer_sdp, /*auto_handoff_output_as_context*/ None, + ) + .await + } + + async fn start_webrtc_realtime_with_context_output( + &mut self, + offer_sdp: &str, + ) -> Result { + self.start_webrtc_realtime_with_auto_handoff_output_as_context( + offer_sdp, + /*auto_handoff_output_as_context*/ Some(true), + ) + .await + } + + async fn start_webrtc_realtime_with_auto_handoff_output_as_context( + &mut self, + offer_sdp: &str, + auto_handoff_output_as_context: Option, + ) -> Result { // Starts realtime through the public JSON-RPC method, then waits for the same client-visible // notifications a desktop app needs: started first, SDP answer second. let start_request_id = self .mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { thread_id: self.thread_id.clone(), + auto_handoff_output_as_context, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -580,6 +603,7 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { + auto_handoff_output_as_context: None, thread_id: thread_start.thread.id.clone(), model: Some("realtime-treatment-model".to_string()), output_modality: RealtimeOutputModality::Audio, @@ -835,6 +859,7 @@ async fn realtime_text_output_modality_requests_text_output_and_final_transcript let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { + auto_handoff_output_as_context: None, thread_id: thread_start.thread.id.clone(), model: None, output_modality: RealtimeOutputModality::Text, @@ -1011,6 +1036,7 @@ async fn realtime_conversation_stop_emits_closed_notification() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { + auto_handoff_output_as_context: None, thread_id: thread_start.thread.id.clone(), model: None, output_modality: RealtimeOutputModality::Audio, @@ -1110,6 +1136,7 @@ async fn realtime_webrtc_start_emits_sdp_notification() -> Result<()> { let thread_id = thread_start.thread.id; let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { + auto_handoff_output_as_context: None, thread_id: thread_id.clone(), model: None, output_modality: RealtimeOutputModality::Audio, @@ -1284,7 +1311,64 @@ async fn webrtc_v1_start_posts_offer_returns_sdp_and_joins_sideband() -> Result< } #[tokio::test] -async fn webrtc_v1_handoff_request_delegates_and_appends_result() -> Result<()> { +async fn webrtc_v1_default_automatic_output_uses_handoff_append() -> Result<()> { + skip_if_no_network!(Ok(())); + + let mut harness = RealtimeE2eHarness::new( + RealtimeTestVersion::V1, + main_loop_responses(vec![create_final_assistant_message_sse_response( + "legacy automatic speech", + )?]), + realtime_sideband(vec![realtime_sideband_connection(vec![ + vec![session_updated("sess_v1_default_handoff")], + vec![], + vec![], + ])]), + ) + .await?; + + let started = harness.start_webrtc_realtime("v=offer\r\n").await?; + assert_eq!(started.started.version, RealtimeConversationVersion::V1); + assert_v1_session_update(&harness.sideband_outbound_request(/*request_index*/ 0).await)?; + + let turn_request_id = harness + .mcp + .send_turn_start_request(TurnStartParams { + thread_id: harness.thread_id.clone(), + input: vec![V2UserInput::Text { + text: "say the default output".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let turn_response: JSONRPCResponse = timeout( + DEFAULT_TIMEOUT, + harness + .mcp + .read_stream_until_response_message(RequestId::Integer(turn_request_id)), + ) + .await??; + let _: TurnStartResponse = to_response(turn_response)?; + let _ = harness + .read_notification::("turn/completed") + .await?; + + assert_eq!( + harness.sideband_outbound_request(/*request_index*/ 1).await, + json!({ + "type": "conversation.handoff.append", + "handoff_id": "codex", + "output_text": "legacy automatic speech", + }) + ); + + harness.shutdown().await; + Ok(()) +} + +#[tokio::test] +async fn webrtc_v1_handoff_request_delegates_context_and_manual_append_speaks() -> Result<()> { skip_if_no_network!(Ok(())); // Phase 1: script one v1 handoff request on the sideband and one delegated Responses turn. @@ -1316,11 +1400,14 @@ async fn webrtc_v1_handoff_request_delegates_and_appends_result() -> Result<()> }), ], vec![], + vec![], ])]), ) .await?; - let started = harness.start_webrtc_realtime("v=offer\r\n").await?; + let started = harness + .start_webrtc_realtime_with_context_output("v=offer\r\n") + .await?; assert_eq!(started.started.version, RealtimeConversationVersion::V1); assert_call_create_multipart( harness.call_capture.single_request(), @@ -1339,8 +1426,8 @@ async fn webrtc_v1_handoff_request_delegates_and_appends_result() -> Result<()> .await?; assert_eq!(turn_completed.thread_id, harness.thread_id); - // Phase 3: assert the delegated prompt went to Responses, then the v1 handoff append went back - // over the existing sideband connection. + // Phase 3: assert the delegated prompt went to Responses, then the automatic v1 output went + // back over the existing sideband connection as silent context. let requests = harness.main_loop_responses_requests().await?; assert_eq!(requests.len(), 1); assert!( @@ -1351,13 +1438,32 @@ async fn webrtc_v1_handoff_request_delegates_and_appends_result() -> Result<()> "delegated Responses request should contain realtime delegation envelope: {}", requests[0] ); - let handoff_append = harness.sideband_outbound_request(/*request_index*/ 1).await; + let context_update = harness.sideband_outbound_request(/*request_index*/ 1).await; assert_eq!( - handoff_append, + context_update, + json!({ + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "developer", + "content": [{ + "type": "input_text", + "text": "delegated from v1" + }] + } + }) + ); + + harness + .append_handoff(harness.thread_id.clone(), "manual spoken v1 update") + .await?; + let spoken_append = harness.sideband_outbound_request(/*request_index*/ 2).await; + assert_eq!( + spoken_append, json!({ "type": "conversation.handoff.append", - "handoff_id": "handoff_v1", - "output_text": "\"Agent Final Message\":\n\ndelegated from v1", + "handoff_id": "codex", + "output_text": "manual spoken v1 update", }) ); @@ -1383,7 +1489,9 @@ async fn realtime_automatic_standalone_output_is_context_and_append_speaks() -> ) .await?; - let started = harness.start_webrtc_realtime("v=offer\r\n").await?; + let started = harness + .start_webrtc_realtime_with_context_output("v=offer\r\n") + .await?; assert_eq!(started.started.version, RealtimeConversationVersion::V2); assert_eq!( harness.sideband_outbound_request(/*request_index*/ 0).await["type"].as_str(), @@ -1464,7 +1572,9 @@ async fn realtime_automatic_handoff_output_is_context_and_append_speaks() -> Res ) .await?; - let started = harness.start_webrtc_realtime("v=offer\r\n").await?; + let started = harness + .start_webrtc_realtime_with_context_output("v=offer\r\n") + .await?; assert_eq!(started.started.version, RealtimeConversationVersion::V2); assert_eq!( harness.sideband_outbound_request(/*request_index*/ 0).await["type"].as_str(), @@ -1545,7 +1655,9 @@ async fn webrtc_v2_assistant_output_without_handoff_reaches_realtime_context() - ) .await?; - let started = harness.start_webrtc_realtime("v=offer\r\n").await?; + let started = harness + .start_webrtc_realtime_with_context_output("v=offer\r\n") + .await?; assert_eq!(started.started.version, RealtimeConversationVersion::V2); let request_id = harness @@ -2254,6 +2366,7 @@ async fn realtime_webrtc_start_surfaces_backend_error() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { + auto_handoff_output_as_context: None, thread_id: thread_start.thread.id, model: None, output_modality: RealtimeOutputModality::Audio, @@ -2315,6 +2428,7 @@ async fn realtime_conversation_requires_feature_flag() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { + auto_handoff_output_as_context: None, thread_id: thread_start.thread.id.clone(), model: None, output_modality: RealtimeOutputModality::Audio, diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs index 3b9bcbab5ecc..7d3585bd1e0e 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs @@ -296,10 +296,7 @@ impl RealtimeWebsocketWriter { &self, text: String, ) -> Result<(), ApiError> { - let Some(message) = conversation_context_item_create_message(self.event_parser, text) - else { - return Ok(()); - }; + let message = conversation_context_item_create_message(self.event_parser, text); self.send_json(&message).await } diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_common.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_common.rs index 8e98df6f437d..4cebb27661a7 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_common.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_common.rs @@ -1,3 +1,4 @@ +use crate::endpoint::realtime_websocket::methods_v1::conversation_context_item_create_message as v1_conversation_context_item_create_message; use crate::endpoint::realtime_websocket::methods_v1::conversation_handoff_append_message as v1_conversation_handoff_append_message; use crate::endpoint::realtime_websocket::methods_v1::conversation_item_create_message as v1_conversation_item_create_message; use crate::endpoint::realtime_websocket::methods_v1::session_update_session as v1_session_update_session; @@ -44,10 +45,10 @@ pub(super) fn conversation_item_create_message( pub(super) fn conversation_context_item_create_message( event_parser: RealtimeEventParser, text: String, -) -> Option { +) -> RealtimeOutboundMessage { match event_parser { - RealtimeEventParser::V1 => None, - RealtimeEventParser::RealtimeV2 => Some(v2_conversation_context_item_create_message(text)), + RealtimeEventParser::V1 => v1_conversation_context_item_create_message(text), + RealtimeEventParser::RealtimeV2 => v2_conversation_context_item_create_message(text), } } diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v1.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v1.rs index 0f1a26908232..d1dccaa4d735 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v1.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v1.rs @@ -16,10 +16,21 @@ use crate::endpoint::realtime_websocket::protocol::SessionType; use crate::endpoint::realtime_websocket::protocol::SessionUpdateSession; pub(super) fn conversation_item_create_message(text: String) -> RealtimeOutboundMessage { + conversation_message_item_create_message(ConversationRole::User, text) +} + +pub(super) fn conversation_context_item_create_message(text: String) -> RealtimeOutboundMessage { + conversation_message_item_create_message(ConversationRole::Developer, text) +} + +fn conversation_message_item_create_message( + role: ConversationRole, + text: String, +) -> RealtimeOutboundMessage { RealtimeOutboundMessage::ConversationItemCreate { item: ConversationItemPayload::Message(ConversationMessageItem { r#type: ConversationItemType::Message, - role: ConversationRole::User, + role, content: vec![ConversationItemContent { r#type: ConversationContentType::InputText, text, diff --git a/codex-rs/core/src/realtime_conversation.rs b/codex-rs/core/src/realtime_conversation.rs index 8ab1c547bde3..e3d2e082682f 100644 --- a/codex-rs/core/src/realtime_conversation.rs +++ b/codex-rs/core/src/realtime_conversation.rs @@ -102,12 +102,14 @@ enum RealtimeSessionKind { struct RealtimeHandoffState { output_tx: Sender, active_handoff: Arc>>, + auto_handoff_output_as_context: bool, session_kind: RealtimeSessionKind, } #[derive(Debug, PartialEq, Eq)] enum HandoffOutput { ContextUpdate { + as_context: bool, handoff_id: Option, output_text: String, }, @@ -206,10 +208,15 @@ struct RealtimeInputChannels { } impl RealtimeHandoffState { - fn new(output_tx: Sender, session_kind: RealtimeSessionKind) -> Self { + fn new( + output_tx: Sender, + auto_handoff_output_as_context: bool, + session_kind: RealtimeSessionKind, + ) -> Self { Self { output_tx, active_handoff: Arc::new(Mutex::new(None)), + auto_handoff_output_as_context, session_kind, } } @@ -229,6 +236,7 @@ struct ConversationState { struct RealtimeStart { api_provider: ApiProvider, extra_headers: Option, + auto_handoff_output_as_context: bool, session_config: RealtimeSessionConfig, model_client: ModelClient, sdp: Option, @@ -281,6 +289,7 @@ impl RealtimeConversationManager { let RealtimeStart { api_provider, extra_headers, + auto_handoff_output_as_context, session_config, model_client, sdp, @@ -301,7 +310,11 @@ impl RealtimeConversationManager { async_channel::bounded::(OUTPUT_EVENTS_QUEUE_CAPACITY); let realtime_active = Arc::new(AtomicBool::new(true)); - let handoff = RealtimeHandoffState::new(handoff_output_tx, session_kind); + let handoff = RealtimeHandoffState::new( + handoff_output_tx, + auto_handoff_output_as_context, + session_kind, + ); let input_channels = RealtimeInputChannels { user_text_rx, handoff_output_rx, @@ -469,6 +482,7 @@ impl RealtimeConversationManager { handoff .output_tx .send(HandoffOutput::ContextUpdate { + as_context: handoff.auto_handoff_output_as_context, handoff_id, output_text: realtime_backend_output(output_text, handoff.session_kind), }) @@ -606,6 +620,7 @@ pub(crate) async fn handle_start( struct PreparedRealtimeConversationStart { api_provider: ApiProvider, extra_headers: Option, + auto_handoff_output_as_context: bool, requested_realtime_session_id: Option, version: RealtimeWsVersion, session_config: RealtimeSessionConfig, @@ -663,6 +678,7 @@ async fn prepare_realtime_start( Ok(PreparedRealtimeConversationStart { api_provider, extra_headers, + auto_handoff_output_as_context: params.auto_handoff_output_as_context, requested_realtime_session_id, version, session_config, @@ -784,6 +800,7 @@ async fn handle_start_inner( let PreparedRealtimeConversationStart { api_provider, extra_headers, + auto_handoff_output_as_context, requested_realtime_session_id, version, session_config, @@ -797,6 +814,7 @@ async fn handle_start_inner( let start = RealtimeStart { api_provider, extra_headers, + auto_handoff_output_as_context, session_config, model_client: sess.services.model_client.clone(), sdp, @@ -1215,10 +1233,15 @@ async fn handle_handoff_output( let result = match event_parser { RealtimeEventParser::V1 => match handoff_output { HandoffOutput::ContextUpdate { + as_context, handoff_id, output_text, } => { - if let Some(handoff_id) = handoff_id { + if as_context { + writer + .send_conversation_context_item_create(output_text) + .await + } else if let Some(handoff_id) = handoff_id { writer .send_conversation_function_call_output(handoff_id, output_text) .await @@ -1244,12 +1267,21 @@ async fn handle_handoff_output( }, RealtimeEventParser::RealtimeV2 => match handoff_output { HandoffOutput::ContextUpdate { + as_context, handoff_id: _, output_text, } => { - writer - .send_conversation_context_item_create(output_text) - .await + if as_context { + writer + .send_conversation_context_item_create(output_text) + .await + } else if let Err(err) = writer.send_conversation_item_create(output_text).await { + Err(err) + } else { + return response_create_queue + .request_create(writer, events_tx, "handoff output") + .await; + } } HandoffOutput::HandoffComplete { handoff_id } => { writer diff --git a/codex-rs/core/src/realtime_conversation_tests.rs b/codex-rs/core/src/realtime_conversation_tests.rs index b67205ef8fc6..3517f9aed9fc 100644 --- a/codex-rs/core/src/realtime_conversation_tests.rs +++ b/codex-rs/core/src/realtime_conversation_tests.rs @@ -128,7 +128,11 @@ fn wraps_realtime_delegation_input_with_xml_escaping_without_transcript() { #[tokio::test] async fn clears_active_handoff_explicitly() { let (tx, _rx) = bounded(1); - let state = RealtimeHandoffState::new(tx, RealtimeSessionKind::V1); + let state = RealtimeHandoffState::new( + tx, + /*auto_handoff_output_as_context*/ false, + RealtimeSessionKind::V1, + ); *state.active_handoff.lock().await = Some("handoff_1".to_string()); assert_eq!( diff --git a/codex-rs/core/tests/suite/compact_remote.rs b/codex-rs/core/tests/suite/compact_remote.rs index eee445ea9fa1..dfec70fc92b3 100644 --- a/codex-rs/core/tests/suite/compact_remote.rs +++ b/codex-rs/core/tests/suite/compact_remote.rs @@ -194,6 +194,7 @@ async fn start_remote_realtime_server() -> responses::WebSocketTestServer { async fn start_realtime_conversation(codex: &codex_core::CodexThread) -> Result<()> { codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), diff --git a/codex-rs/core/tests/suite/realtime_conversation.rs b/codex-rs/core/tests/suite/realtime_conversation.rs index 25f0516b0d91..70a19d7e76c4 100644 --- a/codex-rs/core/tests/suite/realtime_conversation.rs +++ b/codex-rs/core/tests/suite/realtime_conversation.rs @@ -282,6 +282,7 @@ async fn conversation_start_audio_text_close_round_trip() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -422,6 +423,7 @@ async fn conversation_start_defaults_to_v2_and_gpt_realtime_1_5() -> Result<()> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -512,6 +514,7 @@ async fn conversation_webrtc_start_posts_generated_session() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: Some("session-override-model".to_string()), output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -684,6 +687,7 @@ async fn conversation_webrtc_close_while_sideband_connecting_drops_pending_join( test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -770,6 +774,7 @@ async fn conversation_webrtc_sideband_connect_failure_closes_with_error() -> Res test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -857,6 +862,7 @@ async fn conversation_start_uses_openai_env_key_fallback_with_chatgpt_auth() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -925,6 +931,7 @@ async fn conversation_transport_close_emits_closed_event() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1017,6 +1024,7 @@ async fn conversation_start_preflight_failure_emits_realtime_error_only() -> Res test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1063,6 +1071,7 @@ async fn conversation_start_connect_failure_emits_realtime_error_only() -> Resul test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1156,6 +1165,7 @@ async fn conversation_second_start_replaces_runtime() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("old".to_string())), @@ -1181,6 +1191,7 @@ async fn conversation_second_start_replaces_runtime() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("new".to_string())), @@ -1277,6 +1288,7 @@ async fn conversation_uses_experimental_realtime_ws_base_url_override() -> Resul test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1341,6 +1353,7 @@ async fn conversation_uses_default_realtime_backend_prompt() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: None, @@ -1413,6 +1426,7 @@ async fn conversation_uses_empty_instructions_for_null_or_empty_prompt() -> Resu ] { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt, @@ -1478,6 +1492,7 @@ async fn conversation_uses_explicit_start_voice() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1535,6 +1550,7 @@ async fn conversation_uses_configured_realtime_voice() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1580,6 +1596,7 @@ async fn conversation_rejects_voice_for_wrong_realtime_version() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1626,6 +1643,7 @@ async fn conversation_uses_experimental_realtime_ws_backend_prompt_override() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("prompt from op".to_string())), @@ -1698,6 +1716,7 @@ async fn conversation_uses_experimental_realtime_ws_startup_context_override() - test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("prompt from op".to_string())), @@ -1764,6 +1783,7 @@ async fn conversation_disables_realtime_startup_context_with_empty_override() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("prompt from op".to_string())), @@ -1823,6 +1843,7 @@ async fn conversation_start_injects_startup_context_from_thread_history() -> Res test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1934,6 +1955,7 @@ async fn conversation_startup_context_current_thread_selects_many_turns_by_budge codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2041,6 +2063,7 @@ async fn conversation_startup_context_falls_back_to_workspace_map() -> Result<() test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2100,6 +2123,7 @@ async fn conversation_startup_context_is_truncated_and_sent_once_per_start() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2179,6 +2203,7 @@ async fn conversation_user_text_turn_is_not_sent_to_realtime() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2275,6 +2300,7 @@ async fn realtime_v2_noop_tool_call_returns_empty_function_output_without_respon test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2373,6 +2399,7 @@ async fn conversation_mirrors_assistant_message_text_to_realtime_handoff() -> Re test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2509,6 +2536,7 @@ async fn conversation_handoff_persists_across_item_done_until_turn_complete() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2660,6 +2688,7 @@ async fn inbound_handoff_request_starts_turn() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2761,6 +2790,7 @@ async fn inbound_handoff_request_uses_active_transcript() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2863,6 +2893,7 @@ async fn inbound_handoff_request_sends_transcript_delta_after_each_handoff() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2963,6 +2994,7 @@ async fn inbound_conversation_item_does_not_start_turn_and_still_forwards_audio( test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -3085,6 +3117,7 @@ async fn delegated_turn_user_role_echo_does_not_redelegate_and_still_forwards_au test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -3237,6 +3270,7 @@ async fn inbound_handoff_request_does_not_block_realtime_event_forwarding() -> R test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -3373,6 +3407,7 @@ async fn inbound_handoff_request_steers_active_turn() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -3525,6 +3560,7 @@ async fn inbound_handoff_request_starts_turn_and_does_not_block_realtime_audio() test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + auto_handoff_output_as_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 192dbdd2c4a7..743b123be5e9 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -183,6 +183,9 @@ pub struct McpServerRefreshConfig { #[derive(Debug, Clone, PartialEq)] pub struct ConversationStartParams { + /// Sends automatic backend Codex output as silent realtime context instead of speakable + /// handoff output. + pub auto_handoff_output_as_context: bool, /// Overrides the configured realtime model for this session only. pub model: Option, /// Selects whether the realtime session should produce text or audio output. From 58474a5868bc36efed6f7352d3fcffd7afafb4dc Mon Sep 17 00:00:00 2001 From: Guinness Chen Date: Sun, 14 Jun 2026 10:46:25 -0700 Subject: [PATCH 05/11] Refine realtime speech context API --- .../src/protocol/common.rs | 63 +++-- .../src/protocol/v2/realtime.rs | 30 +- codex-rs/app-server/README.md | 14 +- codex-rs/app-server/src/message_processor.rs | 9 +- codex-rs/app-server/src/request_processors.rs | 9 +- .../src/request_processors/turn_processor.rs | 59 +++- .../tests/common/test_app_server.rs | 21 +- .../tests/suite/v2/experimental_api.rs | 4 +- .../tests/suite/v2/realtime_conversation.rs | 131 +++++++-- codex-rs/core/src/realtime_conversation.rs | 259 +++++++++++------- .../core/src/realtime_conversation_tests.rs | 2 +- codex-rs/core/src/session/handlers.rs | 12 +- codex-rs/core/tests/suite/compact_remote.rs | 2 +- .../core/tests/suite/realtime_conversation.rs | 76 ++--- codex-rs/protocol/src/protocol.rs | 24 +- 15 files changed, 478 insertions(+), 237 deletions(-) diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index e457fb267a57..8b9d6b353f24 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -819,11 +819,17 @@ client_request_definitions! { serialization: thread_id(params.thread_id), response: v2::ThreadRealtimeAppendTextResponse, }, - #[experimental("thread/realtime/appendHandoff")] - ThreadRealtimeAppendHandoff => "thread/realtime/appendHandoff" { - params: v2::ThreadRealtimeAppendHandoffParams, + #[experimental("thread/realtime/appendSilentContext")] + ThreadRealtimeAppendSilentContext => "thread/realtime/appendSilentContext" { + params: v2::ThreadRealtimeAppendSilentContextParams, serialization: thread_id(params.thread_id), - response: v2::ThreadRealtimeAppendHandoffResponse, + response: v2::ThreadRealtimeAppendSilentContextResponse, + }, + #[experimental("thread/realtime/appendSpeech")] + ThreadRealtimeAppendSpeech => "thread/realtime/appendSpeech" { + params: v2::ThreadRealtimeAppendSpeechParams, + serialization: thread_id(params.thread_id), + response: v2::ThreadRealtimeAppendSpeechResponse, }, #[experimental("thread/realtime/stop")] ThreadRealtimeStop => "thread/realtime/stop" { @@ -3020,7 +3026,7 @@ mod tests { request_id: RequestId::Integer(9), params: v2::ThreadRealtimeStartParams { architecture: Some(RealtimeConversationArchitecture::Avas), - auto_handoff_output_as_context: None, + codex_responses_as_silent_context: None, thread_id: "thr_123".to_string(), model: Some("realtime-treatment-model".to_string()), output_modality: RealtimeOutputModality::Audio, @@ -3038,7 +3044,7 @@ mod tests { "params": { "architecture": "avas", "threadId": "thr_123", - "autoHandoffOutputAsContext": null, + "codexResponsesAsSilentContext": null, "model": "realtime-treatment-model", "outputModality": "audio", "prompt": "You are on a call", @@ -3059,7 +3065,7 @@ mod tests { request_id: RequestId::Integer(9), params: v2::ThreadRealtimeStartParams { architecture: None, - auto_handoff_output_as_context: None, + codex_responses_as_silent_context: None, thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, @@ -3077,7 +3083,7 @@ mod tests { "params": { "architecture": null, "threadId": "thr_123", - "autoHandoffOutputAsContext": null, + "codexResponsesAsSilentContext": null, "model": null, "outputModality": "audio", "realtimeSessionId": null, @@ -3093,7 +3099,7 @@ mod tests { request_id: RequestId::Integer(9), params: v2::ThreadRealtimeStartParams { architecture: None, - auto_handoff_output_as_context: None, + codex_responses_as_silent_context: None, thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, @@ -3111,7 +3117,7 @@ mod tests { "params": { "architecture": null, "threadId": "thr_123", - "autoHandoffOutputAsContext": null, + "codexResponsesAsSilentContext": null, "model": null, "outputModality": "audio", "prompt": null, @@ -3161,21 +3167,44 @@ mod tests { } #[test] - fn serialize_thread_realtime_append_handoff() -> Result<()> { - let request = ClientRequest::ThreadRealtimeAppendHandoff { + fn serialize_thread_realtime_append_silent_context() -> Result<()> { + let request = ClientRequest::ThreadRealtimeAppendSilentContext { + request_id: RequestId::Integer(10), + params: v2::ThreadRealtimeAppendSilentContextParams { + thread_id: "thr_123".to_string(), + text: "Quiet context".to_string(), + }, + }; + assert_eq!( + json!({ + "method": "thread/realtime/appendSilentContext", + "id": 10, + "params": { + "threadId": "thr_123", + "text": "Quiet context" + } + }), + serde_json::to_value(&request)?, + ); + Ok(()) + } + + #[test] + fn serialize_thread_realtime_append_speech() -> Result<()> { + let request = ClientRequest::ThreadRealtimeAppendSpeech { request_id: RequestId::Integer(10), - params: v2::ThreadRealtimeAppendHandoffParams { + params: v2::ThreadRealtimeAppendSpeechParams { thread_id: "thr_123".to_string(), - output_text: "Short voice update".to_string(), + text: "Short voice update".to_string(), }, }; assert_eq!( json!({ - "method": "thread/realtime/appendHandoff", + "method": "thread/realtime/appendSpeech", "id": 10, "params": { "threadId": "thr_123", - "outputText": "Short voice update" + "text": "Short voice update" } }), serde_json::to_value(&request)?, @@ -3293,7 +3322,7 @@ mod tests { request_id: RequestId::Integer(1), params: v2::ThreadRealtimeStartParams { architecture: None, - auto_handoff_output_as_context: None, + codex_responses_as_silent_context: None, thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, diff --git a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs index d63e96340f81..770fb35d01f9 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs @@ -70,10 +70,9 @@ pub struct ThreadRealtimeStartParams { /// Overrides the configured realtime architecture for this session only. #[ts(optional = nullable)] pub architecture: Option, - /// Sends automatic backend Codex output as silent realtime context instead of speakable - /// handoff output. + /// Inserts automatic Codex responses as silent realtime context instead of speakable output. #[ts(optional = nullable)] - pub auto_handoff_output_as_context: Option, + pub codex_responses_as_silent_context: Option, /// Overrides the configured realtime model for this session only. #[ts(optional = nullable)] pub model: Option, @@ -150,20 +149,35 @@ pub struct ThreadRealtimeAppendTextParams { #[ts(export_to = "v2/")] pub struct ThreadRealtimeAppendTextResponse {} -/// EXPERIMENTAL - append assistant output to thread realtime. +/// EXPERIMENTAL - append silent context to thread realtime. #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] -pub struct ThreadRealtimeAppendHandoffParams { +pub struct ThreadRealtimeAppendSilentContextParams { pub thread_id: String, - pub output_text: String, + pub text: String, +} + +/// EXPERIMENTAL - response for appending realtime silent context. +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadRealtimeAppendSilentContextResponse {} + +/// EXPERIMENTAL - append speakable text to thread realtime. +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadRealtimeAppendSpeechParams { + pub thread_id: String, + pub text: String, } -/// EXPERIMENTAL - response for appending realtime assistant output. +/// EXPERIMENTAL - response for appending realtime speech. #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] -pub struct ThreadRealtimeAppendHandoffResponse {} +pub struct ThreadRealtimeAppendSpeechResponse {} /// EXPERIMENTAL - stop thread realtime. #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 8996bc1e0d59..1b8b9393d7fd 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -165,10 +165,11 @@ Example with notification opt-out: - `thread/inject_items` — append raw Responses API items to a loaded thread’s model-visible history without starting a user turn; returns `{}` on success. - `turn/steer` — add user input to an already in-flight regular turn without starting a new turn; returns the active `turnId` that accepted the input. `clientUserMessageId` is optional; when supplied, the corresponding `userMessage` item echoes it as `clientId`. Review and manual compaction turns reject `turn/steer`. - `turn/interrupt` — request cancellation of an in-flight turn by `(thread_id, turn_id)`; success is an empty `{}` response and the turn finishes with `status: "interrupted"`. -- `thread/realtime/start` — start a thread-scoped realtime session (experimental); pass `outputModality: "text"` or `outputModality: "audio"` to choose model output, and optionally pass `model` and `version` to override configured realtime selection for this session only. By default, automatic backend Codex text follows the legacy speakable handoff path. Pass `autoHandoffOutputAsContext: true` to send automatic backend Codex text as silent developer context, then use `thread/realtime/appendHandoff` when the app wants realtime to speak a backend update. Returns `{}` and streams `thread/realtime/*` notifications. Omit `transport` for the websocket transport, or pass `{ "type": "webrtc", "sdp": "..." }` to create a WebRTC session from a browser-generated SDP offer; the remote answer SDP is emitted as `thread/realtime/sdp`. +- `thread/realtime/start` — start a thread-scoped realtime session (experimental); pass `outputModality: "text"` or `outputModality: "audio"` to choose model output, and optionally pass `model` and `version` to override configured realtime selection for this session only. By default, automatic Codex text follows the legacy speakable handoff path. Pass `codexResponsesAsSilentContext: true` to insert automatic Codex responses as silent developer context instead. Returns `{}` and streams `thread/realtime/*` notifications. Omit `transport` for the websocket transport, or pass `{ "type": "webrtc", "sdp": "..." }` to create a WebRTC session from a browser-generated SDP offer; the remote answer SDP is emitted as `thread/realtime/sdp`. - `thread/realtime/appendAudio` — append an input audio chunk to the active realtime session (experimental); returns `{}`. - `thread/realtime/appendText` — append text input to the active realtime session with a required `role` of `user` or `developer` (experimental); returns `{}`. Older clients that omit `role` default to `user`. -- `thread/realtime/appendHandoff` — append assistant output to the active realtime session (experimental); returns `{}`. For v1 sessions this sends `conversation.handoff.append`. +- `thread/realtime/appendSilentContext` — append silent developer context to the active realtime session (experimental); returns `{}` and does not request a realtime response. +- `thread/realtime/appendSpeech` — append text that the realtime model should speak to the user (experimental); returns `{}`. For v1 sessions this sends `conversation.handoff.append`. - `thread/realtime/stop` — stop the active realtime session for the thread (experimental); returns `{}`. - `review/start` — kick off Codex’s automated reviewer for a thread; responds like `turn/start` and emits `item/started`/`item/completed` notifications with `enteredReviewMode` and `exitedReviewMode` items, plus a final assistant `agentMessage` containing the review. - `command/exec` — run a single command under the server sandbox without starting a thread/turn (handy for utilities and validation). @@ -853,12 +854,13 @@ Omit `prompt` to use Codex's default realtime backend prompt. Send `prompt: null `prompt: ""` when the session should start without that default backend prompt. Clients may also pass `model` and `version` on `thread/realtime/start` to select a different realtime session configuration without changing thread or user config. -Pass `autoHandoffOutputAsContext: true` to inject automatic backend Codex output -as silent developer context so the realtime model stays coherent without +Pass `codexResponsesAsSilentContext: true` to inject automatic Codex responses as +silent developer context so the realtime model stays coherent without automatically speaking backend preambles, progress, or final assistant text. Omit the field, or pass `false`, to preserve the legacy speakable handoff behavior. -Call `thread/realtime/appendHandoff` when the app decides a realtime update -should be spoken. +Call `thread/realtime/appendSilentContext` to add app-provided silent context, or +`thread/realtime/appendSpeech` when the app decides a realtime update should be +spoken. ```javascript await pc.setRemoteDescription({ diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index b295b2d900d9..6c3be0e78d93 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -1310,9 +1310,14 @@ impl MessageProcessor { .thread_realtime_append_text(&request_id, params) .await } - ClientRequest::ThreadRealtimeAppendHandoff { params, .. } => { + ClientRequest::ThreadRealtimeAppendSilentContext { params, .. } => { self.turn_processor - .thread_realtime_append_handoff(&request_id, params) + .thread_realtime_append_silent_context(&request_id, params) + .await + } + ClientRequest::ThreadRealtimeAppendSpeech { params, .. } => { + self.turn_processor + .thread_realtime_append_speech(&request_id, params) .await } ClientRequest::ThreadRealtimeStop { params, .. } => { diff --git a/codex-rs/app-server/src/request_processors.rs b/codex-rs/app-server/src/request_processors.rs index 4899249003d6..f1e41862a376 100644 --- a/codex-rs/app-server/src/request_processors.rs +++ b/codex-rs/app-server/src/request_processors.rs @@ -219,8 +219,10 @@ use codex_app_server_protocol::ThreadReadParams; use codex_app_server_protocol::ThreadReadResponse; use codex_app_server_protocol::ThreadRealtimeAppendAudioParams; use codex_app_server_protocol::ThreadRealtimeAppendAudioResponse; -use codex_app_server_protocol::ThreadRealtimeAppendHandoffParams; -use codex_app_server_protocol::ThreadRealtimeAppendHandoffResponse; +use codex_app_server_protocol::ThreadRealtimeAppendSilentContextParams; +use codex_app_server_protocol::ThreadRealtimeAppendSilentContextResponse; +use codex_app_server_protocol::ThreadRealtimeAppendSpeechParams; +use codex_app_server_protocol::ThreadRealtimeAppendSpeechResponse; use codex_app_server_protocol::ThreadRealtimeAppendTextParams; use codex_app_server_protocol::ThreadRealtimeAppendTextResponse; use codex_app_server_protocol::ThreadRealtimeListVoicesResponse; @@ -392,7 +394,8 @@ use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::permissions::FileSystemSandboxPolicy; use codex_protocol::protocol::AgentStatus; use codex_protocol::protocol::ConversationAudioParams; -use codex_protocol::protocol::ConversationHandoffParams; +use codex_protocol::protocol::ConversationSilentContextParams; +use codex_protocol::protocol::ConversationSpeechParams; use codex_protocol::protocol::ConversationStartParams; use codex_protocol::protocol::ConversationStartTransport; use codex_protocol::protocol::ConversationTextParams; diff --git a/codex-rs/app-server/src/request_processors/turn_processor.rs b/codex-rs/app-server/src/request_processors/turn_processor.rs index a7980b31ab50..f60b4246ca88 100644 --- a/codex-rs/app-server/src/request_processors/turn_processor.rs +++ b/codex-rs/app-server/src/request_processors/turn_processor.rs @@ -181,12 +181,22 @@ impl TurnRequestProcessor { .map(|response| response.map(Into::into)) } - pub(crate) async fn thread_realtime_append_handoff( + pub(crate) async fn thread_realtime_append_silent_context( &self, request_id: &ConnectionRequestId, - params: ThreadRealtimeAppendHandoffParams, + params: ThreadRealtimeAppendSilentContextParams, ) -> Result, JSONRPCErrorError> { - self.thread_realtime_append_handoff_inner(request_id, params) + self.thread_realtime_append_silent_context_inner(request_id, params) + .await + .map(|response| response.map(Into::into)) + } + + pub(crate) async fn thread_realtime_append_speech( + &self, + request_id: &ConnectionRequestId, + params: ThreadRealtimeAppendSpeechParams, + ) -> Result, JSONRPCErrorError> { + self.thread_realtime_append_speech_inner(request_id, params) .await .map(|response| response.map(Into::into)) } @@ -945,8 +955,8 @@ impl TurnRequestProcessor { thread.as_ref(), Op::RealtimeConversationStart(ConversationStartParams { architecture: params.architecture, - auto_handoff_output_as_context: params - .auto_handoff_output_as_context + codex_responses_as_silent_context: params + .codex_responses_as_silent_context .unwrap_or(false), model: params.model, output_modality: params.output_modality, @@ -1024,11 +1034,11 @@ impl TurnRequestProcessor { Ok(Some(ThreadRealtimeAppendTextResponse::default())) } - async fn thread_realtime_append_handoff_inner( + async fn thread_realtime_append_silent_context_inner( &self, request_id: &ConnectionRequestId, - params: ThreadRealtimeAppendHandoffParams, - ) -> Result, JSONRPCErrorError> { + params: ThreadRealtimeAppendSilentContextParams, + ) -> Result, JSONRPCErrorError> { let Some((_, thread)) = self .prepare_realtime_conversation_thread(request_id, ¶ms.thread_id) .await? @@ -1038,17 +1048,42 @@ impl TurnRequestProcessor { self.submit_core_op( request_id, thread.as_ref(), - Op::RealtimeConversationHandoff(ConversationHandoffParams { - output_text: params.output_text, + Op::RealtimeConversationSilentContext(ConversationSilentContextParams { + text: params.text, }), ) .await .map_err(|err| { internal_error(format!( - "failed to append realtime conversation handoff output: {err}" + "failed to append realtime conversation silent context: {err}" + )) + })?; + Ok(Some(ThreadRealtimeAppendSilentContextResponse::default())) + } + + async fn thread_realtime_append_speech_inner( + &self, + request_id: &ConnectionRequestId, + params: ThreadRealtimeAppendSpeechParams, + ) -> Result, JSONRPCErrorError> { + let Some((_, thread)) = self + .prepare_realtime_conversation_thread(request_id, ¶ms.thread_id) + .await? + else { + return Ok(None); + }; + self.submit_core_op( + request_id, + thread.as_ref(), + Op::RealtimeConversationSpeech(ConversationSpeechParams { text: params.text }), + ) + .await + .map_err(|err| { + internal_error(format!( + "failed to append realtime conversation speech: {err}" )) })?; - Ok(Some(ThreadRealtimeAppendHandoffResponse::default())) + Ok(Some(ThreadRealtimeAppendSpeechResponse::default())) } async fn thread_realtime_stop_inner( diff --git a/codex-rs/app-server/tests/common/test_app_server.rs b/codex-rs/app-server/tests/common/test_app_server.rs index 85914fd9a76b..7f0926f22148 100644 --- a/codex-rs/app-server/tests/common/test_app_server.rs +++ b/codex-rs/app-server/tests/common/test_app_server.rs @@ -89,7 +89,8 @@ use codex_app_server_protocol::ThreadMemoryModeSetParams; use codex_app_server_protocol::ThreadMetadataUpdateParams; use codex_app_server_protocol::ThreadReadParams; use codex_app_server_protocol::ThreadRealtimeAppendAudioParams; -use codex_app_server_protocol::ThreadRealtimeAppendHandoffParams; +use codex_app_server_protocol::ThreadRealtimeAppendSilentContextParams; +use codex_app_server_protocol::ThreadRealtimeAppendSpeechParams; use codex_app_server_protocol::ThreadRealtimeAppendTextParams; use codex_app_server_protocol::ThreadRealtimeListVoicesParams; use codex_app_server_protocol::ThreadRealtimeStartParams; @@ -1024,13 +1025,23 @@ impl TestAppServer { .await } - /// Send a `thread/realtime/appendHandoff` JSON-RPC request (v2). - pub async fn send_thread_realtime_append_handoff_request( + /// Send a `thread/realtime/appendSilentContext` JSON-RPC request (v2). + pub async fn send_thread_realtime_append_silent_context_request( &mut self, - params: ThreadRealtimeAppendHandoffParams, + params: ThreadRealtimeAppendSilentContextParams, ) -> anyhow::Result { let params = Some(serde_json::to_value(params)?); - self.send_request("thread/realtime/appendHandoff", params) + self.send_request("thread/realtime/appendSilentContext", params) + .await + } + + /// Send a `thread/realtime/appendSpeech` JSON-RPC request (v2). + pub async fn send_thread_realtime_append_speech_request( + &mut self, + params: ThreadRealtimeAppendSpeechParams, + ) -> anyhow::Result { + let params = Some(serde_json::to_value(params)?); + self.send_request("thread/realtime/appendSpeech", params) .await } diff --git a/codex-rs/app-server/tests/suite/v2/experimental_api.rs b/codex-rs/app-server/tests/suite/v2/experimental_api.rs index c23ce789a5d9..5436cb95a166 100644 --- a/codex-rs/app-server/tests/suite/v2/experimental_api.rs +++ b/codex-rs/app-server/tests/suite/v2/experimental_api.rs @@ -80,7 +80,7 @@ async fn realtime_conversation_start_requires_experimental_api_capability() -> R let request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { architecture: None, - auto_handoff_output_as_context: None, + codex_responses_as_silent_context: None, thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, @@ -190,7 +190,7 @@ async fn realtime_webrtc_start_requires_experimental_api_capability() -> Result< let request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { architecture: None, - auto_handoff_output_as_context: None, + codex_responses_as_silent_context: None, thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, diff --git a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs index 661585c9eadb..7a621b5679a8 100644 --- a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs +++ b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs @@ -15,8 +15,10 @@ use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::ThreadRealtimeAppendAudioParams; use codex_app_server_protocol::ThreadRealtimeAppendAudioResponse; -use codex_app_server_protocol::ThreadRealtimeAppendHandoffParams; -use codex_app_server_protocol::ThreadRealtimeAppendHandoffResponse; +use codex_app_server_protocol::ThreadRealtimeAppendSilentContextParams; +use codex_app_server_protocol::ThreadRealtimeAppendSilentContextResponse; +use codex_app_server_protocol::ThreadRealtimeAppendSpeechParams; +use codex_app_server_protocol::ThreadRealtimeAppendSpeechResponse; use codex_app_server_protocol::ThreadRealtimeAppendTextParams; use codex_app_server_protocol::ThreadRealtimeAppendTextResponse; use codex_app_server_protocol::ThreadRealtimeAudioChunk; @@ -84,6 +86,8 @@ const V2_STEERING_ACKNOWLEDGEMENT: &str = "This was sent to steer the previous background agent task."; const V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT: &str = "Background agent finished. Use the preceding [BACKEND] messages as the result."; +const SILENT_CONTEXT_PREFIX: &str = + "Use the following context to inform future responses, but do not speak it to the user."; #[derive(Debug, Clone, Copy)] enum StartupContextConfig<'a> { @@ -311,8 +315,8 @@ impl RealtimeE2eHarness { } async fn start_webrtc_realtime(&mut self, offer_sdp: &str) -> Result { - self.start_webrtc_realtime_with_auto_handoff_output_as_context( - offer_sdp, /*auto_handoff_output_as_context*/ None, + self.start_webrtc_realtime_with_codex_responses_as_silent_context( + offer_sdp, /*codex_responses_as_silent_context*/ None, ) .await } @@ -321,17 +325,17 @@ impl RealtimeE2eHarness { &mut self, offer_sdp: &str, ) -> Result { - self.start_webrtc_realtime_with_auto_handoff_output_as_context( + self.start_webrtc_realtime_with_codex_responses_as_silent_context( offer_sdp, - /*auto_handoff_output_as_context*/ Some(true), + /*codex_responses_as_silent_context*/ Some(true), ) .await } - async fn start_webrtc_realtime_with_auto_handoff_output_as_context( + async fn start_webrtc_realtime_with_codex_responses_as_silent_context( &mut self, offer_sdp: &str, - auto_handoff_output_as_context: Option, + codex_responses_as_silent_context: Option, ) -> Result { // Starts realtime through the public JSON-RPC method, then waits for the same client-visible // notifications a desktop app needs: started first, SDP answer second. @@ -340,7 +344,7 @@ impl RealtimeE2eHarness { .send_thread_realtime_start_request(ThreadRealtimeStartParams { architecture: None, thread_id: self.thread_id.clone(), - auto_handoff_output_as_context, + codex_responses_as_silent_context, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -432,12 +436,32 @@ impl RealtimeE2eHarness { Ok(()) } - async fn append_handoff(&mut self, thread_id: String, output_text: &str) -> Result<()> { + async fn append_silent_context(&mut self, thread_id: String, text: &str) -> Result<()> { let request_id = self .mcp - .send_thread_realtime_append_handoff_request(ThreadRealtimeAppendHandoffParams { + .send_thread_realtime_append_silent_context_request( + ThreadRealtimeAppendSilentContextParams { + thread_id, + text: text.to_string(), + }, + ) + .await?; + let response: JSONRPCResponse = timeout( + DEFAULT_TIMEOUT, + self.mcp + .read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let _: ThreadRealtimeAppendSilentContextResponse = to_response(response)?; + Ok(()) + } + + async fn append_speech(&mut self, thread_id: String, text: &str) -> Result<()> { + let request_id = self + .mcp + .send_thread_realtime_append_speech_request(ThreadRealtimeAppendSpeechParams { thread_id, - output_text: output_text.to_string(), + text: text.to_string(), }) .await?; let response: JSONRPCResponse = timeout( @@ -446,7 +470,7 @@ impl RealtimeE2eHarness { .read_stream_until_response_message(RequestId::Integer(request_id)), ) .await??; - let _: ThreadRealtimeAppendHandoffResponse = to_response(response)?; + let _: ThreadRealtimeAppendSpeechResponse = to_response(response)?; Ok(()) } @@ -607,7 +631,7 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { architecture: None, - auto_handoff_output_as_context: None, + codex_responses_as_silent_context: None, thread_id: thread_start.thread.id.clone(), model: Some("realtime-treatment-model".to_string()), output_modality: RealtimeOutputModality::Audio, @@ -884,7 +908,7 @@ async fn realtime_text_output_modality_requests_text_output_and_final_transcript let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { architecture: None, - auto_handoff_output_as_context: None, + codex_responses_as_silent_context: None, thread_id: thread_start.thread.id.clone(), model: None, output_modality: RealtimeOutputModality::Text, @@ -1062,7 +1086,7 @@ async fn realtime_conversation_stop_emits_closed_notification() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { architecture: None, - auto_handoff_output_as_context: None, + codex_responses_as_silent_context: None, thread_id: thread_start.thread.id.clone(), model: None, output_modality: RealtimeOutputModality::Audio, @@ -1163,7 +1187,7 @@ async fn realtime_webrtc_start_emits_sdp_notification() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { architecture: None, - auto_handoff_output_as_context: None, + codex_responses_as_silent_context: None, thread_id: thread_id.clone(), model: None, output_modality: RealtimeOutputModality::Audio, @@ -1475,14 +1499,14 @@ async fn webrtc_v1_handoff_request_delegates_context_and_manual_append_speaks() "role": "developer", "content": [{ "type": "input_text", - "text": "delegated from v1" + "text": format!("{SILENT_CONTEXT_PREFIX}\n\ndelegated from v1") }] } }) ); harness - .append_handoff(harness.thread_id.clone(), "manual spoken v1 update") + .append_speech(harness.thread_id.clone(), "manual spoken v1 update") .await?; let spoken_append = harness.sideband_outbound_request(/*request_index*/ 2).await; assert_eq!( @@ -1548,7 +1572,7 @@ async fn realtime_automatic_standalone_output_is_context_and_append_speaks() -> .read_notification::("turn/completed") .await?; - assert_v2_context_update( + assert_v2_backend_context_update( &harness.sideband_outbound_request(/*request_index*/ 1).await, "automatic output", ); @@ -1565,7 +1589,7 @@ async fn realtime_automatic_standalone_output_is_context_and_append_speaks() -> ); harness - .append_handoff(harness.thread_id.clone(), "manual voice update") + .append_speech(harness.thread_id.clone(), "manual voice update") .await?; assert_v2_progress_update( &harness.sideband_outbound_request(/*request_index*/ 2).await, @@ -1577,6 +1601,51 @@ async fn realtime_automatic_standalone_output_is_context_and_append_speaks() -> Ok(()) } +#[tokio::test] +async fn realtime_append_silent_context_does_not_speak() -> Result<()> { + skip_if_no_network!(Ok(())); + + let mut harness = RealtimeE2eHarness::new( + RealtimeTestVersion::V2, + no_main_loop_responses(), + realtime_sideband(vec![realtime_sideband_connection(vec![ + vec![session_updated("sess_append_silent_context")], + vec![], + vec![], + ])]), + ) + .await?; + + let started = harness.start_webrtc_realtime("v=offer\r\n").await?; + assert_eq!(started.started.version, RealtimeConversationVersion::V2); + assert_eq!( + harness.sideband_outbound_request(/*request_index*/ 0).await["type"].as_str(), + Some("session.update") + ); + + harness + .append_silent_context(harness.thread_id.clone(), "client-provided context") + .await?; + assert_v2_silent_context_update( + &harness.sideband_outbound_request(/*request_index*/ 1).await, + "client-provided context", + ); + let response_create = timeout( + Duration::from_millis(200), + harness + .realtime_server + .wait_for_request(/*connection_index*/ 0, /*request_index*/ 2), + ) + .await; + assert!( + response_create.is_err(), + "explicit silent context should not request a realtime response" + ); + + harness.shutdown().await; + Ok(()) +} + #[tokio::test] async fn realtime_automatic_handoff_output_is_context_and_append_speaks() -> Result<()> { skip_if_no_network!(Ok(())); @@ -1617,7 +1686,7 @@ async fn realtime_automatic_handoff_output_is_context_and_append_speaks() -> Res .await?; assert_eq!(turn_completed.thread_id, harness.thread_id); - assert_v2_context_update( + assert_v2_backend_context_update( &harness.sideband_outbound_request(/*request_index*/ 1).await, "automatic final response", ); @@ -1639,7 +1708,7 @@ async fn realtime_automatic_handoff_output_is_context_and_append_speaks() -> Res ); harness - .append_handoff(harness.thread_id.clone(), "manual spoken update") + .append_speech(harness.thread_id.clone(), "manual spoken update") .await?; assert_v2_progress_update( &harness.sideband_outbound_request(/*request_index*/ 3).await, @@ -1710,7 +1779,7 @@ async fn webrtc_v2_assistant_output_without_handoff_reaches_realtime_context() - .read_notification::("turn/completed") .await?; - assert_v2_context_update( + assert_v2_backend_context_update( &harness.sideband_outbound_request(/*request_index*/ 1).await, preamble, ); @@ -1722,7 +1791,7 @@ async fn webrtc_v2_assistant_output_without_handoff_reaches_realtime_context() - let output_text = final_request["item"]["content"][0]["text"] .as_str() .expect("output text"); - assert!(output_text.starts_with("[BACKEND] ")); + assert!(output_text.starts_with(&format!("{SILENT_CONTEXT_PREFIX}\n\n[BACKEND] "))); assert!(output_text.contains("tokens truncated")); assert!(output_text.len() <= 4_000); @@ -2394,7 +2463,7 @@ async fn realtime_webrtc_start_surfaces_backend_error() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { architecture: None, - auto_handoff_output_as_context: None, + codex_responses_as_silent_context: None, thread_id: thread_start.thread.id, model: None, output_modality: RealtimeOutputModality::Audio, @@ -2457,7 +2526,7 @@ async fn realtime_conversation_requires_feature_flag() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { architecture: None, - auto_handoff_output_as_context: None, + codex_responses_as_silent_context: None, thread_id: thread_start.thread.id.clone(), model: None, output_modality: RealtimeOutputModality::Audio, @@ -2624,7 +2693,11 @@ fn assert_v2_progress_update(request: &Value, expected_text: &str) { ); } -fn assert_v2_context_update(request: &Value, expected_text: &str) { +fn assert_v2_backend_context_update(request: &Value, expected_text: &str) { + assert_v2_silent_context_update(request, &format!("[BACKEND] {expected_text}")); +} + +fn assert_v2_silent_context_update(request: &Value, expected_text: &str) { assert_eq!( request, &json!({ @@ -2634,7 +2707,7 @@ fn assert_v2_context_update(request: &Value, expected_text: &str) { "role": "developer", "content": [{ "type": "input_text", - "text": format!("[BACKEND] {expected_text}") + "text": format!("{SILENT_CONTEXT_PREFIX}\n\n{expected_text}") }] } }) diff --git a/codex-rs/core/src/realtime_conversation.rs b/codex-rs/core/src/realtime_conversation.rs index 04b1f675003b..1031725d71ce 100644 --- a/codex-rs/core/src/realtime_conversation.rs +++ b/codex-rs/core/src/realtime_conversation.rs @@ -32,7 +32,8 @@ use codex_protocol::error::CodexErr; use codex_protocol::error::Result as CodexResult; use codex_protocol::protocol::CodexErrorInfo; use codex_protocol::protocol::ConversationAudioParams; -use codex_protocol::protocol::ConversationHandoffParams; +use codex_protocol::protocol::ConversationSilentContextParams; +use codex_protocol::protocol::ConversationSpeechParams; use codex_protocol::protocol::ConversationStartParams; use codex_protocol::protocol::ConversationStartTransport; use codex_protocol::protocol::ConversationTextParams; @@ -73,8 +74,12 @@ const STANDALONE_HANDOFF_ID: &str = "codex"; const DEFAULT_REALTIME_MODEL: &str = "gpt-realtime-1.5"; pub(crate) const REALTIME_USER_TEXT_PREFIX: &str = "[USER] "; pub(crate) const REALTIME_BACKEND_TEXT_PREFIX: &str = "[BACKEND] "; +const REALTIME_SILENT_CONTEXT_PREFIX: &str = + "Use the following context to inform future responses, but do not speak it to the user."; const REALTIME_V2_STEER_ACKNOWLEDGEMENT: &str = "This was sent to steer the previous background agent task."; +const REALTIME_V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT: &str = + "Background agent finished. Use the preceding [BACKEND] messages as the result."; const REALTIME_ACTIVE_RESPONSE_ERROR_PREFIX: &str = "Conversation already has an active response in progress:"; @@ -105,28 +110,35 @@ struct RealtimeHandoffState { output_tx: Sender, active_handoff: Arc>>, last_output_text: Arc>>, - auto_handoff_output_as_context: bool, + codex_responses_as_silent_context: bool, session_kind: RealtimeSessionKind, } #[derive(Debug, PartialEq, Eq)] enum HandoffOutput { StandaloneAssistantOutput { - as_context: bool, output_text: String, }, ProgressUpdate { - as_context: bool, handoff_id: String, output_text: String, }, FinalUpdate { - as_context: bool, handoff_id: String, output_text: String, }, - SpokenAppend { - output_text: String, + SilentContext { + text: String, + }, + SilentHandoffProgress { + handoff_id: String, + text: String, + }, + SilentHandoffComplete { + handoff_id: String, + }, + Speech { + text: String, }, } @@ -219,14 +231,14 @@ struct RealtimeInputChannels { impl RealtimeHandoffState { fn new( output_tx: Sender, - auto_handoff_output_as_context: bool, + codex_responses_as_silent_context: bool, session_kind: RealtimeSessionKind, ) -> Self { Self { output_tx, active_handoff: Arc::new(Mutex::new(None)), last_output_text: Arc::new(Mutex::new(None)), - auto_handoff_output_as_context, + codex_responses_as_silent_context, session_kind, } } @@ -247,7 +259,7 @@ struct RealtimeStart { api_provider: ApiProvider, architecture: RealtimeConversationArchitecture, extra_headers: Option, - auto_handoff_output_as_context: bool, + codex_responses_as_silent_context: bool, realtime_call_api_provider: Option, session_config: RealtimeSessionConfig, model_client: ModelClient, @@ -302,7 +314,7 @@ impl RealtimeConversationManager { api_provider, architecture, extra_headers, - auto_handoff_output_as_context, + codex_responses_as_silent_context, realtime_call_api_provider, session_config, model_client, @@ -326,7 +338,7 @@ impl RealtimeConversationManager { let realtime_active = Arc::new(AtomicBool::new(true)); let handoff = RealtimeHandoffState::new( handoff_output_tx, - auto_handoff_output_as_context, + codex_responses_as_silent_context, session_kind, ); let input_channels = RealtimeInputChannels { @@ -499,17 +511,29 @@ impl RealtimeConversationManager { Some(handoff_id) => { let output_text = realtime_backend_output(output_text, handoff.session_kind); *handoff.last_output_text.lock().await = Some(output_text.clone()); - HandoffOutput::ProgressUpdate { - as_context: handoff.auto_handoff_output_as_context, - handoff_id, - output_text, + if handoff.codex_responses_as_silent_context { + HandoffOutput::SilentHandoffProgress { + handoff_id, + text: realtime_silent_context(output_text), + } + } else { + HandoffOutput::ProgressUpdate { + handoff_id, + output_text, + } } } None if output_text.trim().is_empty() => return Ok(()), - None => HandoffOutput::StandaloneAssistantOutput { - as_context: handoff.auto_handoff_output_as_context, - output_text: realtime_backend_output(output_text, handoff.session_kind), - }, + None => { + let output_text = realtime_backend_output(output_text, handoff.session_kind); + if handoff.codex_responses_as_silent_context { + HandoffOutput::SilentContext { + text: realtime_silent_context(output_text), + } + } else { + HandoffOutput::StandaloneAssistantOutput { output_text } + } + } }; handoff .output_tx @@ -519,8 +543,32 @@ impl RealtimeConversationManager { Ok(()) } - pub(crate) async fn handoff_append(&self, output_text: String) -> CodexResult<()> { - if output_text.trim().is_empty() { + pub(crate) async fn append_silent_context(&self, text: String) -> CodexResult<()> { + if text.trim().is_empty() { + return Ok(()); + } + + let sender = { + let guard = self.state.lock().await; + let Some(state) = guard.as_ref() else { + return Err(CodexErr::InvalidRequest( + "conversation is not running".to_string(), + )); + }; + state.handoff.output_tx.clone() + }; + + sender + .send(HandoffOutput::SilentContext { + text: realtime_silent_context(text), + }) + .await + .map_err(|_| CodexErr::InvalidRequest("conversation is not running".to_string()))?; + Ok(()) + } + + pub(crate) async fn append_speech(&self, text: String) -> CodexResult<()> { + if text.trim().is_empty() { return Ok(()); } @@ -536,8 +584,8 @@ impl RealtimeConversationManager { handoff .output_tx - .send(HandoffOutput::SpokenAppend { - output_text: realtime_backend_output(output_text, handoff.session_kind), + .send(HandoffOutput::Speech { + text: realtime_backend_output(text, handoff.session_kind), }) .await .map_err(|_| CodexErr::InvalidRequest("conversation is not running".to_string()))?; @@ -564,13 +612,18 @@ impl RealtimeConversationManager { return Ok(()); }; - handoff - .output_tx - .send(HandoffOutput::FinalUpdate { - as_context: handoff.auto_handoff_output_as_context, + let output = if handoff.codex_responses_as_silent_context { + HandoffOutput::SilentHandoffComplete { handoff_id } + } else { + HandoffOutput::FinalUpdate { handoff_id, output_text, - }) + } + }; + + handoff + .output_tx + .send(output) .await .map_err(|_| CodexErr::InvalidRequest("conversation is not running".to_string())) } @@ -657,7 +710,7 @@ struct PreparedRealtimeConversationStart { api_provider: ApiProvider, architecture: RealtimeConversationArchitecture, extra_headers: Option, - auto_handoff_output_as_context: bool, + codex_responses_as_silent_context: bool, realtime_call_api_provider: Option, requested_realtime_session_id: Option, version: RealtimeWsVersion, @@ -733,7 +786,7 @@ async fn prepare_realtime_start( api_provider, architecture, extra_headers, - auto_handoff_output_as_context: params.auto_handoff_output_as_context, + codex_responses_as_silent_context: params.codex_responses_as_silent_context, realtime_call_api_provider, requested_realtime_session_id, version, @@ -850,6 +903,11 @@ fn realtime_backend_output(output_text: String, session_kind: RealtimeSessionKin truncate_realtime_text_to_token_budget(&output_text, REALTIME_ASSISTANT_OUTPUT_TOKEN_BUDGET) } +fn realtime_silent_context(text: String) -> String { + let text = format!("{REALTIME_SILENT_CONTEXT_PREFIX}\n\n{text}"); + truncate_realtime_text_to_token_budget(&text, REALTIME_ASSISTANT_OUTPUT_TOKEN_BUDGET) +} + fn validate_realtime_voice(version: RealtimeWsVersion, voice: RealtimeVoice) -> CodexResult<()> { let voices = RealtimeVoicesList::builtin(); let allowed = match version { @@ -884,7 +942,7 @@ async fn handle_start_inner( api_provider, architecture, extra_headers, - auto_handoff_output_as_context, + codex_responses_as_silent_context, realtime_call_api_provider, requested_realtime_session_id, version, @@ -900,7 +958,7 @@ async fn handle_start_inner( api_provider, architecture, extra_headers, - auto_handoff_output_as_context, + codex_responses_as_silent_context, realtime_call_api_provider, session_config, model_client: sess.services.model_client.clone(), @@ -1129,16 +1187,33 @@ pub(crate) async fn handle_text( } } -pub(crate) async fn handle_handoff( +pub(crate) async fn handle_silent_context( + sess: &Arc, + sub_id: String, + params: ConversationSilentContextParams, +) { + debug!(text = %params.text, "[realtime-text] appending realtime silent context"); + if let Err(err) = sess.conversation.append_silent_context(params.text).await { + error!("failed to append realtime silent context: {err}"); + if sess.conversation.running_state().await.is_some() { + warn!("realtime silent context append failed while the session was already ending"); + } else { + send_conversation_error(sess, sub_id, err.to_string(), CodexErrorInfo::BadRequest) + .await; + } + } +} + +pub(crate) async fn handle_speech( sess: &Arc, sub_id: String, - params: ConversationHandoffParams, + params: ConversationSpeechParams, ) { - debug!(text = %params.output_text, "[realtime-text] appending realtime handoff output"); - if let Err(err) = sess.conversation.handoff_append(params.output_text).await { - error!("failed to append realtime handoff output: {err}"); + debug!(text = %params.text, "[realtime-text] appending realtime speech"); + if let Err(err) = sess.conversation.append_speech(params.text).await { + error!("failed to append realtime speech: {err}"); if sess.conversation.running_state().await.is_some() { - warn!("realtime handoff append failed while the session was already ending"); + warn!("realtime speech append failed while the session was already ending"); } else { send_conversation_error(sess, sub_id, err.to_string(), CodexErrorInfo::BadRequest) .await; @@ -1324,63 +1399,36 @@ async fn handle_handoff_output( let result = match event_parser { RealtimeEventParser::V1 => match handoff_output { - HandoffOutput::StandaloneAssistantOutput { - as_context, - output_text, - } => { - if as_context { - writer - .send_conversation_context_item_create(output_text) - .await - } else { - writer - .send_conversation_handoff_append( - STANDALONE_HANDOFF_ID.to_string(), - output_text, - ) - .await - } + HandoffOutput::StandaloneAssistantOutput { output_text } + | HandoffOutput::Speech { text: output_text } => { + writer + .send_conversation_handoff_append( + STANDALONE_HANDOFF_ID.to_string(), + output_text, + ) + .await } HandoffOutput::ProgressUpdate { - as_context, handoff_id, output_text, } | HandoffOutput::FinalUpdate { - as_context, handoff_id, output_text, } => { - if as_context { - writer - .send_conversation_context_item_create(output_text) - .await - } else { - writer - .send_conversation_function_call_output(handoff_id, output_text) - .await - } - } - HandoffOutput::SpokenAppend { output_text } => { - // TODO(guinness): Use the new client event for standalone handoffs once the API changes are complete. writer - .send_conversation_handoff_append( - STANDALONE_HANDOFF_ID.to_string(), - output_text, - ) + .send_conversation_function_call_output(handoff_id, output_text) .await } + HandoffOutput::SilentContext { text } + | HandoffOutput::SilentHandoffProgress { text, .. } => { + writer.send_conversation_context_item_create(text).await + } + HandoffOutput::SilentHandoffComplete { .. } => Ok(()), }, RealtimeEventParser::RealtimeV2 => match handoff_output { - HandoffOutput::StandaloneAssistantOutput { - as_context, - output_text, - } => { - if as_context { - writer - .send_conversation_context_item_create(output_text) - .await - } else if let Err(err) = writer + HandoffOutput::StandaloneAssistantOutput { output_text } => { + if let Err(err) = writer .send_conversation_item_create(output_text, ConversationTextRole::User) .await { @@ -1392,7 +1440,6 @@ async fn handle_handoff_output( } } HandoffOutput::ProgressUpdate { - as_context, handoff_id, output_text, } => { @@ -1404,29 +1451,18 @@ async fn handle_handoff_output( return Ok(()); } } - if as_context { - writer - .send_conversation_context_item_create(output_text) - .await - } else { - writer - .send_conversation_item_create(output_text, ConversationTextRole::User) - .await - } + writer + .send_conversation_item_create(output_text, ConversationTextRole::User) + .await } HandoffOutput::FinalUpdate { - as_context, handoff_id, output_text: _, } => { - if as_context { - writer - .send_conversation_function_call_output(handoff_id, String::new()) - .await - } else if let Err(err) = writer + if let Err(err) = writer .send_conversation_function_call_output( handoff_id, - REALTIME_V2_STEER_ACKNOWLEDGEMENT.to_string(), + REALTIME_V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT.to_string(), ) .await { @@ -1437,15 +1473,34 @@ async fn handle_handoff_output( .await; } } - HandoffOutput::SpokenAppend { output_text } => { + HandoffOutput::SilentContext { text } => { + writer.send_conversation_context_item_create(text).await + } + HandoffOutput::SilentHandoffProgress { handoff_id, text } => { + let active_handoff = handoff_state.active_handoff.lock().await.clone(); + match active_handoff { + Some(active_handoff) if active_handoff == handoff_id => {} + Some(_) | None => { + debug!("dropping stale realtime silent handoff progress update"); + return Ok(()); + } + } + writer.send_conversation_context_item_create(text).await + } + HandoffOutput::SilentHandoffComplete { handoff_id } => { + writer + .send_conversation_function_call_output(handoff_id, String::new()) + .await + } + HandoffOutput::Speech { text } => { if let Err(err) = writer - .send_conversation_item_create(output_text, ConversationTextRole::User) + .send_conversation_item_create(text, ConversationTextRole::User) .await { Err(err) } else { return response_create_queue - .request_create(writer, events_tx, "handoff append") + .request_create(writer, events_tx, "speech append") .await; } } diff --git a/codex-rs/core/src/realtime_conversation_tests.rs b/codex-rs/core/src/realtime_conversation_tests.rs index 3517f9aed9fc..b6ecc41ffcd6 100644 --- a/codex-rs/core/src/realtime_conversation_tests.rs +++ b/codex-rs/core/src/realtime_conversation_tests.rs @@ -130,7 +130,7 @@ async fn clears_active_handoff_explicitly() { let (tx, _rx) = bounded(1); let state = RealtimeHandoffState::new( tx, - /*auto_handoff_output_as_context*/ false, + /*codex_responses_as_silent_context*/ false, RealtimeSessionKind::V1, ); diff --git a/codex-rs/core/src/session/handlers.rs b/codex-rs/core/src/session/handlers.rs index c088bdb36447..a2267393f1fc 100644 --- a/codex-rs/core/src/session/handlers.rs +++ b/codex-rs/core/src/session/handlers.rs @@ -1,6 +1,7 @@ use crate::realtime_conversation::handle_audio as handle_realtime_conversation_audio; use crate::realtime_conversation::handle_close as handle_realtime_conversation_close; -use crate::realtime_conversation::handle_handoff as handle_realtime_conversation_handoff; +use crate::realtime_conversation::handle_silent_context as handle_realtime_conversation_silent_context; +use crate::realtime_conversation::handle_speech as handle_realtime_conversation_speech; use crate::realtime_conversation::handle_start as handle_realtime_conversation_start; use crate::realtime_conversation::handle_text as handle_realtime_conversation_text; use async_channel::Receiver; @@ -738,8 +739,13 @@ pub(super) async fn submission_loop( handle_realtime_conversation_text(&sess, sub.id.clone(), params).await; false } - Op::RealtimeConversationHandoff(params) => { - handle_realtime_conversation_handoff(&sess, sub.id.clone(), params).await; + Op::RealtimeConversationSilentContext(params) => { + handle_realtime_conversation_silent_context(&sess, sub.id.clone(), params) + .await; + false + } + Op::RealtimeConversationSpeech(params) => { + handle_realtime_conversation_speech(&sess, sub.id.clone(), params).await; false } Op::RealtimeConversationClose => { diff --git a/codex-rs/core/tests/suite/compact_remote.rs b/codex-rs/core/tests/suite/compact_remote.rs index 7cdec21bcf23..55b05da4cef6 100644 --- a/codex-rs/core/tests/suite/compact_remote.rs +++ b/codex-rs/core/tests/suite/compact_remote.rs @@ -201,7 +201,7 @@ async fn start_realtime_conversation(codex: &codex_core::CodexThread) -> Result< codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), diff --git a/codex-rs/core/tests/suite/realtime_conversation.rs b/codex-rs/core/tests/suite/realtime_conversation.rs index 3c66a0900931..49f8cb6de507 100644 --- a/codex-rs/core/tests/suite/realtime_conversation.rs +++ b/codex-rs/core/tests/suite/realtime_conversation.rs @@ -285,7 +285,7 @@ async fn conversation_start_audio_text_close_round_trip() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -428,7 +428,7 @@ async fn conversation_start_defaults_to_v2_and_gpt_realtime_1_5() -> Result<()> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -520,7 +520,7 @@ async fn conversation_webrtc_start_posts_generated_session() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: Some("session-override-model".to_string()), output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -701,7 +701,7 @@ async fn conversation_webrtc_start_uses_avas_architecture_query() -> Result<()> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: Some(RealtimeConversationArchitecture::Avas), - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -800,7 +800,7 @@ async fn conversation_webrtc_start_uses_configured_call_base_url_for_avas() -> R test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: Some(RealtimeConversationArchitecture::Avas), - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -891,7 +891,7 @@ async fn conversation_webrtc_close_while_sideband_connecting_drops_pending_join( test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -979,7 +979,7 @@ async fn conversation_webrtc_sideband_connect_failure_closes_with_error() -> Res test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1069,7 +1069,7 @@ async fn conversation_start_uses_openai_env_key_fallback_with_chatgpt_auth() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1139,7 +1139,7 @@ async fn conversation_transport_close_emits_closed_event() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1233,7 +1233,7 @@ async fn conversation_start_preflight_failure_emits_realtime_error_only() -> Res test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1281,7 +1281,7 @@ async fn conversation_start_connect_failure_emits_realtime_error_only() -> Resul test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1377,7 +1377,7 @@ async fn conversation_second_start_replaces_runtime() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("old".to_string())), @@ -1404,7 +1404,7 @@ async fn conversation_second_start_replaces_runtime() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("new".to_string())), @@ -1502,7 +1502,7 @@ async fn conversation_uses_experimental_realtime_ws_base_url_override() -> Resul test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1568,7 +1568,7 @@ async fn conversation_uses_default_realtime_backend_prompt() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: None, @@ -1642,7 +1642,7 @@ async fn conversation_uses_empty_instructions_for_null_or_empty_prompt() -> Resu test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt, @@ -1709,7 +1709,7 @@ async fn conversation_uses_explicit_start_voice() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1768,7 +1768,7 @@ async fn conversation_uses_configured_realtime_voice() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1815,7 +1815,7 @@ async fn conversation_rejects_voice_for_wrong_realtime_version() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1863,7 +1863,7 @@ async fn conversation_uses_experimental_realtime_ws_backend_prompt_override() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("prompt from op".to_string())), @@ -1937,7 +1937,7 @@ async fn conversation_uses_experimental_realtime_ws_startup_context_override() - test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("prompt from op".to_string())), @@ -2005,7 +2005,7 @@ async fn conversation_disables_realtime_startup_context_with_empty_override() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("prompt from op".to_string())), @@ -2066,7 +2066,7 @@ async fn conversation_start_injects_startup_context_from_thread_history() -> Res test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2179,7 +2179,7 @@ async fn conversation_startup_context_current_thread_selects_many_turns_by_budge codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2288,7 +2288,7 @@ async fn conversation_startup_context_falls_back_to_workspace_map() -> Result<() test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2349,7 +2349,7 @@ async fn conversation_startup_context_is_truncated_and_sent_once_per_start() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2431,7 +2431,7 @@ async fn conversation_user_text_turn_is_not_sent_to_realtime() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2529,7 +2529,7 @@ async fn realtime_v2_noop_tool_call_returns_empty_function_output_without_respon test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2629,7 +2629,7 @@ async fn conversation_mirrors_assistant_message_text_to_realtime_handoff() -> Re test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2767,7 +2767,7 @@ async fn conversation_handoff_persists_across_item_done_until_turn_complete() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2920,7 +2920,7 @@ async fn inbound_handoff_request_starts_turn() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -3023,7 +3023,7 @@ async fn inbound_handoff_request_uses_active_transcript() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -3127,7 +3127,7 @@ async fn inbound_handoff_request_sends_transcript_delta_after_each_handoff() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -3229,7 +3229,7 @@ async fn inbound_conversation_item_does_not_start_turn_and_still_forwards_audio( test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -3353,7 +3353,7 @@ async fn delegated_turn_user_role_echo_does_not_redelegate_and_still_forwards_au test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -3507,7 +3507,7 @@ async fn inbound_handoff_request_does_not_block_realtime_event_forwarding() -> R test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -3645,7 +3645,7 @@ async fn inbound_handoff_request_steers_active_turn() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -3799,7 +3799,7 @@ async fn inbound_handoff_request_starts_turn_and_does_not_block_realtime_audio() test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - auto_handoff_output_as_context: false, + codex_responses_as_silent_context: false, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index a2643ecba081..f46d9fe7d23d 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -188,9 +188,8 @@ pub struct McpServerRefreshConfig { pub struct ConversationStartParams { /// Overrides the configured realtime architecture for this session only. pub architecture: Option, - /// Sends automatic backend Codex output as silent realtime context instead of speakable - /// handoff output. - pub auto_handoff_output_as_context: bool, + /// Inserts automatic Codex responses as silent realtime context instead of speakable output. + pub codex_responses_as_silent_context: bool, /// Overrides the configured realtime model for this session only. pub model: Option, /// Selects whether the realtime session should produce text or audio output. @@ -417,8 +416,13 @@ pub enum ConversationTextRole { } #[derive(Debug, Clone, PartialEq)] -pub struct ConversationHandoffParams { - pub output_text: String, +pub struct ConversationSilentContextParams { + pub text: String, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct ConversationSpeechParams { + pub text: String, } /// Persistent thread-settings overrides that can be applied before user input or @@ -517,8 +521,11 @@ pub enum Op { /// Send text input to the running realtime conversation stream. RealtimeConversationText(ConversationTextParams), - /// Append assistant output to the running realtime conversation stream. - RealtimeConversationHandoff(ConversationHandoffParams), + /// Append silent context to the running realtime conversation stream. + RealtimeConversationSilentContext(ConversationSilentContextParams), + + /// Append speakable text to the running realtime conversation stream. + RealtimeConversationSpeech(ConversationSpeechParams), /// Close the running realtime conversation stream. RealtimeConversationClose, @@ -773,7 +780,8 @@ impl Op { Self::RealtimeConversationStart(_) => "realtime_conversation_start", Self::RealtimeConversationAudio(_) => "realtime_conversation_audio", Self::RealtimeConversationText(_) => "realtime_conversation_text", - Self::RealtimeConversationHandoff(_) => "realtime_conversation_handoff", + Self::RealtimeConversationSilentContext(_) => "realtime_conversation_silent_context", + Self::RealtimeConversationSpeech(_) => "realtime_conversation_speech", Self::RealtimeConversationClose => "realtime_conversation_close", Self::RealtimeConversationListVoices => "realtime_conversation_list_voices", Self::UserInput { .. } => "user_input", From ab5d0d0282817d0353090ec414b2b30ea1498b86 Mon Sep 17 00:00:00 2001 From: Guinness Chen Date: Mon, 15 Jun 2026 12:09:48 -0700 Subject: [PATCH 06/11] Simplify realtime speech handoff API --- .../src/protocol/common.rs | 50 ++-- .../src/protocol/v2/realtime.rs | 22 +- codex-rs/app-server/README.md | 16 +- codex-rs/app-server/src/message_processor.rs | 5 - codex-rs/app-server/src/request_processors.rs | 3 - .../src/request_processors/turn_processor.rs | 42 +--- .../tests/common/test_app_server.rs | 11 - .../tests/suite/v2/experimental_api.rs | 6 +- .../tests/suite/v2/realtime_conversation.rs | 217 +++++------------- codex-rs/core/src/realtime_conversation.rs | 169 +++++--------- .../core/src/realtime_conversation_tests.rs | 3 +- codex-rs/core/src/session/handlers.rs | 6 - codex-rs/core/tests/suite/compact_remote.rs | 3 +- .../core/tests/suite/realtime_conversation.rs | 114 ++++++--- codex-rs/protocol/src/protocol.rs | 15 +- 15 files changed, 233 insertions(+), 449 deletions(-) diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index afa79a8b6c04..b407c55ac313 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -831,12 +831,6 @@ client_request_definitions! { serialization: thread_id(params.thread_id), response: v2::ThreadRealtimeAppendTextResponse, }, - #[experimental("thread/realtime/appendSilentContext")] - ThreadRealtimeAppendSilentContext => "thread/realtime/appendSilentContext" { - params: v2::ThreadRealtimeAppendSilentContextParams, - serialization: thread_id(params.thread_id), - response: v2::ThreadRealtimeAppendSilentContextResponse, - }, #[experimental("thread/realtime/appendSpeech")] ThreadRealtimeAppendSpeech => "thread/realtime/appendSpeech" { params: v2::ThreadRealtimeAppendSpeechParams, @@ -3038,7 +3032,8 @@ mod tests { request_id: RequestId::Integer(9), params: v2::ThreadRealtimeStartParams { architecture: Some(RealtimeConversationArchitecture::Avas), - codex_responses_as_silent_context: None, + codex_responses_as_items: None, + codex_response_item_prefix: None, thread_id: "thr_123".to_string(), model: Some("realtime-treatment-model".to_string()), output_modality: RealtimeOutputModality::Audio, @@ -3056,7 +3051,8 @@ mod tests { "params": { "architecture": "avas", "threadId": "thr_123", - "codexResponsesAsSilentContext": null, + "codexResponsesAsItems": null, + "codexResponseItemPrefix": null, "model": "realtime-treatment-model", "outputModality": "audio", "prompt": "You are on a call", @@ -3077,7 +3073,8 @@ mod tests { request_id: RequestId::Integer(9), params: v2::ThreadRealtimeStartParams { architecture: None, - codex_responses_as_silent_context: None, + codex_responses_as_items: None, + codex_response_item_prefix: None, thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, @@ -3095,7 +3092,8 @@ mod tests { "params": { "architecture": null, "threadId": "thr_123", - "codexResponsesAsSilentContext": null, + "codexResponsesAsItems": null, + "codexResponseItemPrefix": null, "model": null, "outputModality": "audio", "realtimeSessionId": null, @@ -3111,7 +3109,8 @@ mod tests { request_id: RequestId::Integer(9), params: v2::ThreadRealtimeStartParams { architecture: None, - codex_responses_as_silent_context: None, + codex_responses_as_items: None, + codex_response_item_prefix: None, thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, @@ -3129,7 +3128,8 @@ mod tests { "params": { "architecture": null, "threadId": "thr_123", - "codexResponsesAsSilentContext": null, + "codexResponsesAsItems": null, + "codexResponseItemPrefix": null, "model": null, "outputModality": "audio", "prompt": null, @@ -3178,29 +3178,6 @@ mod tests { Ok(()) } - #[test] - fn serialize_thread_realtime_append_silent_context() -> Result<()> { - let request = ClientRequest::ThreadRealtimeAppendSilentContext { - request_id: RequestId::Integer(10), - params: v2::ThreadRealtimeAppendSilentContextParams { - thread_id: "thr_123".to_string(), - text: "Quiet context".to_string(), - }, - }; - assert_eq!( - json!({ - "method": "thread/realtime/appendSilentContext", - "id": 10, - "params": { - "threadId": "thr_123", - "text": "Quiet context" - } - }), - serde_json::to_value(&request)?, - ); - Ok(()) - } - #[test] fn serialize_thread_realtime_append_speech() -> Result<()> { let request = ClientRequest::ThreadRealtimeAppendSpeech { @@ -3334,7 +3311,8 @@ mod tests { request_id: RequestId::Integer(1), params: v2::ThreadRealtimeStartParams { architecture: None, - codex_responses_as_silent_context: None, + codex_responses_as_items: None, + codex_response_item_prefix: None, thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, diff --git a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs index 770fb35d01f9..5bf69285f310 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs @@ -70,9 +70,12 @@ pub struct ThreadRealtimeStartParams { /// Overrides the configured realtime architecture for this session only. #[ts(optional = nullable)] pub architecture: Option, - /// Inserts automatic Codex responses as silent realtime context instead of speakable output. + /// Sends automatic Codex responses as realtime conversation items instead of handoff appends. #[ts(optional = nullable)] - pub codex_responses_as_silent_context: Option, + pub codex_responses_as_items: Option, + /// Optional prefix added to automatic Codex response items when `codexResponsesAsItems` is true. + #[ts(optional = nullable)] + pub codex_response_item_prefix: Option, /// Overrides the configured realtime model for this session only. #[ts(optional = nullable)] pub model: Option, @@ -149,21 +152,6 @@ pub struct ThreadRealtimeAppendTextParams { #[ts(export_to = "v2/")] pub struct ThreadRealtimeAppendTextResponse {} -/// EXPERIMENTAL - append silent context to thread realtime. -#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] -#[serde(rename_all = "camelCase")] -#[ts(export_to = "v2/")] -pub struct ThreadRealtimeAppendSilentContextParams { - pub thread_id: String, - pub text: String, -} - -/// EXPERIMENTAL - response for appending realtime silent context. -#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] -#[serde(rename_all = "camelCase")] -#[ts(export_to = "v2/")] -pub struct ThreadRealtimeAppendSilentContextResponse {} - /// EXPERIMENTAL - append speakable text to thread realtime. #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 32536cb039b2..35cadfe0ca39 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -165,11 +165,10 @@ Example with notification opt-out: - `thread/inject_items` — append raw Responses API items to a loaded thread’s model-visible history without starting a user turn; returns `{}` on success. - `turn/steer` — add user input to an already in-flight regular turn without starting a new turn; returns the active `turnId` that accepted the input. `clientUserMessageId` is optional; when supplied, the corresponding `userMessage` item echoes it as `clientId`. Review and manual compaction turns reject `turn/steer`. - `turn/interrupt` — request cancellation of an in-flight turn by `(thread_id, turn_id)`; success is an empty `{}` response and the turn finishes with `status: "interrupted"`. -- `thread/realtime/start` — start a thread-scoped realtime session (experimental); pass `outputModality: "text"` or `outputModality: "audio"` to choose model output, and optionally pass `model` and `version` to override configured realtime selection for this session only. By default, automatic Codex text follows the legacy speakable handoff path. Pass `codexResponsesAsSilentContext: true` to insert automatic Codex responses as silent developer context instead. Returns `{}` and streams `thread/realtime/*` notifications. Omit `transport` for the websocket transport, or pass `{ "type": "webrtc", "sdp": "..." }` to create a WebRTC session from a browser-generated SDP offer; the remote answer SDP is emitted as `thread/realtime/sdp`. +- `thread/realtime/start` — start a thread-scoped realtime session (experimental); pass `outputModality: "text"` or `outputModality: "audio"` to choose model output, and optionally pass `model` and `version` to override configured realtime selection for this session only. By default, automatic Codex text follows the legacy speakable handoff path. Pass `codexResponsesAsItems: true` to send automatic Codex responses as realtime conversation items instead, and optionally pass `codexResponseItemPrefix` to prepend experiment instructions to those items. Returns `{}` and streams `thread/realtime/*` notifications. Omit `transport` for the websocket transport, or pass `{ "type": "webrtc", "sdp": "..." }` to create a WebRTC session from a browser-generated SDP offer; the remote answer SDP is emitted as `thread/realtime/sdp`. - `thread/realtime/appendAudio` — append an input audio chunk to the active realtime session (experimental); returns `{}`. - `thread/realtime/appendText` — append text input to the active realtime session with a required `role` of `user` or `developer` (experimental); returns `{}`. Older clients that omit `role` default to `user`. -- `thread/realtime/appendSilentContext` — append silent developer context to the active realtime session (experimental); returns `{}` and does not request a realtime response. -- `thread/realtime/appendSpeech` — append text that the realtime model should speak to the user (experimental); returns `{}`. For v1 sessions this sends `conversation.handoff.append`. +- `thread/realtime/appendSpeech` — append text that the realtime model should speak to the user (experimental); returns `{}` and sends `conversation.handoff.append`. - `thread/realtime/stop` — stop the active realtime session for the thread (experimental); returns `{}`. - `review/start` — kick off Codex’s automated reviewer for a thread; responds like `turn/start` and emits `item/started`/`item/completed` notifications with `enteredReviewMode` and `exitedReviewMode` items, plus a final assistant `agentMessage` containing the review. - `command/exec` — run a single command under the server sandbox without starting a thread/turn (handy for utilities and validation). @@ -872,11 +871,12 @@ Omit `prompt` to use Codex's default realtime backend prompt. Send `prompt: null `prompt: ""` when the session should start without that default backend prompt. Clients may also pass `model` and `version` on `thread/realtime/start` to select a different realtime session configuration without changing thread or user config. -Pass `codexResponsesAsSilentContext: true` to inject automatic Codex responses as -silent developer context so the realtime model stays coherent without -automatically speaking backend preambles, progress, or final assistant text. Omit -the field, or pass `false`, to preserve the legacy speakable handoff behavior. -Call `thread/realtime/appendSilentContext` to add app-provided silent context, or +Pass `codexResponsesAsItems: true` to inject automatic Codex responses with +`conversation.item.create` instead of the legacy handoff append path. When using +that mode, `codexResponseItemPrefix` can prepend short experiment instructions +to each automatic Codex response item. Omit `codexResponsesAsItems`, or pass +`false`, to preserve the legacy speakable handoff behavior. Call +`thread/realtime/appendText` to append app-provided realtime text items, or `thread/realtime/appendSpeech` when the app decides a realtime update should be spoken. diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index cda40904cdfb..2ff0d2ae022e 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -1316,11 +1316,6 @@ impl MessageProcessor { .thread_realtime_append_text(&request_id, params) .await } - ClientRequest::ThreadRealtimeAppendSilentContext { params, .. } => { - self.turn_processor - .thread_realtime_append_silent_context(&request_id, params) - .await - } ClientRequest::ThreadRealtimeAppendSpeech { params, .. } => { self.turn_processor .thread_realtime_append_speech(&request_id, params) diff --git a/codex-rs/app-server/src/request_processors.rs b/codex-rs/app-server/src/request_processors.rs index f1e41862a376..2087df17e604 100644 --- a/codex-rs/app-server/src/request_processors.rs +++ b/codex-rs/app-server/src/request_processors.rs @@ -219,8 +219,6 @@ use codex_app_server_protocol::ThreadReadParams; use codex_app_server_protocol::ThreadReadResponse; use codex_app_server_protocol::ThreadRealtimeAppendAudioParams; use codex_app_server_protocol::ThreadRealtimeAppendAudioResponse; -use codex_app_server_protocol::ThreadRealtimeAppendSilentContextParams; -use codex_app_server_protocol::ThreadRealtimeAppendSilentContextResponse; use codex_app_server_protocol::ThreadRealtimeAppendSpeechParams; use codex_app_server_protocol::ThreadRealtimeAppendSpeechResponse; use codex_app_server_protocol::ThreadRealtimeAppendTextParams; @@ -394,7 +392,6 @@ use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::permissions::FileSystemSandboxPolicy; use codex_protocol::protocol::AgentStatus; use codex_protocol::protocol::ConversationAudioParams; -use codex_protocol::protocol::ConversationSilentContextParams; use codex_protocol::protocol::ConversationSpeechParams; use codex_protocol::protocol::ConversationStartParams; use codex_protocol::protocol::ConversationStartTransport; diff --git a/codex-rs/app-server/src/request_processors/turn_processor.rs b/codex-rs/app-server/src/request_processors/turn_processor.rs index b67e653fcc8f..c771ff6d46a7 100644 --- a/codex-rs/app-server/src/request_processors/turn_processor.rs +++ b/codex-rs/app-server/src/request_processors/turn_processor.rs @@ -182,16 +182,6 @@ impl TurnRequestProcessor { .map(|response| response.map(Into::into)) } - pub(crate) async fn thread_realtime_append_silent_context( - &self, - request_id: &ConnectionRequestId, - params: ThreadRealtimeAppendSilentContextParams, - ) -> Result, JSONRPCErrorError> { - self.thread_realtime_append_silent_context_inner(request_id, params) - .await - .map(|response| response.map(Into::into)) - } - pub(crate) async fn thread_realtime_append_speech( &self, request_id: &ConnectionRequestId, @@ -956,9 +946,8 @@ impl TurnRequestProcessor { thread.as_ref(), Op::RealtimeConversationStart(ConversationStartParams { architecture: params.architecture, - codex_responses_as_silent_context: params - .codex_responses_as_silent_context - .unwrap_or(false), + codex_responses_as_items: params.codex_responses_as_items.unwrap_or(false), + codex_response_item_prefix: params.codex_response_item_prefix, model: params.model, output_modality: params.output_modality, prompt: params.prompt, @@ -1035,33 +1024,6 @@ impl TurnRequestProcessor { Ok(Some(ThreadRealtimeAppendTextResponse::default())) } - async fn thread_realtime_append_silent_context_inner( - &self, - request_id: &ConnectionRequestId, - params: ThreadRealtimeAppendSilentContextParams, - ) -> Result, JSONRPCErrorError> { - let Some((_, thread)) = self - .prepare_realtime_conversation_thread(request_id, ¶ms.thread_id) - .await? - else { - return Ok(None); - }; - self.submit_core_op( - request_id, - thread.as_ref(), - Op::RealtimeConversationSilentContext(ConversationSilentContextParams { - text: params.text, - }), - ) - .await - .map_err(|err| { - internal_error(format!( - "failed to append realtime conversation silent context: {err}" - )) - })?; - Ok(Some(ThreadRealtimeAppendSilentContextResponse::default())) - } - async fn thread_realtime_append_speech_inner( &self, request_id: &ConnectionRequestId, diff --git a/codex-rs/app-server/tests/common/test_app_server.rs b/codex-rs/app-server/tests/common/test_app_server.rs index 34ede8678a27..d552520654b5 100644 --- a/codex-rs/app-server/tests/common/test_app_server.rs +++ b/codex-rs/app-server/tests/common/test_app_server.rs @@ -89,7 +89,6 @@ use codex_app_server_protocol::ThreadMemoryModeSetParams; use codex_app_server_protocol::ThreadMetadataUpdateParams; use codex_app_server_protocol::ThreadReadParams; use codex_app_server_protocol::ThreadRealtimeAppendAudioParams; -use codex_app_server_protocol::ThreadRealtimeAppendSilentContextParams; use codex_app_server_protocol::ThreadRealtimeAppendSpeechParams; use codex_app_server_protocol::ThreadRealtimeAppendTextParams; use codex_app_server_protocol::ThreadRealtimeListVoicesParams; @@ -1025,16 +1024,6 @@ impl TestAppServer { .await } - /// Send a `thread/realtime/appendSilentContext` JSON-RPC request (v2). - pub async fn send_thread_realtime_append_silent_context_request( - &mut self, - params: ThreadRealtimeAppendSilentContextParams, - ) -> anyhow::Result { - let params = Some(serde_json::to_value(params)?); - self.send_request("thread/realtime/appendSilentContext", params) - .await - } - /// Send a `thread/realtime/appendSpeech` JSON-RPC request (v2). pub async fn send_thread_realtime_append_speech_request( &mut self, diff --git a/codex-rs/app-server/tests/suite/v2/experimental_api.rs b/codex-rs/app-server/tests/suite/v2/experimental_api.rs index 5436cb95a166..417fa5a39954 100644 --- a/codex-rs/app-server/tests/suite/v2/experimental_api.rs +++ b/codex-rs/app-server/tests/suite/v2/experimental_api.rs @@ -80,7 +80,8 @@ async fn realtime_conversation_start_requires_experimental_api_capability() -> R let request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { architecture: None, - codex_responses_as_silent_context: None, + codex_responses_as_items: None, + codex_response_item_prefix: None, thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, @@ -190,7 +191,8 @@ async fn realtime_webrtc_start_requires_experimental_api_capability() -> Result< let request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { architecture: None, - codex_responses_as_silent_context: None, + codex_responses_as_items: None, + codex_response_item_prefix: None, thread_id: "thr_123".to_string(), model: None, output_modality: RealtimeOutputModality::Audio, diff --git a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs index 7a621b5679a8..35498ae7d1ee 100644 --- a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs +++ b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs @@ -15,8 +15,6 @@ use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::ThreadRealtimeAppendAudioParams; use codex_app_server_protocol::ThreadRealtimeAppendAudioResponse; -use codex_app_server_protocol::ThreadRealtimeAppendSilentContextParams; -use codex_app_server_protocol::ThreadRealtimeAppendSilentContextResponse; use codex_app_server_protocol::ThreadRealtimeAppendSpeechParams; use codex_app_server_protocol::ThreadRealtimeAppendSpeechResponse; use codex_app_server_protocol::ThreadRealtimeAppendTextParams; @@ -84,9 +82,7 @@ const DELEGATED_SHELL_TOOL_TIMEOUT_MS: u64 = 30_000; const STARTUP_CONTEXT_HEADER: &str = "Startup context from Codex."; const V2_STEERING_ACKNOWLEDGEMENT: &str = "This was sent to steer the previous background agent task."; -const V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT: &str = - "Background agent finished. Use the preceding [BACKEND] messages as the result."; -const SILENT_CONTEXT_PREFIX: &str = +const RESPONSE_ITEM_PREFIX: &str = "Use the following context to inform future responses, but do not speak it to the user."; #[derive(Debug, Clone, Copy)] @@ -315,27 +311,27 @@ impl RealtimeE2eHarness { } async fn start_webrtc_realtime(&mut self, offer_sdp: &str) -> Result { - self.start_webrtc_realtime_with_codex_responses_as_silent_context( - offer_sdp, /*codex_responses_as_silent_context*/ None, + self.start_webrtc_realtime_with_codex_responses_as_items( + offer_sdp, /*codex_responses_as_items*/ None, ) .await } - async fn start_webrtc_realtime_with_context_output( + async fn start_webrtc_realtime_with_codex_response_items( &mut self, offer_sdp: &str, ) -> Result { - self.start_webrtc_realtime_with_codex_responses_as_silent_context( + self.start_webrtc_realtime_with_codex_responses_as_items( offer_sdp, - /*codex_responses_as_silent_context*/ Some(true), + /*codex_responses_as_items*/ Some(true), ) .await } - async fn start_webrtc_realtime_with_codex_responses_as_silent_context( + async fn start_webrtc_realtime_with_codex_responses_as_items( &mut self, offer_sdp: &str, - codex_responses_as_silent_context: Option, + codex_responses_as_items: Option, ) -> Result { // Starts realtime through the public JSON-RPC method, then waits for the same client-visible // notifications a desktop app needs: started first, SDP answer second. @@ -344,7 +340,10 @@ impl RealtimeE2eHarness { .send_thread_realtime_start_request(ThreadRealtimeStartParams { architecture: None, thread_id: self.thread_id.clone(), - codex_responses_as_silent_context, + codex_response_item_prefix: codex_responses_as_items + .unwrap_or(false) + .then(|| RESPONSE_ITEM_PREFIX.to_string()), + codex_responses_as_items, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -436,26 +435,6 @@ impl RealtimeE2eHarness { Ok(()) } - async fn append_silent_context(&mut self, thread_id: String, text: &str) -> Result<()> { - let request_id = self - .mcp - .send_thread_realtime_append_silent_context_request( - ThreadRealtimeAppendSilentContextParams { - thread_id, - text: text.to_string(), - }, - ) - .await?; - let response: JSONRPCResponse = timeout( - DEFAULT_TIMEOUT, - self.mcp - .read_stream_until_response_message(RequestId::Integer(request_id)), - ) - .await??; - let _: ThreadRealtimeAppendSilentContextResponse = to_response(response)?; - Ok(()) - } - async fn append_speech(&mut self, thread_id: String, text: &str) -> Result<()> { let request_id = self .mcp @@ -631,7 +610,8 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { architecture: None, - codex_responses_as_silent_context: None, + codex_responses_as_items: None, + codex_response_item_prefix: None, thread_id: thread_start.thread.id.clone(), model: Some("realtime-treatment-model".to_string()), output_modality: RealtimeOutputModality::Audio, @@ -908,7 +888,8 @@ async fn realtime_text_output_modality_requests_text_output_and_final_transcript let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { architecture: None, - codex_responses_as_silent_context: None, + codex_responses_as_items: None, + codex_response_item_prefix: None, thread_id: thread_start.thread.id.clone(), model: None, output_modality: RealtimeOutputModality::Text, @@ -1086,7 +1067,8 @@ async fn realtime_conversation_stop_emits_closed_notification() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { architecture: None, - codex_responses_as_silent_context: None, + codex_responses_as_items: None, + codex_response_item_prefix: None, thread_id: thread_start.thread.id.clone(), model: None, output_modality: RealtimeOutputModality::Audio, @@ -1187,7 +1169,8 @@ async fn realtime_webrtc_start_emits_sdp_notification() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { architecture: None, - codex_responses_as_silent_context: None, + codex_responses_as_items: None, + codex_response_item_prefix: None, thread_id: thread_id.clone(), model: None, output_modality: RealtimeOutputModality::Audio, @@ -1457,7 +1440,7 @@ async fn webrtc_v1_handoff_request_delegates_context_and_manual_append_speaks() .await?; let started = harness - .start_webrtc_realtime_with_context_output("v=offer\r\n") + .start_webrtc_realtime_with_codex_response_items("v=offer\r\n") .await?; assert_eq!(started.started.version, RealtimeConversationVersion::V1); assert_call_create_multipart( @@ -1478,7 +1461,7 @@ async fn webrtc_v1_handoff_request_delegates_context_and_manual_append_speaks() assert_eq!(turn_completed.thread_id, harness.thread_id); // Phase 3: assert the delegated prompt went to Responses, then the automatic v1 output went - // back over the existing sideband connection as silent context. + // back over the existing sideband connection as a conversation item. let requests = harness.main_loop_responses_requests().await?; assert_eq!(requests.len(), 1); assert!( @@ -1499,7 +1482,7 @@ async fn webrtc_v1_handoff_request_delegates_context_and_manual_append_speaks() "role": "developer", "content": [{ "type": "input_text", - "text": format!("{SILENT_CONTEXT_PREFIX}\n\ndelegated from v1") + "text": format!("{RESPONSE_ITEM_PREFIX}\n\ndelegated from v1") }] } }) @@ -1523,7 +1506,7 @@ async fn webrtc_v1_handoff_request_delegates_context_and_manual_append_speaks() } #[tokio::test] -async fn realtime_automatic_standalone_output_is_context_and_append_speaks() -> Result<()> { +async fn realtime_automatic_standalone_output_is_item_and_append_speaks() -> Result<()> { skip_if_no_network!(Ok(())); let mut harness = RealtimeE2eHarness::new( @@ -1541,7 +1524,7 @@ async fn realtime_automatic_standalone_output_is_context_and_append_speaks() -> .await?; let started = harness - .start_webrtc_realtime_with_context_output("v=offer\r\n") + .start_webrtc_realtime_with_codex_response_items("v=offer\r\n") .await?; assert_eq!(started.started.version, RealtimeConversationVersion::V2); assert_eq!( @@ -1572,7 +1555,7 @@ async fn realtime_automatic_standalone_output_is_context_and_append_speaks() -> .read_notification::("turn/completed") .await?; - assert_v2_backend_context_update( + assert_v2_backend_item_update( &harness.sideband_outbound_request(/*request_index*/ 1).await, "automatic output", ); @@ -1585,7 +1568,7 @@ async fn realtime_automatic_standalone_output_is_context_and_append_speaks() -> .await; assert!( automatic_response_create.is_err(), - "automatic context should not request a realtime response" + "automatic item should not request a realtime response" ); harness @@ -1595,59 +1578,13 @@ async fn realtime_automatic_standalone_output_is_context_and_append_speaks() -> &harness.sideband_outbound_request(/*request_index*/ 2).await, "manual voice update", ); - assert_v2_response_create(&harness.sideband_outbound_request(/*request_index*/ 3).await); harness.shutdown().await; Ok(()) } #[tokio::test] -async fn realtime_append_silent_context_does_not_speak() -> Result<()> { - skip_if_no_network!(Ok(())); - - let mut harness = RealtimeE2eHarness::new( - RealtimeTestVersion::V2, - no_main_loop_responses(), - realtime_sideband(vec![realtime_sideband_connection(vec![ - vec![session_updated("sess_append_silent_context")], - vec![], - vec![], - ])]), - ) - .await?; - - let started = harness.start_webrtc_realtime("v=offer\r\n").await?; - assert_eq!(started.started.version, RealtimeConversationVersion::V2); - assert_eq!( - harness.sideband_outbound_request(/*request_index*/ 0).await["type"].as_str(), - Some("session.update") - ); - - harness - .append_silent_context(harness.thread_id.clone(), "client-provided context") - .await?; - assert_v2_silent_context_update( - &harness.sideband_outbound_request(/*request_index*/ 1).await, - "client-provided context", - ); - let response_create = timeout( - Duration::from_millis(200), - harness - .realtime_server - .wait_for_request(/*connection_index*/ 0, /*request_index*/ 2), - ) - .await; - assert!( - response_create.is_err(), - "explicit silent context should not request a realtime response" - ); - - harness.shutdown().await; - Ok(()) -} - -#[tokio::test] -async fn realtime_automatic_handoff_output_is_context_and_append_speaks() -> Result<()> { +async fn realtime_automatic_handoff_output_is_item_and_append_speaks() -> Result<()> { skip_if_no_network!(Ok(())); let mut harness = RealtimeE2eHarness::new( @@ -1669,7 +1606,7 @@ async fn realtime_automatic_handoff_output_is_context_and_append_speaks() -> Res .await?; let started = harness - .start_webrtc_realtime_with_context_output("v=offer\r\n") + .start_webrtc_realtime_with_codex_response_items("v=offer\r\n") .await?; assert_eq!(started.started.version, RealtimeConversationVersion::V2); assert_eq!( @@ -1686,7 +1623,7 @@ async fn realtime_automatic_handoff_output_is_context_and_append_speaks() -> Res .await?; assert_eq!(turn_completed.thread_id, harness.thread_id); - assert_v2_backend_context_update( + assert_v2_backend_item_update( &harness.sideband_outbound_request(/*request_index*/ 1).await, "automatic final response", ); @@ -1704,7 +1641,7 @@ async fn realtime_automatic_handoff_output_is_context_and_append_speaks() -> Res .await; assert!( automatic_response_create.is_err(), - "automatic handoff context should not request a realtime response" + "automatic handoff item should not request a realtime response" ); harness @@ -1714,7 +1651,6 @@ async fn realtime_automatic_handoff_output_is_context_and_append_speaks() -> Res &harness.sideband_outbound_request(/*request_index*/ 3).await, "manual spoken update", ); - assert_v2_response_create(&harness.sideband_outbound_request(/*request_index*/ 4).await); harness.shutdown().await; Ok(()) @@ -1752,7 +1688,7 @@ async fn webrtc_v2_assistant_output_without_handoff_reaches_realtime_context() - .await?; let started = harness - .start_webrtc_realtime_with_context_output("v=offer\r\n") + .start_webrtc_realtime_with_codex_response_items("v=offer\r\n") .await?; assert_eq!(started.started.version, RealtimeConversationVersion::V2); @@ -1779,7 +1715,7 @@ async fn webrtc_v2_assistant_output_without_handoff_reaches_realtime_context() - .read_notification::("turn/completed") .await?; - assert_v2_backend_context_update( + assert_v2_backend_item_update( &harness.sideband_outbound_request(/*request_index*/ 1).await, preamble, ); @@ -1791,7 +1727,7 @@ async fn webrtc_v2_assistant_output_without_handoff_reaches_realtime_context() - let output_text = final_request["item"]["content"][0]["text"] .as_str() .expect("output text"); - assert!(output_text.starts_with(&format!("{SILENT_CONTEXT_PREFIX}\n\n[BACKEND] "))); + assert!(output_text.starts_with(&format!("{RESPONSE_ITEM_PREFIX}\n\n[BACKEND] "))); assert!(output_text.contains("tokens truncated")); assert!(output_text.len() <= 4_000); @@ -2103,16 +2039,8 @@ async fn webrtc_v2_background_agent_tool_call_delegates_and_returns_function_out let progress = harness.sideband_outbound_request(/*request_index*/ 1).await; assert_v2_progress_update(&progress, "delegated from v2"); - let tool_output = harness.sideband_outbound_request(/*request_index*/ 2).await; - assert_v2_function_call_output(&tool_output, "call_v2", V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT); - assert_eq!( - function_call_output_sideband_requests(&harness.realtime_server).len(), - 1 - ); - - // Phase 4: after the final function-call output, realtime needs an explicit - // `response.create` to produce the next user-visible response. - assert_v2_response_create(&harness.sideband_outbound_request(/*request_index*/ 3).await); + let final_update = harness.sideband_outbound_request(/*request_index*/ 2).await; + assert_v2_progress_update(&final_update, "delegated from v2"); harness.shutdown().await; Ok(()) @@ -2238,12 +2166,8 @@ async fn webrtc_v2_background_agent_progress_is_sent_before_function_output() -> let progress = harness.sideband_outbound_request(/*request_index*/ 1).await; assert_v2_progress_update(&progress, "progress before final"); - let tool_output = harness.sideband_outbound_request(/*request_index*/ 2).await; - assert_v2_function_call_output( - &tool_output, - "call_progress_order", - V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT, - ); + let final_update = harness.sideband_outbound_request(/*request_index*/ 2).await; + assert_v2_progress_update(&final_update, "progress before final"); harness.shutdown().await; Ok(()) @@ -2328,16 +2252,8 @@ async fn webrtc_v2_tool_call_delegated_turn_can_execute_shell_tool() -> Result<( let progress = harness.sideband_outbound_request(/*request_index*/ 1).await; assert_v2_progress_update(&progress, "shell tool finished"); - let tool_output = harness.sideband_outbound_request(/*request_index*/ 2).await; - assert_v2_function_call_output( - &tool_output, - "call_shell", - V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT, - ); - assert_eq!( - function_call_output_sideband_requests(&harness.realtime_server).len(), - 1 - ); + let final_update = harness.sideband_outbound_request(/*request_index*/ 2).await; + assert_v2_progress_update(&final_update, "shell tool finished"); harness.shutdown().await; Ok(()) @@ -2412,12 +2328,8 @@ async fn webrtc_v2_tool_call_does_not_block_sideband_audio() -> Result<()> { let progress = harness.sideband_outbound_request(/*request_index*/ 1).await; assert_v2_progress_update(&progress, "late delegated result"); - let tool_output = harness.sideband_outbound_request(/*request_index*/ 2).await; - assert_v2_function_call_output( - &tool_output, - "call_audio", - V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT, - ); + let final_update = harness.sideband_outbound_request(/*request_index*/ 2).await; + assert_v2_progress_update(&final_update, "late delegated result"); harness.shutdown().await; Ok(()) @@ -2463,7 +2375,8 @@ async fn realtime_webrtc_start_surfaces_backend_error() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { architecture: None, - codex_responses_as_silent_context: None, + codex_responses_as_items: None, + codex_response_item_prefix: None, thread_id: thread_start.thread.id, model: None, output_modality: RealtimeOutputModality::Audio, @@ -2526,7 +2439,8 @@ async fn realtime_conversation_requires_feature_flag() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { architecture: None, - codex_responses_as_silent_context: None, + codex_responses_as_items: None, + codex_response_item_prefix: None, thread_id: thread_start.thread.id.clone(), model: None, output_modality: RealtimeOutputModality::Audio, @@ -2650,18 +2564,6 @@ fn realtime_tool_ok_command() -> Vec { } } -fn function_call_output_sideband_requests(server: &WebSocketTestServer) -> Vec { - server - .single_connection() - .iter() - .map(WebSocketRequest::body_json) - .filter(|request| { - request["type"] == "conversation.item.create" - && request["item"]["type"] == "function_call_output" - }) - .collect() -} - fn assert_v2_function_call_output(request: &Value, call_id: &str, expected_output: &str) { assert_eq!( request, @@ -2677,27 +2579,22 @@ fn assert_v2_function_call_output(request: &Value, call_id: &str, expected_outpu } fn assert_v2_progress_update(request: &Value, expected_text: &str) { + let expected_output_text = format!("[BACKEND] {expected_text}"); assert_eq!( - request, - &json!({ - "type": "conversation.item.create", - "item": { - "type": "message", - "role": "user", - "content": [{ - "type": "input_text", - "text": format!("[BACKEND] {expected_text}") - }] - } - }) + request["type"].as_str(), + Some("conversation.handoff.append") + ); + assert_eq!( + request["output_text"].as_str(), + Some(expected_output_text.as_str()) ); } -fn assert_v2_backend_context_update(request: &Value, expected_text: &str) { - assert_v2_silent_context_update(request, &format!("[BACKEND] {expected_text}")); +fn assert_v2_backend_item_update(request: &Value, expected_text: &str) { + assert_v2_items_update(request, &format!("[BACKEND] {expected_text}")); } -fn assert_v2_silent_context_update(request: &Value, expected_text: &str) { +fn assert_v2_items_update(request: &Value, expected_text: &str) { assert_eq!( request, &json!({ @@ -2707,7 +2604,7 @@ fn assert_v2_silent_context_update(request: &Value, expected_text: &str) { "role": "developer", "content": [{ "type": "input_text", - "text": format!("{SILENT_CONTEXT_PREFIX}\n\n{expected_text}") + "text": format!("{RESPONSE_ITEM_PREFIX}\n\n{expected_text}") }] } }) diff --git a/codex-rs/core/src/realtime_conversation.rs b/codex-rs/core/src/realtime_conversation.rs index 1031725d71ce..2e4267954d70 100644 --- a/codex-rs/core/src/realtime_conversation.rs +++ b/codex-rs/core/src/realtime_conversation.rs @@ -32,7 +32,6 @@ use codex_protocol::error::CodexErr; use codex_protocol::error::Result as CodexResult; use codex_protocol::protocol::CodexErrorInfo; use codex_protocol::protocol::ConversationAudioParams; -use codex_protocol::protocol::ConversationSilentContextParams; use codex_protocol::protocol::ConversationSpeechParams; use codex_protocol::protocol::ConversationStartParams; use codex_protocol::protocol::ConversationStartTransport; @@ -74,12 +73,8 @@ const STANDALONE_HANDOFF_ID: &str = "codex"; const DEFAULT_REALTIME_MODEL: &str = "gpt-realtime-1.5"; pub(crate) const REALTIME_USER_TEXT_PREFIX: &str = "[USER] "; pub(crate) const REALTIME_BACKEND_TEXT_PREFIX: &str = "[BACKEND] "; -const REALTIME_SILENT_CONTEXT_PREFIX: &str = - "Use the following context to inform future responses, but do not speak it to the user."; const REALTIME_V2_STEER_ACKNOWLEDGEMENT: &str = "This was sent to steer the previous background agent task."; -const REALTIME_V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT: &str = - "Background agent finished. Use the preceding [BACKEND] messages as the result."; const REALTIME_ACTIVE_RESPONSE_ERROR_PREFIX: &str = "Conversation already has an active response in progress:"; @@ -110,7 +105,8 @@ struct RealtimeHandoffState { output_tx: Sender, active_handoff: Arc>>, last_output_text: Arc>>, - codex_responses_as_silent_context: bool, + codex_responses_as_items: bool, + codex_response_item_prefix: Option, session_kind: RealtimeSessionKind, } @@ -127,14 +123,14 @@ enum HandoffOutput { handoff_id: String, output_text: String, }, - SilentContext { + Item { text: String, }, - SilentHandoffProgress { + ItemProgress { handoff_id: String, text: String, }, - SilentHandoffComplete { + ItemHandoffComplete { handoff_id: String, }, Speech { @@ -231,14 +227,16 @@ struct RealtimeInputChannels { impl RealtimeHandoffState { fn new( output_tx: Sender, - codex_responses_as_silent_context: bool, + codex_responses_as_items: bool, + codex_response_item_prefix: Option, session_kind: RealtimeSessionKind, ) -> Self { Self { output_tx, active_handoff: Arc::new(Mutex::new(None)), last_output_text: Arc::new(Mutex::new(None)), - codex_responses_as_silent_context, + codex_responses_as_items, + codex_response_item_prefix, session_kind, } } @@ -259,7 +257,8 @@ struct RealtimeStart { api_provider: ApiProvider, architecture: RealtimeConversationArchitecture, extra_headers: Option, - codex_responses_as_silent_context: bool, + codex_responses_as_items: bool, + codex_response_item_prefix: Option, realtime_call_api_provider: Option, session_config: RealtimeSessionConfig, model_client: ModelClient, @@ -314,7 +313,8 @@ impl RealtimeConversationManager { api_provider, architecture, extra_headers, - codex_responses_as_silent_context, + codex_responses_as_items, + codex_response_item_prefix, realtime_call_api_provider, session_config, model_client, @@ -338,7 +338,8 @@ impl RealtimeConversationManager { let realtime_active = Arc::new(AtomicBool::new(true)); let handoff = RealtimeHandoffState::new( handoff_output_tx, - codex_responses_as_silent_context, + codex_responses_as_items, + codex_response_item_prefix, session_kind, ); let input_channels = RealtimeInputChannels { @@ -511,10 +512,13 @@ impl RealtimeConversationManager { Some(handoff_id) => { let output_text = realtime_backend_output(output_text, handoff.session_kind); *handoff.last_output_text.lock().await = Some(output_text.clone()); - if handoff.codex_responses_as_silent_context { - HandoffOutput::SilentHandoffProgress { + if handoff.codex_responses_as_items { + HandoffOutput::ItemProgress { handoff_id, - text: realtime_silent_context(output_text), + text: realtime_backend_item( + output_text, + handoff.codex_response_item_prefix.as_deref(), + ), } } else { HandoffOutput::ProgressUpdate { @@ -526,9 +530,12 @@ impl RealtimeConversationManager { None if output_text.trim().is_empty() => return Ok(()), None => { let output_text = realtime_backend_output(output_text, handoff.session_kind); - if handoff.codex_responses_as_silent_context { - HandoffOutput::SilentContext { - text: realtime_silent_context(output_text), + if handoff.codex_responses_as_items { + HandoffOutput::Item { + text: realtime_backend_item( + output_text, + handoff.codex_response_item_prefix.as_deref(), + ), } } else { HandoffOutput::StandaloneAssistantOutput { output_text } @@ -543,30 +550,6 @@ impl RealtimeConversationManager { Ok(()) } - pub(crate) async fn append_silent_context(&self, text: String) -> CodexResult<()> { - if text.trim().is_empty() { - return Ok(()); - } - - let sender = { - let guard = self.state.lock().await; - let Some(state) = guard.as_ref() else { - return Err(CodexErr::InvalidRequest( - "conversation is not running".to_string(), - )); - }; - state.handoff.output_tx.clone() - }; - - sender - .send(HandoffOutput::SilentContext { - text: realtime_silent_context(text), - }) - .await - .map_err(|_| CodexErr::InvalidRequest("conversation is not running".to_string()))?; - Ok(()) - } - pub(crate) async fn append_speech(&self, text: String) -> CodexResult<()> { if text.trim().is_empty() { return Ok(()); @@ -612,8 +595,8 @@ impl RealtimeConversationManager { return Ok(()); }; - let output = if handoff.codex_responses_as_silent_context { - HandoffOutput::SilentHandoffComplete { handoff_id } + let output = if handoff.codex_responses_as_items { + HandoffOutput::ItemHandoffComplete { handoff_id } } else { HandoffOutput::FinalUpdate { handoff_id, @@ -710,7 +693,8 @@ struct PreparedRealtimeConversationStart { api_provider: ApiProvider, architecture: RealtimeConversationArchitecture, extra_headers: Option, - codex_responses_as_silent_context: bool, + codex_responses_as_items: bool, + codex_response_item_prefix: Option, realtime_call_api_provider: Option, requested_realtime_session_id: Option, version: RealtimeWsVersion, @@ -786,7 +770,8 @@ async fn prepare_realtime_start( api_provider, architecture, extra_headers, - codex_responses_as_silent_context: params.codex_responses_as_silent_context, + codex_responses_as_items: params.codex_responses_as_items, + codex_response_item_prefix: params.codex_response_item_prefix, realtime_call_api_provider, requested_realtime_session_id, version, @@ -903,8 +888,11 @@ fn realtime_backend_output(output_text: String, session_kind: RealtimeSessionKin truncate_realtime_text_to_token_budget(&output_text, REALTIME_ASSISTANT_OUTPUT_TOKEN_BUDGET) } -fn realtime_silent_context(text: String) -> String { - let text = format!("{REALTIME_SILENT_CONTEXT_PREFIX}\n\n{text}"); +fn realtime_backend_item(text: String, prefix: Option<&str>) -> String { + let text = match prefix.filter(|prefix| !prefix.is_empty()) { + Some(prefix) => format!("{prefix}\n\n{text}"), + None => text, + }; truncate_realtime_text_to_token_budget(&text, REALTIME_ASSISTANT_OUTPUT_TOKEN_BUDGET) } @@ -942,7 +930,8 @@ async fn handle_start_inner( api_provider, architecture, extra_headers, - codex_responses_as_silent_context, + codex_responses_as_items, + codex_response_item_prefix, realtime_call_api_provider, requested_realtime_session_id, version, @@ -958,7 +947,8 @@ async fn handle_start_inner( api_provider, architecture, extra_headers, - codex_responses_as_silent_context, + codex_responses_as_items, + codex_response_item_prefix, realtime_call_api_provider, session_config, model_client: sess.services.model_client.clone(), @@ -1187,23 +1177,6 @@ pub(crate) async fn handle_text( } } -pub(crate) async fn handle_silent_context( - sess: &Arc, - sub_id: String, - params: ConversationSilentContextParams, -) { - debug!(text = %params.text, "[realtime-text] appending realtime silent context"); - if let Err(err) = sess.conversation.append_silent_context(params.text).await { - error!("failed to append realtime silent context: {err}"); - if sess.conversation.running_state().await.is_some() { - warn!("realtime silent context append failed while the session was already ending"); - } else { - send_conversation_error(sess, sub_id, err.to_string(), CodexErrorInfo::BadRequest) - .await; - } - } -} - pub(crate) async fn handle_speech( sess: &Arc, sub_id: String, @@ -1337,7 +1310,6 @@ async fn run_realtime_input_task(input: RealtimeInputTask) { &events_tx, &handoff_state, event_parser, - &mut response_create_queue, ) .await } @@ -1393,7 +1365,6 @@ async fn handle_handoff_output( events_tx: &Sender, handoff_state: &RealtimeHandoffState, event_parser: RealtimeEventParser, - response_create_queue: &mut RealtimeResponseCreateQueue, ) -> anyhow::Result<()> { let handoff_output = handoff_output.context("handoff output channel closed")?; @@ -1420,24 +1391,19 @@ async fn handle_handoff_output( .send_conversation_function_call_output(handoff_id, output_text) .await } - HandoffOutput::SilentContext { text } - | HandoffOutput::SilentHandoffProgress { text, .. } => { + HandoffOutput::Item { text } | HandoffOutput::ItemProgress { text, .. } => { writer.send_conversation_context_item_create(text).await } - HandoffOutput::SilentHandoffComplete { .. } => Ok(()), + HandoffOutput::ItemHandoffComplete { .. } => Ok(()), }, RealtimeEventParser::RealtimeV2 => match handoff_output { HandoffOutput::StandaloneAssistantOutput { output_text } => { - if let Err(err) = writer - .send_conversation_item_create(output_text, ConversationTextRole::User) + writer + .send_conversation_handoff_append( + STANDALONE_HANDOFF_ID.to_string(), + output_text, + ) .await - { - Err(err) - } else { - return response_create_queue - .request_create(writer, events_tx, "standalone assistant output") - .await; - } } HandoffOutput::ProgressUpdate { handoff_id, @@ -1452,57 +1418,40 @@ async fn handle_handoff_output( } } writer - .send_conversation_item_create(output_text, ConversationTextRole::User) + .send_conversation_handoff_append(handoff_id, output_text) .await } HandoffOutput::FinalUpdate { handoff_id, - output_text: _, + output_text, } => { - if let Err(err) = writer - .send_conversation_function_call_output( - handoff_id, - REALTIME_V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT.to_string(), - ) + writer + .send_conversation_handoff_append(handoff_id, output_text) .await - { - Err(err) - } else { - return response_create_queue - .request_create(writer, events_tx, "handoff") - .await; - } } - HandoffOutput::SilentContext { text } => { + HandoffOutput::Item { text } => { writer.send_conversation_context_item_create(text).await } - HandoffOutput::SilentHandoffProgress { handoff_id, text } => { + HandoffOutput::ItemProgress { handoff_id, text } => { let active_handoff = handoff_state.active_handoff.lock().await.clone(); match active_handoff { Some(active_handoff) if active_handoff == handoff_id => {} Some(_) | None => { - debug!("dropping stale realtime silent handoff progress update"); + debug!("dropping stale realtime handoff item update"); return Ok(()); } } writer.send_conversation_context_item_create(text).await } - HandoffOutput::SilentHandoffComplete { handoff_id } => { + HandoffOutput::ItemHandoffComplete { handoff_id } => { writer .send_conversation_function_call_output(handoff_id, String::new()) .await } HandoffOutput::Speech { text } => { - if let Err(err) = writer - .send_conversation_item_create(text, ConversationTextRole::User) + writer + .send_conversation_handoff_append(STANDALONE_HANDOFF_ID.to_string(), text) .await - { - Err(err) - } else { - return response_create_queue - .request_create(writer, events_tx, "speech append") - .await; - } } }, }; diff --git a/codex-rs/core/src/realtime_conversation_tests.rs b/codex-rs/core/src/realtime_conversation_tests.rs index b6ecc41ffcd6..147e0cc1236b 100644 --- a/codex-rs/core/src/realtime_conversation_tests.rs +++ b/codex-rs/core/src/realtime_conversation_tests.rs @@ -130,7 +130,8 @@ async fn clears_active_handoff_explicitly() { let (tx, _rx) = bounded(1); let state = RealtimeHandoffState::new( tx, - /*codex_responses_as_silent_context*/ false, + /*codex_responses_as_items*/ false, + /*codex_response_item_prefix*/ None, RealtimeSessionKind::V1, ); diff --git a/codex-rs/core/src/session/handlers.rs b/codex-rs/core/src/session/handlers.rs index a2267393f1fc..4ed79d04e383 100644 --- a/codex-rs/core/src/session/handlers.rs +++ b/codex-rs/core/src/session/handlers.rs @@ -1,6 +1,5 @@ use crate::realtime_conversation::handle_audio as handle_realtime_conversation_audio; use crate::realtime_conversation::handle_close as handle_realtime_conversation_close; -use crate::realtime_conversation::handle_silent_context as handle_realtime_conversation_silent_context; use crate::realtime_conversation::handle_speech as handle_realtime_conversation_speech; use crate::realtime_conversation::handle_start as handle_realtime_conversation_start; use crate::realtime_conversation::handle_text as handle_realtime_conversation_text; @@ -739,11 +738,6 @@ pub(super) async fn submission_loop( handle_realtime_conversation_text(&sess, sub.id.clone(), params).await; false } - Op::RealtimeConversationSilentContext(params) => { - handle_realtime_conversation_silent_context(&sess, sub.id.clone(), params) - .await; - false - } Op::RealtimeConversationSpeech(params) => { handle_realtime_conversation_speech(&sess, sub.id.clone(), params).await; false diff --git a/codex-rs/core/tests/suite/compact_remote.rs b/codex-rs/core/tests/suite/compact_remote.rs index 6fd52f47c181..6245efab8452 100644 --- a/codex-rs/core/tests/suite/compact_remote.rs +++ b/codex-rs/core/tests/suite/compact_remote.rs @@ -202,7 +202,8 @@ async fn start_realtime_conversation(codex: &codex_core::CodexThread) -> Result< codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), diff --git a/codex-rs/core/tests/suite/realtime_conversation.rs b/codex-rs/core/tests/suite/realtime_conversation.rs index 49f8cb6de507..62b7551ee3ac 100644 --- a/codex-rs/core/tests/suite/realtime_conversation.rs +++ b/codex-rs/core/tests/suite/realtime_conversation.rs @@ -285,7 +285,8 @@ async fn conversation_start_audio_text_close_round_trip() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -428,7 +429,8 @@ async fn conversation_start_defaults_to_v2_and_gpt_realtime_1_5() -> Result<()> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -520,7 +522,8 @@ async fn conversation_webrtc_start_posts_generated_session() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: Some("session-override-model".to_string()), output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -701,7 +704,8 @@ async fn conversation_webrtc_start_uses_avas_architecture_query() -> Result<()> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: Some(RealtimeConversationArchitecture::Avas), - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -800,7 +804,8 @@ async fn conversation_webrtc_start_uses_configured_call_base_url_for_avas() -> R test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: Some(RealtimeConversationArchitecture::Avas), - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -891,7 +896,8 @@ async fn conversation_webrtc_close_while_sideband_connecting_drops_pending_join( test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -979,7 +985,8 @@ async fn conversation_webrtc_sideband_connect_failure_closes_with_error() -> Res test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1069,7 +1076,8 @@ async fn conversation_start_uses_openai_env_key_fallback_with_chatgpt_auth() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1139,7 +1147,8 @@ async fn conversation_transport_close_emits_closed_event() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1233,7 +1242,8 @@ async fn conversation_start_preflight_failure_emits_realtime_error_only() -> Res test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1281,7 +1291,8 @@ async fn conversation_start_connect_failure_emits_realtime_error_only() -> Resul test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1377,7 +1388,8 @@ async fn conversation_second_start_replaces_runtime() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("old".to_string())), @@ -1404,7 +1416,8 @@ async fn conversation_second_start_replaces_runtime() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("new".to_string())), @@ -1502,7 +1515,8 @@ async fn conversation_uses_experimental_realtime_ws_base_url_override() -> Resul test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1568,7 +1582,8 @@ async fn conversation_uses_default_realtime_backend_prompt() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: None, @@ -1642,7 +1657,8 @@ async fn conversation_uses_empty_instructions_for_null_or_empty_prompt() -> Resu test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt, @@ -1709,7 +1725,8 @@ async fn conversation_uses_explicit_start_voice() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1768,7 +1785,8 @@ async fn conversation_uses_configured_realtime_voice() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1815,7 +1833,8 @@ async fn conversation_rejects_voice_for_wrong_realtime_version() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -1863,7 +1882,8 @@ async fn conversation_uses_experimental_realtime_ws_backend_prompt_override() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("prompt from op".to_string())), @@ -1937,7 +1957,8 @@ async fn conversation_uses_experimental_realtime_ws_startup_context_override() - test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("prompt from op".to_string())), @@ -2005,7 +2026,8 @@ async fn conversation_disables_realtime_startup_context_with_empty_override() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("prompt from op".to_string())), @@ -2066,7 +2088,8 @@ async fn conversation_start_injects_startup_context_from_thread_history() -> Res test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2179,7 +2202,8 @@ async fn conversation_startup_context_current_thread_selects_many_turns_by_budge codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2288,7 +2312,8 @@ async fn conversation_startup_context_falls_back_to_workspace_map() -> Result<() test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2349,7 +2374,8 @@ async fn conversation_startup_context_is_truncated_and_sent_once_per_start() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2431,7 +2457,8 @@ async fn conversation_user_text_turn_is_not_sent_to_realtime() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2529,7 +2556,8 @@ async fn realtime_v2_noop_tool_call_returns_empty_function_output_without_respon test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2629,7 +2657,8 @@ async fn conversation_mirrors_assistant_message_text_to_realtime_handoff() -> Re test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2767,7 +2796,8 @@ async fn conversation_handoff_persists_across_item_done_until_turn_complete() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -2920,7 +2950,8 @@ async fn inbound_handoff_request_starts_turn() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -3023,7 +3054,8 @@ async fn inbound_handoff_request_uses_active_transcript() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -3127,7 +3159,8 @@ async fn inbound_handoff_request_sends_transcript_delta_after_each_handoff() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -3229,7 +3262,8 @@ async fn inbound_conversation_item_does_not_start_turn_and_still_forwards_audio( test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -3353,7 +3387,8 @@ async fn delegated_turn_user_role_echo_does_not_redelegate_and_still_forwards_au test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -3507,7 +3542,8 @@ async fn inbound_handoff_request_does_not_block_realtime_event_forwarding() -> R test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -3645,7 +3681,8 @@ async fn inbound_handoff_request_steers_active_turn() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), @@ -3799,7 +3836,8 @@ async fn inbound_handoff_request_starts_turn_and_does_not_block_realtime_audio() test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { architecture: None, - codex_responses_as_silent_context: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 416da00b7cbb..285e8eebed6f 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -195,8 +195,10 @@ pub struct McpServerRefreshConfig { pub struct ConversationStartParams { /// Overrides the configured realtime architecture for this session only. pub architecture: Option, - /// Inserts automatic Codex responses as silent realtime context instead of speakable output. - pub codex_responses_as_silent_context: bool, + /// Sends automatic Codex responses as realtime conversation items instead of handoff appends. + pub codex_responses_as_items: bool, + /// Optional prefix added to automatic Codex response items when `codex_responses_as_items` is set. + pub codex_response_item_prefix: Option, /// Overrides the configured realtime model for this session only. pub model: Option, /// Selects whether the realtime session should produce text or audio output. @@ -422,11 +424,6 @@ pub enum ConversationTextRole { Developer, } -#[derive(Debug, Clone, PartialEq)] -pub struct ConversationSilentContextParams { - pub text: String, -} - #[derive(Debug, Clone, PartialEq)] pub struct ConversationSpeechParams { pub text: String, @@ -528,9 +525,6 @@ pub enum Op { /// Send text input to the running realtime conversation stream. RealtimeConversationText(ConversationTextParams), - /// Append silent context to the running realtime conversation stream. - RealtimeConversationSilentContext(ConversationSilentContextParams), - /// Append speakable text to the running realtime conversation stream. RealtimeConversationSpeech(ConversationSpeechParams), @@ -787,7 +781,6 @@ impl Op { Self::RealtimeConversationStart(_) => "realtime_conversation_start", Self::RealtimeConversationAudio(_) => "realtime_conversation_audio", Self::RealtimeConversationText(_) => "realtime_conversation_text", - Self::RealtimeConversationSilentContext(_) => "realtime_conversation_silent_context", Self::RealtimeConversationSpeech(_) => "realtime_conversation_speech", Self::RealtimeConversationClose => "realtime_conversation_close", Self::RealtimeConversationListVoices => "realtime_conversation_list_voices", From 63c228e9f3b6fb131451e72ab6ba3716ba2bac1e Mon Sep 17 00:00:00 2001 From: Guinness Chen Date: Mon, 15 Jun 2026 13:14:31 -0700 Subject: [PATCH 07/11] Tighten realtime outbound routing types --- codex-rs/core/src/realtime_conversation.rs | 181 ++++++------------ .../core/src/realtime_conversation_tests.rs | 7 +- 2 files changed, 66 insertions(+), 122 deletions(-) diff --git a/codex-rs/core/src/realtime_conversation.rs b/codex-rs/core/src/realtime_conversation.rs index 2e4267954d70..1db6b331aa9f 100644 --- a/codex-rs/core/src/realtime_conversation.rs +++ b/codex-rs/core/src/realtime_conversation.rs @@ -102,40 +102,26 @@ enum RealtimeSessionKind { #[derive(Clone, Debug)] struct RealtimeHandoffState { - output_tx: Sender, + output_tx: Sender, active_handoff: Arc>>, last_output_text: Arc>>, - codex_responses_as_items: bool, - codex_response_item_prefix: Option, + codex_output: RealtimeCodexOutputConfig, session_kind: RealtimeSessionKind, } #[derive(Debug, PartialEq, Eq)] -enum HandoffOutput { - StandaloneAssistantOutput { - output_text: String, - }, - ProgressUpdate { - handoff_id: String, - output_text: String, - }, - FinalUpdate { - handoff_id: String, - output_text: String, - }, - Item { - text: String, - }, - ItemProgress { - handoff_id: String, - text: String, - }, - ItemHandoffComplete { - handoff_id: String, - }, - Speech { - text: String, - }, +enum RealtimeOutbound { + StandaloneHandoff { text: String }, + HandoffUpdate { handoff_id: String, text: String }, + CompletedHandoff { handoff_id: String, text: String }, + ConversationItem { text: String }, + HandoffCompleteAck { handoff_id: String }, +} + +#[derive(Clone, Debug)] +struct RealtimeCodexOutputConfig { + responses_as_items: bool, + response_item_prefix: Option, } #[derive(Debug, PartialEq, Eq)] @@ -210,7 +196,7 @@ struct RealtimeInputTask { writer: RealtimeWebsocketWriter, events: RealtimeWebsocketEvents, text_rx: Receiver, - handoff_output_rx: Receiver, + handoff_output_rx: Receiver, audio_rx: Receiver, events_tx: Sender, handoff_state: RealtimeHandoffState, @@ -220,23 +206,21 @@ struct RealtimeInputTask { struct RealtimeInputChannels { text_rx: Receiver, - handoff_output_rx: Receiver, + handoff_output_rx: Receiver, audio_rx: Receiver, } impl RealtimeHandoffState { fn new( - output_tx: Sender, - codex_responses_as_items: bool, - codex_response_item_prefix: Option, + output_tx: Sender, + codex_output: RealtimeCodexOutputConfig, session_kind: RealtimeSessionKind, ) -> Self { Self { output_tx, active_handoff: Arc::new(Mutex::new(None)), last_output_text: Arc::new(Mutex::new(None)), - codex_responses_as_items, - codex_response_item_prefix, + codex_output, session_kind, } } @@ -257,8 +241,7 @@ struct RealtimeStart { api_provider: ApiProvider, architecture: RealtimeConversationArchitecture, extra_headers: Option, - codex_responses_as_items: bool, - codex_response_item_prefix: Option, + codex_output: RealtimeCodexOutputConfig, realtime_call_api_provider: Option, session_config: RealtimeSessionConfig, model_client: ModelClient, @@ -313,8 +296,7 @@ impl RealtimeConversationManager { api_provider, architecture, extra_headers, - codex_responses_as_items, - codex_response_item_prefix, + codex_output, realtime_call_api_provider, session_config, model_client, @@ -331,17 +313,12 @@ impl RealtimeConversationManager { let (text_tx, text_rx) = async_channel::bounded::(TEXT_IN_QUEUE_CAPACITY); let (handoff_output_tx, handoff_output_rx) = - async_channel::bounded::(HANDOFF_OUT_QUEUE_CAPACITY); + async_channel::bounded::(HANDOFF_OUT_QUEUE_CAPACITY); let (events_tx, events_rx) = async_channel::bounded::(OUTPUT_EVENTS_QUEUE_CAPACITY); let realtime_active = Arc::new(AtomicBool::new(true)); - let handoff = RealtimeHandoffState::new( - handoff_output_tx, - codex_responses_as_items, - codex_response_item_prefix, - session_kind, - ); + let handoff = RealtimeHandoffState::new(handoff_output_tx, codex_output, session_kind); let input_channels = RealtimeInputChannels { text_rx, handoff_output_rx, @@ -512,33 +489,32 @@ impl RealtimeConversationManager { Some(handoff_id) => { let output_text = realtime_backend_output(output_text, handoff.session_kind); *handoff.last_output_text.lock().await = Some(output_text.clone()); - if handoff.codex_responses_as_items { - HandoffOutput::ItemProgress { - handoff_id, + if handoff.codex_output.responses_as_items { + RealtimeOutbound::ConversationItem { text: realtime_backend_item( output_text, - handoff.codex_response_item_prefix.as_deref(), + handoff.codex_output.response_item_prefix.as_deref(), ), } } else { - HandoffOutput::ProgressUpdate { + RealtimeOutbound::HandoffUpdate { handoff_id, - output_text, + text: output_text, } } } None if output_text.trim().is_empty() => return Ok(()), None => { let output_text = realtime_backend_output(output_text, handoff.session_kind); - if handoff.codex_responses_as_items { - HandoffOutput::Item { + if handoff.codex_output.responses_as_items { + RealtimeOutbound::ConversationItem { text: realtime_backend_item( output_text, - handoff.codex_response_item_prefix.as_deref(), + handoff.codex_output.response_item_prefix.as_deref(), ), } } else { - HandoffOutput::StandaloneAssistantOutput { output_text } + RealtimeOutbound::StandaloneHandoff { text: output_text } } } }; @@ -567,7 +543,7 @@ impl RealtimeConversationManager { handoff .output_tx - .send(HandoffOutput::Speech { + .send(RealtimeOutbound::StandaloneHandoff { text: realtime_backend_output(text, handoff.session_kind), }) .await @@ -595,12 +571,12 @@ impl RealtimeConversationManager { return Ok(()); }; - let output = if handoff.codex_responses_as_items { - HandoffOutput::ItemHandoffComplete { handoff_id } + let output = if handoff.codex_output.responses_as_items { + RealtimeOutbound::HandoffCompleteAck { handoff_id } } else { - HandoffOutput::FinalUpdate { + RealtimeOutbound::CompletedHandoff { handoff_id, - output_text, + text: output_text, } }; @@ -693,8 +669,7 @@ struct PreparedRealtimeConversationStart { api_provider: ApiProvider, architecture: RealtimeConversationArchitecture, extra_headers: Option, - codex_responses_as_items: bool, - codex_response_item_prefix: Option, + codex_output: RealtimeCodexOutputConfig, realtime_call_api_provider: Option, requested_realtime_session_id: Option, version: RealtimeWsVersion, @@ -770,8 +745,10 @@ async fn prepare_realtime_start( api_provider, architecture, extra_headers, - codex_responses_as_items: params.codex_responses_as_items, - codex_response_item_prefix: params.codex_response_item_prefix, + codex_output: RealtimeCodexOutputConfig { + responses_as_items: params.codex_responses_as_items, + response_item_prefix: params.codex_response_item_prefix, + }, realtime_call_api_provider, requested_realtime_session_id, version, @@ -930,8 +907,7 @@ async fn handle_start_inner( api_provider, architecture, extra_headers, - codex_responses_as_items, - codex_response_item_prefix, + codex_output, realtime_call_api_provider, requested_realtime_session_id, version, @@ -947,8 +923,7 @@ async fn handle_start_inner( api_provider, architecture, extra_headers, - codex_responses_as_items, - codex_response_item_prefix, + codex_output, realtime_call_api_provider, session_config, model_client: sess.services.model_client.clone(), @@ -1360,7 +1335,7 @@ async fn handle_text_input( } async fn handle_handoff_output( - handoff_output: Result, + handoff_output: Result, writer: &RealtimeWebsocketWriter, events_tx: &Sender, handoff_state: &RealtimeHandoffState, @@ -1370,45 +1345,30 @@ async fn handle_handoff_output( let result = match event_parser { RealtimeEventParser::V1 => match handoff_output { - HandoffOutput::StandaloneAssistantOutput { output_text } - | HandoffOutput::Speech { text: output_text } => { + RealtimeOutbound::StandaloneHandoff { text } => { + // TODO(guinness): Use the new client event for standalone handoffs once the API changes are complete. writer - .send_conversation_handoff_append( - STANDALONE_HANDOFF_ID.to_string(), - output_text, - ) + .send_conversation_handoff_append(STANDALONE_HANDOFF_ID.to_string(), text) .await } - HandoffOutput::ProgressUpdate { - handoff_id, - output_text, - } - | HandoffOutput::FinalUpdate { - handoff_id, - output_text, - } => { + RealtimeOutbound::HandoffUpdate { handoff_id, text } + | RealtimeOutbound::CompletedHandoff { handoff_id, text } => { writer - .send_conversation_function_call_output(handoff_id, output_text) + .send_conversation_function_call_output(handoff_id, text) .await } - HandoffOutput::Item { text } | HandoffOutput::ItemProgress { text, .. } => { + RealtimeOutbound::ConversationItem { text } => { writer.send_conversation_context_item_create(text).await } - HandoffOutput::ItemHandoffComplete { .. } => Ok(()), + RealtimeOutbound::HandoffCompleteAck { .. } => Ok(()), }, RealtimeEventParser::RealtimeV2 => match handoff_output { - HandoffOutput::StandaloneAssistantOutput { output_text } => { + RealtimeOutbound::StandaloneHandoff { text } => { writer - .send_conversation_handoff_append( - STANDALONE_HANDOFF_ID.to_string(), - output_text, - ) + .send_conversation_handoff_append(STANDALONE_HANDOFF_ID.to_string(), text) .await } - HandoffOutput::ProgressUpdate { - handoff_id, - output_text, - } => { + RealtimeOutbound::HandoffUpdate { handoff_id, text } => { let active_handoff = handoff_state.active_handoff.lock().await.clone(); match active_handoff { Some(active_handoff) if active_handoff == handoff_id => {} @@ -1418,41 +1378,22 @@ async fn handle_handoff_output( } } writer - .send_conversation_handoff_append(handoff_id, output_text) + .send_conversation_handoff_append(handoff_id, text) .await } - HandoffOutput::FinalUpdate { - handoff_id, - output_text, - } => { + RealtimeOutbound::CompletedHandoff { handoff_id, text } => { writer - .send_conversation_handoff_append(handoff_id, output_text) + .send_conversation_handoff_append(handoff_id, text) .await } - HandoffOutput::Item { text } => { + RealtimeOutbound::ConversationItem { text } => { writer.send_conversation_context_item_create(text).await } - HandoffOutput::ItemProgress { handoff_id, text } => { - let active_handoff = handoff_state.active_handoff.lock().await.clone(); - match active_handoff { - Some(active_handoff) if active_handoff == handoff_id => {} - Some(_) | None => { - debug!("dropping stale realtime handoff item update"); - return Ok(()); - } - } - writer.send_conversation_context_item_create(text).await - } - HandoffOutput::ItemHandoffComplete { handoff_id } => { + RealtimeOutbound::HandoffCompleteAck { handoff_id } => { writer .send_conversation_function_call_output(handoff_id, String::new()) .await } - HandoffOutput::Speech { text } => { - writer - .send_conversation_handoff_append(STANDALONE_HANDOFF_ID.to_string(), text) - .await - } }, }; if let Err(err) = result { diff --git a/codex-rs/core/src/realtime_conversation_tests.rs b/codex-rs/core/src/realtime_conversation_tests.rs index 147e0cc1236b..f15012aeecd8 100644 --- a/codex-rs/core/src/realtime_conversation_tests.rs +++ b/codex-rs/core/src/realtime_conversation_tests.rs @@ -1,3 +1,4 @@ +use super::RealtimeCodexOutputConfig; use super::RealtimeHandoffState; use super::RealtimeSessionKind; use super::realtime_delegation_from_handoff; @@ -130,8 +131,10 @@ async fn clears_active_handoff_explicitly() { let (tx, _rx) = bounded(1); let state = RealtimeHandoffState::new( tx, - /*codex_responses_as_items*/ false, - /*codex_response_item_prefix*/ None, + RealtimeCodexOutputConfig { + responses_as_items: false, + response_item_prefix: None, + }, RealtimeSessionKind::V1, ); From ebabc004ff039fb065b0309f3ba30600f12a376f Mon Sep 17 00:00:00 2001 From: Guinness Chen Date: Mon, 15 Jun 2026 13:48:19 -0700 Subject: [PATCH 08/11] Reuse generic realtime item create helper --- .../src/endpoint/realtime_websocket/methods.rs | 5 ++--- .../endpoint/realtime_websocket/methods_common.rs | 12 ------------ .../src/endpoint/realtime_websocket/methods_v1.rs | 4 ---- .../src/endpoint/realtime_websocket/methods_v2.rs | 4 ---- 4 files changed, 2 insertions(+), 23 deletions(-) diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs index 5470953be1a9..f0308f5fe596 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs @@ -1,4 +1,3 @@ -use crate::endpoint::realtime_websocket::methods_common::conversation_context_item_create_message; use crate::endpoint::realtime_websocket::methods_common::conversation_function_call_output_message; use crate::endpoint::realtime_websocket::methods_common::conversation_item_create_message; use crate::endpoint::realtime_websocket::methods_common::normalized_session_mode; @@ -309,8 +308,8 @@ impl RealtimeWebsocketWriter { &self, text: String, ) -> Result<(), ApiError> { - let message = conversation_context_item_create_message(self.event_parser, text); - self.send_json(&message).await + self.send_conversation_item_create(text, ConversationTextRole::Developer) + .await } pub async fn send_conversation_handoff_append( diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_common.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_common.rs index c96b9ac0b05c..131cf27a9447 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_common.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_common.rs @@ -1,9 +1,7 @@ -use crate::endpoint::realtime_websocket::methods_v1::conversation_context_item_create_message as v1_conversation_context_item_create_message; use crate::endpoint::realtime_websocket::methods_v1::conversation_handoff_append_message as v1_conversation_handoff_append_message; use crate::endpoint::realtime_websocket::methods_v1::conversation_item_create_message as v1_conversation_item_create_message; use crate::endpoint::realtime_websocket::methods_v1::session_update_session as v1_session_update_session; use crate::endpoint::realtime_websocket::methods_v1::websocket_intent as v1_websocket_intent; -use crate::endpoint::realtime_websocket::methods_v2::conversation_context_item_create_message as v2_conversation_context_item_create_message; use crate::endpoint::realtime_websocket::methods_v2::conversation_function_call_output_message as v2_conversation_function_call_output_message; use crate::endpoint::realtime_websocket::methods_v2::conversation_item_create_message as v2_conversation_item_create_message; use crate::endpoint::realtime_websocket::methods_v2::session_update_session as v2_session_update_session; @@ -44,16 +42,6 @@ pub(super) fn conversation_item_create_message( } } -pub(super) fn conversation_context_item_create_message( - event_parser: RealtimeEventParser, - text: String, -) -> RealtimeOutboundMessage { - match event_parser { - RealtimeEventParser::V1 => v1_conversation_context_item_create_message(text), - RealtimeEventParser::RealtimeV2 => v2_conversation_context_item_create_message(text), - } -} - pub(super) fn conversation_function_call_output_message( event_parser: RealtimeEventParser, call_id: String, diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v1.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v1.rs index 223e5d431c43..a7f73d82f3c6 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v1.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v1.rs @@ -15,10 +15,6 @@ use crate::endpoint::realtime_websocket::protocol::SessionType; use crate::endpoint::realtime_websocket::protocol::SessionUpdateSession; use codex_protocol::protocol::ConversationTextRole; -pub(super) fn conversation_context_item_create_message(text: String) -> RealtimeOutboundMessage { - conversation_item_create_message(text, ConversationTextRole::Developer) -} - pub(super) fn conversation_item_create_message( text: String, role: ConversationTextRole, diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v2.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v2.rs index f2947c4f07ef..702b4be8692e 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v2.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v2.rs @@ -36,10 +36,6 @@ const REALTIME_V2_SILENCE_TOOL_NAME: &str = "remain_silent"; const REALTIME_V2_SILENCE_TOOL_DESCRIPTION: &str = "Call this when the best response is to say nothing. Use it instead of speaking after hidden system/control messages, after background agent updates in silent modes, or whenever acknowledging aloud would be distracting. This tool has no user-visible effect."; const REALTIME_V2_INPUT_TRANSCRIPTION_MODEL: &str = "gpt-4o-mini-transcribe"; -pub(super) fn conversation_context_item_create_message(text: String) -> RealtimeOutboundMessage { - conversation_item_create_message(text, ConversationTextRole::Developer) -} - pub(super) fn conversation_item_create_message( text: String, role: ConversationTextRole, From 47b863388a7b9f17d1c6cad1301ada846b9f8fba Mon Sep 17 00:00:00 2001 From: Guinness Chen Date: Mon, 15 Jun 2026 13:54:54 -0700 Subject: [PATCH 09/11] Remove realtime context item writer --- .../codex-api/src/endpoint/realtime_websocket/methods.rs | 8 -------- codex-rs/core/src/realtime_conversation.rs | 8 ++++++-- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs index f0308f5fe596..0aa2feb04e76 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs @@ -304,14 +304,6 @@ impl RealtimeWebsocketWriter { .await } - pub async fn send_conversation_context_item_create( - &self, - text: String, - ) -> Result<(), ApiError> { - self.send_conversation_item_create(text, ConversationTextRole::Developer) - .await - } - pub async fn send_conversation_handoff_append( &self, handoff_id: String, diff --git a/codex-rs/core/src/realtime_conversation.rs b/codex-rs/core/src/realtime_conversation.rs index 1db6b331aa9f..46bf61881f9c 100644 --- a/codex-rs/core/src/realtime_conversation.rs +++ b/codex-rs/core/src/realtime_conversation.rs @@ -1358,7 +1358,9 @@ async fn handle_handoff_output( .await } RealtimeOutbound::ConversationItem { text } => { - writer.send_conversation_context_item_create(text).await + writer + .send_conversation_item_create(text, ConversationTextRole::Developer) + .await } RealtimeOutbound::HandoffCompleteAck { .. } => Ok(()), }, @@ -1387,7 +1389,9 @@ async fn handle_handoff_output( .await } RealtimeOutbound::ConversationItem { text } => { - writer.send_conversation_context_item_create(text).await + writer + .send_conversation_item_create(text, ConversationTextRole::Developer) + .await } RealtimeOutbound::HandoffCompleteAck { handoff_id } => { writer From 25cdef98895a025d1e87c771be927e5ca0a82ee7 Mon Sep 17 00:00:00 2001 From: Guinness Chen Date: Mon, 15 Jun 2026 14:54:33 -0700 Subject: [PATCH 10/11] Keep v2 realtime output off handoff append --- codex-rs/app-server/README.md | 13 ++--- .../tests/suite/v2/realtime_conversation.rs | 51 +++++++++++++------ codex-rs/core/src/realtime_conversation.rs | 36 ++++++++++--- 3 files changed, 73 insertions(+), 27 deletions(-) diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 35cadfe0ca39..abf5d1085722 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -165,10 +165,10 @@ Example with notification opt-out: - `thread/inject_items` — append raw Responses API items to a loaded thread’s model-visible history without starting a user turn; returns `{}` on success. - `turn/steer` — add user input to an already in-flight regular turn without starting a new turn; returns the active `turnId` that accepted the input. `clientUserMessageId` is optional; when supplied, the corresponding `userMessage` item echoes it as `clientId`. Review and manual compaction turns reject `turn/steer`. - `turn/interrupt` — request cancellation of an in-flight turn by `(thread_id, turn_id)`; success is an empty `{}` response and the turn finishes with `status: "interrupted"`. -- `thread/realtime/start` — start a thread-scoped realtime session (experimental); pass `outputModality: "text"` or `outputModality: "audio"` to choose model output, and optionally pass `model` and `version` to override configured realtime selection for this session only. By default, automatic Codex text follows the legacy speakable handoff path. Pass `codexResponsesAsItems: true` to send automatic Codex responses as realtime conversation items instead, and optionally pass `codexResponseItemPrefix` to prepend experiment instructions to those items. Returns `{}` and streams `thread/realtime/*` notifications. Omit `transport` for the websocket transport, or pass `{ "type": "webrtc", "sdp": "..." }` to create a WebRTC session from a browser-generated SDP offer; the remote answer SDP is emitted as `thread/realtime/sdp`. +- `thread/realtime/start` — start a thread-scoped realtime session (experimental); pass `outputModality: "text"` or `outputModality: "audio"` to choose model output, and optionally pass `model` and `version` to override configured realtime selection for this session only. By default, automatic Codex text follows the protocol's speakable output path. Pass `codexResponsesAsItems: true` to send automatic Codex responses as realtime conversation items instead, and optionally pass `codexResponseItemPrefix` to prepend experiment instructions to those items. Returns `{}` and streams `thread/realtime/*` notifications. Omit `transport` for the websocket transport, or pass `{ "type": "webrtc", "sdp": "..." }` to create a WebRTC session from a browser-generated SDP offer; the remote answer SDP is emitted as `thread/realtime/sdp`. - `thread/realtime/appendAudio` — append an input audio chunk to the active realtime session (experimental); returns `{}`. - `thread/realtime/appendText` — append text input to the active realtime session with a required `role` of `user` or `developer` (experimental); returns `{}`. Older clients that omit `role` default to `user`. -- `thread/realtime/appendSpeech` — append text that the realtime model should speak to the user (experimental); returns `{}` and sends `conversation.handoff.append`. +- `thread/realtime/appendSpeech` — append text that the realtime model should speak to the user (experimental); returns `{}`. - `thread/realtime/stop` — stop the active realtime session for the thread (experimental); returns `{}`. - `review/start` — kick off Codex’s automated reviewer for a thread; responds like `turn/start` and emits `item/started`/`item/completed` notifications with `enteredReviewMode` and `exitedReviewMode` items, plus a final assistant `agentMessage` containing the review. - `command/exec` — run a single command under the server sandbox without starting a thread/turn (handy for utilities and validation). @@ -872,10 +872,11 @@ Omit `prompt` to use Codex's default realtime backend prompt. Send `prompt: null Clients may also pass `model` and `version` on `thread/realtime/start` to select a different realtime session configuration without changing thread or user config. Pass `codexResponsesAsItems: true` to inject automatic Codex responses with -`conversation.item.create` instead of the legacy handoff append path. When using -that mode, `codexResponseItemPrefix` can prepend short experiment instructions -to each automatic Codex response item. Omit `codexResponsesAsItems`, or pass -`false`, to preserve the legacy speakable handoff behavior. Call +`conversation.item.create` instead of the protocol's default speakable output +path. When using that mode, `codexResponseItemPrefix` can prepend short +experiment instructions to each automatic Codex response item. Omit +`codexResponsesAsItems`, or pass `false`, to preserve the default speakable +behavior. Call `thread/realtime/appendText` to append app-provided realtime text items, or `thread/realtime/appendSpeech` when the app decides a realtime update should be spoken. diff --git a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs index 35498ae7d1ee..6f9ed7f67488 100644 --- a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs +++ b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs @@ -82,6 +82,8 @@ const DELEGATED_SHELL_TOOL_TIMEOUT_MS: u64 = 30_000; const STARTUP_CONTEXT_HEADER: &str = "Startup context from Codex."; const V2_STEERING_ACKNOWLEDGEMENT: &str = "This was sent to steer the previous background agent task."; +const V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT: &str = + "Background agent finished. Use the preceding [BACKEND] messages as the result."; const RESPONSE_ITEM_PREFIX: &str = "Use the following context to inform future responses, but do not speak it to the user."; @@ -1578,6 +1580,7 @@ async fn realtime_automatic_standalone_output_is_item_and_append_speaks() -> Res &harness.sideband_outbound_request(/*request_index*/ 2).await, "manual voice update", ); + assert_v2_response_create(&harness.sideband_outbound_request(/*request_index*/ 3).await); harness.shutdown().await; Ok(()) @@ -1651,6 +1654,7 @@ async fn realtime_automatic_handoff_output_is_item_and_append_speaks() -> Result &harness.sideband_outbound_request(/*request_index*/ 3).await, "manual spoken update", ); + assert_v2_response_create(&harness.sideband_outbound_request(/*request_index*/ 4).await); harness.shutdown().await; Ok(()) @@ -2039,8 +2043,8 @@ async fn webrtc_v2_background_agent_tool_call_delegates_and_returns_function_out let progress = harness.sideband_outbound_request(/*request_index*/ 1).await; assert_v2_progress_update(&progress, "delegated from v2"); - let final_update = harness.sideband_outbound_request(/*request_index*/ 2).await; - assert_v2_progress_update(&final_update, "delegated from v2"); + let tool_output = harness.sideband_outbound_request(/*request_index*/ 2).await; + assert_v2_function_call_output(&tool_output, "call_v2", V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT); harness.shutdown().await; Ok(()) @@ -2166,8 +2170,12 @@ async fn webrtc_v2_background_agent_progress_is_sent_before_function_output() -> let progress = harness.sideband_outbound_request(/*request_index*/ 1).await; assert_v2_progress_update(&progress, "progress before final"); - let final_update = harness.sideband_outbound_request(/*request_index*/ 2).await; - assert_v2_progress_update(&final_update, "progress before final"); + let tool_output = harness.sideband_outbound_request(/*request_index*/ 2).await; + assert_v2_function_call_output( + &tool_output, + "call_progress_order", + V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT, + ); harness.shutdown().await; Ok(()) @@ -2252,8 +2260,12 @@ async fn webrtc_v2_tool_call_delegated_turn_can_execute_shell_tool() -> Result<( let progress = harness.sideband_outbound_request(/*request_index*/ 1).await; assert_v2_progress_update(&progress, "shell tool finished"); - let final_update = harness.sideband_outbound_request(/*request_index*/ 2).await; - assert_v2_progress_update(&final_update, "shell tool finished"); + let tool_output = harness.sideband_outbound_request(/*request_index*/ 2).await; + assert_v2_function_call_output( + &tool_output, + "call_shell", + V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT, + ); harness.shutdown().await; Ok(()) @@ -2328,8 +2340,12 @@ async fn webrtc_v2_tool_call_does_not_block_sideband_audio() -> Result<()> { let progress = harness.sideband_outbound_request(/*request_index*/ 1).await; assert_v2_progress_update(&progress, "late delegated result"); - let final_update = harness.sideband_outbound_request(/*request_index*/ 2).await; - assert_v2_progress_update(&final_update, "late delegated result"); + let tool_output = harness.sideband_outbound_request(/*request_index*/ 2).await; + assert_v2_function_call_output( + &tool_output, + "call_audio", + V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT, + ); harness.shutdown().await; Ok(()) @@ -2579,14 +2595,19 @@ fn assert_v2_function_call_output(request: &Value, call_id: &str, expected_outpu } fn assert_v2_progress_update(request: &Value, expected_text: &str) { - let expected_output_text = format!("[BACKEND] {expected_text}"); - assert_eq!( - request["type"].as_str(), - Some("conversation.handoff.append") - ); assert_eq!( - request["output_text"].as_str(), - Some(expected_output_text.as_str()) + request, + &json!({ + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "user", + "content": [{ + "type": "input_text", + "text": format!("[BACKEND] {expected_text}") + }] + } + }) ); } diff --git a/codex-rs/core/src/realtime_conversation.rs b/codex-rs/core/src/realtime_conversation.rs index 46bf61881f9c..c40b71961fca 100644 --- a/codex-rs/core/src/realtime_conversation.rs +++ b/codex-rs/core/src/realtime_conversation.rs @@ -73,6 +73,8 @@ const STANDALONE_HANDOFF_ID: &str = "codex"; const DEFAULT_REALTIME_MODEL: &str = "gpt-realtime-1.5"; pub(crate) const REALTIME_USER_TEXT_PREFIX: &str = "[USER] "; pub(crate) const REALTIME_BACKEND_TEXT_PREFIX: &str = "[BACKEND] "; +const REALTIME_V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT: &str = + "Background agent finished. Use the preceding [BACKEND] messages as the result."; const REALTIME_V2_STEER_ACKNOWLEDGEMENT: &str = "This was sent to steer the previous background agent task."; const REALTIME_ACTIVE_RESPONSE_ERROR_PREFIX: &str = @@ -1285,6 +1287,7 @@ async fn run_realtime_input_task(input: RealtimeInputTask) { &events_tx, &handoff_state, event_parser, + &mut response_create_queue, ) .await } @@ -1340,6 +1343,7 @@ async fn handle_handoff_output( events_tx: &Sender, handoff_state: &RealtimeHandoffState, event_parser: RealtimeEventParser, + response_create_queue: &mut RealtimeResponseCreateQueue, ) -> anyhow::Result<()> { let handoff_output = handoff_output.context("handoff output channel closed")?; @@ -1366,9 +1370,16 @@ async fn handle_handoff_output( }, RealtimeEventParser::RealtimeV2 => match handoff_output { RealtimeOutbound::StandaloneHandoff { text } => { - writer - .send_conversation_handoff_append(STANDALONE_HANDOFF_ID.to_string(), text) + if let Err(err) = writer + .send_conversation_item_create(text, ConversationTextRole::User) .await + { + Err(err) + } else { + return response_create_queue + .request_create(writer, events_tx, "standalone handoff") + .await; + } } RealtimeOutbound::HandoffUpdate { handoff_id, text } => { let active_handoff = handoff_state.active_handoff.lock().await.clone(); @@ -1380,13 +1391,26 @@ async fn handle_handoff_output( } } writer - .send_conversation_handoff_append(handoff_id, text) + .send_conversation_item_create(text, ConversationTextRole::User) .await } - RealtimeOutbound::CompletedHandoff { handoff_id, text } => { - writer - .send_conversation_handoff_append(handoff_id, text) + RealtimeOutbound::CompletedHandoff { + handoff_id, + text: _, + } => { + if let Err(err) = writer + .send_conversation_function_call_output( + handoff_id, + REALTIME_V2_HANDOFF_COMPLETE_ACKNOWLEDGEMENT.to_string(), + ) .await + { + Err(err) + } else { + return response_create_queue + .request_create(writer, events_tx, "handoff") + .await; + } } RealtimeOutbound::ConversationItem { text } => { writer From 6acf653491ae1d21449072be224fc5977730564d Mon Sep 17 00:00:00 2001 From: Guinness Chen Date: Mon, 15 Jun 2026 15:41:20 -0700 Subject: [PATCH 11/11] Flatten realtime Codex output config --- codex-rs/core/src/realtime_conversation.rs | 53 ++++++++++--------- .../core/src/realtime_conversation_tests.rs | 7 +-- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/codex-rs/core/src/realtime_conversation.rs b/codex-rs/core/src/realtime_conversation.rs index c40b71961fca..76f7e3437ead 100644 --- a/codex-rs/core/src/realtime_conversation.rs +++ b/codex-rs/core/src/realtime_conversation.rs @@ -107,7 +107,8 @@ struct RealtimeHandoffState { output_tx: Sender, active_handoff: Arc>>, last_output_text: Arc>>, - codex_output: RealtimeCodexOutputConfig, + codex_responses_as_items: bool, + codex_response_item_prefix: Option, session_kind: RealtimeSessionKind, } @@ -120,12 +121,6 @@ enum RealtimeOutbound { HandoffCompleteAck { handoff_id: String }, } -#[derive(Clone, Debug)] -struct RealtimeCodexOutputConfig { - responses_as_items: bool, - response_item_prefix: Option, -} - #[derive(Debug, PartialEq, Eq)] struct OutputAudioState { item_id: String, @@ -215,14 +210,16 @@ struct RealtimeInputChannels { impl RealtimeHandoffState { fn new( output_tx: Sender, - codex_output: RealtimeCodexOutputConfig, + codex_responses_as_items: bool, + codex_response_item_prefix: Option, session_kind: RealtimeSessionKind, ) -> Self { Self { output_tx, active_handoff: Arc::new(Mutex::new(None)), last_output_text: Arc::new(Mutex::new(None)), - codex_output, + codex_responses_as_items, + codex_response_item_prefix, session_kind, } } @@ -243,7 +240,8 @@ struct RealtimeStart { api_provider: ApiProvider, architecture: RealtimeConversationArchitecture, extra_headers: Option, - codex_output: RealtimeCodexOutputConfig, + codex_responses_as_items: bool, + codex_response_item_prefix: Option, realtime_call_api_provider: Option, session_config: RealtimeSessionConfig, model_client: ModelClient, @@ -298,7 +296,8 @@ impl RealtimeConversationManager { api_provider, architecture, extra_headers, - codex_output, + codex_responses_as_items, + codex_response_item_prefix, realtime_call_api_provider, session_config, model_client, @@ -320,7 +319,12 @@ impl RealtimeConversationManager { async_channel::bounded::(OUTPUT_EVENTS_QUEUE_CAPACITY); let realtime_active = Arc::new(AtomicBool::new(true)); - let handoff = RealtimeHandoffState::new(handoff_output_tx, codex_output, session_kind); + let handoff = RealtimeHandoffState::new( + handoff_output_tx, + codex_responses_as_items, + codex_response_item_prefix, + session_kind, + ); let input_channels = RealtimeInputChannels { text_rx, handoff_output_rx, @@ -491,11 +495,11 @@ impl RealtimeConversationManager { Some(handoff_id) => { let output_text = realtime_backend_output(output_text, handoff.session_kind); *handoff.last_output_text.lock().await = Some(output_text.clone()); - if handoff.codex_output.responses_as_items { + if handoff.codex_responses_as_items { RealtimeOutbound::ConversationItem { text: realtime_backend_item( output_text, - handoff.codex_output.response_item_prefix.as_deref(), + handoff.codex_response_item_prefix.as_deref(), ), } } else { @@ -508,11 +512,11 @@ impl RealtimeConversationManager { None if output_text.trim().is_empty() => return Ok(()), None => { let output_text = realtime_backend_output(output_text, handoff.session_kind); - if handoff.codex_output.responses_as_items { + if handoff.codex_responses_as_items { RealtimeOutbound::ConversationItem { text: realtime_backend_item( output_text, - handoff.codex_output.response_item_prefix.as_deref(), + handoff.codex_response_item_prefix.as_deref(), ), } } else { @@ -573,7 +577,7 @@ impl RealtimeConversationManager { return Ok(()); }; - let output = if handoff.codex_output.responses_as_items { + let output = if handoff.codex_responses_as_items { RealtimeOutbound::HandoffCompleteAck { handoff_id } } else { RealtimeOutbound::CompletedHandoff { @@ -671,7 +675,8 @@ struct PreparedRealtimeConversationStart { api_provider: ApiProvider, architecture: RealtimeConversationArchitecture, extra_headers: Option, - codex_output: RealtimeCodexOutputConfig, + codex_responses_as_items: bool, + codex_response_item_prefix: Option, realtime_call_api_provider: Option, requested_realtime_session_id: Option, version: RealtimeWsVersion, @@ -747,10 +752,8 @@ async fn prepare_realtime_start( api_provider, architecture, extra_headers, - codex_output: RealtimeCodexOutputConfig { - responses_as_items: params.codex_responses_as_items, - response_item_prefix: params.codex_response_item_prefix, - }, + codex_responses_as_items: params.codex_responses_as_items, + codex_response_item_prefix: params.codex_response_item_prefix, realtime_call_api_provider, requested_realtime_session_id, version, @@ -909,7 +912,8 @@ async fn handle_start_inner( api_provider, architecture, extra_headers, - codex_output, + codex_responses_as_items, + codex_response_item_prefix, realtime_call_api_provider, requested_realtime_session_id, version, @@ -925,7 +929,8 @@ async fn handle_start_inner( api_provider, architecture, extra_headers, - codex_output, + codex_responses_as_items, + codex_response_item_prefix, realtime_call_api_provider, session_config, model_client: sess.services.model_client.clone(), diff --git a/codex-rs/core/src/realtime_conversation_tests.rs b/codex-rs/core/src/realtime_conversation_tests.rs index f15012aeecd8..147e0cc1236b 100644 --- a/codex-rs/core/src/realtime_conversation_tests.rs +++ b/codex-rs/core/src/realtime_conversation_tests.rs @@ -1,4 +1,3 @@ -use super::RealtimeCodexOutputConfig; use super::RealtimeHandoffState; use super::RealtimeSessionKind; use super::realtime_delegation_from_handoff; @@ -131,10 +130,8 @@ async fn clears_active_handoff_explicitly() { let (tx, _rx) = bounded(1); let state = RealtimeHandoffState::new( tx, - RealtimeCodexOutputConfig { - responses_as_items: false, - response_item_prefix: None, - }, + /*codex_responses_as_items*/ false, + /*codex_response_item_prefix*/ None, RealtimeSessionKind::V1, );