Skip to content

Add per-session realtime model and version overrides #24999

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
guinness-oai merged 1 commit into from
Jun 10, 2026
Merged

Add per-session realtime model and version overrides #24999

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions codex-rs/app-server-protocol/schema/json/ClientRequest.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 14 additions & 0 deletions codex-rs/app-server-protocol/src/protocol/common.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2942,10 +2942,12 @@ mod tests {
request_id: RequestId::Integer(9),
params: v2::ThreadRealtimeStartParams {
thread_id: "thr_123".to_string(),
model: Some("realtime-treatment-model".to_string()),
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("You are on a call".to_string())),
realtime_session_id: Some("sess_456".to_string()),
transport: None,
version: Some(RealtimeConversationVersion::V1),
voice: Some(RealtimeVoice::Marin),
},
};
Expand All @@ -2955,10 +2957,12 @@ mod tests {
"id": 9,
"params": {
"threadId": "thr_123",
"model": "realtime-treatment-model",
"outputModality": "audio",
"prompt": "You are on a call",
"realtimeSessionId": "sess_456",
"transport": null,
"version": "v1",
"voice": "marin"
}
}),
Expand All @@ -2973,10 +2977,12 @@ mod tests {
request_id: RequestId::Integer(9),
params: v2::ThreadRealtimeStartParams {
thread_id: "thr_123".to_string(),
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: None,
realtime_session_id: None,
transport: None,
version: None,
voice: None,
},
};
Expand All @@ -2986,9 +2992,11 @@ mod tests {
"id": 9,
"params": {
"threadId": "thr_123",
"model": null,
"outputModality": "audio",
"realtimeSessionId": null,
"transport": null,
"version": null,
"voice": null
}
}),
Expand All @@ -2999,10 +3007,12 @@ mod tests {
request_id: RequestId::Integer(9),
params: v2::ThreadRealtimeStartParams {
thread_id: "thr_123".to_string(),
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(None),
realtime_session_id: None,
transport: None,
version: None,
voice: None,
},
};
Expand All @@ -3012,10 +3022,12 @@ mod tests {
"id": 9,
"params": {
"threadId": "thr_123",
"model": null,
"outputModality": "audio",
"prompt": null,
"realtimeSessionId": null,
"transport": null,
"version": null,
"voice": null
}
}),
Expand Down Expand Up @@ -3168,10 +3180,12 @@ mod tests {
request_id: RequestId::Integer(1),
params: v2::ThreadRealtimeStartParams {
thread_id: "thr_123".to_string(),
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("You are on a call".to_string())),
realtime_session_id: None,
transport: None,
version: None,
voice: None,
},
};
Expand Down
6 changes: 6 additions & 0 deletions codex-rs/app-server-protocol/src/protocol/v2/realtime.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,9 @@ impl From<ThreadRealtimeAudioChunk> for CoreRealtimeAudioFrame {
#[ts(export_to = "v2/")]
pub struct ThreadRealtimeStartParams {
pub thread_id: String,
/// Overrides the configured realtime model for this session only.
#[ts(optional = nullable)]
pub model: Option<String>,
/// Selects text or audio output for the realtime session. Transport and voice stay
/// independent so clients can choose how they connect separately from what the model emits.
pub output_modality: RealtimeOutputModality,
Expand All @@ -80,6 +83,9 @@ pub struct ThreadRealtimeStartParams {
pub realtime_session_id: Option<String>,
#[ts(optional = nullable)]
pub transport: Option<ThreadRealtimeStartTransport>,
/// Overrides the configured realtime protocol version for this session only.
#[ts(optional = nullable)]
pub version: Option<RealtimeConversationVersion>,
#[ts(optional = nullable)]
pub voice: Option<RealtimeVoice>,
}
Expand Down
4 changes: 3 additions & 1 deletion codex-rs/app-server/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ Example with notification opt-out:
- `thread/inject_items` — append raw Responses API items to a loaded thread’s model-visible history without starting a user turn; returns `{}` on success.
- `turn/steer` — add user input to an already in-flight regular turn without starting a new turn; returns the active `turnId` that accepted the input. `clientUserMessageId` is optional; when supplied, the corresponding `userMessage` item echoes it as `clientId`. Review and manual compaction turns reject `turn/steer`.
- `turn/interrupt` — request cancellation of an in-flight turn by `(thread_id, turn_id)`; success is an empty `{}` response and the turn finishes with `status: "interrupted"`.
- `thread/realtime/start` — start a thread-scoped realtime session (experimental); pass `outputModality: "text"` or `outputModality: "audio"` to choose model output, returns `{}` and streams `thread/realtime/*` notifications. Omit `transport` for the websocket transport, or pass `{ "type": "webrtc", "sdp": "..." }` to create a WebRTC session from a browser-generated SDP offer; the remote answer SDP is emitted as `thread/realtime/sdp`.
- `thread/realtime/start` — start a thread-scoped realtime session (experimental); pass `outputModality: "text"` or `outputModality: "audio"` to choose model output, and optionally pass `model` and `version` to override configured realtime selection for this session only. Returns `{}` and streams `thread/realtime/*` notifications. Omit `transport` for the websocket transport, or pass `{ "type": "webrtc", "sdp": "..." }` to create a WebRTC session from a browser-generated SDP offer; the remote answer SDP is emitted as `thread/realtime/sdp`.
- `thread/realtime/appendAudio` — append an input audio chunk to the active realtime session (experimental); returns `{}`.
- `thread/realtime/appendText` — append text input to the active realtime session (experimental); returns `{}`.
- `thread/realtime/stop` — stop the active realtime session for the thread (experimental); returns `{}`.
Expand Down Expand Up @@ -837,6 +837,8 @@ Then send `offer.sdp` to app-server. Core uses `experimental_realtime_ws_backend

Omit `prompt` to use Codex's default realtime backend prompt. Send `prompt: null` or
`prompt: ""` when the session should start without that default backend prompt.
Clients may also pass `model` and `version` on `thread/realtime/start` to select a
different realtime session configuration without changing thread or user config.

```javascript
await pc.setRemoteDescription({
Expand Down
2 changes: 2 additions & 0 deletions codex-rs/app-server/src/request_processors/turn_processor.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -934,6 +934,7 @@ impl TurnRequestProcessor {
request_id,
thread.as_ref(),
Op::RealtimeConversationStart(ConversationStartParams {
model: params.model,
output_modality: params.output_modality,
prompt: params.prompt,
realtime_session_id: params.realtime_session_id,
Expand All @@ -945,6 +946,7 @@ impl TurnRequestProcessor {
ConversationStartTransport::Webrtc { sdp }
}
}),
version: params.version,
voice: params.voice,
}),
)
Expand Down
4 changes: 4 additions & 0 deletions codex-rs/app-server/tests/suite/v2/experimental_api.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,10 +80,12 @@ async fn realtime_conversation_start_requires_experimental_api_capability() -> R
let request_id = mcp
.send_thread_realtime_start_request(ThreadRealtimeStartParams {
thread_id: "thr_123".to_string(),
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("hello".to_string())),
realtime_session_id: None,
transport: None,
version: None,
voice: None,
})
.await?;
Expand Down Expand Up @@ -186,12 +188,14 @@ async fn realtime_webrtc_start_requires_experimental_api_capability() -> Result<
let request_id = mcp
.send_thread_realtime_start_request(ThreadRealtimeStartParams {
thread_id: "thr_123".to_string(),
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("hello".to_string())),
realtime_session_id: None,
transport: Some(ThreadRealtimeStartTransport::Webrtc {
sdp: "v=offer\r\n".to_string(),
}),
version: None,
voice: None,
})
.await?;
Expand Down
18 changes: 18 additions & 0 deletions codex-rs/app-server/tests/suite/v2/realtime_conversation.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,12 +311,14 @@ impl RealtimeE2eHarness {
.mcp
.send_thread_realtime_start_request(ThreadRealtimeStartParams {
thread_id: self.thread_id.clone(),
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
realtime_session_id: None,
transport: Some(ThreadRealtimeStartTransport::Webrtc {
sdp: offer_sdp.to_string(),
}),
version: None,
voice: None,
})
.await?;
Expand Down Expand Up @@ -556,10 +558,12 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> {
let start_request_id = mcp
.send_thread_realtime_start_request(ThreadRealtimeStartParams {
thread_id: thread_start.thread.id.clone(),
model: Some("realtime-treatment-model".to_string()),
output_modality: RealtimeOutputModality::Audio,
prompt: None,
realtime_session_id: None,
transport: None,
version: None,
voice: Some(RealtimeVoice::Cedar),
})
.await?;
Expand Down Expand Up @@ -588,6 +592,10 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> {
startup_context_request.body_json()["session"]["audio"]["output"]["voice"],
"cedar"
);
assert_eq!(
realtime_server.single_handshake().uri(),
"/v1/realtime?model=realtime-treatment-model"
);
assert_eq!(
startup_context_request.body_json()["session"]["output_modalities"],
json!(["audio"])
Expand Down Expand Up @@ -805,10 +813,12 @@ async fn realtime_text_output_modality_requests_text_output_and_final_transcript
let start_request_id = mcp
.send_thread_realtime_start_request(ThreadRealtimeStartParams {
thread_id: thread_start.thread.id.clone(),
model: None,
output_modality: RealtimeOutputModality::Text,
prompt: None,
realtime_session_id: None,
transport: None,
version: None,
voice: None,
})
.await?;
Expand Down Expand Up @@ -979,10 +989,12 @@ async fn realtime_conversation_stop_emits_closed_notification() -> Result<()> {
let start_request_id = mcp
.send_thread_realtime_start_request(ThreadRealtimeStartParams {
thread_id: thread_start.thread.id.clone(),
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
realtime_session_id: None,
transport: None,
version: None,
voice: None,
})
.await?;
Expand Down Expand Up @@ -1076,12 +1088,14 @@ async fn realtime_webrtc_start_emits_sdp_notification() -> Result<()> {
let start_request_id = mcp
.send_thread_realtime_start_request(ThreadRealtimeStartParams {
thread_id: thread_id.clone(),
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
realtime_session_id: None,
transport: Some(ThreadRealtimeStartTransport::Webrtc {
sdp: "v=offer\r\n".to_string(),
}),
version: None,
voice: None,
})
.await?;
Expand Down Expand Up @@ -1991,12 +2005,14 @@ async fn realtime_webrtc_start_surfaces_backend_error() -> Result<()> {
let start_request_id = mcp
.send_thread_realtime_start_request(ThreadRealtimeStartParams {
thread_id: thread_start.thread.id,
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
realtime_session_id: None,
transport: Some(ThreadRealtimeStartTransport::Webrtc {
sdp: "v=offer\r\n".to_string(),
}),
version: None,
voice: None,
})
.await?;
Expand Down Expand Up @@ -2050,10 +2066,12 @@ async fn realtime_conversation_requires_feature_flag() -> Result<()> {
let start_request_id = mcp
.send_thread_realtime_start_request(ThreadRealtimeStartParams {
thread_id: thread_start.thread.id.clone(),
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
realtime_session_id: None,
transport: None,
version: None,
voice: None,
})
.await?;
Expand Down
21 changes: 11 additions & 10 deletions codex-rs/core/src/realtime_conversation.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -625,12 +625,14 @@ async fn prepare_realtime_start(
if let Some(realtime_ws_base_url) = &config.experimental_realtime_ws_base_url {
api_provider.base_url = realtime_ws_base_url.clone();
}
let version = config.realtime.version;
let version = params.version.unwrap_or(config.realtime.version);
let session_config = build_realtime_session_config(
sess,
params.model,
params.prompt,
params.realtime_session_id,
params.output_modality,
version,
params.voice,
)
.await?;
Expand Down Expand Up @@ -664,9 +666,11 @@ async fn prepare_realtime_start(

pub(crate) async fn build_realtime_session_config(
sess: &Arc<Session>,
model: Option<String>,
prompt: Option<Option<String>>,
realtime_session_id: Option<String>,
output_modality: RealtimeOutputModality,
version: RealtimeWsVersion,
voice: Option<RealtimeVoice>,
) -> CodexResult<RealtimeSessionConfig> {
let config = sess.get_config().await;
Expand All @@ -689,18 +693,15 @@ pub(crate) async fn build_realtime_session_config(
(false, false) => format!("{prompt}\n\n{startup_context}"),
};
let model = Some(
config
.experimental_realtime_ws_model
.clone()
model
.or_else(|| config.experimental_realtime_ws_model.clone())
.unwrap_or_else(|| DEFAULT_REALTIME_MODEL.to_string()),
);
let event_parser = match config.realtime.version {
let event_parser = match version {
RealtimeWsVersion::V1 => RealtimeEventParser::V1,
RealtimeWsVersion::V2 => RealtimeEventParser::RealtimeV2,
};
if config.realtime.version == RealtimeWsVersion::V1
&& matches!(output_modality, RealtimeOutputModality::Text)
{
if version == RealtimeWsVersion::V1 && matches!(output_modality, RealtimeOutputModality::Text) {
return Err(CodexErr::InvalidRequest(
"text realtime output modality requires realtime v2".to_string(),
));
Expand All @@ -711,8 +712,8 @@ pub(crate) async fn build_realtime_session_config(
};
let voice = voice
.or(config.realtime.voice)
.unwrap_or_else(|| default_realtime_voice(config.realtime.version));
validate_realtime_voice(config.realtime.version, voice)?;
.unwrap_or_else(|| default_realtime_voice(version));
validate_realtime_voice(version, voice)?;
Ok(RealtimeSessionConfig {
instructions: prompt,
model,
Expand Down
2 changes: 2 additions & 0 deletions codex-rs/core/tests/suite/compact_remote.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,10 +194,12 @@ async fn start_remote_realtime_server() -> responses::WebSocketTestServer {
async fn start_realtime_conversation(codex: &codex_core::CodexThread) -> Result<()> {
codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
model: None,
output_modality: RealtimeOutputModality::Audio,
prompt: Some(Some("backend prompt".to_string())),
realtime_session_id: None,
transport: None,
version: None,
voice: None,
}))
.await?;
Expand Down
Loading
Loading