Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions codex-rs/app-server-protocol/src/protocol/common.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3043,6 +3043,7 @@ mod tests {
thread_id: "thr_123".to_string(),
model: Some("realtime-treatment-model".to_string()),
output_modality: RealtimeOutputModality::Audio,
include_startup_context: Some(false),
prompt: Some(Some("You are on a call".to_string())),
realtime_session_id: Some("sess_456".to_string()),
transport: None,
Expand All @@ -3061,6 +3062,7 @@ mod tests {
"codexResponseItemPrefix": null,
"model": "realtime-treatment-model",
"outputModality": "audio",
"includeStartupContext": false,
"prompt": "You are on a call",
"realtimeSessionId": "sess_456",
"transport": null,
Expand All @@ -3084,6 +3086,7 @@ mod tests {
thread_id: "thr_123".to_string(),
model: None,
output_modality: RealtimeOutputModality::Audio,
include_startup_context: None,
prompt: None,
realtime_session_id: None,
transport: None,
Expand All @@ -3102,6 +3105,7 @@ mod tests {
"codexResponseItemPrefix": null,
"model": null,
"outputModality": "audio",
"includeStartupContext": null,
"realtimeSessionId": null,
"transport": null,
"version": null,
Expand All @@ -3120,6 +3124,7 @@ mod tests {
thread_id: "thr_123".to_string(),
model: None,
output_modality: RealtimeOutputModality::Audio,
include_startup_context: None,
prompt: Some(None),
realtime_session_id: None,
transport: None,
Expand All @@ -3138,6 +3143,7 @@ mod tests {
"codexResponseItemPrefix": null,
"model": null,
"outputModality": "audio",
"includeStartupContext": null,
"prompt": null,
"realtimeSessionId": null,
"transport": null,
Expand Down Expand Up @@ -3322,6 +3328,7 @@ mod tests {
thread_id: "thr_123".to_string(),
model: None,
output_modality: RealtimeOutputModality::Audio,
include_startup_context: None,
prompt: Some(Some("You are on a call".to_string())),
realtime_session_id: None,
transport: None,
Expand Down
3 changes: 3 additions & 0 deletions codex-rs/app-server-protocol/src/protocol/v2/realtime.rs
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,9 @@ pub struct ThreadRealtimeStartParams {
/// Selects text or audio output for the realtime session. Transport and voice stay
/// independent so clients can choose how they connect separately from what the model emits.
pub output_modality: RealtimeOutputModality,
/// Set to false to start without Codex's startup context. Omitted or null includes it.
#[ts(optional = nullable)]
pub include_startup_context: Option<bool>,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Regenerate app-server schema fixtures

Adding includeStartupContext changes the v2 app-server request shape, but the checked-in schema fixtures still lack this field/ThreadRealtimeStartParams, so generated TS/JSON clients cannot discover the new flag. Please regenerate the experimental schema fixtures. guidance

Useful? React with 👍 / 👎.

#[serde(
default,
deserialize_with = "crate::protocol::serde_helpers::deserialize_double_option",
Expand Down
4 changes: 3 additions & 1 deletion codex-rs/app-server/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ Example with notification opt-out:
- `thread/inject_items` — append raw Responses API items to a loaded thread’s model-visible history without starting a user turn; returns `{}` on success.
- `turn/steer` — add user input to an already in-flight regular turn without starting a new turn; returns the active `turnId` that accepted the input. `clientUserMessageId` is optional; when supplied, the corresponding `userMessage` item echoes it as `clientId`. Review and manual compaction turns reject `turn/steer`.
- `turn/interrupt` — request cancellation of an in-flight turn by `(thread_id, turn_id)`; success is an empty `{}` response and the turn finishes with `status: "interrupted"`.
- `thread/realtime/start` — start a thread-scoped realtime session (experimental); pass `outputModality: "text"` or `outputModality: "audio"` to choose model output, and optionally pass `model` and `version` to override configured realtime selection for this session only. By default, automatic Codex text follows the protocol's speakable output path. Pass `codexResponsesAsItems: true` to send automatic Codex responses as realtime conversation items instead, and optionally pass `codexResponseItemPrefix` to prepend experiment instructions to those items. Returns `{}` and streams `thread/realtime/*` notifications. Omit `transport` for the websocket transport, or pass `{ "type": "webrtc", "sdp": "..." }` to create a WebRTC session from a browser-generated SDP offer; the remote answer SDP is emitted as `thread/realtime/sdp`.
- `thread/realtime/start` — start a thread-scoped realtime session (experimental); pass `outputModality: "text"` or `outputModality: "audio"` to choose model output, optionally pass `model` and `version` to override configured realtime selection for this session only, and pass `includeStartupContext: false` to omit Codex's generated startup context. By default, automatic Codex text follows the protocol's speakable output path. Pass `codexResponsesAsItems: true` to send automatic Codex responses as realtime conversation items instead, and optionally pass `codexResponseItemPrefix` to prepend experiment instructions to those items. Returns `{}` and streams `thread/realtime/*` notifications. Omit `transport` for the websocket transport, or pass `{ "type": "webrtc", "sdp": "..." }` to create a WebRTC session from a browser-generated SDP offer; the remote answer SDP is emitted as `thread/realtime/sdp`.
- `thread/realtime/appendAudio` — append an input audio chunk to the active realtime session (experimental); returns `{}`.
- `thread/realtime/appendText` — append text input to the active realtime session with a required `role` of `user` or `developer` (experimental); returns `{}`. Older clients that omit `role` default to `user`.
- `thread/realtime/appendSpeech` — append text that the realtime model should speak to the user (experimental); returns `{}`.
Expand Down Expand Up @@ -879,6 +879,8 @@ Omit `prompt` to use Codex's default realtime backend prompt. Send `prompt: null
`prompt: ""` when the session should start without that default backend prompt.
Clients may also pass `model` and `version` on `thread/realtime/start` to select a
different realtime session configuration without changing thread or user config.
Pass `includeStartupContext: false` to skip Codex's startup context for this
session while still using the selected backend prompt.
Pass `codexResponsesAsItems: true` to inject automatic Codex responses with
`conversation.item.create` instead of the protocol's default speakable output
path. When using that mode, `codexResponseItemPrefix` can prepend short
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -956,6 +956,7 @@ impl TurnRequestProcessor {
codex_response_item_prefix: params.codex_response_item_prefix,
model: params.model,
output_modality: params.output_modality,
include_startup_context: params.include_startup_context.unwrap_or(true),
prompt: params.prompt,
realtime_session_id: params.realtime_session_id,
transport: params.transport.map(|transport| match transport {
Expand Down
2 changes: 2 additions & 0 deletions codex-rs/app-server/tests/suite/v2/experimental_api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ async fn realtime_conversation_start_requires_experimental_api_capability() -> R
thread_id: "thr_123".to_string(),
model: None,
output_modality: RealtimeOutputModality::Audio,
include_startup_context: None,
prompt: Some(Some("hello".to_string())),
realtime_session_id: None,
transport: None,
Expand Down Expand Up @@ -196,6 +197,7 @@ async fn realtime_webrtc_start_requires_experimental_api_capability() -> Result<
thread_id: "thr_123".to_string(),
model: None,
output_modality: RealtimeOutputModality::Audio,
include_startup_context: None,
prompt: Some(Some("hello".to_string())),
realtime_session_id: None,
transport: Some(ThreadRealtimeStartTransport::Webrtc {
Expand Down
81 changes: 81 additions & 0 deletions codex-rs/app-server/tests/suite/v2/realtime_conversation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -348,6 +348,7 @@ impl RealtimeE2eHarness {
codex_responses_as_items,
model: None,
output_modality: RealtimeOutputModality::Audio,
include_startup_context: None,
prompt: Some(Some("backend prompt".to_string())),
realtime_session_id: None,
transport: Some(ThreadRealtimeStartTransport::Webrtc {
Expand Down Expand Up @@ -617,6 +618,7 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> {
thread_id: thread_start.thread.id.clone(),
model: Some("realtime-treatment-model".to_string()),
output_modality: RealtimeOutputModality::Audio,
include_startup_context: None,
prompt: None,
realtime_session_id: None,
transport: None,
Expand Down Expand Up @@ -830,6 +832,80 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> {
Ok(())
}

#[tokio::test]
async fn realtime_start_can_skip_startup_context() -> Result<()> {
skip_if_no_network!(Ok(()));

let responses_server = create_mock_responses_server_sequence_unchecked(Vec::new()).await;
let realtime_server = start_websocket_server(vec![vec![vec![json!({
"type": "session.updated",
"session": { "id": "sess_backend", "instructions": "backend prompt" }
})]]])
.await;

let codex_home = TempDir::new()?;
create_config_toml(
codex_home.path(),
&responses_server.uri(),
realtime_server.uri(),
/*realtime_enabled*/ true,
StartupContextConfig::Generated,
)?;

let mut mcp = TestAppServer::new(codex_home.path()).await?;
timeout(DEFAULT_TIMEOUT, mcp.initialize()).await??;
login_with_api_key(&mut mcp, "sk-test-key").await?;

let thread_start_request_id = mcp
.send_thread_start_request(ThreadStartParams::default())
.await?;
let thread_start_response: JSONRPCResponse = timeout(
DEFAULT_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(thread_start_request_id)),
)
.await??;
let thread_start: ThreadStartResponse = to_response(thread_start_response)?;

let start_request_id = mcp
.send_thread_realtime_start_request(ThreadRealtimeStartParams {
architecture: None,
codex_responses_as_items: None,
codex_response_item_prefix: None,
thread_id: thread_start.thread.id.clone(),
model: None,
output_modality: RealtimeOutputModality::Audio,
include_startup_context: Some(false),
prompt: None,
realtime_session_id: None,
transport: None,
version: None,
voice: None,
})
.await?;
let start_response: JSONRPCResponse = timeout(
DEFAULT_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(start_request_id)),
)
.await??;
let _: ThreadRealtimeStartResponse = to_response(start_response)?;

read_notification::<ThreadRealtimeStartedNotification>(&mut mcp, "thread/realtime/started")
.await?;

let startup_context_request = realtime_server
.wait_for_request(/*connection_index*/ 0, /*request_index*/ 0)
.await;
let startup_context_body = startup_context_request.body_json();
let instructions = startup_context_body["session"]["instructions"]
.as_str()
.context("expected realtime instructions")?;
assert_eq!(instructions, "backend prompt");
assert!(!instructions.contains(STARTUP_CONTEXT_HEADER));

realtime_server.shutdown().await;
Ok(())
}

#[tokio::test]
async fn realtime_text_output_modality_requests_text_output_and_final_transcript() -> Result<()> {
skip_if_no_network!(Ok(()));
Expand Down Expand Up @@ -895,6 +971,7 @@ async fn realtime_text_output_modality_requests_text_output_and_final_transcript
thread_id: thread_start.thread.id.clone(),
model: None,
output_modality: RealtimeOutputModality::Text,
include_startup_context: None,
prompt: None,
realtime_session_id: None,
transport: None,
Expand Down Expand Up @@ -1074,6 +1151,7 @@ async fn realtime_conversation_stop_emits_closed_notification() -> Result<()> {
thread_id: thread_start.thread.id.clone(),
model: None,
output_modality: RealtimeOutputModality::Audio,
include_startup_context: None,
prompt: Some(Some("backend prompt".to_string())),
realtime_session_id: None,
transport: None,
Expand Down Expand Up @@ -1176,6 +1254,7 @@ async fn realtime_webrtc_start_emits_sdp_notification() -> Result<()> {
thread_id: thread_id.clone(),
model: None,
output_modality: RealtimeOutputModality::Audio,
include_startup_context: None,
prompt: Some(Some("backend prompt".to_string())),
realtime_session_id: None,
transport: Some(ThreadRealtimeStartTransport::Webrtc {
Expand Down Expand Up @@ -2396,6 +2475,7 @@ async fn realtime_webrtc_start_surfaces_backend_error() -> Result<()> {
thread_id: thread_start.thread.id,
model: None,
output_modality: RealtimeOutputModality::Audio,
include_startup_context: None,
prompt: Some(Some("backend prompt".to_string())),
realtime_session_id: None,
transport: Some(ThreadRealtimeStartTransport::Webrtc {
Expand Down Expand Up @@ -2460,6 +2540,7 @@ async fn realtime_conversation_requires_feature_flag() -> Result<()> {
thread_id: thread_start.thread.id.clone(),
model: None,
output_modality: RealtimeOutputModality::Audio,
include_startup_context: None,
prompt: Some(Some("backend prompt".to_string())),
realtime_session_id: None,
transport: None,
Expand Down
1 change: 1 addition & 0 deletions codex-rs/codex-api/tests/clients.rs
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,7 @@ async fn responses_client_stream_request_preserves_exact_json_body() -> Result<(
role: "user".into(),
content: vec![ContentItem::InputText { text: "hi".into() }],
phase: None,
metadata: None,
}],
tools: Vec::new(),
tool_choice: "auto".into(),
Expand Down
1 change: 1 addition & 0 deletions codex-rs/core/src/guardian/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1696,6 +1696,7 @@ async fn guardian_review_request_layout_matches_model_visible_request_snapshot()
),
}],
phase: None,
metadata: None,
}],
)
.await;
Expand Down
56 changes: 29 additions & 27 deletions codex-rs/core/src/realtime_conversation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -698,6 +698,7 @@ async fn prepare_realtime_start(
let config = sess.get_config().await;
let transport = params
.transport
.clone()
.unwrap_or(ConversationStartTransport::Websocket);
let mut api_provider = provider.to_api_provider(Some(AuthMode::ApiKey))?;
if let Some(realtime_ws_base_url) = &config.experimental_realtime_ws_base_url {
Expand All @@ -720,16 +721,7 @@ async fn prepare_realtime_start(
&transport,
config.realtime.session_type,
)?;
let session_config = build_realtime_session_config(
sess,
params.model,
params.prompt,
params.realtime_session_id,
params.output_modality,
version,
params.voice,
)
.await?;
let session_config = build_realtime_session_config(sess, &params, version).await?;
let requested_realtime_session_id = session_config.session_id.clone();
let extra_headers = match transport {
ConversationStartTransport::Websocket => {
Expand Down Expand Up @@ -791,25 +783,25 @@ fn validate_realtime_architecture(

pub(crate) async fn build_realtime_session_config(
sess: &Arc<Session>,
model: Option<String>,
prompt: Option<Option<String>>,
realtime_session_id: Option<String>,
output_modality: RealtimeOutputModality,
params: &ConversationStartParams,
version: RealtimeWsVersion,
voice: Option<RealtimeVoice>,
) -> CodexResult<RealtimeSessionConfig> {
let config = sess.get_config().await;
let prompt = prepare_realtime_backend_prompt(
prompt,
params.prompt.clone(),
config.experimental_realtime_ws_backend_prompt.clone(),
);
let startup_context = match config.experimental_realtime_ws_startup_context.clone() {
Some(startup_context) => startup_context,
None => {
build_realtime_startup_context(sess.as_ref(), REALTIME_STARTUP_CONTEXT_TOKEN_BUDGET)
.await
.unwrap_or_default()
let startup_context = if params.include_startup_context {
match config.experimental_realtime_ws_startup_context.clone() {
Some(startup_context) => startup_context,
None => {
build_realtime_startup_context(sess.as_ref(), REALTIME_STARTUP_CONTEXT_TOKEN_BUDGET)
.await
.unwrap_or_default()
}
}
} else {
String::new()
};
let prompt = match (prompt.is_empty(), startup_context.is_empty()) {
(true, true) => String::new(),
Expand All @@ -818,15 +810,19 @@ pub(crate) async fn build_realtime_session_config(
(false, false) => format!("{prompt}\n\n{startup_context}"),
};
let model = Some(
model
params
.model
.clone()
.or_else(|| config.experimental_realtime_ws_model.clone())
.unwrap_or_else(|| DEFAULT_REALTIME_MODEL.to_string()),
);
let event_parser = match version {
RealtimeWsVersion::V1 => RealtimeEventParser::V1,
RealtimeWsVersion::V2 => RealtimeEventParser::RealtimeV2,
};
if version == RealtimeWsVersion::V1 && matches!(output_modality, RealtimeOutputModality::Text) {
if version == RealtimeWsVersion::V1
&& matches!(params.output_modality, RealtimeOutputModality::Text)
{
return Err(CodexErr::InvalidRequest(
"text realtime output modality requires realtime v2".to_string(),
));
Expand All @@ -835,17 +831,23 @@ pub(crate) async fn build_realtime_session_config(
RealtimeWsMode::Conversational => RealtimeSessionMode::Conversational,
RealtimeWsMode::Transcription => RealtimeSessionMode::Transcription,
};
let voice = voice
let voice = params
.voice
.or(config.realtime.voice)
.unwrap_or_else(|| default_realtime_voice(version));
validate_realtime_voice(version, voice)?;
Ok(RealtimeSessionConfig {
instructions: prompt,
model,
session_id: Some(realtime_session_id.unwrap_or_else(|| sess.thread_id.to_string())),
session_id: Some(
params
.realtime_session_id
.clone()
.unwrap_or_else(|| sess.thread_id.to_string()),
),
event_parser,
session_mode,
output_modality,
output_modality: params.output_modality,
voice,
})
}
Expand Down
1 change: 1 addition & 0 deletions codex-rs/core/tests/suite/compact_remote.rs
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,7 @@ async fn start_realtime_conversation(codex: &codex_core::CodexThread) -> Result<
codex_response_item_prefix: None,
model: None,
output_modality: RealtimeOutputModality::Audio,
include_startup_context: true,
prompt: Some(Some("backend prompt".to_string())),
realtime_session_id: None,
transport: None,
Expand Down
Loading
Loading