Track realtime conversation state in world state

codex-rs/core/src/context/realtime_end_instructions.rs

@@ -32,6 +32,10 @@ impl ContextualUserFragment for RealtimeEndInstructions {
         )
     }
 
+    fn matches_text(text: &str) -> bool {
+        text.contains(END_INSTRUCTIONS.trim())
+    }
+
     fn body(&self) -> String {
         format!("\n{}\n\nReason: {}\n", END_INSTRUCTIONS.trim(), self.reason)
     }

codex-rs/core/src/context/world_state/mod.rs

@@ -4,6 +4,7 @@ mod collaboration_mode;
 mod environment;
 mod environments_instructions;
 mod plugins_instructions;
+mod realtime;
 #[cfg(test)]
 mod test_support;
 
@@ -27,6 +28,7 @@ pub(crate) use collaboration_mode::CollaborationModeState;
 pub(crate) use environment::EnvironmentsState;
 pub(crate) use environments_instructions::EnvironmentsInstructionsState;
 pub(crate) use plugins_instructions::PluginsInstructionsState;
+pub(crate) use realtime::RealtimeState;
 
 trait ErasedWorldStateSection: Send + Sync {
     fn snapshot(&self) -> Option<Value>;

codex-rs/core/src/context/world_state/realtime.rs

@@ -0,0 +1,87 @@
+use super::PreviousSectionState;
+use super::WorldStateSection;
+use crate::context::ContextualUserFragment;
+use crate::context::RealtimeEndInstructions;
+use crate::context::RealtimeStartInstructions;
+use crate::context::RealtimeStartWithInstructions;
+use serde::Deserialize;
+use serde::Serialize;
+
+/// The realtime conversation state currently visible to the model.
+#[derive(Clone, Debug)]
+pub(crate) struct RealtimeState {
+    snapshot: RealtimeSnapshot,
+    start_instructions: Option<String>,
+}
+
+#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
+pub(crate) struct RealtimeSnapshot {
+    active: bool,
+}
+
+impl RealtimeState {
+    pub(crate) fn new(active: bool, start_instructions: Option<&str>) -> Self {
+        Self {
+            snapshot: RealtimeSnapshot { active },
+            start_instructions: start_instructions.map(str::to_string),
+        }
+    }
+
+    fn render_start(&self) -> Box<dyn ContextualUserFragment> {
+        match self.start_instructions.as_deref() {
+            Some(instructions) => Box::new(RealtimeStartWithInstructions::new(instructions)),
+            None => Box::new(RealtimeStartInstructions),
+        }
+    }
+
+    fn render_transition(&self, previous_active: bool) -> Option<Box<dyn ContextualUserFragment>> {
+        match (previous_active, self.snapshot.active) {
+            (false, true) => Some(self.render_start()),
+            (true, false) => Some(Box::new(RealtimeEndInstructions::new("inactive"))),
+            (false, false) | (true, true) => None,
+        }
+    }
+}
+
+impl WorldStateSection for RealtimeState {
+    const ID: &'static str = "realtime";
+    type Snapshot = RealtimeSnapshot;
+
+    fn snapshot(&self) -> Self::Snapshot {
+        self.snapshot.clone()
+    }
+
+    fn matches_legacy_fragment(role: &str, text: &str) -> bool {
+        role == "developer"
+            && RealtimeStartInstructions::matches_text(text)
+            && !RealtimeEndInstructions::matches_text(text)
+    }
+
+    fn has_retained_fragment_matcher() -> bool {
+        true
+    }
+
+    fn matches_retained_fragment(role: &str, text: &str) -> bool {
+        Self::matches_legacy_fragment(role, text)
+    }
+
+    fn render_diff(
+        &self,
+        previous: PreviousSectionState<'_, Self::Snapshot>,
+    ) -> Option<Box<dyn ContextualUserFragment>> {
+        match previous {
+            PreviousSectionState::Known(previous) if previous == &self.snapshot => None,
+            PreviousSectionState::Known(previous) => self.render_transition(previous.active),
+            PreviousSectionState::Absent | PreviousSectionState::Unknown
+                if self.snapshot.active =>
+            {
+                Some(self.render_start())
+            }
+            PreviousSectionState::Absent | PreviousSectionState::Unknown => None,
+        }
+    }
+}
+
+#[cfg(test)]
+#[path = "realtime_tests.rs"]
+mod tests;

codex-rs/core/src/context/world_state/realtime_tests.rs

@@ -0,0 +1,43 @@
+use super::super::test_support::render_section_cases;
+use super::*;
+
+fn state(active: bool, start_instructions: Option<&str>) -> RealtimeState {
+    RealtimeState::new(active, start_instructions)
+}
+
+#[test]
+fn snapshots() {
+    use PreviousSectionState::Absent;
+    use PreviousSectionState::Known;
+    use PreviousSectionState::Unknown;
+
+    let inactive = state(/*active*/ false, /*start_instructions*/ None);
+    let active = state(/*active*/ true, /*start_instructions*/ None);
+    let custom_active = state(/*active*/ true, Some("custom realtime instructions"));
+    let changed_custom_active = state(
+        /*active*/ true,
+        Some("changed custom realtime instructions"),
+    );
+
+    insta::assert_snapshot!(render_section_cases(&[
+        (Absent, Absent),
+        (Absent, Known(&inactive)),
+        (Absent, Known(&active)),
+        (Known(&inactive), Known(&active)),
+        (Known(&inactive), Known(&custom_active)),
+        (Known(&active), Known(&active)),
+        (Known(&custom_active), Known(&changed_custom_active)),
+        (Known(&active), Known(&inactive)),
+        (Unknown, Known(&active)),
+        (Unknown, Known(&inactive)),
+    ]));
+}
+
+#[test]
+fn retained_fragment_matcher_only_matches_starts() {
+    let start = RealtimeStartWithInstructions::new("custom instructions").render();
+    let end = RealtimeEndInstructions::new("inactive").render();
+
+    assert!(RealtimeState::matches_legacy_fragment("developer", &start));
+    assert!(!RealtimeState::matches_legacy_fragment("developer", &end));
+}

codex-rs/core/src/context/world_state/snapshots/codex_core__context__world_state__realtime__tests__snapshots.snap

@@ -0,0 +1,71 @@
+---
+source: core/src/context/world_state/realtime_tests.rs
+expression: "render_section_cases(&[(Absent, Absent), (Absent, Known(&inactive)),\n(Absent, Known(&active)), (Known(&inactive), Known(&active)),\n(Known(&inactive), Known(&custom_active)), (Known(&active), Known(&active)),\n(Known(&custom_active), Known(&changed_custom_active)),\n(Known(&active), Known(&inactive)), (Unknown, Known(&active)),\n(Unknown, Known(&inactive)),])"
+---
+Absent -> Absent
+None
+
+Absent -> {"active":false}
+None
+
+Absent -> {"active":true} (role - developer)
+<realtime_conversation>
+Realtime conversation started.
+
+You are operating as a backend executor behind an intermediary. The user does not talk to you directly. Any response you produce will be consumed by the intermediary and may be summarized before the user sees it.
+
+When invoked, you receive the latest conversation transcript and any relevant mode or metadata. The intermediary may invoke you even when backend help is not actually needed. Use the transcript to decide whether you should do work. If backend help is unnecessary, avoid verbose responses that add user-visible latency.
+
+When user text is routed from realtime, treat it as a transcript. It may be unpunctuated or contain recognition errors.
+
+- Keep responses concise and action-oriented. Your updates should help the intermediary respond to the user.
+</realtime_conversation>
+
+{"active":false} -> {"active":true} (role - developer)
+<realtime_conversation>
+Realtime conversation started.
+
+You are operating as a backend executor behind an intermediary. The user does not talk to you directly. Any response you produce will be consumed by the intermediary and may be summarized before the user sees it.
+
+When invoked, you receive the latest conversation transcript and any relevant mode or metadata. The intermediary may invoke you even when backend help is not actually needed. Use the transcript to decide whether you should do work. If backend help is unnecessary, avoid verbose responses that add user-visible latency.
+
+When user text is routed from realtime, treat it as a transcript. It may be unpunctuated or contain recognition errors.
+
+- Keep responses concise and action-oriented. Your updates should help the intermediary respond to the user.
+</realtime_conversation>
+
+{"active":false} -> {"active":true} (role - developer)
+<realtime_conversation>
+custom realtime instructions
+</realtime_conversation>
+
+{"active":true} -> {"active":true}
+None
+
+{"active":true} -> {"active":true}
+None
+
+{"active":true} -> {"active":false} (role - developer)
+<realtime_conversation>
+Realtime conversation ended.
+
+Subsequent user input will return to typed text rather than transcript-style text. Do not assume recognition errors or missing punctuation once realtime has ended. Resume normal chat behavior.
+
+Reason: inactive
+</realtime_conversation>
+
+Unknown -> {"active":true} (role - developer)
+<realtime_conversation>
+Realtime conversation started.
+
+You are operating as a backend executor behind an intermediary. The user does not talk to you directly. Any response you produce will be consumed by the intermediary and may be summarized before the user sees it.
+
+When invoked, you receive the latest conversation transcript and any relevant mode or metadata. The intermediary may invoke you even when backend help is not actually needed. Use the transcript to decide whether you should do work. If backend help is unnecessary, avoid verbose responses that add user-visible latency.
+
+When user text is routed from realtime, treat it as a transcript. It may be unpunctuated or contain recognition errors.
+
+- Keep responses concise and action-oriented. Your updates should help the intermediary respond to the user.
+</realtime_conversation>
+
+Unknown -> {"active":false}
+None

codex-rs/core/src/context_manager/updates.rs

@@ -4,9 +4,6 @@ use crate::context::ModelSwitchInstructions;
 use crate::context::MultiAgentModeInstructions;
 use crate::context::PermissionsInstructions;
 use crate::context::PersonalitySpecInstructions;
-use crate::context::RealtimeEndInstructions;
-use crate::context::RealtimeStartInstructions;
-use crate::context::RealtimeStartWithInstructions;
 use crate::session::PreviousTurnSettings;
 use crate::session::turn_context::TurnContext;
 use codex_execpolicy::Policy;
@@ -85,43 +82,6 @@ fn build_multi_agent_mode_update_item(
     }
 }
 
-pub(crate) fn build_realtime_update_item(
-    previous: Option<&TurnContextItem>,
-    previous_turn_settings: Option<&PreviousTurnSettings>,
-    next: &TurnContext,
-) -> Option<String> {
-    match (
-        previous.and_then(|item| item.realtime_active),
-        next.realtime_active,
-    ) {
-        (Some(true), false) => Some(RealtimeEndInstructions::new("inactive").render()),
-        (Some(false), true) | (None, true) => Some(
-            if let Some(instructions) = next
-                .config
-                .experimental_realtime_start_instructions
-                .as_deref()
-            {
-                RealtimeStartWithInstructions::new(instructions).render()
-            } else {
-                RealtimeStartInstructions.render()
-            },
-        ),
-        (Some(true), true) | (Some(false), false) => None,
-        (None, false) => previous_turn_settings
-            .and_then(|settings| settings.realtime_active)
-            .filter(|realtime_active| *realtime_active)
-            .map(|_| RealtimeEndInstructions::new("inactive").render()),
-    }
-}
-
-pub(crate) fn build_initial_realtime_item(
-    previous: Option<&TurnContextItem>,
-    previous_turn_settings: Option<&PreviousTurnSettings>,
-    next: &TurnContext,
-) -> Option<String> {
-    build_realtime_update_item(previous, previous_turn_settings, next)
-}
-
 fn build_personality_update_item(
     previous: Option<&TurnContextItem>,
     next: &TurnContext,
@@ -238,7 +198,6 @@ pub(crate) fn build_settings_update_items(
         build_model_instructions_update_item(previous_turn_settings, next),
         build_permissions_update_item(previous, next, exec_policy),
         build_multi_agent_mode_update_item(previous, next),
-        build_realtime_update_item(previous, previous_turn_settings, next),
         build_personality_update_item(previous, next, personality_feature_enabled),
     ]
     .into_iter()

codex-rs/core/src/session/mod.rs

@@ -3193,16 +3193,9 @@ impl Session {
         let mut developer_sections = Vec::<String>::with_capacity(8);
         let mut contextual_user_sections = Vec::<String>::with_capacity(2);
         let mut separate_developer_sections = Vec::<String>::new();
-        let (
-            reference_context_item,
-            previous_turn_settings,
-            base_instructions,
-            session_source,
-            auto_compact_window_ids,
-        ) = {
+        let (previous_turn_settings, base_instructions, session_source, auto_compact_window_ids) = {
             let state = self.state.lock().await;
             (
-                state.reference_context_item(),
                 state.previous_turn_settings(),
                 state.session_configuration.base_instructions.clone(),
                 state.session_configuration.session_source.clone(),
@@ -3260,13 +3253,6 @@ impl Session {
         {
             developer_sections.push(developer_instructions.to_string());
         }
-        if let Some(realtime_update) = crate::context_manager::updates::build_initial_realtime_item(
-            reference_context_item.as_ref(),
-            previous_turn_settings.as_ref(),
-            turn_context,
-        ) {
-            developer_sections.push(realtime_update);
-        }
         if self.features.enabled(Feature::Personality)
             && let Some(personality) = turn_context.personality
         {