From c10010928075babeebfc65b824db8778f10a1c56 Mon Sep 17 00:00:00 2001 From: jif Date: Mon, 13 Jul 2026 10:34:04 +0000 Subject: [PATCH] Add shadow metrics for lexical skill selection (#32761) ## What changed - Add an opt-in `skill_search` feature that ranks prompt-visible skills against each turn's user input with a bounded weighted lexical selector. - Keep the ranked selection out of model-visible context and record metrics for selection cost, catalog reduction, and whether later implicit or `skills.read` invocations matched the ranked candidates. - Include host-provided skills in the experiment catalog without changing the rendered skill catalog. ## Testing - Add selector unit tests covering ranking, limits, truncation, stop words, and deterministic tie-breaking. - Add extension tests covering turn-local invocation recording and host-skill shadow selection. GitOrigin-RevId: 4d00a1c805ea8b391d6c6ac6a8450afa88ca3e25 --- codex-rs/Cargo.lock | 2 + codex-rs/app-server/src/extensions.rs | 6 +- codex-rs/core/config.schema.json | 6 + codex-rs/ext/skills/Cargo.toml | 3 +- codex-rs/ext/skills/src/config.rs | 2 + .../ext/skills/src/dynamic_skill_selector.rs | 37 ++ .../weighted_lexical.rs | 208 ++++++++++ .../weighted_lexical_tests.rs | 166 ++++++++ codex-rs/ext/skills/src/extension.rs | 71 ++++ codex-rs/ext/skills/src/lib.rs | 3 + .../skills/src/shadow_selection_experiment.rs | 356 ++++++++++++++++++ codex-rs/ext/skills/src/state.rs | 35 ++ codex-rs/ext/skills/src/tools/mod.rs | 4 + codex-rs/ext/skills/src/tools/read.rs | 18 +- .../ext/skills/tests/implicit_invocation.rs | 331 ++++++++++++++++ codex-rs/ext/skills/tests/skills_extension.rs | 6 +- codex-rs/features/src/lib.rs | 8 + 17 files changed, 1255 insertions(+), 7 deletions(-) create mode 100644 codex-rs/ext/skills/src/dynamic_skill_selector.rs create mode 100644 codex-rs/ext/skills/src/dynamic_skill_selector/weighted_lexical.rs create mode 100644 codex-rs/ext/skills/src/dynamic_skill_selector/weighted_lexical_tests.rs create mode 100644 codex-rs/ext/skills/src/shadow_selection_experiment.rs create mode 100644 codex-rs/ext/skills/tests/implicit_invocation.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 6ea0242251d3..cf3d0a3b4792 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -3979,11 +3979,13 @@ dependencies = [ "codex-exec-server", "codex-extension-api", "codex-mcp", + "codex-otel", "codex-protocol", "codex-tools", "codex-utils-absolute-path", "codex-utils-path-uri", "codex-utils-string", + "opentelemetry_sdk", "pretty_assertions", "schemars 0.8.22", "serde", diff --git a/codex-rs/app-server/src/extensions.rs b/codex-rs/app-server/src/extensions.rs index ed040aa8d9e9..31af1d650718 100644 --- a/codex-rs/app-server/src/extensions.rs +++ b/codex-rs/app-server/src/extensions.rs @@ -84,13 +84,17 @@ where .with_orchestrator_provider(Arc::new( codex_skills_extension::OrchestratorSkillProvider::new(), )); - codex_skills_extension::install_with_providers( + codex_skills_extension::install_with_providers_and_metrics( &mut builder, skill_providers, + codex_otel::global(), |config: &Config| codex_skills_extension::SkillsExtensionConfig { include_instructions: config.include_skill_instructions, bundled_skills_enabled: config.bundled_skills_enabled(), orchestrator_skills_enabled: config.orchestrator_skills_enabled, + shadow_selection_enabled: config + .features + .enabled(codex_features::Feature::SkillSearch), }, ); Arc::new(builder.build()) diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index b624dc2bd789..eed232864ce4 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -650,6 +650,9 @@ "skill_mcp_dependency_install": { "type": "boolean" }, + "skill_search": { + "type": "boolean" + }, "sqlite": { "type": "boolean" }, @@ -5026,6 +5029,9 @@ "skill_mcp_dependency_install": { "type": "boolean" }, + "skill_search": { + "type": "boolean" + }, "sqlite": { "type": "boolean" }, diff --git a/codex-rs/ext/skills/Cargo.toml b/codex-rs/ext/skills/Cargo.toml index 35827ac5788d..df32e02f39ca 100644 --- a/codex-rs/ext/skills/Cargo.toml +++ b/codex-rs/ext/skills/Cargo.toml @@ -7,7 +7,6 @@ version.workspace = true [lib] name = "codex_skills_extension" path = "src/lib.rs" -test = false doctest = false [lints] @@ -18,6 +17,7 @@ codex-core-skills = { workspace = true } codex-exec-server = { workspace = true } codex-extension-api = { workspace = true } codex-mcp = { workspace = true } +codex-otel = { workspace = true } codex-protocol = { workspace = true } codex-tools = { workspace = true } codex-utils-path-uri = { workspace = true } @@ -31,5 +31,6 @@ url = { workspace = true } [dev-dependencies] codex-utils-absolute-path = { workspace = true } +opentelemetry_sdk = { workspace = true } pretty_assertions = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/codex-rs/ext/skills/src/config.rs b/codex-rs/ext/skills/src/config.rs index 3d903cafb848..f0198145dec5 100644 --- a/codex-rs/ext/skills/src/config.rs +++ b/codex-rs/ext/skills/src/config.rs @@ -7,4 +7,6 @@ pub struct SkillsExtensionConfig { pub bundled_skills_enabled: bool, /// Whether orchestrator-owned skills are eligible for discovery. pub orchestrator_skills_enabled: bool, + /// Whether cheap skill selectors run in shadow mode without changing prompt contents. + pub shadow_selection_enabled: bool, } diff --git a/codex-rs/ext/skills/src/dynamic_skill_selector.rs b/codex-rs/ext/skills/src/dynamic_skill_selector.rs new file mode 100644 index 000000000000..948c815f1a53 --- /dev/null +++ b/codex-rs/ext/skills/src/dynamic_skill_selector.rs @@ -0,0 +1,37 @@ +mod weighted_lexical; +pub(crate) use weighted_lexical::WeightedLexicalSkillSelector; + +/// Metadata searched by a cheap skill selector. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct SkillSelectionDocument<'a> { + /// Caller-owned identifier returned in [`CheapSkillSelection::candidate_ids`]. + pub id: usize, + pub name: &'a str, + pub short_description: Option<&'a str>, + pub description: &'a str, +} + +/// Bounded output from one cheap skill-selection method. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct CheapSkillSelection { + pub candidate_ids: Vec, + pub query_term_count: usize, + pub query_truncated: bool, + pub candidate_set_truncated: bool, +} + +/// Selects likely-relevant skills without changing the model-visible catalog. +/// +/// Implementations must be deterministic, side-effect free, and cheap enough to run in shadow +/// mode on every turn. Callers must validate returned IDs against the supplied documents. +pub(crate) trait CheapSkillSelector: Send + Sync { + /// Low-cardinality identifier suitable for experiment metrics. + fn method(&self) -> &'static str; + + fn select( + &self, + query: &str, + documents: &[SkillSelectionDocument<'_>], + limit: usize, + ) -> CheapSkillSelection; +} diff --git a/codex-rs/ext/skills/src/dynamic_skill_selector/weighted_lexical.rs b/codex-rs/ext/skills/src/dynamic_skill_selector/weighted_lexical.rs new file mode 100644 index 000000000000..4ee79011f4a4 --- /dev/null +++ b/codex-rs/ext/skills/src/dynamic_skill_selector/weighted_lexical.rs @@ -0,0 +1,208 @@ +use std::collections::HashSet; + +use super::CheapSkillSelection; +use super::CheapSkillSelector; +use super::SkillSelectionDocument; + +const MAX_QUERY_BYTES: usize = 4 * 1024; +const MAX_QUERY_TERMS: usize = 64; +const MAX_DOCUMENT_BYTES: usize = 4 * 1024; +const MAX_DOCUMENT_TERMS: usize = 256; +const MAX_CANDIDATES: usize = 1_000; +const MAX_RESULTS: usize = 50; + +const STOP_WORDS: &[&str] = &[ + "a", "an", "and", "are", "as", "at", "be", "by", "do", "for", "from", "how", "i", "in", "is", + "it", "me", "my", "of", "on", "or", "please", "that", "the", "this", "to", "use", "we", "what", + "when", "where", "which", "with", "you", "your", +]; + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct WeightedLexicalSkillSelector; + +impl CheapSkillSelector for WeightedLexicalSkillSelector { + fn method(&self) -> &'static str { + "weighted_lexical_v1" + } + + fn select( + &self, + query: &str, + documents: &[SkillSelectionDocument<'_>], + limit: usize, + ) -> CheapSkillSelection { + let (query, query_bytes_truncated) = bounded(query, MAX_QUERY_BYTES); + let query_phrase = normalize_phrase(query); + let (query_terms, query_terms_truncated) = query_terms(&query_phrase); + let query_truncated = query_bytes_truncated || query_terms_truncated; + let candidate_set_truncated = documents.len() > MAX_CANDIDATES; + if query_terms.is_empty() || limit == 0 { + return CheapSkillSelection { + query_term_count: query_terms.len(), + query_truncated, + candidate_set_truncated, + ..Default::default() + }; + } + + let mut scored = documents + .iter() + .take(MAX_CANDIDATES) + .filter_map(|document| { + let score = score_document(&query_phrase, &query_terms, document); + (score > 0).then_some((score, document.id, document.name)) + }) + .collect::>(); + scored.sort_by(|left, right| { + right + .0 + .cmp(&left.0) + .then_with(|| left.2.cmp(right.2)) + .then_with(|| left.1.cmp(&right.1)) + }); + + CheapSkillSelection { + candidate_ids: scored + .into_iter() + .take(limit.min(MAX_RESULTS)) + .map(|(_, id, _)| id) + .collect(), + query_term_count: query_terms.len(), + query_truncated, + candidate_set_truncated, + } + } +} + +fn score_document( + query_phrase: &str, + query_terms: &[&str], + document: &SkillSelectionDocument<'_>, +) -> u32 { + let name = normalize_bounded(document.name); + let short_description = document + .short_description + .map(normalize_bounded) + .unwrap_or_default(); + let description = normalize_bounded(document.description); + let name_terms = phrase_terms(&name); + let short_description_terms = phrase_terms(&short_description); + let description_terms = phrase_terms(&description); + + let mut score = 0u32; + if !name.is_empty() && contains_phrase(query_phrase, &name) { + score = score.saturating_add(256); + } + + let mut matched_query_terms = 0u32; + for query_term in query_terms { + let mut matched = false; + if name == *query_term { + score = score.saturating_add(128); + matched = true; + } else if name_terms.contains(query_term) { + score = score.saturating_add(64); + matched = true; + } else if contains_related_term(&name_terms, query_term) { + score = score.saturating_add(24); + matched = true; + } + + if short_description_terms.contains(query_term) { + score = score.saturating_add(16); + matched = true; + } else if contains_related_term(&short_description_terms, query_term) { + score = score.saturating_add(6); + matched = true; + } + + if description_terms.contains(query_term) { + score = score.saturating_add(4); + matched = true; + } else if contains_related_term(&description_terms, query_term) { + score = score.saturating_add(1); + matched = true; + } + + if matched { + matched_query_terms = matched_query_terms.saturating_add(1); + } + } + + score.saturating_add(matched_query_terms.saturating_mul(matched_query_terms)) +} + +fn normalize_bounded(value: &str) -> String { + normalize_phrase(bounded(value, MAX_DOCUMENT_BYTES).0) +} + +fn normalize_phrase(value: &str) -> String { + let mut normalized = String::with_capacity(value.len()); + let mut previous_was_separator = true; + for character in value.chars() { + if character.is_alphanumeric() { + normalized.extend(character.to_lowercase()); + previous_was_separator = false; + } else if !previous_was_separator { + normalized.push(' '); + previous_was_separator = true; + } + } + if previous_was_separator { + normalized.pop(); + } + normalized +} + +fn query_terms(query_phrase: &str) -> (Vec<&str>, bool) { + let mut seen = HashSet::new(); + let mut terms = Vec::new(); + for term in query_phrase + .split_whitespace() + .filter(|term| term.chars().count() >= 2 && !STOP_WORDS.contains(term)) + { + if !seen.insert(term) { + continue; + } + if terms.len() == MAX_QUERY_TERMS { + return (terms, true); + } + terms.push(term); + } + (terms, false) +} + +fn phrase_terms(phrase: &str) -> HashSet<&str> { + phrase.split_whitespace().take(MAX_DOCUMENT_TERMS).collect() +} + +fn contains_phrase(haystack: &str, needle: &str) -> bool { + haystack == needle + || haystack.starts_with(&format!("{needle} ")) + || haystack.ends_with(&format!(" {needle}")) + || haystack.contains(&format!(" {needle} ")) +} + +fn contains_related_term(terms: &HashSet<&str>, query_term: &str) -> bool { + if query_term.chars().count() < 4 { + return false; + } + terms.iter().any(|term| { + term.chars().count() >= 4 && (term.starts_with(query_term) || query_term.starts_with(*term)) + }) +} + +fn bounded(value: &str, max_bytes: usize) -> (&str, bool) { + if value.len() <= max_bytes { + return (value, false); + } + let mut end = max_bytes; + while !value.is_char_boundary(end) { + end = end.saturating_sub(1); + } + (&value[..end], true) +} + +#[cfg(test)] +#[path = "weighted_lexical_tests.rs"] +mod tests; diff --git a/codex-rs/ext/skills/src/dynamic_skill_selector/weighted_lexical_tests.rs b/codex-rs/ext/skills/src/dynamic_skill_selector/weighted_lexical_tests.rs new file mode 100644 index 000000000000..696c160cd906 --- /dev/null +++ b/codex-rs/ext/skills/src/dynamic_skill_selector/weighted_lexical_tests.rs @@ -0,0 +1,166 @@ +use super::*; +use pretty_assertions::assert_eq; + +#[test] +fn lexical_selector_prioritizes_an_exact_skill_name() { + let documents = [ + SkillSelectionDocument { + id: 10, + name: "slides-helper", + short_description: None, + description: "Create presentations and visual decks.", + }, + SkillSelectionDocument { + id: 20, + name: "presentations", + short_description: None, + description: "Create or edit PowerPoint presentations.", + }, + SkillSelectionDocument { + id: 30, + name: "spreadsheets", + short_description: None, + description: "Analyze tabular data.", + }, + ]; + + let selection = WeightedLexicalSkillSelector.select( + "Use presentations to create a deck", + &documents, + /*limit*/ 20, + ); + + assert_eq!(vec![20, 10], selection.candidate_ids); + assert!(!selection.query_truncated); + assert!(!selection.candidate_set_truncated); +} + +#[test] +fn lexical_selector_uses_descriptions_and_drops_zero_score_candidates() { + let documents = [ + SkillSelectionDocument { + id: 1, + name: "ci-helper", + short_description: Some("Diagnose continuous integration failures."), + description: "Inspect failing GitHub Actions checks and logs.", + }, + SkillSelectionDocument { + id: 2, + name: "document-editor", + short_description: None, + description: "Edit Word documents.", + }, + ]; + + let selection = WeightedLexicalSkillSelector.select( + "Please diagnose the failing GitHub Actions check", + &documents, + /*limit*/ 20, + ); + + assert_eq!(vec![1], selection.candidate_ids); +} + +#[test] +fn lexical_selector_respects_requested_limit() { + let names = (0..10) + .map(|index| format!("lint-{index}")) + .collect::>(); + let documents = names + .iter() + .enumerate() + .map(|(id, name)| SkillSelectionDocument { + id, + name, + short_description: None, + description: "Fix lint errors.", + }) + .collect::>(); + + let selection = + WeightedLexicalSkillSelector.select("fix lint errors", &documents, /*limit*/ 3); + + assert_eq!(3, selection.candidate_ids.len()); +} + +#[test] +fn lexical_selector_reports_bounded_inputs() { + let long_query = "match ".repeat(MAX_QUERY_BYTES); + let names = (0..=MAX_CANDIDATES) + .map(|index| format!("candidate-{index}")) + .collect::>(); + let documents = names + .iter() + .enumerate() + .map(|(id, name)| SkillSelectionDocument { + id, + name, + short_description: None, + description: "match", + }) + .collect::>(); + + let selection = WeightedLexicalSkillSelector.select(&long_query, &documents, /*limit*/ 20); + + assert!(selection.query_truncated); + assert!(selection.candidate_set_truncated); + assert_eq!(20, selection.candidate_ids.len()); + assert!( + selection + .candidate_ids + .iter() + .all(|id| *id < MAX_CANDIDATES) + ); +} + +#[test] +fn lexical_selector_caps_query_terms() { + let query = (0..=MAX_QUERY_TERMS) + .map(|index| format!("term{index}")) + .collect::>() + .join(" "); + let documents = [SkillSelectionDocument { + id: 1, + name: "term0", + short_description: None, + description: "term0", + }]; + + let selection = WeightedLexicalSkillSelector.select(&query, &documents, /*limit*/ 20); + + assert_eq!(MAX_QUERY_TERMS, selection.query_term_count); + assert!(selection.query_truncated); +} + +#[test] +fn lexical_selector_returns_nothing_for_stop_words_only() { + let documents = [SkillSelectionDocument { + id: 1, + name: "anything", + short_description: None, + description: "Do anything.", + }]; + + let selection = + WeightedLexicalSkillSelector.select("please use the", &documents, /*limit*/ 20); + + assert_eq!(CheapSkillSelection::default(), selection); +} + +#[test] +fn selector_can_be_used_behind_the_shared_trait() { + fn run(selector: &dyn CheapSkillSelector) -> CheapSkillSelection { + selector.select( + "review code", + &[SkillSelectionDocument { + id: 7, + name: "code-review", + short_description: None, + description: "Review code.", + }], + /*limit*/ 20, + ) + } + + assert_eq!(vec![7], run(&WeightedLexicalSkillSelector).candidate_ids); +} diff --git a/codex-rs/ext/skills/src/extension.rs b/codex-rs/ext/skills/src/extension.rs index a632ab682407..8e877ac8a520 100644 --- a/codex-rs/ext/skills/src/extension.rs +++ b/codex-rs/ext/skills/src/extension.rs @@ -11,6 +11,9 @@ use codex_extension_api::ExtensionEventSink; use codex_extension_api::ExtensionFuture; use codex_extension_api::ExtensionRegistryBuilder; use codex_extension_api::PromptFragment; +use codex_extension_api::SkillInvocationContributor; +use codex_extension_api::SkillInvocationInput; +use codex_extension_api::SkillInvocationKind; use codex_extension_api::ThreadLifecycleContributor; use codex_extension_api::ThreadStartInput; use codex_extension_api::ToolCall; @@ -21,6 +24,7 @@ use codex_extension_api::TurnInputContributor; use codex_extension_api::WorldStateContributionInput; use codex_extension_api::WorldStateSectionContribution; use codex_mcp::McpResourceClient; +use codex_otel::MetricsClient; use codex_protocol::openai_models::ModelInfo; use codex_protocol::protocol::Event; use codex_protocol::protocol::EventMsg; @@ -34,6 +38,7 @@ use crate::catalog::SkillSourceKind; use crate::fragments::SkillInstructions; use crate::provider::HostSkillProvider; use crate::provider::SkillListQuery; +use crate::provider::SkillProvider; use crate::provider::SkillReadRequest; use crate::render::MAX_SKILL_NAME_BYTES; use crate::render::MAX_SKILL_PATH_BYTES; @@ -41,6 +46,7 @@ use crate::render::available_skills_fragment; use crate::render::truncate_main_prompt_contents; use crate::render::truncate_utf8_to_bytes; use crate::selection::collect_explicit_skill_mentions; +use crate::shadow_selection_experiment::ShadowSelectionExperiment; use crate::sources::SkillProviders; use crate::state::ExecutorSkillsStepState; use crate::state::SkillsThreadState; @@ -52,6 +58,7 @@ struct SkillsExtension { providers: SkillProviders, event_sink: Arc, config_from_host: Arc SkillsExtensionConfig + Send + Sync>, + shadow_selection: Arc, } impl ThreadLifecycleContributor for SkillsExtension @@ -201,10 +208,37 @@ where self.providers.clone(), session_store.get::(), thread_state, + Arc::clone(&self.shadow_selection), ) } } +impl SkillInvocationContributor for SkillsExtension +where + C: Send + Sync + 'static, +{ + fn on_skill_invocation<'a>( + &'a self, + input: SkillInvocationInput<'a>, + ) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + match input.kind { + SkillInvocationKind::Implicit => { + if let Some(state) = input + .thread_store + .get::() + .and_then(|state| state.shadow_selection_turn(input.turn_id)) + { + self.shadow_selection + .record_invocation(&state, input.skill_resource); + } + } + SkillInvocationKind::Explicit => {} + } + }) + } +} + impl TurnInputContributor for SkillsExtension where C: Send + Sync + 'static, @@ -232,6 +266,7 @@ where include_orchestrator_skills: thread_state.orchestrator_skills_enabled(), mcp_resources: session_store.get::(), }; + let host_query = query.clone(); let mut catalog = self.list_skills(query, &thread_state).await; if let Some(executor_skills) = turn_store.get::() { catalog.extend(executor_skills.0.clone()); @@ -241,6 +276,24 @@ where } let selected_entries = collect_explicit_skill_mentions(&input.user_input, &catalog); + let shadow_selection_turn = if config.shadow_selection_enabled { + // App-server leaves host discovery in core, so its configured providers omit the + // host snapshot. Extend only the experiment catalog to keep rendered context intact. + let mut shadow_catalog = catalog.clone(); + if host_snapshot.is_some() + && let Ok(host_catalog) = HostSkillProvider::new().list(host_query).await + { + shadow_catalog.extend(host_catalog); + } + Some( + self.shadow_selection + .run(&input.user_input, &shadow_catalog), + ) + } else { + None + }; + thread_state + .replace_shadow_selection_turn(input.turn_id.clone(), shadow_selection_turn); let mut fragments: Vec> = Vec::new(); if config.include_instructions { let mut turn_catalog = catalog.clone(); @@ -406,15 +459,33 @@ pub fn install_with_providers( config_from_host: impl Fn(&C) -> SkillsExtensionConfig + Send + Sync + 'static, ) where C: Send + Sync + 'static, +{ + install_with_providers_and_metrics( + registry, + providers, + /*metrics_client*/ None, + config_from_host, + ); +} + +pub fn install_with_providers_and_metrics( + registry: &mut ExtensionRegistryBuilder, + providers: SkillProviders, + metrics_client: Option, + config_from_host: impl Fn(&C) -> SkillsExtensionConfig + Send + Sync + 'static, +) where + C: Send + Sync + 'static, { let extension = Arc::new(SkillsExtension { providers, event_sink: registry.event_sink(), config_from_host: Arc::new(config_from_host), + shadow_selection: Arc::new(ShadowSelectionExperiment::new(metrics_client)), }); registry.thread_lifecycle_contributor(extension.clone()); registry.config_contributor(extension.clone()); registry.prompt_contributor(extension.clone()); registry.turn_input_contributor(extension.clone()); + registry.skill_invocation_contributor(extension.clone()); registry.tool_contributor(extension); } diff --git a/codex-rs/ext/skills/src/lib.rs b/codex-rs/ext/skills/src/lib.rs index 4a004ff9ac0c..a01ff2bfdc91 100644 --- a/codex-rs/ext/skills/src/lib.rs +++ b/codex-rs/ext/skills/src/lib.rs @@ -1,10 +1,12 @@ pub mod catalog; mod config; +mod dynamic_skill_selector; mod extension; mod fragments; pub mod provider; mod render; mod selection; +mod shadow_selection_experiment; mod sources; mod state; mod tools; @@ -13,6 +15,7 @@ mod world_state; pub use config::SkillsExtensionConfig; pub use extension::install; pub use extension::install_with_providers; +pub use extension::install_with_providers_and_metrics; pub use provider::ExecutorSkillProvider; pub use provider::HostSkillProvider; pub use provider::OrchestratorSkillProvider; diff --git a/codex-rs/ext/skills/src/shadow_selection_experiment.rs b/codex-rs/ext/skills/src/shadow_selection_experiment.rs new file mode 100644 index 000000000000..39a6bb9dc588 --- /dev/null +++ b/codex-rs/ext/skills/src/shadow_selection_experiment.rs @@ -0,0 +1,356 @@ +// This shadow-selection experiment is temporary and should be removed after evaluation. + +use std::collections::HashSet; +use std::sync::Mutex; +use std::sync::PoisonError; +use std::time::Duration; +use std::time::Instant; + +use codex_otel::MetricsClient; +use codex_protocol::user_input::UserInput; + +use crate::catalog::SkillCatalog; +use crate::dynamic_skill_selector::CheapSkillSelection; +use crate::dynamic_skill_selector::CheapSkillSelector; +use crate::dynamic_skill_selector::SkillSelectionDocument; +use crate::dynamic_skill_selector::WeightedLexicalSkillSelector; + +const MAX_SHADOW_QUERY_BYTES: usize = 16 * 1024; +const MAX_SHADOW_RESULTS: usize = 20; + +const RUN_METRIC: &str = "codex.skills.shadow_selection"; +const DURATION_METRIC: &str = "codex.skills.shadow_selection.duration_ms"; +const CATALOG_ENTRY_COUNT_METRIC: &str = "codex.skills.shadow_selection.catalog_entries"; +const SELECTED_ENTRY_COUNT_METRIC: &str = "codex.skills.shadow_selection.selected_entries"; +const QUERY_TERM_COUNT_METRIC: &str = "codex.skills.shadow_selection.query_terms"; +const REDUCTION_BPS_METRIC: &str = "codex.skills.shadow_selection.reduction_bps"; +const INVOCATION_METRIC: &str = "codex.skills.shadow_selection.invocation"; + +pub(crate) struct ShadowSelectionExperiment { + selectors: Vec>, + metrics_client: Option, +} + +impl ShadowSelectionExperiment { + pub(crate) fn new(metrics_client: Option) -> Self { + Self { + selectors: vec![Box::new(WeightedLexicalSkillSelector)], + metrics_client, + } + } + + pub(crate) fn run( + &self, + inputs: &[UserInput], + catalog: &SkillCatalog, + ) -> ShadowSelectionTurnState { + let query = build_shadow_query(inputs); + let query_script = query_script_tag(&query.text); + let documents = catalog + .entries + .iter() + .enumerate() + .filter(|(_, entry)| entry.enabled && entry.prompt_visible) + .map(|(id, entry)| SkillSelectionDocument { + id, + name: entry.name.as_str(), + short_description: entry.short_description.as_deref(), + description: entry.description.as_str(), + }) + .collect::>(); + let eligible_ids = documents + .iter() + .map(|document| document.id) + .collect::>(); + let mut ranked_selections = Vec::with_capacity(self.selectors.len()); + + for selector in &self.selectors { + let start = Instant::now(); + let selection = + selector.select(&query.text, &documents, /*limit*/ MAX_SHADOW_RESULTS); + let duration = start.elapsed(); + let selected_ids = sanitize_selected_ids(&selection, &eligible_ids); + self.record_metrics(ShadowSelectionObservation { + method: selector.method(), + selection: &selection, + query_truncated_before_selection: query.truncated, + query_script, + catalog_entry_count: documents.len(), + selected_entry_count: selected_ids.len(), + duration, + }); + ranked_selections.push(RankedSelection { + method: selector.method(), + skill_resources: selected_ids + .iter() + .map(|id| normalize_skill_resource(catalog.entries[*id].main_prompt.as_str())) + .collect(), + }); + tracing::debug!( + method = selector.method(), + catalog_entries = documents.len(), + selected_entries = selected_ids.len(), + query_terms = selection.query_term_count, + query_script, + query_truncated = query.truncated || selection.query_truncated, + candidate_set_truncated = selection.candidate_set_truncated, + "ran shadow skill selection" + ); + } + + ShadowSelectionTurnState { + ranked_selections, + query_script, + seen_skill_resources: Mutex::new(HashSet::new()), + } + } + + pub(crate) fn record_invocation(&self, state: &ShadowSelectionTurnState, skill_resource: &str) { + let skill_resource = normalize_skill_resource(skill_resource); + if !state + .seen_skill_resources + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(skill_resource.clone()) + { + return; + } + let Some(metrics_client) = self.metrics_client.as_ref() else { + return; + }; + for selection in &state.ranked_selections { + let rank = selection + .skill_resources + .iter() + .position(|candidate| candidate == &skill_resource) + .map(|index| index + 1); + let tags = [ + ("method", selection.method), + ("hit", bool_tag(rank.is_some())), + ("rank", rank_bucket(rank)), + ("query_script", state.query_script), + ]; + let _ = metrics_client.counter(INVOCATION_METRIC, /*inc*/ 1, &tags); + } + } + + fn record_metrics(&self, observation: ShadowSelectionObservation<'_>) { + let Some(metrics_client) = self.metrics_client.as_ref() else { + return; + }; + let ShadowSelectionObservation { + method, + selection, + query_truncated_before_selection, + query_script, + catalog_entry_count, + selected_entry_count, + duration, + } = observation; + let status = selection_status(selection, selected_entry_count); + let query_truncated = + bool_tag(query_truncated_before_selection || selection.query_truncated); + let candidate_set_truncated = bool_tag(selection.candidate_set_truncated); + let tags = [ + ("method", method), + ("status", status), + ("query_script", query_script), + ("query_truncated", query_truncated), + ("candidate_set_truncated", candidate_set_truncated), + ]; + let _ = metrics_client.counter(RUN_METRIC, /*inc*/ 1, &tags); + let _ = metrics_client.record_duration(DURATION_METRIC, duration, &tags); + let _ = metrics_client.histogram( + CATALOG_ENTRY_COUNT_METRIC, + metric_value(catalog_entry_count), + &tags, + ); + let _ = metrics_client.histogram( + SELECTED_ENTRY_COUNT_METRIC, + metric_value(selected_entry_count), + &tags, + ); + let _ = metrics_client.histogram( + QUERY_TERM_COUNT_METRIC, + metric_value(selection.query_term_count), + &tags, + ); + let _ = metrics_client.histogram( + REDUCTION_BPS_METRIC, + reduction_bps(catalog_entry_count, selected_entry_count), + &tags, + ); + } +} + +pub(crate) struct ShadowSelectionTurnState { + ranked_selections: Vec, + query_script: &'static str, + seen_skill_resources: Mutex>, +} + +struct RankedSelection { + method: &'static str, + skill_resources: Vec, +} + +struct ShadowSelectionObservation<'a> { + method: &'static str, + selection: &'a CheapSkillSelection, + query_truncated_before_selection: bool, + query_script: &'static str, + catalog_entry_count: usize, + selected_entry_count: usize, + duration: Duration, +} + +fn sanitize_selected_ids( + selection: &CheapSkillSelection, + eligible_ids: &HashSet, +) -> Vec { + let mut seen = HashSet::new(); + selection + .candidate_ids + .iter() + .copied() + .filter(|id| eligible_ids.contains(id) && seen.insert(*id)) + .take(MAX_SHADOW_RESULTS) + .collect() +} + +fn selection_status(selection: &CheapSkillSelection, selected_entry_count: usize) -> &'static str { + if selection.query_term_count == 0 { + "no_query_terms" + } else if selected_entry_count == 0 { + "no_matches" + } else { + "selected" + } +} + +fn reduction_bps(catalog_entry_count: usize, selected_entry_count: usize) -> i64 { + if catalog_entry_count == 0 { + return 0; + } + 10_000i64.saturating_sub(ratio_bps(selected_entry_count, catalog_entry_count)) +} + +fn ratio_bps(numerator: usize, denominator: usize) -> i64 { + if denominator == 0 { + return 0; + } + let numerator = u128::try_from(numerator).unwrap_or(u128::MAX); + let denominator = u128::try_from(denominator).unwrap_or(u128::MAX); + let basis_points = numerator.saturating_mul(10_000) / denominator; + i64::try_from(basis_points).unwrap_or(i64::MAX) +} + +fn metric_value(value: usize) -> i64 { + i64::try_from(value).unwrap_or(i64::MAX) +} + +fn bool_tag(value: bool) -> &'static str { + if value { "true" } else { "false" } +} + +fn rank_bucket(rank: Option) -> &'static str { + match rank { + Some(1) => "1", + Some(2..=5) => "2_5", + Some(6..=10) => "6_10", + Some(11..=MAX_SHADOW_RESULTS) => "11_20", + Some(_) | None => "miss", + } +} + +fn normalize_skill_resource(skill_resource: &str) -> String { + skill_resource.replace('\\', "/") +} + +fn query_script_tag(query: &str) -> &'static str { + let mut has_ascii_latin = false; + let mut has_cjk = false; + let mut has_other = false; + + for character in query.chars().filter(|character| character.is_alphabetic()) { + if character.is_ascii_alphabetic() { + has_ascii_latin = true; + } else if is_cjk(character) { + has_cjk = true; + } else { + has_other = true; + } + } + + match (has_ascii_latin, has_cjk, has_other) { + (false, false, false) => "none", + (true, false, false) => "ascii_latin", + (false, true, false) => "cjk", + (false, false, true) => "other", + (true, true, false) | (true, false, true) | (false, true, true) | (true, true, true) => { + "mixed" + } + } +} + +fn is_cjk(character: char) -> bool { + matches!( + character, + '\u{1100}'..='\u{11ff}' + | '\u{3040}'..='\u{30ff}' + | '\u{3100}'..='\u{312f}' + | '\u{3130}'..='\u{318f}' + | '\u{31a0}'..='\u{31bf}' + | '\u{31f0}'..='\u{31ff}' + | '\u{3400}'..='\u{4dbf}' + | '\u{4e00}'..='\u{9fff}' + | '\u{a960}'..='\u{a97f}' + | '\u{ac00}'..='\u{d7af}' + | '\u{d7b0}'..='\u{d7ff}' + | '\u{f900}'..='\u{faff}' + | '\u{20000}'..='\u{2fa1f}' + ) +} + +struct ShadowQuery { + text: String, + truncated: bool, +} + +fn build_shadow_query(inputs: &[UserInput]) -> ShadowQuery { + let mut text = String::new(); + let mut truncated = false; + for input in inputs { + let part = match input { + UserInput::Text { text, .. } => text.as_str(), + UserInput::Skill { name, .. } | UserInput::Mention { name, .. } => name.as_str(), + _ => continue, + }; + if part.is_empty() { + continue; + } + if !text.is_empty() && !push_bounded(&mut text, " ") { + truncated = true; + break; + } + if !push_bounded(&mut text, part) { + truncated = true; + break; + } + } + ShadowQuery { text, truncated } +} + +fn push_bounded(destination: &mut String, value: &str) -> bool { + let remaining = MAX_SHADOW_QUERY_BYTES.saturating_sub(destination.len()); + if value.len() <= remaining { + destination.push_str(value); + return true; + } + let mut end = remaining; + while !value.is_char_boundary(end) { + end = end.saturating_sub(1); + } + destination.push_str(&value[..end]); + false +} diff --git a/codex-rs/ext/skills/src/state.rs b/codex-rs/ext/skills/src/state.rs index 2094f344962c..db6fae949495 100644 --- a/codex-rs/ext/skills/src/state.rs +++ b/codex-rs/ext/skills/src/state.rs @@ -20,6 +20,7 @@ use crate::catalog::SkillResourceId; use crate::catalog::SkillSourceKind; use crate::provider::SkillListQuery; use crate::provider::SkillReadRequest; +use crate::shadow_selection_experiment::ShadowSelectionTurnState; use crate::sources::SkillProviders; const MAX_CACHED_ORCHESTRATOR_RESOURCES: usize = 100; @@ -30,6 +31,7 @@ pub(crate) struct SkillsThreadState { orchestrator_skills_available: bool, executor_cache: Mutex>, orchestrator_cache: Mutex>>, + shadow_selection_turn: Mutex>, } impl SkillsThreadState { @@ -39,6 +41,7 @@ impl SkillsThreadState { orchestrator_skills_available, executor_cache: Mutex::new(Vec::new()), orchestrator_cache: Mutex::new(None), + shadow_selection_turn: Mutex::new(None), } } @@ -60,6 +63,33 @@ impl SkillsThreadState { self.orchestrator_skills_available && self.config().orchestrator_skills_enabled } + pub(crate) fn replace_shadow_selection_turn( + &self, + turn_id: String, + state: Option, + ) { + *self + .shadow_selection_turn + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + state.map(|state| ShadowSelectionTurn { + turn_id, + state: Arc::new(state), + }); + } + + pub(crate) fn shadow_selection_turn( + &self, + turn_id: &str, + ) -> Option> { + self.shadow_selection_turn + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + .filter(|turn| turn.turn_id == turn_id) + .map(|turn| Arc::clone(&turn.state)) + } + /// Returns catalogs for stable selected roots. /// /// The first catalog returned for a root remains cached until this thread state is dropped. @@ -196,6 +226,11 @@ impl SkillsThreadState { } } +struct ShadowSelectionTurn { + turn_id: String, + state: Arc, +} + struct CachedExecutorCatalog { root: SelectedCapabilityRoot, catalog: SkillCatalog, diff --git a/codex-rs/ext/skills/src/tools/mod.rs b/codex-rs/ext/skills/src/tools/mod.rs index 0d05064afb89..286b043cd8d2 100644 --- a/codex-rs/ext/skills/src/tools/mod.rs +++ b/codex-rs/ext/skills/src/tools/mod.rs @@ -23,6 +23,7 @@ use crate::catalog::SkillAuthority; use crate::catalog::SkillCatalog; use crate::catalog::SkillSourceKind; use crate::provider::SkillListQuery; +use crate::shadow_selection_experiment::ShadowSelectionExperiment; use crate::sources::SkillProviders; use crate::state::SkillsThreadState; @@ -37,11 +38,13 @@ pub(crate) fn skill_tools( providers: SkillProviders, mcp_resources: Option>, thread_state: Arc, + shadow_selection: Arc, ) -> Vec>> { let context = SkillToolContext { providers, mcp_resources, thread_state, + shadow_selection, }; vec![ Arc::new(list::ListTool { @@ -56,6 +59,7 @@ struct SkillToolContext { providers: SkillProviders, mcp_resources: Option>, thread_state: Arc, + shadow_selection: Arc, } impl SkillToolContext { diff --git a/codex-rs/ext/skills/src/tools/read.rs b/codex-rs/ext/skills/src/tools/read.rs index 64587cab95b6..d2971db19212 100644 --- a/codex-rs/ext/skills/src/tools/read.rs +++ b/codex-rs/ext/skills/src/tools/read.rs @@ -63,14 +63,14 @@ impl ToolExecutor for ReadTool { validate_handle("resource", &args.resource, MAX_HANDLE_BYTES)?; let catalog = self.context.catalog(&call.turn_id, args.authority).await; - let package_is_available = catalog.entries.iter().any(|entry| { + let Some(skill_entry) = catalog.entries.iter().find(|entry| { entry.enabled && entry.authority == authority && entry.id.0 == args.package - }); - if !package_is_available { + }) else { return Err(FunctionCallError::RespondToModel( "skill package is not available from the requested authority".to_string(), )); - } + }; + let main_prompt = skill_entry.main_prompt.clone(); let requested_resource = SkillResourceId::new(args.resource); let result = self @@ -103,6 +103,16 @@ impl ToolExecutor for ReadTool { )); } + if let Some(state) = self + .context + .thread_state + .shadow_selection_turn(&call.turn_id) + { + self.context + .shadow_selection + .record_invocation(&state, main_prompt.as_str()); + } + external_json_output(&ReadResponse { resource: result.resource.as_str().to_string(), contents: result.contents, diff --git a/codex-rs/ext/skills/tests/implicit_invocation.rs b/codex-rs/ext/skills/tests/implicit_invocation.rs new file mode 100644 index 000000000000..7b5999766cf5 --- /dev/null +++ b/codex-rs/ext/skills/tests/implicit_invocation.rs @@ -0,0 +1,331 @@ +use std::collections::BTreeMap; +use std::sync::Arc; + +use codex_core_skills::HostSkillsSnapshot; +use codex_core_skills::SkillLoadOutcome; +use codex_core_skills::SkillMetadata; +use codex_extension_api::ConversationHistory; +use codex_extension_api::ExtensionData; +use codex_extension_api::ExtensionRegistry; +use codex_extension_api::ExtensionRegistryBuilder; +use codex_extension_api::NoopTurnItemEmitter; +use codex_extension_api::SkillInvocationInput; +use codex_extension_api::SkillInvocationKind; +use codex_extension_api::ThreadStartInput; +use codex_extension_api::ToolCall; +use codex_extension_api::ToolPayload; +use codex_extension_api::TurnInputContext; +use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; +use codex_otel::MetricsClient; +use codex_otel::MetricsConfig; +use codex_protocol::protocol::SessionSource; +use codex_protocol::protocol::SkillScope; +use codex_protocol::protocol::TruncationPolicy; +use codex_protocol::user_input::UserInput; +use codex_skills_extension::SkillProviders; +use codex_skills_extension::SkillsExtensionConfig; +use codex_skills_extension::catalog::SkillAuthority; +use codex_skills_extension::catalog::SkillCatalog; +use codex_skills_extension::catalog::SkillCatalogEntry; +use codex_skills_extension::catalog::SkillPackageId; +use codex_skills_extension::catalog::SkillReadResult; +use codex_skills_extension::catalog::SkillResourceId; +use codex_skills_extension::catalog::SkillSearchResult; +use codex_skills_extension::catalog::SkillSourceKind; +use codex_skills_extension::install_with_providers_and_metrics; +use codex_skills_extension::provider::SkillListQuery; +use codex_skills_extension::provider::SkillProvider; +use codex_skills_extension::provider::SkillProviderFuture; +use codex_skills_extension::provider::SkillReadRequest; +use codex_skills_extension::provider::SkillSearchRequest; +use codex_utils_absolute_path::AbsolutePathBuf; +use opentelemetry_sdk::metrics::InMemoryMetricExporter; +use opentelemetry_sdk::metrics::data::AggregatedMetrics; +use opentelemetry_sdk::metrics::data::MetricData; +use pretty_assertions::assert_eq; + +type TestResult = Result<(), Box>; +type InvocationPoint = (BTreeMap, u64); + +const INVOCATION_METRIC: &str = "codex.skills.shadow_selection.invocation"; +const PACKAGE: &str = "orchestrator/demo"; +const MAIN_RESOURCE: &str = "skill://orchestrator/demo/SKILL.md"; + +#[derive(Clone)] +struct OrchestratorProvider; + +impl SkillProvider for OrchestratorProvider { + fn list(&self, _query: SkillListQuery) -> SkillProviderFuture<'_, SkillCatalog> { + Box::pin(std::future::ready(Ok(SkillCatalog { + entries: vec![SkillCatalogEntry::new( + SkillPackageId(PACKAGE.to_string()), + SkillAuthority::new(SkillSourceKind::Orchestrator, CODEX_APPS_MCP_SERVER_NAME), + "demo", + "Use the demo skill.", + SkillResourceId::new(MAIN_RESOURCE), + )], + warnings: Vec::new(), + }))) + } + + fn read(&self, request: SkillReadRequest) -> SkillProviderFuture<'_, SkillReadResult> { + Box::pin(std::future::ready(Ok(SkillReadResult { + resource: request.resource, + contents: "# Demo\n\nUse the demo skill.".to_string(), + }))) + } + + fn search(&self, _request: SkillSearchRequest) -> SkillProviderFuture<'_, SkillSearchResult> { + Box::pin(std::future::ready(Ok(SkillSearchResult::default()))) + } +} + +#[tokio::test] +async fn implicit_core_and_native_read_invocations_share_turn_local_recording() -> TestResult { + let metrics = MetricsClient::new( + MetricsConfig::in_memory( + "test", + "codex-skills-extension", + env!("CARGO_PKG_VERSION"), + InMemoryMetricExporter::default(), + ) + .with_runtime_reader(), + )?; + let mut builder = ExtensionRegistryBuilder::<()>::new(); + install_with_providers_and_metrics( + &mut builder, + SkillProviders::new().with_orchestrator_provider(Arc::new(OrchestratorProvider)), + Some(metrics.clone()), + |_| SkillsExtensionConfig { + include_instructions: false, + bundled_skills_enabled: false, + orchestrator_skills_enabled: true, + shadow_selection_enabled: true, + }, + ); + let registry = builder.build(); + let session_store = ExtensionData::new("session"); + let thread_store = ExtensionData::new("thread"); + registry.thread_lifecycle_contributors()[0] + .on_thread_start(ThreadStartInput { + config: &(), + session_source: &SessionSource::Cli, + persistent_thread_state_available: true, + environments: &[], + session_store: &session_store, + thread_store: &thread_store, + }) + .await; + + let core_turn = start_turn(®istry, &session_store, &thread_store, "turn-core").await; + registry.skill_invocation_contributors()[0] + .on_skill_invocation(SkillInvocationInput { + session_store: &session_store, + thread_store: &thread_store, + turn_store: core_turn.as_ref(), + turn_id: "turn-core", + skill_resource: MAIN_RESOURCE, + kind: SkillInvocationKind::Implicit, + }) + .await; + + start_turn(®istry, &session_store, &thread_store, "turn-tool").await; + let tools = registry.tool_contributors()[0].tools(&session_store, &thread_store); + let read_tool = tools + .iter() + .find(|tool| tool.tool_name().name == "read") + .ok_or("skills.read tool should be registered")?; + for call_id in ["call-1", "call-2"] { + read_tool + .handle(ToolCall { + turn_id: "turn-tool".to_string(), + call_id: call_id.to_string(), + tool_name: read_tool.tool_name(), + model: "gpt-test".to_string(), + truncation_policy: TruncationPolicy::Bytes(1_024), + conversation_history: ConversationHistory::default(), + turn_item_emitter: Arc::new(NoopTurnItemEmitter), + environments: Vec::new(), + payload: ToolPayload::Function { + arguments: serde_json::json!({ + "authority": {"kind": "orchestrator"}, + "package": PACKAGE, + "resource": MAIN_RESOURCE, + }) + .to_string(), + }, + }) + .await?; + } + + assert_eq!( + vec![( + BTreeMap::from([ + ("hit".to_string(), "true".to_string()), + ("method".to_string(), "weighted_lexical_v1".to_string()), + ("query_script".to_string(), "ascii_latin".to_string()), + ("rank".to_string(), "1".to_string()), + ]), + 2, + )], + invocation_points(&metrics)?, + ); + Ok(()) +} + +#[tokio::test] +async fn host_snapshot_is_available_only_to_shadow_selection() -> TestResult { + let metrics = MetricsClient::new( + MetricsConfig::in_memory( + "test", + "codex-skills-extension", + env!("CARGO_PKG_VERSION"), + InMemoryMetricExporter::default(), + ) + .with_runtime_reader(), + )?; + let mut builder = ExtensionRegistryBuilder::<()>::new(); + install_with_providers_and_metrics( + &mut builder, + SkillProviders::new(), + Some(metrics.clone()), + |_| SkillsExtensionConfig { + include_instructions: true, + bundled_skills_enabled: false, + orchestrator_skills_enabled: false, + shadow_selection_enabled: true, + }, + ); + let registry = builder.build(); + let session_store = ExtensionData::new("session"); + let thread_store = ExtensionData::new("thread"); + registry.thread_lifecycle_contributors()[0] + .on_thread_start(ThreadStartInput { + config: &(), + session_source: &SessionSource::Cli, + persistent_thread_state_available: true, + environments: &[], + session_store: &session_store, + thread_store: &thread_store, + }) + .await; + + let skill_path = AbsolutePathBuf::try_from( + std::env::temp_dir() + .join("codex-shadow-host-skill") + .join("SKILL.md"), + )?; + let skill_resource = skill_path.to_string_lossy().into_owned(); + let mut outcome = SkillLoadOutcome::default(); + outcome.skills.push(SkillMetadata { + name: "演示文稿".to_string(), + description: "创建演示文稿。".to_string(), + short_description: None, + interface: None, + dependencies: None, + policy: None, + path_to_skills_md: skill_path, + scope: SkillScope::User, + plugin_id: None, + }); + let turn_store = ExtensionData::new("turn-host"); + turn_store.insert(HostSkillsSnapshot::new(Arc::new(outcome))); + + let fragments = registry.turn_input_contributors()[0] + .contribute( + TurnInputContext { + turn_id: "turn-host".to_string(), + user_input: vec![UserInput::Text { + text: "演示文稿".to_string(), + text_elements: Vec::new(), + }], + environments: Vec::new(), + }, + &session_store, + &thread_store, + &turn_store, + ) + .await; + assert!(fragments.is_empty()); + + registry.skill_invocation_contributors()[0] + .on_skill_invocation(SkillInvocationInput { + session_store: &session_store, + thread_store: &thread_store, + turn_store: &turn_store, + turn_id: "turn-host", + skill_resource: &skill_resource, + kind: SkillInvocationKind::Implicit, + }) + .await; + + assert_eq!( + vec![( + BTreeMap::from([ + ("hit".to_string(), "true".to_string()), + ("method".to_string(), "weighted_lexical_v1".to_string()), + ("query_script".to_string(), "cjk".to_string()), + ("rank".to_string(), "1".to_string()), + ]), + 1, + )], + invocation_points(&metrics)?, + ); + Ok(()) +} + +async fn start_turn( + registry: &ExtensionRegistry<()>, + session_store: &ExtensionData, + thread_store: &ExtensionData, + turn_id: &str, +) -> Arc { + let turn_store = Arc::new(ExtensionData::new(turn_id)); + registry.turn_input_contributors()[0] + .contribute( + TurnInputContext { + turn_id: turn_id.to_string(), + user_input: vec![UserInput::Text { + text: "use demo".to_string(), + text_elements: Vec::new(), + }], + environments: Vec::new(), + }, + session_store, + thread_store, + turn_store.as_ref(), + ) + .await; + turn_store +} + +fn invocation_points( + metrics: &MetricsClient, +) -> Result, Box> { + let snapshot = metrics.snapshot()?; + let metric = snapshot + .scope_metrics() + .flat_map(opentelemetry_sdk::metrics::data::ScopeMetrics::metrics) + .find(|metric| metric.name() == INVOCATION_METRIC) + .ok_or("invocation metric was not emitted")?; + match metric.data() { + AggregatedMetrics::U64(MetricData::Sum(sum)) => Ok(sum + .data_points() + .map(|point| { + ( + point + .attributes() + .map(|attribute| { + ( + attribute.key.as_str().to_string(), + attribute.value.as_str().to_string(), + ) + }) + .collect(), + point.value(), + ) + }) + .collect()), + other => Err(format!("unexpected invocation metric data: {other:?}").into()), + } +} diff --git a/codex-rs/ext/skills/tests/skills_extension.rs b/codex-rs/ext/skills/tests/skills_extension.rs index 76d0df7954b4..da83a3168fb4 100644 --- a/codex-rs/ext/skills/tests/skills_extension.rs +++ b/codex-rs/ext/skills/tests/skills_extension.rs @@ -69,7 +69,8 @@ async fn installed_extension_uses_host_service_snapshot() -> TestResult { .ok_or("skill path should have a parent")?, )?; std::fs::write(&skill_path, DEMO_SKILL_CONTENTS)?; - let config = default_config(); + let mut config = default_config(); + config.shadow_selection_enabled = true; let mut builder = ExtensionRegistryBuilder::new(); install(&mut builder, skills_extension_config); @@ -781,6 +782,7 @@ struct TestConfig { include_instructions: bool, bundled_skills_enabled: bool, orchestrator_skills_enabled: bool, + shadow_selection_enabled: bool, } fn default_config() -> TestConfig { @@ -788,6 +790,7 @@ fn default_config() -> TestConfig { include_instructions: true, bundled_skills_enabled: true, orchestrator_skills_enabled: true, + shadow_selection_enabled: false, } } @@ -796,6 +799,7 @@ fn skills_extension_config(config: &TestConfig) -> SkillsExtensionConfig { include_instructions: config.include_instructions, bundled_skills_enabled: config.bundled_skills_enabled, orchestrator_skills_enabled: config.orchestrator_skills_enabled, + shadow_selection_enabled: config.shadow_selection_enabled, } } diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index 716bb2e44132..ec238b79ff51 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -206,6 +206,8 @@ pub enum Feature { ConcurrentReasoningSummaries, /// Allow prompting and installing missing MCP dependencies. SkillMcpDependencyInstall, + /// Run cheap skill-search methods in shadow mode and emit experiment metrics. + SkillSearch, /// Removed compatibility flag for deleted skill env var dependency prompting. SkillEnvVarDependencyPrompt, /// Enable the unified mention popup used by default in the TUI. @@ -1190,6 +1192,12 @@ pub const FEATURES: &[FeatureSpec] = &[ stage: Stage::Stable, default_enabled: true, }, + FeatureSpec { + id: Feature::SkillSearch, + key: "skill_search", + stage: Stage::UnderDevelopment, + default_enabled: false, + }, FeatureSpec { id: Feature::SkillEnvVarDependencyPrompt, key: "skill_env_var_dependency_prompt",