diff --git a/crates/adaptive/src/acg/ir_builder.rs b/crates/adaptive/src/acg/ir_builder.rs index 8c08e46f6..4cf3af9bb 100644 --- a/crates/adaptive/src/acg/ir_builder.rs +++ b/crates/adaptive/src/acg/ir_builder.rs @@ -35,7 +35,13 @@ use crate::acg::prompt_ir::{ pub fn build_prompt_ir(request: &AnnotatedLlmRequest) -> Result { let mut blocks: Vec = Vec::new(); let mut sequence_index: u32 = 0; - let mut inserted_tool_blocks = false; + let mut inserted_scaffold_blocks = false; + let structured_output = request + .extra + .get("response_format") + .filter(|value| !value.is_null()) + .map(canonicalize_value) + .transpose()?; if let Some(instructions) = &request.instructions { blocks.push(build_text_block( @@ -48,16 +54,28 @@ pub fn build_prompt_ir(request: &AnnotatedLlmRequest) -> Result { } for message in &request.messages { - if should_insert_tool_blocks_before_message(inserted_tool_blocks, request, message) { + if !inserted_scaffold_blocks + && !matches!(message, Message::System { .. } | Message::Developer { .. }) + { append_tool_schema_blocks(&mut blocks, &mut sequence_index, request.tools.as_deref())?; - inserted_tool_blocks = true; + append_structured_output_block( + &mut blocks, + &mut sequence_index, + structured_output.as_deref(), + ); + inserted_scaffold_blocks = true; } append_message_blocks(&mut blocks, &mut sequence_index, message)?; } - if !inserted_tool_blocks { + if !inserted_scaffold_blocks { append_tool_schema_blocks(&mut blocks, &mut sequence_index, request.tools.as_deref())?; + append_structured_output_block( + &mut blocks, + &mut sequence_index, + structured_output.as_deref(), + ); } let tool_schema_hashes = match &request.tools { @@ -70,22 +88,12 @@ pub fn build_prompt_ir(request: &AnnotatedLlmRequest) -> Result { ir_id: Uuid::new_v4(), blocks, tool_schema_hashes, - structured_output_schema_id: None, + structured_output_schema_id: structured_output.as_deref().map(sha256_hex), source_request_hash, created_at: Utc::now(), }) } -fn should_insert_tool_blocks_before_message( - inserted_tool_blocks: bool, - request: &AnnotatedLlmRequest, - message: &Message, -) -> bool { - !inserted_tool_blocks - && !matches!(message, Message::System { .. } | Message::Developer { .. }) - && request.tools.is_some() -} - fn append_message_blocks( blocks: &mut Vec, sequence_index: &mut u32, @@ -323,6 +331,38 @@ fn append_tool_schema_blocks( Ok(()) } +/// Append the canonicalized structured-output contract as a scaffold block. +/// +/// The block sits with the tool schemas ahead of the first non-system message so +/// an output contract that never changes stays inside the stable prefix. +/// +/// # Parameters +/// - `blocks`: Block list being built. +/// - `seq`: Running sequence index shared by every block builder. +/// - `response_format`: Canonicalized `response_format` value, if the request set one. +fn append_structured_output_block( + blocks: &mut Vec, + seq: &mut u32, + response_format: Option<&str>, +) { + let Some(content) = response_format else { + return; + }; + + let index = *seq; + *seq += 1; + blocks.push(PromptBlock { + span_id: generate_span_id(PromptRole::System, index, Some("structured-output")), + sequence_index: index, + role: PromptRole::System, + content: content.to_string(), + content_type: BlockContentType::StructuredOutput, + provenance: ProvenanceLabel::System, + sensitivity: SensitivityLabel::default(), + token_metadata: None, + }); +} + fn build_tool_schema_block(seq: &mut u32, tool: &ToolDefinition) -> Result { let tool_value = serde_json::to_value(tool)?; let canonical = canonicalize_value(&tool_value)?; diff --git a/crates/adaptive/src/acg/stability.rs b/crates/adaptive/src/acg/stability.rs index 6141ffa54..959c59452 100644 --- a/crates/adaptive/src/acg/stability.rs +++ b/crates/adaptive/src/acg/stability.rs @@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize}; use crate::acg::canonicalize::sha256_hex; use crate::acg::profile::{BlockStabilityScore, StabilityClass}; -use crate::acg::prompt_ir::{PromptIR, SpanId}; +use crate::acg::prompt_ir::{PromptBlock, PromptIR, SpanId}; /// Thresholds controlling prompt-block stability classification. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -39,6 +39,11 @@ pub struct StabilityAnalysisResult { pub scores: Vec, /// Number of leading blocks that were classified as stable. pub stable_prefix_length: usize, + /// Fingerprint of the dominant stable prefix. Generic analysis hashes the + /// prompt IR; ACG profile persistence additionally binds it to the learning + /// key and, beyond the leading scaffold, the full source request. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stable_prefix_fingerprint: Option, /// Total number of observations included in the analysis. pub total_observations: u32, } @@ -55,6 +60,20 @@ struct SpanObservations { /// sequence index at which that span appeared, and derives the length of the /// stable prefix at the start of the prompt. /// +/// The result must not depend on the per-process seed `s` that Rust draws for +/// [`HashMap`] iteration, because a profile persisted by one process is read +/// back by another. Over `N` processes producing outcomes with multiplicities +/// `c_1..c_k`, the cross-process agreement rate is +/// +/// ```text +/// A = sum_i c_i * (c_i - 1) / (N * (N - 1)) +/// ``` +/// +/// the probability that two independent processes agree, and equivalently the +/// probability that a persisted profile satisfies the reuse gate elsewhere. +/// `A = 1` exactly when the analysis is seed-independent, which is what +/// [`canonical_scores`] and [`dominant_prefix_length`] each establish. +/// /// # Parameters /// - `observations`: Prompt observations to compare. /// - `thresholds`: Thresholds used for stability classification and confidence. @@ -69,6 +88,7 @@ pub fn analyze_stability( return StabilityAnalysisResult { scores: Vec::new(), stable_prefix_length: 0, + stable_prefix_fingerprint: None, total_observations: 0, }; } @@ -80,23 +100,241 @@ pub fn analyze_stability( record_observation(observation, &mut span_map); } - let mut indexed_scores: Vec<(u32, BlockStabilityScore)> = span_map + let indexed_scores: Vec<(u32, BlockStabilityScore)> = span_map .into_iter() .map(|(span_id, obs)| build_stability_score(span_id, obs, total_observations, thresholds)) .collect(); - indexed_scores.sort_by_key(|(idx, _)| *idx); - let scores: Vec = - indexed_scores.into_iter().map(|(_, score)| score).collect(); - let stable_prefix_length = find_stable_prefix_length(&scores); + let scores = canonical_scores(indexed_scores); + let stable_prefix_length = dominant_prefix_length(observations, thresholds.stable_threshold); + let stable_prefix_fingerprint = + dominant_fingerprint(observations.iter().filter_map(|observation| { + prompt_prefix_fingerprint(observation, stable_prefix_length) + })); StabilityAnalysisResult { scores, stable_prefix_length, + stable_prefix_fingerprint, total_observations, } } +/// Fingerprint the leading `prefix_length` blocks of a single observation. +/// +/// The digest covers each block's span id, role, content type, and normalized +/// content, so any reordering or edit inside the prefix changes the result. +/// +/// # Parameters +/// - `observation`: Normalized prompt IR to fingerprint. +/// - `prefix_length`: Number of leading blocks to include. +/// +/// # Returns +/// The prefix digest, or [`None`] when `prefix_length` is zero or exceeds the +/// number of blocks in the observation. +pub(crate) fn prompt_prefix_fingerprint( + observation: &PromptIR, + prefix_length: usize, +) -> Option { + if prefix_length == 0 || observation.blocks.len() < prefix_length { + return None; + } + + let prefix = observation + .blocks + .iter() + .take(prefix_length) + .map(block_key) + .collect::>>()? + .join("\n"); + Some(sha256_hex(&prefix)) +} + +/// Identify a block by span, role, content type, and normalized content. +/// +/// The prefix length rule and the prefix fingerprint must agree on when two +/// blocks are the same block, so both read this one definition. +fn block_key(block: &PromptBlock) -> Option { + serde_json::to_string(&( + &block.span_id, + block.role, + block.content_type, + &block.content, + )) + .ok() +} + +/// Length of the longest exact block prefix shared by a dominant share of the +/// observation window. +/// +/// Under `d(x, y) = 2^-lcp(x, y)` the window is ultrametric: closed balls are +/// the sets agreeing on a prefix, and every point of a ball is a center, so a +/// ball is fixed by its members rather than by traversal order. Descent also +/// requires a strict majority, which two disjoint children cannot both hold, so +/// the dominant child is unique where it exists and no tie-break is reachable. +/// Where none dominates, the prefix stops. +/// +/// # Parameters +/// - `observations`: Prompt observations to compare. +/// - `threshold`: Share of the window that must share the prefix. +/// +/// # Returns +/// The number of leading blocks shared by the dominant share of the window. +fn dominant_prefix_length(observations: &[PromptIR], threshold: f64) -> usize { + let total = observations.len(); + let required = ((threshold * total as f64).ceil() as usize).clamp(total / 2 + 1, total.max(1)); + let mut members: Vec<&PromptIR> = observations.iter().collect(); + + for depth in 0.. { + let mut children: HashMap> = HashMap::new(); + for observation in &members { + // An observation that ends here joins no child, so its mass stops + // contributing and the descent terminates at the longest window. + if let Some(key) = observation.blocks.get(depth).and_then(block_key) { + children.entry(key).or_default().push(observation); + } + } + let Some(dominant) = children.into_values().find(|child| child.len() >= required) else { + return depth; + }; + members = dominant; + } + + unreachable!("descent returns at the first depth where no child dominates") +} + +/// Fingerprint an observation's stable prefix as stored against a learning key. +/// +/// The digest binds the prefix to its learning key so one profile's stable +/// scaffold can never satisfy the reuse gate for another. A prefix that stops +/// inside the leading scaffold (system, tool-schema, and structured-output +/// blocks) is bound only to that scaffold and therefore stays reusable across +/// turns. A prefix that extends past the scaffold is bound to the complete +/// source request instead, because the normalized IR does not retain a lossless +/// provider-prefix representation and a looser binding could admit a request +/// whose real provider prefix differs. +/// +/// # Parameters +/// - `observation`: Normalized prompt IR to fingerprint. +/// - `prefix_length`: Stable prefix length reported by stability analysis. +/// - `learning_key`: Learning key the observation is bucketed under. +/// +/// # Returns +/// The bound digest, or [`None`] when the prefix cannot be fingerprinted or +/// when a beyond-scaffold prefix has no recorded source request hash. +pub(crate) fn profile_prefix_fingerprint( + observation: &PromptIR, + prefix_length: usize, + learning_key: &str, +) -> Option { + let prefix_fingerprint = prompt_prefix_fingerprint(observation, prefix_length)?; + let scaffold_length = observation + .blocks + .iter() + .take_while(|block| { + block.role == crate::acg::prompt_ir::PromptRole::System + || matches!( + block.content_type, + crate::acg::prompt_ir::BlockContentType::ToolSchema + | crate::acg::prompt_ir::BlockContentType::StructuredOutput + ) + }) + .count(); + // Conservatively bind deeper prefixes to the complete request because the + // normalized IR does not retain a lossless provider-prefix representation. + let request_fingerprint = if prefix_length > scaffold_length { + observation.source_request_hash.as_deref()? + } else { + "stable-scaffold" + }; + + Some(sha256_hex( + &[learning_key, &prefix_fingerprint, request_fingerprint].join("\n"), + )) +} + +/// Pick the most frequent profile prefix fingerprint across an observation window. +/// +/// Ties break on the lexicographically smallest digest so the same window always +/// yields the same fingerprint regardless of iteration order. +/// +/// # Parameters +/// - `observations`: Observation window persisted for the learning key. +/// - `prefix_length`: Stable prefix length reported by stability analysis. +/// - `learning_key`: Learning key the window is bucketed under. +/// +/// # Returns +/// The dominant digest, or [`None`] when no observation can be fingerprinted. +pub(crate) fn dominant_profile_prefix_fingerprint( + observations: &[PromptIR], + prefix_length: usize, + learning_key: &str, +) -> Option { + dominant_fingerprint(observations.iter().filter_map(|observation| { + profile_prefix_fingerprint(observation, prefix_length, learning_key) + })) +} + +/// Pick the most frequent digest, breaking ties on the smallest digest. +/// +/// Counting into a [`HashMap`] loses the input order, so the winner is selected +/// by a total order over `(count, digest)` rather than by iteration position. +/// Digests are unique map keys, so that order has exactly one maximum. +fn dominant_fingerprint(fingerprints: impl Iterator) -> Option { + fingerprints + .fold(HashMap::new(), |mut counts, fingerprint| { + *counts.entry(fingerprint).or_insert(0_u32) += 1; + counts + }) + .into_iter() + .max_by(|(left_hash, left_count), (right_hash, right_count)| { + left_count + .cmp(right_count) + .then_with(|| right_hash.cmp(left_hash)) + }) + .map(|(fingerprint, _)| fingerprint) +} + +/// Rank a classification so the least stable span sorts first. +fn stability_rank(classification: StabilityClass) -> u8 { + match classification { + StabilityClass::Variable => 0, + StabilityClass::SemiStable => 1, + StabilityClass::Stable => 2, + } +} + +/// Order span scores into the one canonical sequence for a given span set. +/// +/// The sequence index alone does not order the set: span ids carry the role and +/// the tool suffix, so `assistant-3-search` and `assistant-3-fetch` share index +/// 3, and the index is a minimum across observations. Tied spans would keep the +/// per-process order [`HashMap`] iteration produced. Extending the key to +/// `(index, stability rank, span id)` makes it injective, because span ids are +/// unique within one analysis, and a total order admits one sorted sequence. +/// +/// Prefix economics walks this vector and stops pricing at the first span that +/// is not stable, so ranking the least stable first ends the priced prefix at a +/// contested position rather than pricing across it. +/// +/// # Parameters +/// - `indexed`: Span scores paired with the sequence index each span was first +/// seen at, in [`HashMap`] iteration order. +/// +/// # Returns +/// The scores in prompt order. +fn canonical_scores(mut indexed: Vec<(u32, BlockStabilityScore)>) -> Vec { + indexed.sort_by(|(left_index, left), (right_index, right)| { + left_index + .cmp(right_index) + .then_with(|| { + stability_rank(left.classification).cmp(&stability_rank(right.classification)) + }) + .then_with(|| left.span_id.0.cmp(&right.span_id.0)) + }); + indexed.into_iter().map(|(_, score)| score).collect() +} + fn record_observation(observation: &PromptIR, span_map: &mut HashMap) { let mut seen_in_observation: HashSet = HashSet::new(); @@ -193,21 +431,6 @@ fn stability_confidence(present_count: u32, thresholds: &StabilityThresholds) -> (present_count as f64 / thresholds.min_observations_for_full_confidence as f64).min(1.0) } -/// Count the number of leading scores classified as stable. -/// -/// # Parameters -/// - `scores`: Span-level stability scores in prompt order. -/// -/// # Returns -/// The number of consecutive leading entries whose classification is -/// [`StabilityClass::Stable`]. -pub fn find_stable_prefix_length(scores: &[BlockStabilityScore]) -> usize { - scores - .iter() - .take_while(|score| score.classification == StabilityClass::Stable) - .count() -} - #[cfg(test)] #[path = "../../tests/unit/acg/stability_internal_tests.rs"] mod tests; diff --git a/crates/adaptive/src/acg_component.rs b/crates/adaptive/src/acg_component.rs index c71ec811d..cea5bfc20 100644 --- a/crates/adaptive/src/acg_component.rs +++ b/crates/adaptive/src/acg_component.rs @@ -107,6 +107,26 @@ fn build_semantic_request_view(request: &LlmRequest) -> Result = None; - for (profile_key, new_observations) in grouped_observations.drain() { - let existing = backend.load_observations(&profile_key).await?; + for (learning_key, new_observations) in grouped_observations.drain() { + let existing = backend.load_observations(&learning_key).await?; let mut window: VecDeque = existing.unwrap_or_default().into_iter().collect(); @@ -102,16 +102,22 @@ impl Learner for AcgLearner { let observations_vec: Vec = window.into_iter().collect(); backend - .store_observations(&profile_key, &observations_vec) + .store_observations(&learning_key, &observations_vec) .await?; - let stability_result = analyze_stability(&observations_vec, &self.thresholds); + let mut stability_result = analyze_stability(&observations_vec, &self.thresholds); + stability_result.stable_prefix_fingerprint = + crate::acg::stability::dominant_profile_prefix_fingerprint( + &observations_vec, + stability_result.stable_prefix_length, + &learning_key, + ); backend - .store_stability(&profile_key, &stability_result) + .store_stability(&learning_key, &stability_result) .await?; - profile_counts.insert(profile_key.clone(), stability_result.total_observations); - profile_stability.insert(profile_key, stability_result.clone()); + profile_counts.insert(learning_key.clone(), stability_result.total_observations); + profile_stability.insert(learning_key, stability_result.clone()); let replace_best = best_profile_seed .as_ref() @@ -130,7 +136,9 @@ impl Learner for AcgLearner { if let Some((aggregate_observations, aggregate_stability)) = best_profile_seed.as_ref() { // Persist the runtime seed entry under plain agent_id so registration can - // rehydrate HotCache without scanning profile-specific keys. + // rehydrate HotCache without scanning learning keys. Its fingerprint stays + // bound to the winning learning key, so a request from a different profile + // fails the reuse gate instead of inheriting another profile's scaffold. backend .store_observations(&self.agent_id, aggregate_observations) .await?; diff --git a/crates/adaptive/src/acg_profile.rs b/crates/adaptive/src/acg_profile.rs index fa3f47c0d..d762a1804 100644 --- a/crates/adaptive/src/acg_profile.rs +++ b/crates/adaptive/src/acg_profile.rs @@ -15,6 +15,8 @@ struct AcgKeyParts<'a> { model: &'a str, system_hash: String, tool_hash: String, + response_format_hash: Option, + has_stable_scaffold: bool, } /// Derive the stable ACG learning key used to bucket observations and hot-cache state. @@ -28,12 +30,20 @@ pub(crate) fn derive_acg_learning_key( annotated_request: &AnnotatedLlmRequest, ) -> String { let parts = derive_key_parts(annotated_request); - let seed_fingerprint = learning_seed_fingerprint(annotated_request); - let seed_hash = short_hash(&seed_fingerprint); - format!( + let seed_fingerprint = + (!parts.has_stable_scaffold).then(|| learning_seed_fingerprint(annotated_request)); + let seed_hash = seed_fingerprint + .as_deref() + .map(short_hash) + .unwrap_or("stable-scaffold"); + let key = format!( "{agent_id}::model={}::seed={seed_hash}::system={}::tools={}", parts.model, parts.system_hash, parts.tool_hash - ) + ); + parts + .response_format_hash + .map(|hash| format!("{key}::response_format={hash}")) + .unwrap_or(key) } /// Derive the exact ACG profile key used for diagnostics and debug output. @@ -58,18 +68,51 @@ pub(crate) fn derive_acg_profile_key( .join("."); format!( "{agent_id}::model={}::roles={role_signature}::system={}::anchor={}::tools={}", - parts.model, parts.system_hash, anchor_hash, parts.tool_hash + parts.model, + short_hash(&parts.system_hash), + anchor_hash, + short_hash(&parts.tool_hash) ) } +/// Derive the shared components behind both the learning key and the profile key. +/// +/// A request has a stable scaffold when it carries a system prompt, at least one +/// tool, or a structured-output contract. Scaffolded requests keep the full +/// system and tool digests so distinct scaffolds can never collide, while +/// scaffold-free requests fall back to short digests because their key is +/// already dominated by the per-turn seed. +/// +/// # Parameters +/// - `annotated_request`: Request to derive key components from. +/// +/// # Returns +/// The borrowed key components plus whether a stable scaffold was found. fn derive_key_parts(annotated_request: &AnnotatedLlmRequest) -> AcgKeyParts<'_> { let system_fingerprint = system_prompt_fingerprint(annotated_request); let tool_fingerprint = tool_schema_fingerprint(annotated_request.tools.as_deref()); + let response_format_fingerprint = response_format_fingerprint(annotated_request); + let has_stable_scaffold = system_fingerprint != "no-system" + || annotated_request + .tools + .as_ref() + .is_some_and(|tools| !tools.is_empty()) + || response_format_fingerprint.is_some(); AcgKeyParts { model: annotated_request.model.as_deref().unwrap_or("unknown"), - system_hash: short_hash(&system_fingerprint).to_string(), - tool_hash: short_hash(&tool_fingerprint).to_string(), + system_hash: if has_stable_scaffold { + system_fingerprint + } else { + short_hash(&system_fingerprint).to_string() + }, + tool_hash: if has_stable_scaffold { + tool_fingerprint + } else { + short_hash(&tool_fingerprint).to_string() + }, + response_format_hash: response_format_fingerprint, + has_stable_scaffold, } } @@ -240,6 +283,25 @@ fn tool_schema_fingerprint(tools: Option<&[ToolDefinition]>) -> String { } } +/// Fingerprint the request's structured-output contract. +/// +/// A null `response_format` is treated as absent so requests that explicitly +/// clear the contract bucket together with requests that never set one. +/// +/// # Parameters +/// - `annotated_request`: Request whose `response_format` extra is inspected. +/// +/// # Returns +/// The canonical digest of the contract, or [`None`] when no contract is set. +fn response_format_fingerprint(annotated_request: &AnnotatedLlmRequest) -> Option { + annotated_request + .extra + .get("response_format") + .filter(|value| !value.is_null()) + .and_then(|value| canonicalize_value(value).ok()) + .map(|value| sha256_hex(&value)) +} + fn extract_text(content: &MessageContent) -> String { match content { MessageContent::Text(text) => text.clone(), diff --git a/crates/adaptive/tests/integration/redis_tests.rs b/crates/adaptive/tests/integration/redis_tests.rs index 8e2eca4f4..d92b42d15 100644 --- a/crates/adaptive/tests/integration/redis_tests.rs +++ b/crates/adaptive/tests/integration/redis_tests.rs @@ -503,3 +503,62 @@ async fn redis_integration_persists_runtime_seed_entries_and_manifest_cleanup() "adaptive manifest should not depend directly on the compatibility shim" ); } + +#[tokio::test] +async fn redis_integration_persists_scaffold_profile_across_backend_restart() { + let Some((backend, prefix)) = get_test_redis_with_prefix().await else { + return; + }; + let agent_id = "agent-redis-scaffold-restart"; + let learner = AcgLearner::new(agent_id, 8, StabilityThresholds::default()); + let first = sample_annotated_request("claude-3-5-sonnet"); + let mut second = first.clone(); + second.messages[1] = Message::User { + content: MessageContent::Text("Plan a different task".to_string()), + name: None, + }; + let hot_cache = empty_hot_cache(); + + for request in [first, second] { + learner + .process_run( + &sample_run_with_request(agent_id, request), + &backend, + &hot_cache, + ) + .await + .unwrap(); + } + let learning_key = { + let guard = hot_cache.read().unwrap(); + assert_eq!(guard.acg_profiles.len(), 1); + guard.acg_profiles.keys().next().unwrap().clone() + }; + // Both requests carry the same scaffold and differ only in the first user + // task, so the surviving record must be the scaffold-keyed profile rather + // than the plain agent seed `process_run` also writes for rehydration. + assert_ne!(learning_key, agent_id); + assert!( + learning_key.contains("::seed=stable-scaffold::"), + "expected a scaffold-keyed profile, got {learning_key}" + ); + drop(backend); + + let restarted = RedisBackend::new("redis://127.0.0.1/", prefix) + .await + .unwrap(); + let observations = restarted + .load_observations(&learning_key) + .await + .unwrap() + .unwrap(); + let stability = restarted + .load_stability(&learning_key) + .await + .unwrap() + .unwrap(); + + assert_eq!(observations.len(), 2); + assert_eq!(stability.total_observations, 2); + assert!(stability.stable_prefix_fingerprint.is_some()); +} diff --git a/crates/adaptive/tests/integration/runtime_integration_tests.rs b/crates/adaptive/tests/integration/runtime_integration_tests.rs index f0e64fb1a..669be1758 100644 --- a/crates/adaptive/tests/integration/runtime_integration_tests.rs +++ b/crates/adaptive/tests/integration/runtime_integration_tests.rs @@ -61,10 +61,6 @@ fn enable_operational_logs() { }); } -fn short_hash(value: &str) -> &str { - value.get(..16).unwrap_or(value) -} - fn reset_global() { enable_operational_logs(); let _ = clear_plugin_configuration(); @@ -525,11 +521,7 @@ async fn runtime_integration_acg_learner_reuses_learning_buckets_across_growing_ let requests = sample_growing_chat_requests("claude-3-5-sonnet"); let learner = AcgLearner::new(agent_id, 8, StabilityThresholds::default()); let learning_key = format!( - "{agent_id}::model=claude-3-5-sonnet::seed={}::system=sha256:3087d8fd4::tools=no-tools", - short_hash(&format!( - "user:{}", - nemo_relay_adaptive::acg::sha256_hex("Summarize the latest findings") - )), + "{agent_id}::model=claude-3-5-sonnet::seed=stable-scaffold::system=sha256:3087d8fd4b98c564984d0f184c06bf6346f0788022d7cb521231e65f673936ac::tools=no-tools" ); learner diff --git a/crates/adaptive/tests/unit/acg/economics_internal_tests.rs b/crates/adaptive/tests/unit/acg/economics_internal_tests.rs index 7a6a01364..69053c425 100644 --- a/crates/adaptive/tests/unit/acg/economics_internal_tests.rs +++ b/crates/adaptive/tests/unit/acg/economics_internal_tests.rs @@ -121,6 +121,7 @@ fn economics_internal_build_prefix_stats_stops_at_first_non_stable_block() { score(1, StabilityClass::Variable, 0.4, 0.9), ], stable_prefix_length: 2, + stable_prefix_fingerprint: None, total_observations: 6, }; diff --git a/crates/adaptive/tests/unit/acg/economics_policy_tests.rs b/crates/adaptive/tests/unit/acg/economics_policy_tests.rs index 1d4a00966..254508ee3 100644 --- a/crates/adaptive/tests/unit/acg/economics_policy_tests.rs +++ b/crates/adaptive/tests/unit/acg/economics_policy_tests.rs @@ -84,6 +84,7 @@ fn stability_result(scores: &[(f64, f64)], observation_count: u32) -> StabilityA }) .collect(), stable_prefix_length: scores.len(), + stable_prefix_fingerprint: None, total_observations: observation_count, } } diff --git a/crates/adaptive/tests/unit/acg/ir_builder_tests.rs b/crates/adaptive/tests/unit/acg/ir_builder_tests.rs index 9d1ba9934..285078cb4 100644 --- a/crates/adaptive/tests/unit/acg/ir_builder_tests.rs +++ b/crates/adaptive/tests/unit/acg/ir_builder_tests.rs @@ -298,3 +298,49 @@ fn build_prompt_ir_covers_extended_request_messages_and_content_parts() { "web_search_preview" ); } + +#[test] +fn build_prompt_ir_canonicalizes_structured_output_before_user_content() { + let mut request = AnnotatedLlmRequest { + messages: vec![Message::User { + content: MessageContent::Text("variable task".to_string()), + name: None, + }], + model: Some("gpt-4o".to_string()), + ..AnnotatedLlmRequest::default() + }; + request.extra.insert( + "response_format".to_string(), + serde_json::json!({"type":"json_schema","json_schema":{"required":["answer"],"type":"object"}}), + ); + + let first = build_prompt_ir(&request).unwrap(); + request.extra.insert( + "response_format".to_string(), + serde_json::json!({"json_schema":{"type":"object","required":["answer"]},"type":"json_schema"}), + ); + let reordered = build_prompt_ir(&request).unwrap(); + + assert_eq!( + first.structured_output_schema_id, + reordered.structured_output_schema_id + ); + assert!(first.structured_output_schema_id.is_some()); + assert_eq!( + first.blocks[0].content_type, + BlockContentType::StructuredOutput + ); + assert_eq!(first.blocks[1].role, PromptRole::User); + + request + .extra + .insert("response_format".to_string(), serde_json::Value::Null); + let null_contract = build_prompt_ir(&request).unwrap(); + assert!(null_contract.structured_output_schema_id.is_none()); + assert!( + null_contract + .blocks + .iter() + .all(|block| block.content_type != BlockContentType::StructuredOutput) + ); +} diff --git a/crates/adaptive/tests/unit/acg/multi_breakpoint_tests.rs b/crates/adaptive/tests/unit/acg/multi_breakpoint_tests.rs index f4c228482..ddf8b0017 100644 --- a/crates/adaptive/tests/unit/acg/multi_breakpoint_tests.rs +++ b/crates/adaptive/tests/unit/acg/multi_breakpoint_tests.rs @@ -81,6 +81,7 @@ fn layered_stability(observation_count: u32, layers: usize) -> StabilityAnalysis }) .collect(), stable_prefix_length: layers, + stable_prefix_fingerprint: None, total_observations: observation_count, } } diff --git a/crates/adaptive/tests/unit/acg/stability_internal_tests.rs b/crates/adaptive/tests/unit/acg/stability_internal_tests.rs index c39843586..77b7294cd 100644 --- a/crates/adaptive/tests/unit/acg/stability_internal_tests.rs +++ b/crates/adaptive/tests/unit/acg/stability_internal_tests.rs @@ -78,3 +78,166 @@ fn stability_internal_effective_score_handles_zero_present_count() { assert_eq!(effective_stability_score(&observations, 3), 0.0); } + +#[test] +fn stability_internal_fingerprints_the_exact_dominant_prefix() { + let first = prompt(vec![ + block("system-0", 0, "stable policy"), + block("user-1", 1, "task alpha"), + ]); + let second = prompt(vec![ + block("system-0", 0, "stable policy"), + block("user-1", 1, "task beta"), + ]); + let changed = prompt(vec![ + block("system-0", 0, "changed policy"), + block("user-1", 1, "task gamma"), + ]); + + let result = analyze_stability(&[first.clone(), second], &StabilityThresholds::default()); + + assert_eq!(result.stable_prefix_length, 1); + assert_eq!( + result.stable_prefix_fingerprint, + prompt_prefix_fingerprint(&first, 1) + ); + assert_ne!( + result.stable_prefix_fingerprint, + prompt_prefix_fingerprint(&changed, 1) + ); + assert_eq!(prompt_prefix_fingerprint(&first, 0), None); +} + +#[test] +fn stability_internal_deserializes_persisted_state_without_a_prefix_fingerprint() { + let restored: StabilityAnalysisResult = serde_json::from_value(serde_json::json!({ + "scores": [], + "stable_prefix_length": 0, + "total_observations": 4 + })) + .unwrap(); + + assert_eq!(restored.stable_prefix_fingerprint, None); +} + +#[test] +fn profile_fingerprint_requires_source_hash_beyond_the_scaffold() { + let mut observation = prompt(vec![ + block("system-0", 0, "stable policy"), + block("user-1", 1, "stable task"), + ]); + observation.blocks[1].role = PromptRole::User; + + assert_eq!( + profile_prefix_fingerprint(&observation, 2, "learning-key"), + None + ); +} + +fn scored(span_id: &str, classification: StabilityClass) -> BlockStabilityScore { + BlockStabilityScore { + span_id: SpanId(span_id.to_string()), + classification, + score: 1.0, + confidence: 1.0, + observation_count: 1, + } +} + +/// Two spans differing only in role or tool suffix share a sequence index, so +/// the index alone is not a total order over the span set. `canonical_scores` +/// must therefore map any permutation of the same span set onto one sequence. +#[test] +fn stability_internal_canonical_scores_are_invariant_under_input_permutation() { + let tied = vec![ + (1_u32, scored("assistant-1-search", StabilityClass::Stable)), + (1, scored("assistant-1-fetch", StabilityClass::Variable)), + (1, scored("assistant-1-write", StabilityClass::SemiStable)), + // Shares the index and the rank of `assistant-1-search`, so only the + // span id separates them. + (1, scored("assistant-1-annotate", StabilityClass::Stable)), + (0, scored("system-0", StabilityClass::Stable)), + (2, scored("tool-2-search", StabilityClass::Stable)), + ]; + + // Every rotation is a distinct HashMap iteration order over the same set. + let expected = canonical_scores(tied.clone()); + for rotation in 1..tied.len() { + let mut permuted = tied.clone(); + permuted.rotate_left(rotation); + assert_eq!(canonical_scores(permuted), expected, "rotation {rotation}"); + } +} + +/// At a contested sequence index the least stable span sorts first, so the +/// stable prefix stops before the position rather than extending across it. +#[test] +fn stability_internal_canonical_scores_put_the_least_stable_span_first() { + let contested = vec![ + (0_u32, scored("system-0", StabilityClass::Stable)), + (1, scored("assistant-1-search", StabilityClass::Stable)), + (1, scored("assistant-1-fetch", StabilityClass::Variable)), + ]; + + let ordered = canonical_scores(contested); + assert_eq!(ordered[1].span_id, SpanId("assistant-1-fetch".to_string())); +} + +/// The observation window is a multiset: the analysis aggregates it with `min`, +/// summation, and multiset counts, all symmetric, so reordering the window must +/// not move the stable prefix or its fingerprint. +#[test] +fn stability_internal_analysis_is_invariant_under_observation_permutation() { + let thresholds = StabilityThresholds::default(); + let turn = |span: &str, content: &str| { + prompt(vec![ + block("system-0", 0, "Follow policy"), + block(span, 1, content), + ]) + }; + let observations = vec![ + turn("assistant-1-search", "search(alpha)"), + turn("assistant-1-search", "search(alpha)"), + turn("assistant-1-fetch", "fetch(beta)"), + turn("assistant-1-search", "search(gamma)"), + ]; + + let expected = analyze_stability(&observations, &thresholds); + for rotation in 1..observations.len() { + let mut permuted = observations.clone(); + permuted.rotate_left(rotation); + assert_eq!( + analyze_stability(&permuted, &thresholds), + expected, + "rotation {rotation}" + ); + } +} + +/// 19 of 20 observations share an exact two-block prefix, so a provider cache +/// would hit on it for those 19. The per-span rule discards it: the minority +/// span at index 1 is Variable and stops the leading run. +#[test] +fn stability_internal_keeps_a_prefix_shared_by_the_dominant_mass() { + let thresholds = StabilityThresholds::default(); + let turn = |span: &str, content: &str| { + prompt(vec![ + block("system-0", 0, "Follow policy"), + block(span, 1, content), + ]) + }; + let observations: Vec<_> = (0..20) + .map(|i| { + if i == 7 { + turn("assistant-1-fetch", "fetch(beta)") + } else { + turn("assistant-1-search", "search(alpha)") + } + }) + .collect(); + + assert_eq!( + analyze_stability(&observations, &thresholds).stable_prefix_length, + 2 + ); +} diff --git a/crates/adaptive/tests/unit/acg_component_tests.rs b/crates/adaptive/tests/unit/acg_component_tests.rs index 69004ee72..dd833b47a 100644 --- a/crates/adaptive/tests/unit/acg_component_tests.rs +++ b/crates/adaptive/tests/unit/acg_component_tests.rs @@ -47,6 +47,7 @@ fn sample_hot_cache() -> Arc> { }, ], stable_prefix_length: 2, + stable_prefix_fingerprint: None, total_observations: 8, }), acg_observation_count: 8, @@ -233,6 +234,7 @@ fn stability_with_prefix(prefix_len: u32, observations: u32) -> StabilityAnalysi }) .collect(), stable_prefix_length: prefix_len as usize, + stable_prefix_fingerprint: None, total_observations: observations, } } @@ -263,6 +265,7 @@ fn layered_stability_result(observation_count: u32) -> StabilityAnalysisResult { }, ], stable_prefix_length: 3, + stable_prefix_fingerprint: None, total_observations: observation_count, } } @@ -287,6 +290,27 @@ fn sample_prompt_ir(span: &str) -> PromptIR { } } +fn stability_for_request( + agent_id: &str, + request: &LlmRequest, + mut stability: StabilityAnalysisResult, +) -> StabilityAnalysisResult { + let view = build_semantic_request_view(request).expect("fixture request should decode"); + let prompt_ir = crate::acg::ir_builder::build_prompt_ir(&view.annotated_request) + .expect("fixture request should build prompt ir"); + stability.stable_prefix_length = stability.stable_prefix_length.min(prompt_ir.blocks.len()); + let learning_key = derive_acg_learning_key(agent_id, &view.annotated_request); + stability.stable_prefix_fingerprint = Some( + crate::acg::stability::profile_prefix_fingerprint( + &prompt_ir, + stability.stable_prefix_length, + &learning_key, + ) + .expect("fixture stable prefix should exist"), + ); + stability +} + #[test] fn acg_component_default_config_and_thresholds_are_stable() { let config = AcgComponentConfig::default(); @@ -429,6 +453,11 @@ fn acg_component_translate_request_degrades_when_provider_semantics_do_not_match fn acg_component_translate_request_applies_openai_semantics_on_resolved_request_surface() { let request = sample_openai_responses_request(); let hot_cache = sample_hot_cache(); + hot_cache.write().unwrap().acg_stability = Some(stability_for_request( + "agent-1", + &request, + stability_with_prefix(2, 8), + )); let plugin = build_provider_plugin("openai").expect("openai plugin should build"); let translated = translate_request(&request, "agent-1", "openai", plugin.as_ref(), &hot_cache) @@ -504,7 +533,11 @@ fn acg_component_translate_request_applies_multiple_ordered_anthropic_breakpoint agent_hints_default: None, acg_profiles: std::collections::HashMap::new(), acg_profile_observation_counts: std::collections::HashMap::new(), - acg_stability: Some(layered_stability_result(6)), + acg_stability: Some(stability_for_request( + "agent-1", + &request, + layered_stability_result(6), + )), acg_observation_count: 6, })); @@ -550,7 +583,11 @@ fn rewrite_request_with_hot_cache_adaptive_placement_differs_by_state() { agent_hints_default: None, acg_profiles: std::collections::HashMap::new(), acg_profile_observation_counts: std::collections::HashMap::new(), - acg_stability: Some(stability_with_prefix(1, 8)), + acg_stability: Some(stability_for_request( + "agent-1", + &request, + stability_with_prefix(1, 8), + )), acg_observation_count: 8, })); let translated_short = @@ -563,7 +600,11 @@ fn rewrite_request_with_hot_cache_adaptive_placement_differs_by_state() { agent_hints_default: None, acg_profiles: std::collections::HashMap::new(), acg_profile_observation_counts: std::collections::HashMap::new(), - acg_stability: Some(stability_with_prefix(3, 8)), + acg_stability: Some(stability_for_request( + "agent-1", + &request, + stability_with_prefix(3, 8), + )), acg_observation_count: 8, })); let translated_long = @@ -604,7 +645,7 @@ fn acg_component_translate_request_uses_learning_key_for_profile_lookup() { agent_hints_default: None, acg_profiles: std::collections::HashMap::from([( learning_key.clone(), - layered_stability_result(6), + stability_for_request("agent-1", &request, layered_stability_result(6)), )]), acg_profile_observation_counts: std::collections::HashMap::from([(learning_key, 6)]), acg_stability: None, @@ -635,7 +676,11 @@ async fn acg_component_stream_execution_intercept_rewrites_streaming_requests() agent_hints_default: None, acg_profiles: std::collections::HashMap::new(), acg_profile_observation_counts: std::collections::HashMap::new(), - acg_stability: Some(layered_stability_result(6)), + acg_stability: Some(stability_for_request( + "agent-1", + &request, + layered_stability_result(6), + )), acg_observation_count: 6, })); @@ -691,6 +736,7 @@ fn acg_component_build_intent_bundle_requires_at_least_two_observations() { }, ], stable_prefix_length: 2, + stable_prefix_fingerprint: None, total_observations: 1, }; @@ -768,6 +814,39 @@ async fn acg_component_load_persisted_state_prefers_stability_and_falls_back_to_ assert_eq!(guard.acg_observation_count, 2); } +#[tokio::test] +async fn acg_component_rehydrates_matching_profile_and_rejects_changed_scaffold() { + let agent_id = "agent-restarted"; + let request = sample_openai_responses_request(); + let stability = stability_for_request( + agent_id, + &request, + stability_with_prefix(2, MIN_ACG_OBSERVATIONS), + ); + let backend = InMemoryBackend::new(); + backend.store_stability(agent_id, &stability).await.unwrap(); + + let hot_cache = Arc::new(RwLock::new(HotCache { + plan: None, + trie: None, + agent_hints_default: None, + acg_profiles: std::collections::HashMap::new(), + acg_profile_observation_counts: std::collections::HashMap::new(), + acg_stability: None, + acg_observation_count: 0, + })); + load_persisted_acg_state(agent_id, &backend, &hot_cache) + .await + .unwrap(); + let plugin = build_provider_plugin("openai").unwrap(); + + assert!(translate_request(&request, agent_id, "openai", plugin.as_ref(), &hot_cache).is_some()); + + let mut changed = request; + changed.content["instructions"] = json!("A different system policy."); + assert!(translate_request(&changed, agent_id, "openai", plugin.as_ref(), &hot_cache).is_none()); +} + #[tokio::test] async fn acg_component_load_persisted_state_handles_empty_backend_and_poisoned_cache() { let backend = InMemoryBackend::new(); @@ -865,7 +944,12 @@ fn acg_component_build_intent_bundle_supports_openai_and_rejects_unknown_provide let request = sample_annotated_request("gpt-4o"); let prompt_ir = crate::acg::ir_builder::build_prompt_ir(&request).unwrap(); let plugin = build_provider_plugin("openai").unwrap(); - let stability = layered_stability_result(4); + let mut stability = stability_with_prefix(2, 4); + stability.stable_prefix_fingerprint = crate::acg::stability::profile_prefix_fingerprint( + &prompt_ir, + stability.stable_prefix_length, + &derive_acg_learning_key("agent-openai", &request), + ); let bundle = build_intent_bundle( "agent-openai", @@ -897,6 +981,157 @@ fn acg_component_build_intent_bundle_supports_openai_and_rejects_unknown_provide ); } +#[test] +fn acg_component_requires_the_exact_learned_prefix_before_reuse() { + let request = sample_annotated_request("gpt-4o"); + let prompt_ir = crate::acg::ir_builder::build_prompt_ir(&request).unwrap(); + let mut stability = crate::acg::stability::analyze_stability( + std::slice::from_ref(&prompt_ir), + &crate::acg::stability::StabilityThresholds::default(), + ); + let learning_key = derive_acg_learning_key("agent-openai", &request); + stability.stable_prefix_fingerprint = crate::acg::stability::profile_prefix_fingerprint( + &prompt_ir, + stability.stable_prefix_length, + &learning_key, + ); + let plugin = build_provider_plugin("openai").unwrap(); + + assert!( + build_intent_bundle( + "agent-openai", + "openai", + plugin.as_ref(), + RequestSurface::OpenAIChat, + &request, + &prompt_ir, + &stability, + MIN_ACG_OBSERVATIONS, + ) + .is_some() + ); + + let mut changed_prefix = prompt_ir.clone(); + changed_prefix.blocks[0].content = "changed policy".to_string(); + assert!( + build_intent_bundle( + "agent-openai", + "openai", + plugin.as_ref(), + RequestSurface::OpenAIChat, + &request, + &changed_prefix, + &stability, + MIN_ACG_OBSERVATIONS, + ) + .is_none() + ); + + let mut legacy_stability = stability; + legacy_stability.stable_prefix_fingerprint = None; + assert!( + build_intent_bundle( + "agent-openai", + "openai", + plugin.as_ref(), + RequestSurface::OpenAIChat, + &request, + &prompt_ir, + &legacy_stability, + MIN_ACG_OBSERVATIONS, + ) + .is_none() + ); + + legacy_stability.stable_prefix_length = prompt_ir.blocks.len() + 1; + assert!( + build_intent_bundle( + "agent-openai", + "openai", + plugin.as_ref(), + RequestSurface::OpenAIChat, + &request, + &prompt_ir, + &legacy_stability, + MIN_ACG_OBSERVATIONS, + ) + .is_none() + ); +} + +#[test] +fn acg_component_rejects_raw_user_changes_beyond_the_stable_scaffold() { + let request = sample_annotated_request("gpt-4o"); + let prompt_ir = crate::acg::ir_builder::build_prompt_ir(&request).unwrap(); + let mut stability = crate::acg::stability::analyze_stability( + std::slice::from_ref(&prompt_ir), + &crate::acg::stability::StabilityThresholds::default(), + ); + let learning_key = derive_acg_learning_key("agent-openai", &request); + stability.stable_prefix_fingerprint = crate::acg::stability::profile_prefix_fingerprint( + &prompt_ir, + stability.stable_prefix_length, + &learning_key, + ); + + let mut changed_request = request.clone(); + changed_request.messages[1] = Message::User { + content: MessageContent::Text("Hello ".to_string()), + name: None, + }; + let changed_ir = crate::acg::ir_builder::build_prompt_ir(&changed_request).unwrap(); + let plugin = build_provider_plugin("openai").unwrap(); + + assert!( + build_intent_bundle( + "agent-openai", + "openai", + plugin.as_ref(), + RequestSurface::OpenAIChat, + &changed_request, + &changed_ir, + &stability, + MIN_ACG_OBSERVATIONS, + ) + .is_none() + ); +} + +#[test] +fn acg_component_rejects_raw_whitespace_changes_inside_the_stable_scaffold() { + let request = sample_annotated_request("gpt-4o"); + let prompt_ir = crate::acg::ir_builder::build_prompt_ir(&request).unwrap(); + let mut stability = stability_with_prefix(1, MIN_ACG_OBSERVATIONS); + stability.stable_prefix_fingerprint = crate::acg::stability::profile_prefix_fingerprint( + &prompt_ir, + 1, + &derive_acg_learning_key("agent-openai", &request), + ); + + let mut changed_request = request.clone(); + changed_request.messages[0] = Message::System { + content: MessageContent::Text("You are helpful. ".to_string()), + name: None, + }; + let changed_ir = crate::acg::ir_builder::build_prompt_ir(&changed_request).unwrap(); + assert_eq!(prompt_ir.blocks[0].content, changed_ir.blocks[0].content); + + let plugin = build_provider_plugin("openai").unwrap(); + assert!( + build_intent_bundle( + "agent-openai", + "openai", + plugin.as_ref(), + RequestSurface::OpenAIChat, + &changed_request, + &changed_ir, + &stability, + MIN_ACG_OBSERVATIONS, + ) + .is_none() + ); +} + #[test] fn acg_component_anthropic_cache_intents_fail_open_when_surface_or_model_do_not_match() { let plugin = build_provider_plugin("anthropic").unwrap(); @@ -949,7 +1184,7 @@ fn acg_component_build_hint_translation_and_apply_hint_translation_cover_passthr let prompt_ir = crate::acg::ir_builder::build_prompt_ir(&semantic_view.annotated_request).unwrap(); let plugin = build_provider_plugin("openai").unwrap(); - let stability = layered_stability_result(4); + let stability = stability_for_request("agent-openai", &request, layered_stability_result(4)); let bundle = build_intent_bundle( "agent-openai", "openai", @@ -1004,7 +1239,7 @@ fn acg_component_translate_request_uses_profile_specific_stability_and_fails_ope agent_hints_default: None, acg_profiles: std::collections::HashMap::from([( learning_key.clone(), - layered_stability_result(6), + stability_for_request("agent-profile", &request, layered_stability_result(6)), )]), acg_profile_observation_counts: std::collections::HashMap::from([(learning_key, 6)]), acg_stability: None, @@ -1051,7 +1286,11 @@ async fn acg_component_execution_intercept_rewrites_non_streaming_requests() { agent_hints_default: None, acg_profiles: std::collections::HashMap::new(), acg_profile_observation_counts: std::collections::HashMap::new(), - acg_stability: Some(layered_stability_result(6)), + acg_stability: Some(stability_for_request( + "agent-1", + &request, + layered_stability_result(6), + )), acg_observation_count: 6, })); @@ -1150,7 +1389,7 @@ fn acg_component_build_hint_translation_succeeds_for_openai_and_anthropic() { RequestSurface::OpenAIChat, &openai_view.annotated_request, &openai_prompt_ir, - &layered_stability_result(4), + &stability_for_request("agent-openai", &openai_request, stability_with_prefix(1, 4)), 4, ) .unwrap(); @@ -1191,7 +1430,11 @@ fn acg_component_build_hint_translation_succeeds_for_openai_and_anthropic() { RequestSurface::AnthropicMessages, &anthropic_view.annotated_request, &anthropic_prompt_ir, - &layered_stability_result(6), + &stability_for_request( + "agent-anthropic", + &anthropic_request, + layered_stability_result(6), + ), 6, ) .unwrap(); @@ -1325,7 +1568,11 @@ fn acg_component_request_intercept_rewrites_annotation_without_mutating_provider agent_hints_default: None, acg_profiles: std::collections::HashMap::new(), acg_profile_observation_counts: std::collections::HashMap::new(), - acg_stability: Some(layered_stability_result(6)), + acg_stability: Some(stability_for_request( + "agent-1", + &request, + layered_stability_result(6), + )), acg_observation_count: 6, })); let intercept = create_acg_llm_request_intercept( diff --git a/crates/adaptive/tests/unit/acg_learner_tests.rs b/crates/adaptive/tests/unit/acg_learner_tests.rs index de23483e2..54c620ee4 100644 --- a/crates/adaptive/tests/unit/acg_learner_tests.rs +++ b/crates/adaptive/tests/unit/acg_learner_tests.rs @@ -230,6 +230,44 @@ async fn acg_learner_returns_early_without_llm_requests() { .unwrap(); } +#[tokio::test(flavor = "current_thread")] +async fn acg_learner_accumulates_same_scaffold_calls_from_one_run_in_one_profile() { + let first = sample_request("gpt-4o", "Stable system", "task alpha"); + let second = sample_request("gpt-4o", "Stable system", "task beta"); + let learning_key = derive_acg_learning_key("agent-a", &first); + assert_eq!(learning_key, derive_acg_learning_key("agent-a", &second)); + + let learner = AcgLearner::new("agent-a", 8, StabilityThresholds::default()); + let backend = crate::storage::memory::InMemoryBackend::new(); + let hot_cache = empty_cache(); + learner + .process_run(&sample_run(vec![first, second]), &backend, &hot_cache) + .await + .unwrap(); + + assert_eq!( + backend + .load_observations(&learning_key) + .await + .unwrap() + .unwrap() + .len(), + 2 + ); + let guard = hot_cache.read().unwrap(); + assert_eq!(guard.acg_profiles.len(), 1); + assert_eq!(guard.acg_profile_observation_counts[&learning_key], 2); + let stored = &guard.acg_profiles[&learning_key]; + assert_eq!( + stored.stable_prefix_fingerprint, + crate::acg::stability::profile_prefix_fingerprint( + &build_prompt_ir(&sample_request("gpt-4o", "Stable system", "task alpha")).unwrap(), + stored.stable_prefix_length, + &learning_key, + ) + ); +} + #[tokio::test(flavor = "current_thread")] async fn acg_learner_trims_observation_windows_and_updates_agent_seed() { let learner = AcgLearner::new("agent-a", 2, StabilityThresholds::default()); @@ -352,3 +390,241 @@ async fn acg_learner_seeds_agent_cache_from_profile_with_more_observations_when_ .any(|count| *count == 1) ); } + +/// Backend that yields between the load and store halves of every learner +/// read-modify-write so concurrent `process_run` futures interleave. +struct InterleavingBackend { + inner: crate::storage::memory::InMemoryBackend, +} + +impl InterleavingBackend { + /// Wrap a fresh in-memory backend with yields around observation and stability writes. + fn new() -> Self { + Self { + inner: crate::storage::memory::InMemoryBackend::new(), + } + } +} + +impl StorageBackendDyn for InterleavingBackend { + fn store_run_dyn<'a>( + &'a self, + record: &'a RunRecord, + ) -> Pin> + Send + 'a>> { + self.inner.store_run_dyn(record) + } + + fn load_plan_dyn<'a>( + &'a self, + agent_id: &'a str, + ) -> Pin>> + Send + 'a>> { + self.inner.load_plan_dyn(agent_id) + } + + fn list_runs_dyn<'a>( + &'a self, + agent_id: &'a str, + ) -> Pin>> + Send + 'a>> { + self.inner.list_runs_dyn(agent_id) + } + + fn store_trie<'a>( + &'a self, + agent_id: &'a str, + envelope: &'a TrieEnvelope, + ) -> Pin> + Send + 'a>> { + self.inner.store_trie(agent_id, envelope) + } + + fn load_trie<'a>( + &'a self, + agent_id: &'a str, + ) -> Pin>> + Send + 'a>> { + self.inner.load_trie(agent_id) + } + + fn store_accumulators<'a>( + &'a self, + agent_id: &'a str, + state: &'a AccumulatorState, + ) -> Pin> + Send + 'a>> { + self.inner.store_accumulators(agent_id, state) + } + + fn load_accumulators<'a>( + &'a self, + agent_id: &'a str, + ) -> Pin>> + Send + 'a>> { + self.inner.load_accumulators(agent_id) + } + + fn store_observations<'a>( + &'a self, + agent_id: &'a str, + observations: &'a [PromptIR], + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + tokio::task::yield_now().await; + self.inner.store_observations(agent_id, observations).await + }) + } + + fn load_observations<'a>( + &'a self, + agent_id: &'a str, + ) -> Pin>>> + Send + 'a>> { + Box::pin(async move { + tokio::task::yield_now().await; + self.inner.load_observations(agent_id).await + }) + } + + fn store_stability<'a>( + &'a self, + agent_id: &'a str, + result: &'a crate::acg::stability::StabilityAnalysisResult, + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + tokio::task::yield_now().await; + self.inner.store_stability(agent_id, result).await + }) + } + + fn load_stability<'a>( + &'a self, + agent_id: &'a str, + ) -> Pin< + Box< + dyn Future>> + + Send + + 'a, + >, + > { + self.inner.load_stability(agent_id) + } +} + +#[tokio::test(flavor = "current_thread")] +async fn acg_learner_keeps_persisted_fingerprints_consistent_under_interleaved_runs() { + let learner = AcgLearner::new("agent-a", 8, StabilityThresholds::default()); + let backend = InterleavingBackend::new(); + let hot_cache = empty_cache(); + + let alpha = sample_request("gpt-4o", "Alpha system", "alpha task"); + let beta = sample_request("gpt-4o", "Beta system", "beta task"); + let alpha_key = derive_acg_learning_key("agent-a", &alpha); + let beta_key = derive_acg_learning_key("agent-a", &beta); + assert_ne!(alpha_key, beta_key); + + let alpha_run = sample_run(vec![alpha.clone()]); + let beta_run = sample_run(vec![beta.clone()]); + let alpha_replay = sample_run(vec![alpha.clone()]); + let (first, second, third) = tokio::join!( + learner.process_run(&alpha_run, &backend, &hot_cache), + learner.process_run(&beta_run, &backend, &hot_cache), + learner.process_run(&alpha_replay, &backend, &hot_cache), + ); + first.unwrap(); + second.unwrap(); + third.unwrap(); + + // The learner's load/store cycle is not atomic, so interleaved runs can drop + // an observation from the window. That is a pre-existing throughput limit, not + // a safety one: what must hold is that every persisted stability record stays + // reproducible from the observations persisted under the same learning key, so + // a fingerprint is never paired with a foreign window. + assert_eq!( + backend + .load_observations(&alpha_key) + .await + .unwrap() + .unwrap() + .len(), + 1, + ); + + for key in [&alpha_key, &beta_key] { + let observations = backend.load_observations(key).await.unwrap().unwrap(); + let stability = backend.load_stability(key).await.unwrap().unwrap(); + assert_eq!( + stability.stable_prefix_fingerprint, + crate::acg::stability::dominant_profile_prefix_fingerprint( + &observations, + stability.stable_prefix_length, + key, + ), + "fingerprint for {key} is not reproducible from its own observation window", + ); + } + + let guard = hot_cache.read().unwrap(); + let alpha_fingerprint = guard.acg_profiles[&alpha_key] + .stable_prefix_fingerprint + .clone(); + let beta_fingerprint = guard.acg_profiles[&beta_key] + .stable_prefix_fingerprint + .clone(); + assert_ne!(alpha_fingerprint, beta_fingerprint); + + // The agent-level seed adopts exactly one profile's fingerprint; it never + // blends the two, so the reuse gate rejects the other profile after restart. + let seed = guard.acg_stability.as_ref().unwrap(); + assert!( + seed.stable_prefix_fingerprint == alpha_fingerprint + || seed.stable_prefix_fingerprint == beta_fingerprint + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn acg_learner_binds_shared_scaffold_profiles_to_one_workflow_beyond_the_scaffold() { + // Two workflows share an identical system prompt and an identical first user + // turn, so they collapse into one learning key and the stable prefix extends + // past the leading scaffold. + let mut requests = Vec::new(); + for tail in ["tail one", "tail two", "tail three", "tail four"] { + let mut request = sample_request("gpt-4o", "Shared system", "Shared anchor turn"); + request.messages.push(Message::Assistant { + content: Some(MessageContent::Text(tail.to_string())), + tool_calls: None, + name: None, + }); + requests.push(request); + } + + let learning_key = derive_acg_learning_key("agent-a", &requests[0]); + assert!( + requests + .iter() + .all(|request| derive_acg_learning_key("agent-a", request) == learning_key) + ); + + let learner = AcgLearner::new("agent-a", 8, StabilityThresholds::default()); + let backend = crate::storage::memory::InMemoryBackend::new(); + let hot_cache = empty_cache(); + learner + .process_run(&sample_run(requests.clone()), &backend, &hot_cache) + .await + .unwrap(); + + let guard = hot_cache.read().unwrap(); + assert_eq!(guard.acg_profiles.len(), 1); + let stability = &guard.acg_profiles[&learning_key]; + assert!( + stability.stable_prefix_length > 1, + "shared system plus shared anchor turn should extend the stable prefix past the scaffold", + ); + + // Beyond the scaffold the fingerprint binds to one complete request, so only + // that workflow is admitted and the sibling tails fail closed. + let admitted = requests + .iter() + .filter(|request| { + crate::acg::stability::profile_prefix_fingerprint( + &build_prompt_ir(request).unwrap(), + stability.stable_prefix_length, + &learning_key, + ) == stability.stable_prefix_fingerprint + }) + .count(); + assert_eq!(admitted, 1); +} diff --git a/crates/adaptive/tests/unit/acg_profile_tests.rs b/crates/adaptive/tests/unit/acg_profile_tests.rs index 08aebcb27..b2e35e9e9 100644 --- a/crates/adaptive/tests/unit/acg_profile_tests.rs +++ b/crates/adaptive/tests/unit/acg_profile_tests.rs @@ -50,6 +50,104 @@ fn sample_tool(name: &str) -> ToolDefinition { } } +fn scaffolded_request(task: &str) -> AnnotatedLlmRequest { + request( + vec![ + Message::System { + content: MessageContent::Text("Follow policy".to_string()), + name: None, + }, + Message::User { + content: MessageContent::Text(task.to_string()), + name: None, + }, + ], + Some(vec![sample_tool("search")]), + ) +} + +#[test] +fn acg_learning_key_groups_variable_tasks_by_stable_scaffold() { + let first = scaffolded_request("find alpha"); + let second = scaffolded_request("find beta"); + + assert_eq!( + derive_acg_learning_key("agent-a", &first), + derive_acg_learning_key("agent-a", &second) + ); +} + +#[test] +fn acg_learning_key_separates_scaffold_and_output_contract_changes() { + let baseline = scaffolded_request("find alpha"); + + let mut changed_system = baseline.clone(); + changed_system.messages[0] = Message::System { + content: MessageContent::Text("Follow a different policy".to_string()), + name: None, + }; + + let mut changed_tool = baseline.clone(); + changed_tool.tools = Some(vec![sample_tool("lookup")]); + + let mut contract_a = baseline.clone(); + contract_a.extra.insert( + "response_format".to_string(), + json!({"type":"json_schema","json_schema":{"type":"object","properties":{"a":{"type":"string"}}}}), + ); + let mut contract_b = baseline.clone(); + contract_b.extra.insert( + "response_format".to_string(), + json!({"json_schema":{"properties":{"b":{"type":"string"}},"type":"object"},"type":"json_schema"}), + ); + + let baseline_key = derive_acg_learning_key("agent-a", &baseline); + assert_ne!( + baseline_key, + derive_acg_learning_key("agent-a", &changed_system) + ); + assert_ne!( + baseline_key, + derive_acg_learning_key("agent-a", &changed_tool) + ); + assert_ne!( + derive_acg_learning_key("agent-a", &contract_a), + derive_acg_learning_key("agent-a", &contract_b) + ); + + let mut null_contract = baseline.clone(); + null_contract + .extra + .insert("response_format".to_string(), serde_json::Value::Null); + assert_eq!( + baseline_key, + derive_acg_learning_key("agent-a", &null_contract) + ); +} + +#[test] +fn acg_learning_key_keeps_task_seed_without_a_stable_scaffold() { + let first = request( + vec![Message::User { + content: MessageContent::Text("alpha".to_string()), + name: None, + }], + None, + ); + let second = request( + vec![Message::User { + content: MessageContent::Text("beta".to_string()), + name: None, + }], + None, + ); + + assert_ne!( + derive_acg_learning_key("agent-a", &first), + derive_acg_learning_key("agent-a", &second) + ); +} + #[test] fn acg_profile_derivation_covers_anchor_hash_system_fallback_and_empty_tools() { let layered = request( diff --git a/crates/adaptive/tests/unit/cache_diagnostics_tests.rs b/crates/adaptive/tests/unit/cache_diagnostics_tests.rs index 433ab4718..136e4fc58 100644 --- a/crates/adaptive/tests/unit/cache_diagnostics_tests.rs +++ b/crates/adaptive/tests/unit/cache_diagnostics_tests.rs @@ -82,6 +82,7 @@ fn make_hot_cache(stable_prefix_length: Option) -> HotCache { }) .collect(), stable_prefix_length, + stable_prefix_fingerprint: None, total_observations: 4, }), acg_observation_count: 4, diff --git a/crates/adaptive/tests/unit/intercepts_tests.rs b/crates/adaptive/tests/unit/intercepts_tests.rs index 971d71c60..f19048d3a 100644 --- a/crates/adaptive/tests/unit/intercepts_tests.rs +++ b/crates/adaptive/tests/unit/intercepts_tests.rs @@ -50,6 +50,7 @@ fn make_hot_cache( acg_stability: Some(StabilityAnalysisResult { scores: vec![], stable_prefix_length, + stable_prefix_fingerprint: None, total_observations: observation_count, }), acg_observation_count: observation_count, diff --git a/crates/adaptive/tests/unit/runtime_features_tests.rs b/crates/adaptive/tests/unit/runtime_features_tests.rs index eaf5b0265..868ddb17a 100644 --- a/crates/adaptive/tests/unit/runtime_features_tests.rs +++ b/crates/adaptive/tests/unit/runtime_features_tests.rs @@ -87,6 +87,14 @@ fn layered_acg_request() -> LlmRequest { } fn layered_acg_stability_result(observation_count: u32) -> StabilityAnalysisResult { + let request = layered_acg_request(); + let annotated_request = nemo_relay::codec::resolve::request_codec( + nemo_relay::codec::resolve::ProviderSurface::AnthropicMessages, + ) + .decode(&request) + .expect("fixture request should decode"); + let prompt_ir = crate::acg::ir_builder::build_prompt_ir(&annotated_request) + .expect("fixture request should build prompt ir"); StabilityAnalysisResult { scores: vec![ BlockStabilityScore { @@ -112,6 +120,11 @@ fn layered_acg_stability_result(observation_count: u32) -> StabilityAnalysisResu }, ], stable_prefix_length: 3, + stable_prefix_fingerprint: crate::acg::stability::profile_prefix_fingerprint( + &prompt_ir, + 3, + &crate::acg_profile::derive_acg_learning_key("agent-acg", &annotated_request), + ), total_observations: observation_count, } } diff --git a/crates/adaptive/tests/unit/runtime_tests.rs b/crates/adaptive/tests/unit/runtime_tests.rs index cfad601bd..cff3c45c4 100644 --- a/crates/adaptive/tests/unit/runtime_tests.rs +++ b/crates/adaptive/tests/unit/runtime_tests.rs @@ -31,10 +31,6 @@ fn reset_runtime_context() { set_thread_scope_stack(create_scope_stack()); } -fn short_hash(value: &str) -> &str { - value.get(..16).unwrap_or(value) -} - fn sample_annotated_request(model: Option<&str>) -> AnnotatedLlmRequest { AnnotatedLlmRequest { instructions: None, @@ -420,15 +416,10 @@ fn adaptive_acg_defaults_and_profile_key_behavior_stay_stable() { "agent-1", &sample_annotated_request(Some("claude-sonnet-4")), ); - let expected_learning_key = format!( - "agent-1::model=claude-sonnet-4::seed={}::system={}::tools=no-tools", - short_hash(&format!( - "user:{}", - crate::acg::sha256_hex("Summarize the latest findings") - )), - "sha256:3087d8fd4", + assert_eq!( + learning_key, + "agent-1::model=claude-sonnet-4::seed=stable-scaffold::system=sha256:3087d8fd4b98c564984d0f184c06bf6346f0788022d7cb521231e65f673936ac::tools=no-tools" ); - assert_eq!(learning_key, expected_learning_key,); let grown_chat_request = AnnotatedLlmRequest { instructions: None, @@ -504,9 +495,9 @@ fn adaptive_acg_defaults_and_profile_key_behavior_stay_stable() { "agent-1", &sample_layered_request(Some("claude-sonnet-4"), "Python review guide"), ); - assert_ne!( + assert_eq!( rust_learning_key, python_learning_key, - "layered requests should still separate learning buckets when the stable anchor differs", + "variable first-user guide content should share the stable system scaffold", ); let rust_bundle_variant = AnnotatedLlmRequest { diff --git a/crates/adaptive/tests/unit/storage_memory_internal_tests.rs b/crates/adaptive/tests/unit/storage_memory_internal_tests.rs index 311f007dc..2a95452f0 100644 --- a/crates/adaptive/tests/unit/storage_memory_internal_tests.rs +++ b/crates/adaptive/tests/unit/storage_memory_internal_tests.rs @@ -84,6 +84,7 @@ fn sample_stability_record() -> StabilityAnalysisResult { observation_count: 1, }], stable_prefix_length: 1, + stable_prefix_fingerprint: None, total_observations: 1, } } diff --git a/crates/adaptive/tests/unit/storage_tests.rs b/crates/adaptive/tests/unit/storage_tests.rs index 955dc8c76..7d48ea3fe 100644 --- a/crates/adaptive/tests/unit/storage_tests.rs +++ b/crates/adaptive/tests/unit/storage_tests.rs @@ -89,6 +89,7 @@ fn sample_stability(agent_id: &str) -> StabilityAnalysisResult { observation_count: 3, }], stable_prefix_length: 1, + stable_prefix_fingerprint: None, total_observations: 3, } } diff --git a/crates/adaptive/tests/unit/types_tests.rs b/crates/adaptive/tests/unit/types_tests.rs index c31b0c58d..e121c5390 100644 --- a/crates/adaptive/tests/unit/types_tests.rs +++ b/crates/adaptive/tests/unit/types_tests.rs @@ -41,6 +41,7 @@ fn sample_stability_result() -> StabilityAnalysisResult { observation_count: 4, }], stable_prefix_length: 1, + stable_prefix_fingerprint: Some("stable-prefix-1".to_string()), total_observations: 4, } } @@ -196,4 +197,13 @@ fn hot_cache_serialization_keeps_acg_field_names_stable() { decoded.acg_stability.as_ref().unwrap().total_observations, 4 ); + assert_eq!( + decoded + .acg_stability + .as_ref() + .unwrap() + .stable_prefix_fingerprint + .as_deref(), + Some("stable-prefix-1") + ); } diff --git a/crates/python/tests/coverage/py_storage_coverage_tests.rs b/crates/python/tests/coverage/py_storage_coverage_tests.rs index 0661dcf90..2a212f8a4 100644 --- a/crates/python/tests/coverage/py_storage_coverage_tests.rs +++ b/crates/python/tests/coverage/py_storage_coverage_tests.rs @@ -140,6 +140,7 @@ fn sample_stability_result() -> StabilityAnalysisResult { observation_count: 2, }], stable_prefix_length: 1, + stable_prefix_fingerprint: Some("stable-prefix-1".to_string()), total_observations: 2, } }