From d3f2f4f7586d1626eaf75e4e0c7c00f3e9fa1507 Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Wed, 15 Jul 2026 10:26:09 -0700 Subject: [PATCH 01/10] Standardize Rust Vally scenarios and remove azure-servicebus-rust Split from #375: restandardizes Rust scenario eval files (adds eval.yaml and skill_effectiveness_eval.yaml, removes older eval-experiment/nobuild variants), updates shared Rust experiment config and cargo-build-failure grader plugin, and removes the azure-servicebus-rust skill and scenarios. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9429ad33-6f31-4ec5-ba00-0f11b5afb82e --- .../skills/azure-servicebus-rust/SKILL.md | 171 ----------- .github/skills/azure-servicebus-rust | 1 - .../azure-sdk-rust-skill-experiment.yaml | 1 - .../rust-cargo-build-failure/index.ts | 3 +- .../rust-cargo-build-failure/package.json | 2 +- .../vally/eval-experiment.yaml | 181 ------------ .../azure-cosmos-rust/vally/eval.yaml | 110 +++---- .../vally/skill_effectiveness_eval.yaml | 19 +- .../vally/eval-experiment.yaml | 162 ----------- .../azure-eventhub-rust/vally/eval.yaml | 105 +++---- .../vally/skill_effectiveness_eval.yaml | 15 +- .../vally/eval-experiment.yaml | 204 ------------- .../azure-identity-rust/vally/eval.yaml | 108 ++++--- .../vally/skill_effectiveness_eval.yaml | 29 +- .../vally/eval-experiment.yaml | 205 -------------- .../vally/eval.yaml | 108 ++++--- .../vally/skill_effectiveness_eval.yaml | 28 +- .../vally/eval-experiment.yaml | 196 ------------- .../azure-keyvault-keys-rust/vally/eval.yaml | 107 ++++--- .../vally/skill_effectiveness_eval.yaml | 21 +- .../vally/eval-experiment.yaml | 178 ------------ .../vally/eval.yaml | 106 +++---- .../vally/skill_effectiveness_eval.yaml | 25 +- .../acceptance-criteria.md | 124 -------- .../azure-servicebus-rust/scenarios.yaml | 125 -------- .../vally/eval-experiment.yaml | 163 ----------- .../azure-servicebus-rust/vally/eval.yaml | 192 ------------- .../vally/skill_effectiveness_eval.yaml | 199 ------------- .../vally/skill_effectiveness_experiment.yaml | 23 -- .../vally/workspace/cargo.toml | 2 - .../vally/eval-keyvault-encryption.yaml | 44 ++- .../vally/eval-nobuild-experiment.yaml | 268 ------------------ .../vally/eval-nobuild.yaml | 268 ------------------ .../azure-storage-blob-rust/vally/eval.yaml | 53 ++-- .../vally/skill_effectiveness_eval.yaml | 70 +---- .../vally/skill_effectiveness_experiment.yaml | 2 - .../vally/eval-experiment.yaml | 165 ----------- .../azure-storage-queue-rust/vally/eval.yaml | 104 +++---- .../vally/skill_effectiveness_eval.yaml | 13 +- 39 files changed, 507 insertions(+), 3393 deletions(-) delete mode 100644 .github/plugins/azure-sdk-rust/skills/azure-servicebus-rust/SKILL.md delete mode 120000 .github/skills/azure-servicebus-rust delete mode 100644 tests/scenarios/azure-cosmos-rust/vally/eval-experiment.yaml delete mode 100644 tests/scenarios/azure-eventhub-rust/vally/eval-experiment.yaml delete mode 100644 tests/scenarios/azure-identity-rust/vally/eval-experiment.yaml delete mode 100644 tests/scenarios/azure-keyvault-certificates-rust/vally/eval-experiment.yaml delete mode 100644 tests/scenarios/azure-keyvault-keys-rust/vally/eval-experiment.yaml delete mode 100644 tests/scenarios/azure-keyvault-secrets-rust/vally/eval-experiment.yaml delete mode 100644 tests/scenarios/azure-servicebus-rust/acceptance-criteria.md delete mode 100644 tests/scenarios/azure-servicebus-rust/scenarios.yaml delete mode 100644 tests/scenarios/azure-servicebus-rust/vally/eval-experiment.yaml delete mode 100644 tests/scenarios/azure-servicebus-rust/vally/eval.yaml delete mode 100644 tests/scenarios/azure-servicebus-rust/vally/skill_effectiveness_eval.yaml delete mode 100644 tests/scenarios/azure-servicebus-rust/vally/skill_effectiveness_experiment.yaml delete mode 100644 tests/scenarios/azure-servicebus-rust/vally/workspace/cargo.toml delete mode 100644 tests/scenarios/azure-storage-blob-rust/vally/eval-nobuild-experiment.yaml delete mode 100644 tests/scenarios/azure-storage-blob-rust/vally/eval-nobuild.yaml delete mode 100644 tests/scenarios/azure-storage-queue-rust/vally/eval-experiment.yaml diff --git a/.github/plugins/azure-sdk-rust/skills/azure-servicebus-rust/SKILL.md b/.github/plugins/azure-sdk-rust/skills/azure-servicebus-rust/SKILL.md deleted file mode 100644 index 365a93a6..00000000 --- a/.github/plugins/azure-sdk-rust/skills/azure-servicebus-rust/SKILL.md +++ /dev/null @@ -1,171 +0,0 @@ ---- -name: azure-servicebus-rust -description: | - Azure Service Bus library for Rust. Send and receive messages using queues, topics, and subscriptions. - Triggers: "service bus rust", "ServiceBusClient rust", "send message servicebus rust", "receive message servicebus rust", "queue rust messaging", "topic subscription rust". -license: MIT -metadata: - author: Microsoft - package: azure_messaging_servicebus ---- - -# Azure Service Bus library for Rust - -Client library for Azure Service Bus — enterprise message broker with queues and publish-subscribe topics. - -> **⚠️ WARNING:** This crate is in early development and **SHOULD NOT** be used in production. APIs may change without notice. - -Use this skill when: - -- An app needs to send or receive messages via Azure Service Bus from Rust -- You need queue-based messaging with competing consumers -- You need publish-subscribe messaging with topics and subscriptions -- You need reliable message delivery with completion semantics - -> **IMPORTANT:** Only use the official `azure_messaging_servicebus` crate published by the [azure-sdk](https://crates.io/users/azure-sdk) crates.io user. Do NOT use unofficial or community crates. Official crates use underscores in names and none have version 0.21.0. - -## Installation - -```sh -cargo add azure_messaging_servicebus azure_identity tokio -``` - -> If your code uses `azure_core` types directly, add `azure_core` to `Cargo.toml`. If you only use `azure_messaging_servicebus` re-exports, direct `azure_core` dependency is optional. - -## Environment Variables - -```bash -SERVICEBUS_NAMESPACE=.servicebus.windows.net # Required — fully qualified namespace -``` - -## Key Concepts - -| Concept | Description | -| ---------------- | --------------------------------------------------------------- | -| **Namespace** | Container for all messaging components | -| **Queue** | Point-to-point messaging with competing consumers | -| **Topic** | Publish-subscribe messaging — one sender, many subscribers | -| **Subscription** | Receives messages from a topic | -| **Message** | Package of data and metadata, with completion/abandon semantics | - -## Authentication - -```rust -use azure_identity::DeveloperToolsCredential; -use azure_messaging_servicebus::ServiceBusClient; - -#[tokio::main] -async fn main() -> Result<(), Box> { - // Local dev: DeveloperToolsCredential. Production: use ManagedIdentityCredential. - let credential = DeveloperToolsCredential::new(None)?; - let client = ServiceBusClient::builder() - .open("your_namespace.servicebus.windows.net", credential.clone()) - .await?; - Ok(()) -} -``` - -## Core Workflow - -### Send a Message to a Queue - -```rust -use azure_identity::DeveloperToolsCredential; -use azure_messaging_servicebus::{ServiceBusClient, Message}; - -#[tokio::main] -async fn main() -> Result<(), Box> { - let credential = DeveloperToolsCredential::new(None)?; - let client = ServiceBusClient::builder() - .open("your_namespace.servicebus.windows.net", credential.clone()) - .await?; - let sender = client.create_sender("my_queue", None).await?; - - let message = Message::from("Hello, Service Bus!"); - sender.send_message(message, None).await?; - Ok(()) -} -``` - -### Receive Messages from a Queue - -```rust -use azure_identity::DeveloperToolsCredential; -use azure_messaging_servicebus::ServiceBusClient; - -#[tokio::main] -async fn main() -> Result<(), Box> { - let credential = DeveloperToolsCredential::new(None)?; - let client = ServiceBusClient::builder() - .open("your_namespace.servicebus.windows.net", credential.clone()) - .await?; - let receiver = client.create_receiver("my_queue", None).await?; - - let messages = receiver.receive_messages(5, None).await?; - for message in messages { - println!("Received: {}", message.body_as_string()?); - receiver.complete_message(&message, None).await?; - } - Ok(()) -} -``` - -### Send a Message to a Topic - -```rust -let sender = client.create_sender("my_topic", None).await?; -let message = Message::from("Hello, Topic subscribers!"); -sender.send_message(message, None).await?; -``` - -### Receive Messages from a Subscription - -```rust -let receiver = client - .create_receiver_for_subscription("my_topic", "my_subscription", None) - .await?; - -let messages = receiver.receive_messages(5, None).await?; -for message in messages { - println!("Received: {}", message.body_as_string()?); - receiver.complete_message(&message, None).await?; -} -``` - -## Message Settlement - -| Action | Purpose | -| ---------- | -------------------------------------------------- | -| `complete` | Remove message from queue — processing succeeded | -| `abandon` | Release lock — message becomes available for retry | - -Always complete messages after successful processing to prevent redelivery. - -## RBAC Roles - -For Entra ID auth, assign one of these roles: - -| Role | Access | -| --------------------------------- | ---------------- | -| `Azure Service Bus Data Sender` | Send messages | -| `Azure Service Bus Data Receiver` | Receive messages | -| `Azure Service Bus Data Owner` | Full access | - -## Best Practices - -1. **Use `cargo add` to manage dependencies, never edit `Cargo.toml` directly.** Add and remove Rust SDK dependencies with cargo commands instead of manual manifest edits. -2. **Add `azure_core` only when importing `azure_core` types directly.** If your code imports `azure_core::http::Url`, `azure_core::http::RequestContent`, or `azure_core::error::ErrorKind`, include `azure_core`; otherwise a direct dependency is optional. -3. **Use `DeveloperToolsCredential`** for local dev, **`ManagedIdentityCredential`** for production — Rust does not provide a single `DefaultAzureCredential` type -4. **Never hardcode credentials** — use environment variables or managed identity -5. **Assign RBAC roles** — ensure the identity has appropriate Service Bus data roles -6. **Always complete messages** — call `complete_message` after processing to remove from queue -7. **Use topics for fan-out** — when multiple consumers need the same messages, use topics with subscriptions -8. **This crate is pre-production** — APIs may change; pin your dependency version with cargo commands in your dependency workflow - -## Reference Links - -| Resource | Link | -| ------------- | ----------------------------------------------------------------------------------------------- | -| API Reference | https://docs.rs/azure_messaging_servicebus/latest/azure_messaging_servicebus | -| crates.io | https://crates.io/crates/azure_messaging_servicebus | -| Source Code | https://github.com/Azure/azure-sdk-for-rust/tree/main/sdk/servicebus/azure_messaging_servicebus | diff --git a/.github/skills/azure-servicebus-rust b/.github/skills/azure-servicebus-rust deleted file mode 120000 index 898d0443..00000000 --- a/.github/skills/azure-servicebus-rust +++ /dev/null @@ -1 +0,0 @@ -../plugins/azure-sdk-rust/skills/azure-servicebus-rust \ No newline at end of file diff --git a/tests/scenarios/_shared/vally/azure-sdk-rust-skill-experiment.yaml b/tests/scenarios/_shared/vally/azure-sdk-rust-skill-experiment.yaml index 3ed92cb2..a1ae8d18 100644 --- a/tests/scenarios/_shared/vally/azure-sdk-rust-skill-experiment.yaml +++ b/tests/scenarios/_shared/vally/azure-sdk-rust-skill-experiment.yaml @@ -7,6 +7,5 @@ evals: - ../../azure-keyvault-certificates-rust/vally/eval.yaml - ../../azure-keyvault-keys-rust/vally/eval.yaml - ../../azure-keyvault-secrets-rust/vally/eval.yaml - - ../../azure-servicebus-rust/vally/eval.yaml - ../../azure-storage-blob-rust/vally/eval.yaml - ../../azure-storage-queue-rust/vally/eval.yaml diff --git a/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/index.ts b/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/index.ts index df07f2e8..786fbb03 100644 --- a/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/index.ts +++ b/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/index.ts @@ -424,9 +424,8 @@ class CargoBuildTrajectoryGrader implements Grader { name: "rust-cargo-build-failure-check", description: "Rust-specific grader: checks trajectory for failed cargo build tool calls, extracts Rust compiler errors (E0XXX), and scales score by failure ratio", - behavior: { execution: "single", requiresWorkspace: false }, + behavior: { requiresWorkspace: false }, determinism: "static", - portability: "t3a-scenario", reference: "reference-free", temporalScope: "trajectory-level", costProfile: "free", diff --git a/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/package.json b/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/package.json index 7caae95e..839e049e 100644 --- a/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/package.json +++ b/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/package.json @@ -10,7 +10,7 @@ "typecheck": "tsc --noEmit" }, "devDependencies": { - "@microsoft/vally": "^0.6.0", + "@microsoft/vally": "^0.7.0", "@types/node": "^20.0.0", "typescript": "^5.4.5" }, diff --git a/tests/scenarios/azure-cosmos-rust/vally/eval-experiment.yaml b/tests/scenarios/azure-cosmos-rust/vally/eval-experiment.yaml deleted file mode 100644 index d0ee97e9..00000000 --- a/tests/scenarios/azure-cosmos-rust/vally/eval-experiment.yaml +++ /dev/null @@ -1,181 +0,0 @@ -name: azure-cosmos-rust-eval -description: Evaluation suite for azure-data-cosmos-rust -version: "1.0" -type: capability -scoring: - weights: {} - threshold: 1.0 -defaults: - runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: - - name: rust-cosmos-crud-operations - prompt: | - Write a Rust program named rust-cosmos-ops that demonstrates complete CRUD operations - and NoSQL document patterns with Azure Cosmos DB: - - Setup operations: - 1. Create a CosmosClient using builder pattern with ProximityTo routing strategy - 2. Get a database client for "myDatabase" - 3. Get a container client for "myContainer" - - CRUD operations: - 1. Create an item with partition key and document fields using create_item() - 2. Read an item by partition key and ID using read_item() - 3. Update an item by replacing it with replace_item() - 4. Delete an item using delete_item() - 5. Handle partition key requirements in all operations - - Advanced operations: - 1. Patch an item using patch_item() with PatchInstructions - 2. Use PatchOperation::set() to update specific fields - 3. Extract model from responses using into_model() - - Data structures: - 1. Define a struct Item with id, partition_key, and value fields - 2. Use serde::Serialize and serde::Deserialize traits - 3. Demonstrate serde_json for JSON operations - - Demonstrate: - - CosmosClient::builder().build(account, routing_strategy).await pattern - - Container hierarchy: client -> database -> container - - Partition key as required parameter in all item operations - - Result-based error handling with ErrorResponse extraction - - TokenCredential authentication (DeveloperToolsCredential/ManagedIdentityCredential) - - Async/await with Tokio runtime - - into_model() for response conversion - - The program should compile and execute successfully, demonstrating the complete - Cosmos DB NoSQL document lifecycle with proper partition key management. - - rubric: - - "Uses official azure_data_cosmos SDK for NoSQL operations." - - "Creates CosmosClient with builder pattern and ProximityTo routing strategy." - - "Navigates client hierarchy: client -> database_client() -> container_client()." - - "Create items with partition key and ID using create_item()." - - "Read items by partition key and ID using read_item()." - - "Update items with replace_item() for full updates." - - "Delete items using delete_item() with partition key." - - "Patch items with PatchInstructions and PatchOperation for partial updates." - - "Extracts partition key and ID from ResourceExt trait methods." - - "Uses TokenCredential for secure authentication." - - "Async/await with Tokio and proper Result-based error propagation." - - "Code compiles without warnings." - workspace_files: - include: - - "**/Cargo.toml" - - "**/Cargo.lock" - - "**/src/**/*.rs" - exclude: - - "**/target/**" - - "**/*.rlib" - - "**/*.rmeta" - environment: - files: - - src: workspace/cargo.toml - dest: Cargo.toml - constraints: - expect_skills: - - azure-cosmos-rust - graders: - - type: prompt - name: "Cosmos DB CRUD completeness" - config: - model: gpt-5.3-codex - prompt: | - - CosmosClient creation with builder and ProximityTo routing - - Database and container client navigation - - Create item with partition key and ID - - Read item by partition key - - Replace item for full updates - - Delete item by partition key - - Patch item with PatchInstructions - - PatchOperation::set() for field updates - - Response conversion with into_model() - - Error handling with ErrorResponse - - TokenCredential authentication - - Async/await with Tokio - scoring: scale_1_5 - - - type: run-command - name: "Cargo Check" - config: - command: cargo - args: - - "check" - timeout: 90s - - name: Rust Clippy Lints - type: run-command - weight: 1.0 - config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" - config: - command: cargo - args: - - "build" - timeout: 120s - - - name: CosmosClient Builder Pattern - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - CosmosClient created via CosmosClient::builder() - .build(account, RoutingStrategy::ProximityTo(region)).await? - Database accessed via client.database_client(name) - Container accessed via database.container_client(name).await - - name: CRUD Operations - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Create: container.create_item(partition_key, id, item, None).await? - Read: container.read_item(partition_key, id, None).await? - Update: container.replace_item(partition_key, id, item, None).await? - Delete: container.delete_item(partition_key, id, None).await? - All require partition_key as first parameter - - name: Partition Key Management - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Partition key is required parameter in all item operations - Partition key value passed as string to operations - Document fields include partition_key field for consistency - Responses contain partition key information - - name: Patch Operations - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Patch operations use PatchInstructions::from(vec![...]) - Operations created with PatchOperation::set(path, value) - Patched items obtained via container.patch_item(...).await? - Patch returns response with into_model() conversion - - name: Response Conversion and Error Handling - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Responses converted with into_model() for deserialization - Error handling with match and error extraction - No .unwrap() on fallible operations - Result propagated through main with ? operator - - name: Async Runtime - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Uses #[tokio::main] on main function - All I/O operations awaited - Serde for serialization/deserialization - TokenCredential passed with Some(credential) diff --git a/tests/scenarios/azure-cosmos-rust/vally/eval.yaml b/tests/scenarios/azure-cosmos-rust/vally/eval.yaml index 097bc098..039aac5a 100644 --- a/tests/scenarios/azure-cosmos-rust/vally/eval.yaml +++ b/tests/scenarios/azure-cosmos-rust/vally/eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-cosmos-crud-operations prompt: | @@ -62,7 +63,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper Result-based error propagation." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -75,6 +76,16 @@ stimuli: skills: - ../../../../.github/plugins/azure-sdk-rust/skills/azure-cosmos-rust files: + - src: ../../_shared/vally/tools/find-cargo-root.js + dest: .vally/tools/find-cargo-root.js + - src: ../../_shared/vally/tools/check-azure-crates.rs + dest: .vally/tools/check-azure-crates.rs + - src: ../../_shared/vally/tools/check-token-credential.rs + dest: .vally/tools/check-token-credential.rs + - src: ../../_shared/vally/tools/check-async-runtime.rs + dest: .vally/tools/check-async-runtime.rs + - src: ../../_shared/vally/tools/check-no-secrets.rs + dest: .vally/tools/check-no-secrets.rs - src: workspace/cargo.toml dest: Cargo.toml constraints: @@ -84,7 +95,6 @@ stimuli: - type: prompt name: "Cosmos DB CRUD completeness" config: - model: gpt-5.3-codex prompt: | - CosmosClient creation with builder and ProximityTo routing - Database and container client navigation @@ -101,72 +111,68 @@ stimuli: scoring: scale_1_5 - name: Official Azure SDK Crate Selection - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs + emit_in_metadata: true - name: TokenCredential Authentication - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs + emit_in_metadata: true - name: Async-First with Tokio Runtime - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs + emit_in_metadata: true - name: No Hardcoded Secrets - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs + emit_in_metadata: true + - name: Rust Cargo Trajectory Build Failure + type: rust-cargo-build-failure-check + weight: 1.0 + config: + collect_trajectory_compiler_errors: true + emit_in_metadata: true + error_exceptions: + - error_code: E0277 + message_pattern: "Option" + action: ignore_for_fail - - type: run-command - name: "Cargo Check" + - name: "Cargo Check" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "check" - timeout: 90s + commands: + - command: cargo check + emit_in_metadata: true - name: Rust Clippy Lints - type: run-command + type: rust-cargo-build-failure-check weight: 1.0 config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" + commands: + - command: cargo clippy -- -D warnings + emit_in_metadata: true + - name: "Cargo Build" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "build" - timeout: 120s + commands: + - command: cargo build + emit_in_metadata: true - name: CosmosClient Builder Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: gpt-5.5 prompt: > CosmosClient created via CosmosClient::builder() .build(account, RoutingStrategy::ProximityTo(region)).await? @@ -175,7 +181,7 @@ stimuli: - name: CRUD Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: gpt-5.5 prompt: > Create: container.create_item(partition_key, id, item, None).await? Read: container.read_item(partition_key, id, None).await? @@ -185,7 +191,7 @@ stimuli: - name: Partition Key Management type: prompt weight: 1.0 - model: gpt-5.3-codex + model: gpt-5.5 prompt: > Partition key is required parameter in all item operations Partition key value passed as string to operations @@ -194,7 +200,7 @@ stimuli: - name: Patch Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: gpt-5.5 prompt: > Patch operations use PatchInstructions::from(vec![...]) Operations created with PatchOperation::set(path, value) @@ -203,7 +209,7 @@ stimuli: - name: Response Conversion and Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: gpt-5.5 prompt: > Responses converted with into_model() for deserialization Error handling with match and error extraction diff --git a/tests/scenarios/azure-cosmos-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-cosmos-rust/vally/skill_effectiveness_eval.yaml index da3af2ca..69239176 100644 --- a/tests/scenarios/azure-cosmos-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-cosmos-rust/vally/skill_effectiveness_eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-cosmos-crud-operations prompt: | @@ -62,7 +63,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper Result-based error propagation." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -90,7 +91,7 @@ stimuli: - type: prompt name: "Cosmos DB CRUD completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - CosmosClient creation with builder and ProximityTo routing - Database and container client navigation @@ -149,7 +150,7 @@ stimuli: command: cargo args: - "check" - timeout: 90s + timeout: 2m - name: Rust Clippy Lints type: run-command weight: 1.0 @@ -172,7 +173,7 @@ stimuli: - name: CosmosClient Builder Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > CosmosClient created via CosmosClient::builder() .build(account, RoutingStrategy::ProximityTo(region)).await? @@ -181,7 +182,7 @@ stimuli: - name: CRUD Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Create: container.create_item(partition_key, id, item, None).await? Read: container.read_item(partition_key, id, None).await? @@ -191,7 +192,7 @@ stimuli: - name: Partition Key Management type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Partition key is required parameter in all item operations Partition key value passed as string to operations @@ -200,7 +201,7 @@ stimuli: - name: Patch Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Patch operations use PatchInstructions::from(vec![...]) Operations created with PatchOperation::set(path, value) @@ -209,7 +210,7 @@ stimuli: - name: Response Conversion and Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Responses converted with into_model() for deserialization Error handling with match and error extraction diff --git a/tests/scenarios/azure-eventhub-rust/vally/eval-experiment.yaml b/tests/scenarios/azure-eventhub-rust/vally/eval-experiment.yaml deleted file mode 100644 index ef73d6e5..00000000 --- a/tests/scenarios/azure-eventhub-rust/vally/eval-experiment.yaml +++ /dev/null @@ -1,162 +0,0 @@ -name: azure-eventhub-rust-eval -description: Evaluation suite for azure-eventhub-rust -version: "1.0" -type: capability -scoring: - weights: {} - threshold: 1.0 -defaults: - runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: - - name: rust-eventhub-producer-consumer - prompt: | - Write a Rust program named rust-eventhub-demo that demonstrates producer and consumer patterns - for Azure Event Hubs: - - Producer operations: - 1. Create a ProducerClient with namespace credentials - 2. Send single events to the hub - 3. Create a batch and add multiple events to it - 4. Send the batch - 5. Handle backpressure and errors during sending - - Consumer operations: - 1. Create a ConsumerClient with the same credentials - 2. Open a receiver on partition "0" - 3. Set the start position to StartLocation::Earliest - 4. Stream events from the receiver - 5. Extract and display event body data - 6. Handle errors per event in the stream - - Program should demonstrate: - - DeveloperToolsCredential or ManagedIdentityCredential usage - - ProducerClient::builder() pattern for producer initialization - - ConsumerClient::builder() pattern for consumer initialization - - Single event sending via producer.send_event(vec![...], None).await - - Batch creation and event addition via create_batch() and try_add_event_data() - - Receiver creation on specific partition with start position configuration - - Stream processing with futures::stream::StreamExt and while let Some(...) pattern - - Proper async/await and error handling with Result types - - The program should compile and execute successfully, demonstrating the complete - event hub producer-consumer workflow with proper credential-based authentication. - - rubric: - - "Implements both ProducerClient and ConsumerClient from official azure_messaging_eventhubs SDK." - - "Uses TokenCredential-based authentication (DeveloperToolsCredential or ManagedIdentityCredential)." - - "ProducerClient sends single events and batch events to Event Hub." - - "ConsumerClient receives events from specific partition with configurable start position." - - "Demonstrates async/await Rust with Tokio runtime." - - "Event stream processing with proper error handling per event." - - "Code compiles with cargo check without warnings." - - "Builder pattern used for both client initialization." - - "StartLocation configuration for consumer start position (Earliest/Latest)." - - "Proper extraction of event body data from stream." - workspace_files: - include: - - "**/Cargo.toml" - - "**/Cargo.lock" - - "**/src/**/*.rs" - - "**/examples/**/*.rs" - exclude: - - "**/target/**" - - "**/*.rlib" - - "**/*.rmeta" - - "**/*.d" - environment: - files: - - src: workspace/cargo.toml - dest: Cargo.toml - constraints: - expect_skills: - - azure-eventhub-rust - graders: - - type: prompt - name: "EventHub producer-consumer completeness" - config: - model: gpt-5.3-codex - prompt: | - - ProducerClient creation with builder pattern - - Single event sending: producer.send_event(vec![...], None).await - - Batch operations: create_batch() and try_add_event_data() - - ConsumerClient creation with builder pattern - - Receiver creation: open_receiver_on_partition() - - StartPosition configuration with StartLocation::Earliest - - Event stream processing with futures::stream::StreamExt - - Error handling for individual events in stream - - TokenCredential authentication - - Async/await patterns throughout - scoring: scale_1_5 - - - type: run-command - name: "Cargo Check" - config: - command: cargo - args: - - "check" - timeout: 90s - - name: Rust Clippy Lints - type: run-command - weight: 1.0 - config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" - config: - command: cargo - args: - - "build" - timeout: 120s - stderr_contains: "Finished" - - - name: ProducerClient Pattern - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - ProducerClient created via ProducerClient::builder() - .open(namespace, eventhub_name, credential).await? - Sends single events via send_event(data, None).await - Creates batches via create_batch(None).await and adds events via try_add_event_data() - - name: ConsumerClient Pattern - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - ConsumerClient created via ConsumerClient::builder() - .open(namespace, eventhub_name, credential).await? - Opens receiver on partition via open_receiver_on_partition() with partition ID as string - Sets StartPosition with StartLocation::Earliest or Latest - - name: Event Stream Processing - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Event stream obtained via receiver.stream_events() - Stream processing with while let Some(event_result) = stream.next().await - Match on Ok/Err for per-event error handling - Uses futures::stream::StreamExt trait for stream operations - - name: Async Runtime - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Code uses #[tokio::main] attribute and async/await throughout - All I/O operations are properly awaited - Result> main signature - - name: Credential Management - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Uses DeveloperToolsCredential for local dev or ManagedIdentityCredential for production - Credential is cloned when passed to multiple clients - No hardcoded credentials in source code diff --git a/tests/scenarios/azure-eventhub-rust/vally/eval.yaml b/tests/scenarios/azure-eventhub-rust/vally/eval.yaml index e1999661..5ba5449c 100644 --- a/tests/scenarios/azure-eventhub-rust/vally/eval.yaml +++ b/tests/scenarios/azure-eventhub-rust/vally/eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-eventhub-producer-consumer prompt: | @@ -54,7 +55,7 @@ stimuli: - "Builder pattern used for both client initialization." - "StartLocation configuration for consumer start position (Earliest/Latest)." - "Proper extraction of event body data from stream." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -69,6 +70,16 @@ stimuli: skills: - ../../../../.github/plugins/azure-sdk-rust/skills/azure-eventhub-rust files: + - src: ../../_shared/vally/tools/find-cargo-root.js + dest: .vally/tools/find-cargo-root.js + - src: ../../_shared/vally/tools/check-azure-crates.rs + dest: .vally/tools/check-azure-crates.rs + - src: ../../_shared/vally/tools/check-token-credential.rs + dest: .vally/tools/check-token-credential.rs + - src: ../../_shared/vally/tools/check-async-runtime.rs + dest: .vally/tools/check-async-runtime.rs + - src: ../../_shared/vally/tools/check-no-secrets.rs + dest: .vally/tools/check-no-secrets.rs - src: workspace/cargo.toml dest: Cargo.toml constraints: @@ -78,7 +89,6 @@ stimuli: - type: prompt name: "EventHub producer-consumer completeness" config: - model: gpt-5.3-codex prompt: | - ProducerClient creation with builder pattern - Single event sending: producer.send_event(vec![...], None).await @@ -93,73 +103,67 @@ stimuli: scoring: scale_1_5 - name: Official Azure SDK Crate Selection - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs + emit_in_metadata: true - name: TokenCredential Authentication - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs + emit_in_metadata: true - name: Async-First with Tokio Runtime - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs + emit_in_metadata: true - name: No Hardcoded Secrets - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs + emit_in_metadata: true + - name: Rust Cargo Trajectory Build Failure + type: rust-cargo-build-failure-check + weight: 1.0 + config: + collect_trajectory_compiler_errors: true + emit_in_metadata: true + error_exceptions: + - error_code: E0277 + message_pattern: "Option" + action: ignore_for_fail - - type: run-command - name: "Cargo Check" + - name: "Cargo Check" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "check" - timeout: 90s + commands: + - command: cargo check + emit_in_metadata: true - name: Rust Clippy Lints - type: run-command + type: rust-cargo-build-failure-check weight: 1.0 config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" + commands: + - command: cargo clippy -- -D warnings + emit_in_metadata: true + - name: "Cargo Build" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "build" - timeout: 120s - stderr_contains: "Finished" + commands: + - command: cargo build + emit_in_metadata: true - name: ProducerClient Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > ProducerClient created via ProducerClient::builder() .open(namespace, eventhub_name, credential).await? @@ -168,7 +172,6 @@ stimuli: - name: ConsumerClient Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > ConsumerClient created via ConsumerClient::builder() .open(namespace, eventhub_name, credential).await? @@ -177,7 +180,6 @@ stimuli: - name: Event Stream Processing type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Event stream obtained via receiver.stream_events() Stream processing with while let Some(event_result) = stream.next().await @@ -186,7 +188,6 @@ stimuli: - name: Credential Management type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Uses DeveloperToolsCredential for local dev or ManagedIdentityCredential for production Credential is cloned when passed to multiple clients diff --git a/tests/scenarios/azure-eventhub-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-eventhub-rust/vally/skill_effectiveness_eval.yaml index 6ea37bd7..04589dd5 100644 --- a/tests/scenarios/azure-eventhub-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-eventhub-rust/vally/skill_effectiveness_eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-eventhub-producer-consumer prompt: | @@ -54,7 +55,7 @@ stimuli: - "Builder pattern used for both client initialization." - "StartLocation configuration for consumer start position (Earliest/Latest)." - "Proper extraction of event body data from stream." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -84,7 +85,7 @@ stimuli: - type: prompt name: "EventHub producer-consumer completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - ProducerClient creation with builder pattern - Single event sending: producer.send_event(vec![...], None).await @@ -165,7 +166,7 @@ stimuli: - name: ProducerClient Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > ProducerClient created via ProducerClient::builder() .open(namespace, eventhub_name, credential).await? @@ -174,7 +175,7 @@ stimuli: - name: ConsumerClient Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > ConsumerClient created via ConsumerClient::builder() .open(namespace, eventhub_name, credential).await? @@ -183,7 +184,7 @@ stimuli: - name: Event Stream Processing type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Event stream obtained via receiver.stream_events() Stream processing with while let Some(event_result) = stream.next().await @@ -192,7 +193,7 @@ stimuli: - name: Credential Management type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Uses DeveloperToolsCredential for local dev or ManagedIdentityCredential for production Credential is cloned when passed to multiple clients diff --git a/tests/scenarios/azure-identity-rust/vally/eval-experiment.yaml b/tests/scenarios/azure-identity-rust/vally/eval-experiment.yaml deleted file mode 100644 index 7f12fdc5..00000000 --- a/tests/scenarios/azure-identity-rust/vally/eval-experiment.yaml +++ /dev/null @@ -1,204 +0,0 @@ -name: azure-identity-rust-eval -description: Evaluation suite for azure-identity-rust -version: "1.0" -type: capability -scoring: - weights: {} - threshold: 1.0 -defaults: - runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: - - name: rust-identity-credentials - prompt: | - Write a Rust program named rust-identity-demo that demonstrates all major credential - types from the Azure Identity SDK: - - Development credentials: - 1. Create a DeveloperToolsCredential with DeveloperToolsCredential::new(None)? - 2. Use it to authenticate (attempts Azure CLI, then Azure Developer CLI) - 3. Clone and pass to multiple Azure SDK clients - - Production credentials: - 1. Create a ManagedIdentityCredential with ManagedIdentityCredential::new(None)? - 2. Demonstrate system-assigned managed identity - 3. Show how to use with Azure-hosted resources (VMs, App Service, Functions, AKS) - - Service principal credentials: - 1. Create a ClientSecretCredential with tenant_id, client_id, and client_secret - 2. Demonstrate configuration from environment variables - 3. Show error handling for missing credentials - - Credential capabilities: - 1. Pass credentials to SecretClient or KeyClient constructors - 2. Clone credentials for use with multiple clients - 3. Handle credential errors (authentication failures, missing env vars) - - Demonstrate: - - DeveloperToolsCredential for local development - - ManagedIdentityCredential for Azure-hosted workloads - - ClientSecretCredential for CI/CD and service accounts - - Credential cloning for multi-client scenarios - - TokenCredential trait abstraction - - Environment variable usage (AZURE_CLIENT_ID, AZURE_TENANT_ID, AZURE_CLIENT_SECRET) - - Error handling with Result-based patterns - - Proper async/await with Tokio runtime - - No hardcoded secrets - - The program should compile and execute successfully, demonstrating the full - credential landscape and how to choose credentials for different deployment scenarios. - - rubric: - - "Uses official azure_identity SDK for all credential types." - - "Demonstrates DeveloperToolsCredential for local development." - - "Demonstrates ManagedIdentityCredential for Azure-hosted resources." - - "Demonstrates ClientSecretCredential with tenant, client, and secret." - - "Credentials properly cloned when passing to multiple clients." - - "Environment variables used for sensitive configuration (never hardcoded)." - - "Credentials passed as Some(credential) to SDK clients." - - "TokenCredential trait used for credential abstraction." - - "Error handling for credential creation and missing env vars." - - "Documentation or comments explain when each credential type is appropriate." - - "Async/await with Tokio runtime." - - "Code compiles without warnings." - - "Result-based error propagation with ? operator." - workspace_files: - include: - - "**/Cargo.toml" - - "**/Cargo.lock" - - "**/src/**/*.rs" - exclude: - - "**/target/**" - - "**/*.rlib" - - "**/*.rmeta" - environment: - files: - - src: workspace/cargo.toml - dest: Cargo.toml - constraints: - expect_skills: - - azure-identity-rust - graders: - - type: prompt - name: "Identity credential completeness" - config: - model: gpt-5.3-codex - prompt: | - - DeveloperToolsCredential creation and usage - - ManagedIdentityCredential creation and usage - - ClientSecretCredential with tenant_id, client_id, client_secret - - Credential cloning for multiple clients - - TokenCredential trait for abstraction - - Environment variable usage (AZURE_CLIENT_ID, AZURE_TENANT_ID, AZURE_CLIENT_SECRET) - - Error handling for credential creation - - Documentation on credential selection per scenario - - Async/await with Tokio runtime - - No hardcoded secrets in source - scoring: scale_1_5 - - - type: run-command - name: "Cargo Check" - config: - command: cargo - args: - - "check" - timeout: 90s - - name: Rust Clippy Lints - type: run-command - weight: 1.0 - config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" - config: - command: cargo - args: - - "build" - timeout: 120s - - - name: DeveloperToolsCredential - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - DeveloperToolsCredential created via DeveloperToolsCredential::new(None)? - Attempts Azure CLI (az login) first, then Azure Developer CLI (azd auth login) - Perfect for local development; never used in production - - name: ManagedIdentityCredential - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - ManagedIdentityCredential created via ManagedIdentityCredential::new(None)? - System-assigned managed identity for Azure-hosted workloads - Used in VMs, App Service, Functions, AKS, Container Instances - No environment variables needed; identity managed by Azure - - name: ClientSecretCredential - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - ClientSecretCredential created with three parameters: - tenant_id (string), client_id (string), client_secret (string) - Typically sourced from environment variables to avoid hardcoding - Used for CI/CD pipelines and service principal authentication - - name: Credential Cloning - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Credentials are Arc-wrapped and support cloning - Pass credential.clone() to multiple client constructors - Single credential instance can authenticate multiple SDK clients - Reduces overhead and maintains consistent auth context - - name: TokenCredential Trait - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - All credential types implement TokenCredential trait - Allows generic code that accepts any credential type - SDK clients accept Some(credential) where credential: impl TokenCredential - - name: Environment Variable Configuration - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - AZURE_TENANT_ID: Tenant ID for service principal auth - AZURE_CLIENT_ID: Client/application ID - AZURE_CLIENT_SECRET: Client secret (never commit to source control) - std::env::var() used to read environment variables - - name: Error Handling - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Credential creation returns Result - Missing or invalid credentials generate errors - Environment variable errors handled with match or ? - No unwrap() on credential creation - - name: Scenario Documentation - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Code includes comments explaining when each credential is appropriate - DeveloperToolsCredential for local development - ManagedIdentityCredential for production on Azure - ClientSecretCredential for CI/CD and service accounts - - name: Async Runtime - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Uses #[tokio::main] on main function - Credential operations are async-compatible - Multiple clients can be created and used concurrently - Result propagated with ? operator diff --git a/tests/scenarios/azure-identity-rust/vally/eval.yaml b/tests/scenarios/azure-identity-rust/vally/eval.yaml index eacdf4f2..eaf46317 100644 --- a/tests/scenarios/azure-identity-rust/vally/eval.yaml +++ b/tests/scenarios/azure-identity-rust/vally/eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-identity-credentials prompt: | @@ -63,7 +64,7 @@ stimuli: - "Async/await with Tokio runtime." - "Code compiles without warnings." - "Result-based error propagation with ? operator." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -76,6 +77,16 @@ stimuli: skills: - ../../../../.github/plugins/azure-sdk-rust/skills/azure-identity-rust files: + - src: ../../_shared/vally/tools/find-cargo-root.js + dest: .vally/tools/find-cargo-root.js + - src: ../../_shared/vally/tools/check-azure-crates.rs + dest: .vally/tools/check-azure-crates.rs + - src: ../../_shared/vally/tools/check-token-credential.rs + dest: .vally/tools/check-token-credential.rs + - src: ../../_shared/vally/tools/check-async-runtime.rs + dest: .vally/tools/check-async-runtime.rs + - src: ../../_shared/vally/tools/check-no-secrets.rs + dest: .vally/tools/check-no-secrets.rs - src: workspace/cargo.toml dest: Cargo.toml constraints: @@ -85,7 +96,6 @@ stimuli: - type: prompt name: "Identity credential completeness" config: - model: gpt-5.3-codex prompt: | - DeveloperToolsCredential creation and usage - ManagedIdentityCredential creation and usage @@ -99,72 +109,67 @@ stimuli: - No hardcoded secrets in source scoring: scale_1_5 - - type: run-command - name: "Cargo Check" + - name: "Cargo Check" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "check" - timeout: 90s + commands: + - command: cargo check + emit_in_metadata: true - name: Rust Clippy Lints - type: run-command + type: rust-cargo-build-failure-check weight: 1.0 config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" + commands: + - command: cargo clippy -- -D warnings + emit_in_metadata: true + - name: "Cargo Build" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "build" - timeout: 120s + commands: + - command: cargo build + emit_in_metadata: true - name: Official Azure SDK Crate Selection - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs + emit_in_metadata: true - name: TokenCredential Authentication - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs + emit_in_metadata: true - name: Async-First with Tokio Runtime - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs + emit_in_metadata: true - name: No Hardcoded Secrets - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs + emit_in_metadata: true + - name: Rust Cargo Trajectory Build Failure + type: rust-cargo-build-failure-check + weight: 1.0 + config: + collect_trajectory_compiler_errors: true + emit_in_metadata: true + error_exceptions: + - error_code: E0277 + message_pattern: "Option" + action: ignore_for_fail - name: DeveloperToolsCredential type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > DeveloperToolsCredential created via DeveloperToolsCredential::new(None)? Attempts Azure CLI (az login) first, then Azure Developer CLI (azd auth login) @@ -172,7 +177,6 @@ stimuli: - name: ManagedIdentityCredential type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > ManagedIdentityCredential created via ManagedIdentityCredential::new(None)? System-assigned managed identity for Azure-hosted workloads @@ -181,7 +185,6 @@ stimuli: - name: ClientSecretCredential type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > ClientSecretCredential created with three parameters: tenant_id (string), client_id (string), client_secret (string) @@ -190,7 +193,6 @@ stimuli: - name: Credential Cloning type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Credentials are Arc-wrapped and support cloning Pass credential.clone() to multiple client constructors @@ -199,7 +201,6 @@ stimuli: - name: TokenCredential Trait type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > All credential types implement TokenCredential trait Allows generic code that accepts any credential type @@ -207,7 +208,6 @@ stimuli: - name: Environment Variable Configuration type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > AZURE_TENANT_ID: Tenant ID for service principal auth AZURE_CLIENT_ID: Client/application ID @@ -216,7 +216,6 @@ stimuli: - name: Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Credential creation returns Result Missing or invalid credentials generate errors @@ -225,7 +224,6 @@ stimuli: - name: Scenario Documentation type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Code includes comments explaining when each credential is appropriate DeveloperToolsCredential for local development diff --git a/tests/scenarios/azure-identity-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-identity-rust/vally/skill_effectiveness_eval.yaml index 07b4383d..4195da52 100644 --- a/tests/scenarios/azure-identity-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-identity-rust/vally/skill_effectiveness_eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-identity-credentials prompt: | @@ -63,7 +64,7 @@ stimuli: - "Async/await with Tokio runtime." - "Code compiles without warnings." - "Result-based error propagation with ? operator." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -91,7 +92,7 @@ stimuli: - type: prompt name: "Identity credential completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - DeveloperToolsCredential creation and usage - ManagedIdentityCredential creation and usage @@ -148,7 +149,7 @@ stimuli: command: cargo args: - "check" - timeout: 90s + timeout: 3m - name: Rust Clippy Lints type: run-command weight: 1.0 @@ -159,19 +160,19 @@ stimuli: - "--" - "-D" - "warnings" - timeout: 120s + timeout: 3m - type: run-command name: "Cargo Build" config: command: cargo args: - "build" - timeout: 120s + timeout: 3m - name: DeveloperToolsCredential type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > DeveloperToolsCredential created via DeveloperToolsCredential::new(None)? Attempts Azure CLI (az login) first, then Azure Developer CLI (azd auth login) @@ -179,7 +180,7 @@ stimuli: - name: ManagedIdentityCredential type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > ManagedIdentityCredential created via ManagedIdentityCredential::new(None)? System-assigned managed identity for Azure-hosted workloads @@ -188,7 +189,7 @@ stimuli: - name: ClientSecretCredential type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > ClientSecretCredential created with three parameters: tenant_id (string), client_id (string), client_secret (string) @@ -197,7 +198,7 @@ stimuli: - name: Credential Cloning type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Credentials are Arc-wrapped and support cloning Pass credential.clone() to multiple client constructors @@ -206,7 +207,7 @@ stimuli: - name: TokenCredential Trait type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > All credential types implement TokenCredential trait Allows generic code that accepts any credential type @@ -214,7 +215,7 @@ stimuli: - name: Environment Variable Configuration type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > AZURE_TENANT_ID: Tenant ID for service principal auth AZURE_CLIENT_ID: Client/application ID @@ -223,7 +224,7 @@ stimuli: - name: Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Credential creation returns Result Missing or invalid credentials generate errors @@ -232,7 +233,7 @@ stimuli: - name: Scenario Documentation type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Code includes comments explaining when each credential is appropriate DeveloperToolsCredential for local development diff --git a/tests/scenarios/azure-keyvault-certificates-rust/vally/eval-experiment.yaml b/tests/scenarios/azure-keyvault-certificates-rust/vally/eval-experiment.yaml deleted file mode 100644 index 811e9217..00000000 --- a/tests/scenarios/azure-keyvault-certificates-rust/vally/eval-experiment.yaml +++ /dev/null @@ -1,205 +0,0 @@ -name: azure-keyvault-certificates-rust-eval -description: Evaluation suite for azure-security-keyvault-certificates-rust -version: "1.0" -type: capability -scoring: - weights: {} - threshold: 1.0 -defaults: - runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: - - name: rust-keyvault-certificates-operations - prompt: | - Write a Rust program named rust-keyvault-certs that demonstrates X.509 certificate - management and long-running operations with Azure Key Vault: - - Setup: - 1. Create a CertificateClient with vault URL and TokenCredential - 2. Use DeveloperToolsCredential for local dev - - Certificate creation (Long-Running Operation): - 1. Define CertificatePolicy with X509CertificateProperties - 2. Set subject to "CN=example.com" - 3. Set IssuerParameters with name "Self" for self-signed - 4. Create certificate with begin_create_certificate() returning Poller - 5. Await the Poller for completion - 6. Extract certificate name and version via ResourceExt - - Certificate management: - 1. Get an existing certificate with get_certificate() - 2. Update certificate properties with UpdateCertificatePropertiesParameters - 3. Add tags to certificates - 4. Delete a certificate with delete_certificate() - 5. List certificates with list_certificate_properties() returning Pager - - Advanced operations: - 1. Use certificate key for signing via KeyClient integration - 2. Create SignParameters with digest and SignatureAlgorithm - 3. Demonstrate that certificates contain embedded keys - - Demonstrate: - - CertificateClient::new() with vault URL and credential - - CertificatePolicy with X509CertificateProperties and IssuerParameters - - CreateCertificateParameters with policy - - Poller from begin_create_certificate() as IntoFuture - - Direct .await on Poller for completion - - GetCertificate and UpdateCertificatePropertiesParameters - - Pager iteration with TryStreamExt::try_next() - - ResourceExt for name and version extraction - - Proper async/await with Tokio runtime - - Result-based error propagation - - The program should compile and execute successfully, demonstrating complete - certificate lifecycle including LRO handling and signing operations. - - rubric: - - "Uses official azure_security_keyvault_certificates SDK." - - "Creates CertificateClient with vault URL and TokenCredential." - - "Defines CertificatePolicy with X509CertificateProperties and subject." - - "Sets IssuerParameters with 'Self' for self-signed certificates." - - "Creates certificates with begin_create_certificate() returning Poller." - - "Awaits Poller directly for LRO completion." - - "Gets certificates with get_certificate() and converts with into_model()." - - "Updates certificate properties with tags." - - "Deletes certificates with delete_certificate()." - - "Lists certificates with pagination using list_certificate_properties()." - - "Uses ResourceExt trait to extract name and version." - - "Demonstrates certificate key access for signing operations." - - "Handles LRO errors gracefully." - - "Uses TokenCredential for secure authentication." - - "Async/await with Tokio and proper error handling." - - "Code compiles without warnings." - workspace_files: - include: - - "**/Cargo.toml" - - "**/Cargo.lock" - - "**/src/**/*.rs" - exclude: - - "**/target/**" - - "**/*.rlib" - - "**/*.rmeta" - environment: - files: - - src: workspace/cargo.toml - dest: Cargo.toml - constraints: - expect_skills: - - azure-keyvault-certificates-rust - graders: - - type: prompt - name: "Key Vault certificates completeness" - config: - model: gpt-5.3-codex - prompt: | - - CertificateClient creation with vault URL and credential - - CertificatePolicy with X509CertificateProperties - - Subject set to CN=example.com format - - IssuerParameters with "Self" for self-signed - - CreateCertificateParameters wrapping policy - - begin_create_certificate() returning Poller - - Direct .await on Poller (IntoFuture pattern) - - Get certificate and into_model() conversion - - Update certificate properties with tags - - Delete certificate operation - - List certificates with Pager pagination - - ResourceExt for name/version extraction - - TokenCredential authentication - - Async/await throughout - scoring: scale_1_5 - - - type: run-command - name: "Cargo Check" - config: - command: cargo - args: - - "check" - timeout: 90s - - name: Rust Clippy Lints - type: run-command - weight: 1.0 - config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" - config: - command: cargo - args: - - "build" - timeout: 120s - - - name: CertificateClient Creation - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - CertificateClient created via CertificateClient::new(vault_url, credential, None) - Vault URL formatted as https://vault_name.vault.azure.net/ - Credential is TokenCredential (DeveloperToolsCredential or ManagedIdentityCredential) - - name: Certificate Policy Definition - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - CertificatePolicy includes x509_certificate_properties and issuer_parameters - X509CertificateProperties has subject: Some("CN=example.com".into()) - IssuerParameters has name: Some("Self".into()) for self-signed certificates - - name: Long-Running Operation Pattern - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - begin_create_certificate() returns Poller - Poller implements IntoFuture trait - Direct await on Poller: client.begin_create_certificate(...)?.await? - Awaiting completes the LRO and returns the certificate - - name: Certificate Management Operations - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Get: client.get_certificate(name, None).await?.into_model()? - Update: client.update_certificate_properties(name, params.try_into()?, None).await? - Delete: client.delete_certificate(name, None).await? - List: client.list_certificate_properties(None)? returns Pager - - name: Certificate Properties Update - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - UpdateCertificatePropertiesParameters with tags: Some(HashMap) - Tags stored as Vec of key-value pairs - Uses #[allow(clippy::needless_update)] with ..Default::default() - Properties persist across certificate versions - - name: Pagination and Resource Extraction - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - List returns Pager, iterate with let mut pager = ...; while let Some(cert) = pager.try_next().await? - ResourceExt provides cert.resource_id()? with name and version - Uses TryStreamExt trait from futures crate - - name: Certificate Key Integration - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Certificates contain embedded keys for signing operations - KeyClient can sign with certificate name as key reference - Demonstrates that certificates are more than just X.509 data - - name: Async Runtime and Error Handling - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Uses #[tokio::main] on main function - All I/O awaited including LRO completion - No .unwrap() on Result types - Proper error propagation with ? operator diff --git a/tests/scenarios/azure-keyvault-certificates-rust/vally/eval.yaml b/tests/scenarios/azure-keyvault-certificates-rust/vally/eval.yaml index 37a4e92b..e2a28c6a 100644 --- a/tests/scenarios/azure-keyvault-certificates-rust/vally/eval.yaml +++ b/tests/scenarios/azure-keyvault-certificates-rust/vally/eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-keyvault-certificates-operations prompt: | @@ -71,7 +72,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper error handling." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -84,6 +85,16 @@ stimuli: skills: - ../../../../.github/plugins/azure-sdk-rust/skills/azure-keyvault-certificates-rust files: + - src: ../../_shared/vally/tools/find-cargo-root.js + dest: .vally/tools/find-cargo-root.js + - src: ../../_shared/vally/tools/check-azure-crates.rs + dest: .vally/tools/check-azure-crates.rs + - src: ../../_shared/vally/tools/check-token-credential.rs + dest: .vally/tools/check-token-credential.rs + - src: ../../_shared/vally/tools/check-async-runtime.rs + dest: .vally/tools/check-async-runtime.rs + - src: ../../_shared/vally/tools/check-no-secrets.rs + dest: .vally/tools/check-no-secrets.rs - src: workspace/cargo.toml dest: Cargo.toml constraints: @@ -93,7 +104,6 @@ stimuli: - type: prompt name: "Key Vault certificates completeness" config: - model: gpt-5.3-codex prompt: | - CertificateClient creation with vault URL and credential - CertificatePolicy with X509CertificateProperties @@ -112,72 +122,67 @@ stimuli: scoring: scale_1_5 - name: Official Azure SDK Crate Selection - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs + emit_in_metadata: true - name: TokenCredential Authentication - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs + emit_in_metadata: true - name: Async-First with Tokio Runtime - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs + emit_in_metadata: true - name: No Hardcoded Secrets - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs + emit_in_metadata: true + - name: Rust Cargo Trajectory Build Failure + type: rust-cargo-build-failure-check + weight: 1.0 + config: + collect_trajectory_compiler_errors: true + emit_in_metadata: true + error_exceptions: + - error_code: E0277 + message_pattern: "Option" + action: ignore_for_fail - - type: run-command - name: "Cargo Check" + - name: "Cargo Check" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "check" - timeout: 90s + commands: + - command: cargo check + emit_in_metadata: true - name: Rust Clippy Lints - type: run-command + type: rust-cargo-build-failure-check weight: 1.0 config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" + commands: + - command: cargo clippy -- -D warnings + emit_in_metadata: true + - name: "Cargo Build" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "build" - timeout: 120s + commands: + - command: cargo build + emit_in_metadata: true - name: CertificateClient Creation type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > CertificateClient created via CertificateClient::new(vault_url, credential, None) Vault URL formatted as https://vault_name.vault.azure.net/ @@ -185,7 +190,6 @@ stimuli: - name: Certificate Policy Definition type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > CertificatePolicy includes x509_certificate_properties and issuer_parameters X509CertificateProperties has subject: Some("CN=example.com".into()) @@ -193,7 +197,6 @@ stimuli: - name: Long-Running Operation Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > begin_create_certificate() returns Poller Poller implements IntoFuture trait @@ -202,7 +205,6 @@ stimuli: - name: Certificate Management Operations type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Get: client.get_certificate(name, None).await?.into_model()? Update: client.update_certificate_properties(name, params.try_into()?, None).await? @@ -211,7 +213,6 @@ stimuli: - name: Certificate Properties Update type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > UpdateCertificatePropertiesParameters with tags: Some(HashMap) Tags stored as Vec of key-value pairs @@ -220,7 +221,6 @@ stimuli: - name: Pagination and Resource Extraction type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > List returns Pager, iterate with let mut pager = ...; while let Some(cert) = pager.try_next().await? ResourceExt provides cert.resource_id()? with name and version @@ -228,7 +228,6 @@ stimuli: - name: Certificate Key Integration type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Certificates contain embedded keys for signing operations KeyClient can sign with certificate name as key reference @@ -236,7 +235,6 @@ stimuli: - name: Async Runtime and Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Uses #[tokio::main] on main function All I/O awaited including LRO completion diff --git a/tests/scenarios/azure-keyvault-certificates-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-keyvault-certificates-rust/vally/skill_effectiveness_eval.yaml index 9ce9fcef..6b1de8c7 100644 --- a/tests/scenarios/azure-keyvault-certificates-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-keyvault-certificates-rust/vally/skill_effectiveness_eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-keyvault-certificates-operations prompt: | @@ -71,7 +72,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper error handling." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -99,7 +100,7 @@ stimuli: - type: prompt name: "Key Vault certificates completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - CertificateClient creation with vault URL and credential - CertificatePolicy with X509CertificateProperties @@ -126,6 +127,7 @@ stimuli: - "+nightly" - "-Zscript" - ".vally/tools/check-azure-crates.rs" + timeout: 3m - name: TokenCredential Authentication type: program weight: 1.0 @@ -160,7 +162,7 @@ stimuli: command: cargo args: - "check" - timeout: 90s + timeout: 3m - name: Rust Clippy Lints type: run-command weight: 1.0 @@ -171,7 +173,7 @@ stimuli: - "--" - "-D" - "warnings" - timeout: 120s + timeout: 3m - type: run-command name: "Cargo Build" config: @@ -183,7 +185,7 @@ stimuli: - name: CertificateClient Creation type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > CertificateClient created via CertificateClient::new(vault_url, credential, None) Vault URL formatted as https://vault_name.vault.azure.net/ @@ -191,7 +193,7 @@ stimuli: - name: Certificate Policy Definition type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > CertificatePolicy includes x509_certificate_properties and issuer_parameters X509CertificateProperties has subject: Some("CN=example.com".into()) @@ -199,7 +201,7 @@ stimuli: - name: Long-Running Operation Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > begin_create_certificate() returns Poller Poller implements IntoFuture trait @@ -208,7 +210,7 @@ stimuli: - name: Certificate Management Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Get: client.get_certificate(name, None).await?.into_model()? Update: client.update_certificate_properties(name, params.try_into()?, None).await? @@ -217,7 +219,7 @@ stimuli: - name: Certificate Properties Update type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > UpdateCertificatePropertiesParameters with tags: Some(HashMap) Tags stored as Vec of key-value pairs @@ -226,7 +228,7 @@ stimuli: - name: Pagination and Resource Extraction type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > List returns Pager, iterate with let mut pager = ...; while let Some(cert) = pager.try_next().await? ResourceExt provides cert.resource_id()? with name and version @@ -234,7 +236,7 @@ stimuli: - name: Certificate Key Integration type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Certificates contain embedded keys for signing operations KeyClient can sign with certificate name as key reference @@ -242,7 +244,7 @@ stimuli: - name: Async Runtime and Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Uses #[tokio::main] on main function All I/O awaited including LRO completion diff --git a/tests/scenarios/azure-keyvault-keys-rust/vally/eval-experiment.yaml b/tests/scenarios/azure-keyvault-keys-rust/vally/eval-experiment.yaml deleted file mode 100644 index 11c078df..00000000 --- a/tests/scenarios/azure-keyvault-keys-rust/vally/eval-experiment.yaml +++ /dev/null @@ -1,196 +0,0 @@ -name: azure-keyvault-keys-rust-eval -description: Evaluation suite for azure-security-keyvault-keys-rust -version: "1.0" -type: capability -scoring: - weights: {} - threshold: 1.0 -defaults: - runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: - - name: rust-keyvault-keys-operations - prompt: | - Write a Rust program named rust-keyvault-keys that demonstrates cryptographic key - management and envelope encryption with Azure Key Vault: - - Key management operations: - 1. Create a KeyClient with vault URL and TokenCredential - 2. Create EC and RSA keys with CreateKeyParameters - 3. Get a key with get_key() - 4. Update key properties with UpdateKeyPropertiesParameters and tags - 5. Delete a key with delete_key() - 6. List keys with list_key_properties() and pagination - - Cryptographic operations: - 1. Wrap a data encryption key (DEK) with a key encryption key (KEK) using wrap_key() - 2. Use EncryptionAlgorithm::RsaOaep256 for wrapping - 3. Unwrap encrypted keys with unwrap_key() using the saved key version - 4. Demonstrate envelope encryption pattern - - Key types demonstrated: - 1. EC keys with CurveName::P256 - 2. RSA keys with key_size: 2048 or 4096 - 3. HSM-protected variants (EcHsm, RsaHsm) - - Advanced patterns: - 1. Extract resource ID components using ResourceExt trait - 2. Retain key versions for unwrap operations - 3. Handle KeyOperationParameters for wrap/unwrap operations - 4. Iterate keys with Pager and TryStreamExt - - Demonstrate: - - KeyClient::new() with vault URL and credential - - CreateKeyParameters with KeyType and curve settings - - UpdateKeyPropertiesParameters with tags - - Wrap/unwrap pattern for envelope encryption - - KeyOperationParameters with algorithm and value - - ResourceExt for name/version extraction - - Proper async/await with Tokio runtime - - Result-based error propagation - - The program should compile and execute successfully, demonstrating complete - key lifecycle management and envelope encryption workflows. - - rubric: - - "Uses official azure_security_keyvault_keys SDK." - - "Creates KeyClient with vault URL and TokenCredential." - - "Creates EC keys with CurveName parameter." - - "Creates RSA keys with key_size parameter." - - "Gets keys with get_key() and converts with into_model()." - - "Updates key properties with tags." - - "Deletes keys with delete_key()." - - "Lists keys with pagination using list_key_properties()." - - "Wraps DEK with KEK using wrap_key() and EncryptionAlgorithm." - - "Unwraps encrypted keys with unwrap_key() using key version." - - "Preserves key versions for encryption/decryption pairs." - - "Uses ResourceExt trait to extract name and version." - - "Handles wrap/unwrap errors gracefully." - - "Uses TokenCredential for secure authentication." - - "Async/await with Tokio and proper error handling." - - "Code compiles without warnings." - workspace_files: - include: - - "**/Cargo.toml" - - "**/Cargo.lock" - - "**/src/**/*.rs" - exclude: - - "**/target/**" - - "**/*.rlib" - - "**/*.rmeta" - environment: - files: - - src: workspace/cargo.toml - dest: Cargo.toml - constraints: - expect_skills: - - azure-keyvault-keys-rust - graders: - - type: prompt - name: "Key Vault keys completeness" - config: - model: gpt-5.3-codex - prompt: | - - KeyClient creation with vault URL and credential - - Create EC keys with CurveName::P256 - - Create RSA keys with key_size 2048 or 4096 - - Get key operation and into_model() conversion - - Update key properties with tags - - Delete key operation - - List keys with Pager pagination - - Wrap operation with EncryptionAlgorithm::RsaOaep256 - - Unwrap operation preserving key versions - - Envelope encryption pattern demonstration - - ResourceExt for name/version extraction - - TokenCredential authentication - - Async/await throughout - scoring: scale_1_5 - - - type: run-command - name: "Cargo Check" - config: - command: cargo - args: - - "check" - timeout: 90s - - name: Rust Clippy Lints - type: run-command - weight: 1.0 - config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" - config: - command: cargo - args: - - "build" - timeout: 120s - - - name: KeyClient Creation - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - KeyClient created via KeyClient::new(vault_url, credential, None) - Vault URL formatted as https://vault_name.vault.azure.net/ - Credential is TokenCredential (DeveloperToolsCredential or ManagedIdentityCredential) - - name: Key Creation Patterns - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - EC keys: CreateKeyParameters with kty: Some(KeyType::Ec), curve: Some(CurveName::P256) - RSA keys: CreateKeyParameters with kty: Some(KeyType::Rsa), key_size: Some(2048) - Keys created via client.create_key(name, body.try_into()?, None).await? - - name: Key Management Operations - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Get: client.get_key(name, None).await?.into_model()? - Update: client.update_key_properties(name, params.try_into()?, None).await? - Delete: client.delete_key(name, None).await? - List: client.list_key_properties(None)? returns Pager - - name: Envelope Encryption Pattern - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Wrap: client.wrap_key(key_name, params.try_into()?, None).await? - Parameters include algorithm: Some(EncryptionAlgorithm::RsaOaep256) and value: Some(dek) - Extract wrapped result and key version for later unwrap - Unwrap: client.unwrap_key(key_name, version, params.try_into()?, None).await? - Compare unwrapped value with original DEK - - name: Key Version Management - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Extract key version from response via resource_id()? - Version stored from wrap operation for later unwrap - Same version used to unwrap (immutable relationship) - Versions allow key rotation without losing decryption capability - - name: Pagination and Resource Extraction - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - List returns Pager, iterate with let mut pager = ...; while let Some(key) = pager.try_next().await? - ResourceExt provides key.resource_id()? with name and version - Use TryStreamExt trait from futures crate - - name: Async Runtime and Error Handling - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Uses #[tokio::main] on main function - All I/O awaited (client operations, wrapping, unwrapping) - No .unwrap() on Result types - Proper error propagation with ? operator diff --git a/tests/scenarios/azure-keyvault-keys-rust/vally/eval.yaml b/tests/scenarios/azure-keyvault-keys-rust/vally/eval.yaml index b927c364..26ad503e 100644 --- a/tests/scenarios/azure-keyvault-keys-rust/vally/eval.yaml +++ b/tests/scenarios/azure-keyvault-keys-rust/vally/eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-keyvault-keys-operations prompt: | @@ -70,7 +71,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper error handling." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -83,6 +84,16 @@ stimuli: skills: - ../../../../.github/plugins/azure-sdk-rust/skills/azure-keyvault-keys-rust files: + - src: ../../_shared/vally/tools/find-cargo-root.js + dest: .vally/tools/find-cargo-root.js + - src: ../../_shared/vally/tools/check-azure-crates.rs + dest: .vally/tools/check-azure-crates.rs + - src: ../../_shared/vally/tools/check-token-credential.rs + dest: .vally/tools/check-token-credential.rs + - src: ../../_shared/vally/tools/check-async-runtime.rs + dest: .vally/tools/check-async-runtime.rs + - src: ../../_shared/vally/tools/check-no-secrets.rs + dest: .vally/tools/check-no-secrets.rs - src: workspace/cargo.toml dest: Cargo.toml constraints: @@ -92,7 +103,6 @@ stimuli: - type: prompt name: "Key Vault keys completeness" config: - model: gpt-5.3-codex prompt: | - KeyClient creation with vault URL and credential - Create EC keys with CurveName::P256 @@ -110,72 +120,67 @@ stimuli: scoring: scale_1_5 - name: Official Azure SDK Crate Selection - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs + emit_in_metadata: true - name: TokenCredential Authentication - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs + emit_in_metadata: true - name: Async-First with Tokio Runtime - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs + emit_in_metadata: true - name: No Hardcoded Secrets - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs + emit_in_metadata: true + - name: Rust Cargo Trajectory Build Failure + type: rust-cargo-build-failure-check + weight: 1.0 + config: + collect_trajectory_compiler_errors: true + emit_in_metadata: true + error_exceptions: + - error_code: E0277 + message_pattern: "Option" + action: ignore_for_fail - - type: run-command - name: "Cargo Check" + - name: "Cargo Check" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "check" - timeout: 90s + commands: + - command: cargo check + emit_in_metadata: true - name: Rust Clippy Lints - type: run-command + type: rust-cargo-build-failure-check weight: 1.0 config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" + commands: + - command: cargo clippy -- -D warnings + emit_in_metadata: true + - name: "Cargo Build" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "build" - timeout: 120s + commands: + - command: cargo build + emit_in_metadata: true - name: KeyClient Creation type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > KeyClient created via KeyClient::new(vault_url, credential, None) Vault URL formatted as https://vault_name.vault.azure.net/ @@ -183,7 +188,6 @@ stimuli: - name: Key Creation Patterns type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > EC keys: CreateKeyParameters with kty: Some(KeyType::Ec), curve: Some(CurveName::P256) RSA keys: CreateKeyParameters with kty: Some(KeyType::Rsa), key_size: Some(2048) @@ -191,7 +195,6 @@ stimuli: - name: Key Management Operations type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Get: client.get_key(name, None).await?.into_model()? Update: client.update_key_properties(name, params.try_into()?, None).await? @@ -200,7 +203,6 @@ stimuli: - name: Envelope Encryption Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Wrap: client.wrap_key(key_name, params.try_into()?, None).await? Parameters include algorithm: Some(EncryptionAlgorithm::RsaOaep256) and value: Some(dek) @@ -210,7 +212,6 @@ stimuli: - name: Key Version Management type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Extract key version from response via resource_id()? Version stored from wrap operation for later unwrap @@ -219,7 +220,6 @@ stimuli: - name: Pagination and Resource Extraction type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > List returns Pager, iterate with let mut pager = ...; while let Some(key) = pager.try_next().await? ResourceExt provides key.resource_id()? with name and version @@ -227,7 +227,6 @@ stimuli: - name: Async Runtime and Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Uses #[tokio::main] on main function All I/O awaited (client operations, wrapping, unwrapping) diff --git a/tests/scenarios/azure-keyvault-keys-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-keyvault-keys-rust/vally/skill_effectiveness_eval.yaml index 3cdaaa30..3b2fd9b8 100644 --- a/tests/scenarios/azure-keyvault-keys-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-keyvault-keys-rust/vally/skill_effectiveness_eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-keyvault-keys-operations prompt: | @@ -70,7 +71,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper error handling." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -98,7 +99,7 @@ stimuli: - type: prompt name: "Key Vault keys completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - KeyClient creation with vault URL and credential - Create EC keys with CurveName::P256 @@ -181,7 +182,7 @@ stimuli: - name: KeyClient Creation type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > KeyClient created via KeyClient::new(vault_url, credential, None) Vault URL formatted as https://vault_name.vault.azure.net/ @@ -189,7 +190,7 @@ stimuli: - name: Key Creation Patterns type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > EC keys: CreateKeyParameters with kty: Some(KeyType::Ec), curve: Some(CurveName::P256) RSA keys: CreateKeyParameters with kty: Some(KeyType::Rsa), key_size: Some(2048) @@ -197,7 +198,7 @@ stimuli: - name: Key Management Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Get: client.get_key(name, None).await?.into_model()? Update: client.update_key_properties(name, params.try_into()?, None).await? @@ -206,7 +207,7 @@ stimuli: - name: Envelope Encryption Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Wrap: client.wrap_key(key_name, params.try_into()?, None).await? Parameters include algorithm: Some(EncryptionAlgorithm::RsaOaep256) and value: Some(dek) @@ -216,7 +217,7 @@ stimuli: - name: Key Version Management type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Extract key version from response via resource_id()? Version stored from wrap operation for later unwrap @@ -225,7 +226,7 @@ stimuli: - name: Pagination and Resource Extraction type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > List returns Pager, iterate with let mut pager = ...; while let Some(key) = pager.try_next().await? ResourceExt provides key.resource_id()? with name and version @@ -233,7 +234,7 @@ stimuli: - name: Async Runtime and Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Uses #[tokio::main] on main function All I/O awaited (client operations, wrapping, unwrapping) diff --git a/tests/scenarios/azure-keyvault-secrets-rust/vally/eval-experiment.yaml b/tests/scenarios/azure-keyvault-secrets-rust/vally/eval-experiment.yaml deleted file mode 100644 index 7be891f3..00000000 --- a/tests/scenarios/azure-keyvault-secrets-rust/vally/eval-experiment.yaml +++ /dev/null @@ -1,178 +0,0 @@ -name: azure-keyvault-secrets-rust-eval -description: Evaluation suite for azure-security-keyvault-secrets-rust -version: "1.0" -type: capability -scoring: - weights: {} - threshold: 1.0 -defaults: - runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: - - name: rust-keyvault-secrets-operations - prompt: | - Write a Rust program named rust-keyvault-secrets that demonstrates secure secret - management with Azure Key Vault: - - Setup: - 1. Create a SecretClient with vault URL and TokenCredential - 2. Use DeveloperToolsCredential for local dev - - CRUD operations: - 1. Set a secret with SetSecretParameters including value and content_type - 2. Get a secret with get_secret() - 3. Update secret properties with UpdateSecretPropertiesParameters - 4. Delete a secret with delete_secret() - 5. List secrets with list_secret_properties() returning Pager - - Advanced operations: - 1. Extract resource ID components (name, version) using ResourceExt trait - 2. Set content_type on secrets (e.g., "text/plain") - 3. Add tags to secrets using HashMap - 4. Handle pagination when listing secrets with try_next() from TryStreamExt - - Error handling: - 1. Match on Ok/Err for secret operations - 2. Extract ErrorResponse from errors - 3. Handle 404 errors for missing secrets gracefully - - Demonstrate: - - SecretClient::new() with vault URL and credential - - SetSecretParameters and try_into() for request conversion - - UpdateSecretPropertiesParameters with content_type and tags - - Pager iteration with TryStreamExt::try_next() - - ResourceExt trait for extracting name and version - - Proper async/await with Tokio runtime - - Result-based error propagation - - The program should compile and execute successfully, demonstrating complete - secret lifecycle management with proper credential-based authentication. - - rubric: - - "Uses official azure_security_keyvault_secrets SDK." - - "Creates SecretClient with vault URL and TokenCredential." - - "Sets secrets with SetSecretParameters including value and content_type." - - "Retrieves secrets with get_secret() and converts with into_model()." - - "Updates secret properties with tags and content_type." - - "Deletes secrets with delete_secret()." - - "Lists secrets returning Pager with pagination support." - - "Uses ResourceExt trait to extract name and version fields." - - "Iterates paginated results with TryStreamExt::try_next()." - - "Handles errors with match patterns and ErrorResponse extraction." - - "Uses TokenCredential for secure authentication." - - "Async/await with Tokio and proper error handling." - - "Code compiles without warnings." - workspace_files: - include: - - "**/Cargo.toml" - - "**/Cargo.lock" - - "**/src/**/*.rs" - exclude: - - "**/target/**" - - "**/*.rlib" - - "**/*.rmeta" - environment: - files: - - src: workspace/cargo.toml - dest: Cargo.toml - constraints: - expect_skills: - - azure-keyvault-secrets-rust - graders: - - type: prompt - name: "Key Vault secrets completeness" - config: - model: gpt-5.3-codex - prompt: | - - SecretClient creation with vault URL and credential - - Set secret with SetSecretParameters including value - - Get secret with into_model() conversion - - Update secret properties with tags and content_type - - Delete secret operation - - List secrets returning Pager - - Pagination with TryStreamExt::try_next() - - ResourceExt for name/version extraction - - Error matching and ErrorResponse handling - - TokenCredential authentication - - Async/await throughout - scoring: scale_1_5 - - - type: run-command - name: "Cargo Check" - config: - command: cargo - args: - - "check" - timeout: 90s - - name: Rust Clippy Lints - type: run-command - weight: 1.0 - config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" - config: - command: cargo - args: - - "build" - timeout: 120s - - - name: SecretClient Creation - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - SecretClient created via SecretClient::new(vault_url, credential, None) - Vault URL is vault_name.vault.azure.net in https protocol - Credential is TokenCredential (DeveloperToolsCredential or ManagedIdentityCredential) - - name: Secret CRUD Operations - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Set: client.set_secret(name, params.try_into()?, None).await? - Get: client.get_secret(name, None).await?.into_model()? - Update: client.update_secret_properties(name, params.try_into()?, None).await? - Delete: client.delete_secret(name, None).await? - - name: SetSecretParameters Pattern - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - SetSecretParameters created with value: Some(value.into()) - Content type can be set via content_type: Some("text/plain".into()) - Uses #[allow(clippy::needless_update)] with ..Default::default() - - name: Listing with Pagination - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - List secrets returns Pager via client.list_secret_properties(None)? - Iterate with let mut pager = ...; while let Some(secret) = pager.try_next().await? - Uses futures::TryStreamExt trait for try_next() - Extract name/version via secret.resource_id()?.name - - name: Resource Extraction - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Use ResourceExt trait via secret.resource_id()? - Returns ResourceId with name and version fields - Stored for audit or reconstruction purposes - - name: Async Runtime and Error Handling - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Uses #[tokio::main] on main function - All I/O operations awaited - No .unwrap() on fallible operations - Result propagated with ? operator - Error extraction with into_inner()? on error types diff --git a/tests/scenarios/azure-keyvault-secrets-rust/vally/eval.yaml b/tests/scenarios/azure-keyvault-secrets-rust/vally/eval.yaml index 632382dd..866150c1 100644 --- a/tests/scenarios/azure-keyvault-secrets-rust/vally/eval.yaml +++ b/tests/scenarios/azure-keyvault-secrets-rust/vally/eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-keyvault-secrets-operations prompt: | @@ -63,7 +64,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper error handling." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -76,6 +77,16 @@ stimuli: skills: - ../../../../.github/plugins/azure-sdk-rust/skills/azure-keyvault-secrets-rust files: + - src: ../../_shared/vally/tools/find-cargo-root.js + dest: .vally/tools/find-cargo-root.js + - src: ../../_shared/vally/tools/check-azure-crates.rs + dest: .vally/tools/check-azure-crates.rs + - src: ../../_shared/vally/tools/check-token-credential.rs + dest: .vally/tools/check-token-credential.rs + - src: ../../_shared/vally/tools/check-async-runtime.rs + dest: .vally/tools/check-async-runtime.rs + - src: ../../_shared/vally/tools/check-no-secrets.rs + dest: .vally/tools/check-no-secrets.rs - src: workspace/cargo.toml dest: Cargo.toml constraints: @@ -85,7 +96,6 @@ stimuli: - type: prompt name: "Key Vault secrets completeness" config: - model: gpt-5.3-codex prompt: | - SecretClient creation with vault URL and credential - Set secret with SetSecretParameters including value @@ -100,72 +110,67 @@ stimuli: - Async/await throughout scoring: scale_1_5 - name: Official Azure SDK Crate Selection - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs + emit_in_metadata: true - name: TokenCredential Authentication - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs + emit_in_metadata: true - name: Async-First with Tokio Runtime - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs + emit_in_metadata: true - name: No Hardcoded Secrets - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs + emit_in_metadata: true + - name: Rust Cargo Trajectory Build Failure + type: rust-cargo-build-failure-check + weight: 1.0 + config: + collect_trajectory_compiler_errors: true + emit_in_metadata: true + error_exceptions: + - error_code: E0277 + message_pattern: "Option" + action: ignore_for_fail - - type: run-command - name: "Cargo Check" + - name: "Cargo Check" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "check" - timeout: 90s + commands: + - command: cargo check + emit_in_metadata: true - name: Rust Clippy Lints - type: run-command + type: rust-cargo-build-failure-check weight: 1.0 config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" + commands: + - command: cargo clippy -- -D warnings + emit_in_metadata: true + - name: "Cargo Build" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "build" - timeout: 120s + commands: + - command: cargo build + emit_in_metadata: true - name: SecretClient Creation type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > SecretClient created via SecretClient::new(vault_url, credential, None) Vault URL is vault_name.vault.azure.net in https protocol @@ -173,7 +178,6 @@ stimuli: - name: Secret CRUD Operations type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Set: client.set_secret(name, params.try_into()?, None).await? Get: client.get_secret(name, None).await?.into_model()? @@ -182,7 +186,6 @@ stimuli: - name: SetSecretParameters Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > SetSecretParameters created with value: Some(value.into()) Content type can be set via content_type: Some("text/plain".into()) @@ -190,7 +193,6 @@ stimuli: - name: Listing with Pagination type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > List secrets returns Pager via client.list_secret_properties(None)? Iterate with let mut pager = ...; while let Some(secret) = pager.try_next().await? @@ -199,7 +201,6 @@ stimuli: - name: Resource Extraction type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Use ResourceExt trait via secret.resource_id()? Returns ResourceId with name and version fields @@ -207,7 +208,6 @@ stimuli: - name: Async Runtime and Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Uses #[tokio::main] on main function All I/O operations awaited diff --git a/tests/scenarios/azure-keyvault-secrets-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-keyvault-secrets-rust/vally/skill_effectiveness_eval.yaml index d12448ac..1278b75a 100644 --- a/tests/scenarios/azure-keyvault-secrets-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-keyvault-secrets-rust/vally/skill_effectiveness_eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-keyvault-secrets-operations prompt: | @@ -63,7 +64,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper error handling." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -91,7 +92,7 @@ stimuli: - type: prompt name: "Key Vault secrets completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - SecretClient creation with vault URL and credential - Set secret with SetSecretParameters including value @@ -149,7 +150,7 @@ stimuli: command: cargo args: - "check" - timeout: 90s + timeout: 180s - name: Rust Clippy Lints type: run-command weight: 1.0 @@ -160,19 +161,19 @@ stimuli: - "--" - "-D" - "warnings" - timeout: 120s + timeout: 300s - type: run-command name: "Cargo Build" config: command: cargo args: - "build" - timeout: 120s + timeout: 300s - name: SecretClient Creation type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > SecretClient created via SecretClient::new(vault_url, credential, None) Vault URL is vault_name.vault.azure.net in https protocol @@ -180,7 +181,7 @@ stimuli: - name: Secret CRUD Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Set: client.set_secret(name, params.try_into()?, None).await? Get: client.get_secret(name, None).await?.into_model()? @@ -189,7 +190,7 @@ stimuli: - name: SetSecretParameters Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > SetSecretParameters created with value: Some(value.into()) Content type can be set via content_type: Some("text/plain".into()) @@ -197,7 +198,7 @@ stimuli: - name: Listing with Pagination type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > List secrets returns Pager via client.list_secret_properties(None)? Iterate with let mut pager = ...; while let Some(secret) = pager.try_next().await? @@ -206,7 +207,7 @@ stimuli: - name: Resource Extraction type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Use ResourceExt trait via secret.resource_id()? Returns ResourceId with name and version fields @@ -214,7 +215,7 @@ stimuli: - name: Async Runtime and Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Uses #[tokio::main] on main function All I/O operations awaited diff --git a/tests/scenarios/azure-servicebus-rust/acceptance-criteria.md b/tests/scenarios/azure-servicebus-rust/acceptance-criteria.md deleted file mode 100644 index 58aa51e9..00000000 --- a/tests/scenarios/azure-servicebus-rust/acceptance-criteria.md +++ /dev/null @@ -1,124 +0,0 @@ -# Azure Service Bus SDK for Rust Acceptance Criteria - -**Crate**: `azure_messaging_servicebus` -**Repository**: -**Purpose**: Skill testing acceptance criteria for validating generated Rust code correctness - ---- - -## 0. Dependency Management Gate (Required) - -### 0.1 ✅ CORRECT: Use cargo commands for dependency changes - -```sh -cargo add azure_messaging_servicebus azure_identity tokio -cargo add azure_core -cargo remove azure_core -``` - -### 0.2 ✅ CORRECT: Add `azure_core` only for direct `azure_core` imports - -```rust -use azure_core::error::ErrorKind; -use azure_messaging_servicebus::ServiceBusClient; -// Direct azure_core import is used, so `azure_core` should be a direct dependency. -``` - -### 0.3 ❌ INCORRECT: Manual Cargo.toml dependency edits in generated guidance - -```toml -# WRONG in generated guidance - use `cargo add` / `cargo remove` commands instead -[dependencies] -azure_core = "*" -``` - -### 0.4 ❌ INCORRECT: Requiring `azure_core` when no direct `azure_core` imports exist - -```rust -use azure_messaging_servicebus::ServiceBusClient; -// No direct azure_core import here, so forcing direct azure_core dependency is unnecessary. -``` - ---- - -## 1. Correct Import Patterns - -### 1.1 ✅ CORRECT: Client and message imports - -```rust -use azure_messaging_servicebus::{ServiceBusClient, Message}; -use azure_identity::DeveloperToolsCredential; -``` - ---- - -## 2. Client Creation - -### 2.1 ✅ CORRECT: ServiceBusClient builder and open - -```rust -let credential = DeveloperToolsCredential::new(None)?; -let client = ServiceBusClient::builder() - .open("your_namespace.servicebus.windows.net", credential.clone()) - .await?; -``` - -### 2.2 ❌ INCORRECT: Direct constructor call instead of builder pattern - -```rust -// WRONG - use ServiceBusClient::builder().open(...) -let client = ServiceBusClient::new("namespace", credential)?; -``` - ---- - -## 3. Queue and Topic Operations - -### 3.1 ✅ CORRECT: Send message to queue - -```rust -let sender = client.create_sender("my_queue", None).await?; -let message = Message::from("Hello, Service Bus!"); -sender.send_message(message, None).await?; -``` - -### 3.2 ✅ CORRECT: Receive and complete queue messages - -```rust -let receiver = client.create_receiver("my_queue", None).await?; -let messages = receiver.receive_messages(5, None).await?; -for message in messages { - println!("Received: {}", message.body_as_string()?); - receiver.complete_message(&message, None).await?; -} -``` - -### 3.3 ✅ CORRECT: Topic/subscription receive - -```rust -let receiver = client - .create_receiver_for_subscription("my_topic", "my_subscription", None) - .await?; - -let messages = receiver.receive_messages(5, None).await?; -for message in messages { - receiver.complete_message(&message, None).await?; -} -``` - -### 3.4 ❌ INCORRECT: Forgetting message settlement - -```rust -// WRONG - leaving messages unsettled causes redelivery -for message in receiver.receive_messages(5, None).await? { - println!("{}", message.body_as_string()?); -} -``` - ---- - -## 4. Package Maturity Warning - -### 4.1 ✅ CORRECT: Include pre-production warning - -Generated guidance should mention this crate is in early development and should not be used in production without caution. diff --git a/tests/scenarios/azure-servicebus-rust/scenarios.yaml b/tests/scenarios/azure-servicebus-rust/scenarios.yaml deleted file mode 100644 index 2fc35c8e..00000000 --- a/tests/scenarios/azure-servicebus-rust/scenarios.yaml +++ /dev/null @@ -1,125 +0,0 @@ -# yaml-language-server: $schema=../skill-scenarios.schema.json -# Test scenarios for azure-servicebus-rust skill evaluation -# Each scenario tests a specific usage pattern against acceptance criteria - -config: - model: claude-sonnet-4.6 - max_tokens: 2000 - temperature: 0.3 - -scenarios: - - name: dependency_gate_cargo_add - prompt: | - For a Rust Azure Service Bus example, explain dependency management rules. - expected_patterns: - - "cargo add" - - "Cargo.toml" - - "azure_core" - forbidden_patterns: - - "edit Cargo.toml manually" - tags: - - dependency - - guardrail - mock_response: | - Use cargo commands for dependency changes, for example: - cargo add azure_messaging_servicebus azure_identity tokio - Add azure_core only when importing azure_core types directly. - Do not manually edit Cargo.toml in generated guidance. - - - name: create_servicebus_client - prompt: | - Create a ServiceBusClient in Rust using DeveloperToolsCredential. - expected_patterns: - - "ServiceBusClient" - - "builder" - - "open" - - "DeveloperToolsCredential" - - "servicebus.windows.net" - forbidden_patterns: - - "ServiceBusClient::new" - tags: - - basic - - authentication - mock_response: | - use azure_identity::DeveloperToolsCredential; - use azure_messaging_servicebus::ServiceBusClient; - - let credential = DeveloperToolsCredential::new(None)?; - let client = ServiceBusClient::builder() - .open("your_namespace.servicebus.windows.net", credential.clone()) - .await?; - - - name: send_queue_message - prompt: | - Send a message to a Service Bus queue in Rust. - expected_patterns: - - "create_sender" - - "Message::from" - - "send_message" - forbidden_patterns: - - "receive_messages" - tags: - - send - - queue - mock_response: | - use azure_messaging_servicebus::Message; - - let sender = client.create_sender("my_queue", None).await?; - let message = Message::from("Hello, Service Bus!"); - sender.send_message(message, None).await?; - - - name: receive_and_complete_queue_messages - prompt: | - Receive queue messages and settle them correctly with complete_message. - expected_patterns: - - "create_receiver" - - "receive_messages" - - "complete_message" - tags: - - receive - - settlement - - queue - mock_response: | - let receiver = client.create_receiver("my_queue", None).await?; - let messages = receiver.receive_messages(5, None).await?; - for message in messages { - println!("Received: {}", message.body_as_string()?); - receiver.complete_message(&message, None).await?; - } - - - name: receive_subscription_messages - prompt: | - Receive messages from a topic subscription in Rust Service Bus. - expected_patterns: - - "create_receiver_for_subscription" - - "receive_messages" - - "complete_message" - forbidden_patterns: - - "create_sender" - tags: - - topic - - subscription - - receive - mock_response: | - let receiver = client - .create_receiver_for_subscription("my_topic", "my_subscription", None) - .await?; - - let messages = receiver.receive_messages(5, None).await?; - for message in messages { - receiver.complete_message(&message, None).await?; - } - - - name: package_maturity_warning - prompt: | - What production caution should be included when using azure_messaging_servicebus in Rust? - expected_patterns: - - "early development" - - "should not be used in production" - forbidden_patterns: - - "production-ready" - tags: - - warning - - maturity - mock_response: | - This crate is in early development and should not be used in production without caution, because APIs may change. diff --git a/tests/scenarios/azure-servicebus-rust/vally/eval-experiment.yaml b/tests/scenarios/azure-servicebus-rust/vally/eval-experiment.yaml deleted file mode 100644 index 98656955..00000000 --- a/tests/scenarios/azure-servicebus-rust/vally/eval-experiment.yaml +++ /dev/null @@ -1,163 +0,0 @@ -name: azure-servicebus-rust-eval -description: Evaluation suite for azure-messaging-servicebus-rust -version: "1.0" -type: capability -scoring: - weights: {} - threshold: 1.0 -defaults: - runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: - - name: rust-servicebus-messaging - prompt: | - Write a Rust program named rust-servicebus-ops that demonstrates Azure Service Bus - messaging patterns including queues, topics, and subscriptions: - - Queue operations: - 1. Create a ServiceBusClient with namespace credentials - 2. Create a sender for a queue named "orders" - 3. Send messages to the queue via sender.send_message() - 4. Create a receiver for the queue - 5. Receive messages (up to 5) via receiver.receive_messages() - 6. Complete messages after processing via receiver.complete_message() - 7. Handle cases where messages must be abandoned for retry - - Topic and subscription operations: - 1. Create a sender for a topic named "notifications" - 2. Send messages to the topic - 3. Create a receiver for a subscription "alerts" on the topic - 4. Receive messages from the subscription - 5. Complete messages after processing - - Demonstrate: - - ServiceBusClient::builder().open(namespace, credential).await pattern - - Message creation via Message::from() - - Sender/receiver creation for both queues and topics - - Message body access via body_as_string() - - Message settlement: complete_message() for success, abandon for retry - - TokenCredential authentication (DeveloperToolsCredential/ManagedIdentityCredential) - - Async/await with Tokio runtime - - Proper Result-based error propagation - - The program should compile and run successfully with proper async message handling, - demonstrating the complete Service Bus workflow for enterprise messaging. - - rubric: - - "Uses official azure_messaging_servicebus SDK with proper client creation." - - "Implements ServiceBusClient with TokenCredential-based authentication." - - "Creates senders and receivers for both queues and topics." - - "Sends messages to queues and topics via sender.send_message()." - - "Receives messages with receiver.receive_messages(count, options)." - - "Completes messages after processing with receiver.complete_message()." - - "Handles message bodies with body_as_string() or equivalent extraction." - - "Demonstrates both queue (point-to-point) and topic (pub-sub) patterns." - - "Async/await with Tokio runtime and proper error handling." - - "Code compiles without warnings and follows Rust idioms." - workspace_files: - include: - - "**/Cargo.toml" - - "**/Cargo.lock" - - "**/src/**/*.rs" - exclude: - - "**/target/**" - - "**/*.rlib" - - "**/*.rmeta" - environment: - files: - - src: workspace/cargo.toml - dest: Cargo.toml - constraints: - expect_skills: - - azure-servicebus-rust - graders: - - type: prompt - name: "Service Bus messaging completeness" - config: - model: gpt-5.3-codex - prompt: | - - ServiceBusClient initialization with builder and open() - - Queue sender/receiver creation - - Topic sender and subscription receiver creation - - Message sending via send_message(message, None).await - - Message receiving via receive_messages(count, None).await - - Message body extraction via body_as_string() - - Message settlement with complete_message() - - Abandon message pattern for retries - - TokenCredential authentication pattern - - Async/await with Tokio #[tokio::main] - scoring: scale_1_5 - - - type: run-command - name: "Cargo Check" - config: - command: cargo - args: - - "check" - timeout: 90s - - name: Rust Clippy Lints - type: run-command - weight: 1.0 - config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" - config: - command: cargo - args: - - "build" - timeout: 120s - - - name: ServiceBusClient Builder Pattern - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - ServiceBusClient created via ServiceBusClient::builder() - .open(namespace, credential).await? - Sender created via client.create_sender(queue_or_topic_name, None).await? - Receiver created via client.create_receiver(queue_name, None).await? - Topic subscription receiver via create_receiver_for_subscription(topic, subscription, None).await? - - name: Message Operations - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Messages created via Message::from(text) or Message::from(bytes) - Send via sender.send_message(message, None).await? - Receive via receiver.receive_messages(count, None).await? returning Vec - Extract body via message.body_as_string()? or equivalent - - name: Message Settlement - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - After processing, call receiver.complete_message(&message, None).await? - For retriable errors, use receiver.abandon_message(&message, None).await? - No message is processed without explicit settlement - Proper Result-based error handling for settlement operations - - name: Queue and Topic Patterns - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Queues demonstrated via create_sender/create_receiver on queue names - Topics demonstrated via create_sender on topic names - Subscriptions accessed via create_receiver_for_subscription(topic_name, subscription_name, None).await? - Both patterns use same Message type and settlement semantics - - name: Async Runtime - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Uses #[tokio::main] attribute on main function - All I/O operations properly awaited - No blocking operations in async context - Result error handling throughout diff --git a/tests/scenarios/azure-servicebus-rust/vally/eval.yaml b/tests/scenarios/azure-servicebus-rust/vally/eval.yaml deleted file mode 100644 index 93e4e130..00000000 --- a/tests/scenarios/azure-servicebus-rust/vally/eval.yaml +++ /dev/null @@ -1,192 +0,0 @@ -name: azure-servicebus-rust-eval -description: Evaluation suite for azure-messaging-servicebus-rust -version: "1.0" -type: capability -scoring: - weights: {} - threshold: 0.75 -defaults: - runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: - - name: rust-servicebus-messaging - prompt: | - Write a Rust program named rust-servicebus-ops that demonstrates Azure Service Bus - messaging patterns including queues, topics, and subscriptions: - - Queue operations: - 1. Create a ServiceBusClient with namespace credentials - 2. Create a sender for a queue named "orders" - 3. Send messages to the queue via sender.send_message() - 4. Create a receiver for the queue - 5. Receive messages (up to 5) via receiver.receive_messages() - 6. Complete messages after processing via receiver.complete_message() - 7. Handle cases where messages must be abandoned for retry - - Topic and subscription operations: - 1. Create a sender for a topic named "notifications" - 2. Send messages to the topic - 3. Create a receiver for a subscription "alerts" on the topic - 4. Receive messages from the subscription - 5. Complete messages after processing - - Demonstrate: - - ServiceBusClient::builder().open(namespace, credential).await pattern - - Message creation via Message::from() - - Sender/receiver creation for both queues and topics - - Message body access via body_as_string() - - Message settlement: complete_message() for success, abandon for retry - - TokenCredential authentication (DeveloperToolsCredential/ManagedIdentityCredential) - - Async/await with Tokio runtime - - Proper Result-based error propagation - - The program should compile and run successfully with proper async message handling, - demonstrating the complete Service Bus workflow for enterprise messaging. - - rubric: - - "Uses official azure_messaging_servicebus SDK with proper client creation." - - "Implements ServiceBusClient with TokenCredential-based authentication." - - "Creates senders and receivers for both queues and topics." - - "Sends messages to queues and topics via sender.send_message()." - - "Receives messages with receiver.receive_messages(count, options)." - - "Completes messages after processing with receiver.complete_message()." - - "Handles message bodies with body_as_string() or equivalent extraction." - - "Demonstrates both queue (point-to-point) and topic (pub-sub) patterns." - - "Async/await with Tokio runtime and proper error handling." - - "Code compiles without warnings and follows Rust idioms." - workspace_files: - include: - - "**/Cargo.toml" - - "**/Cargo.lock" - - "**/src/**/*.rs" - exclude: - - "**/target/**" - - "**/*.rlib" - - "**/*.rmeta" - environment: - skills: - - ../../../../.github/plugins/azure-sdk-rust/skills/azure-servicebus-rust - files: - - src: workspace/cargo.toml - dest: Cargo.toml - constraints: - expect_skills: - - azure-servicebus-rust - graders: - - type: prompt - name: "Service Bus messaging completeness" - config: - model: gpt-5.3-codex - prompt: | - - ServiceBusClient initialization with builder and open() - - Queue sender/receiver creation - - Topic sender and subscription receiver creation - - Message sending via send_message(message, None).await - - Message receiving via receive_messages(count, None).await - - Message body extraction via body_as_string() - - Message settlement with complete_message() - - Abandon message pattern for retries - - TokenCredential authentication pattern - - Async/await with Tokio #[tokio::main] - scoring: scale_1_5 - - name: Official Azure SDK Crate Selection - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" - - name: TokenCredential Authentication - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" - - name: Async-First with Tokio Runtime - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" - - name: No Hardcoded Secrets - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" - - - type: run-command - name: "Cargo Check" - config: - command: cargo - args: - - "check" - timeout: 90s - - name: Rust Clippy Lints - type: run-command - weight: 1.0 - config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" - config: - command: cargo - args: - - "build" - timeout: 120s - - - name: ServiceBusClient Builder Pattern - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - ServiceBusClient created via ServiceBusClient::builder() - .open(namespace, credential).await? - Sender created via client.create_sender(queue_or_topic_name, None).await? - Receiver created via client.create_receiver(queue_name, None).await? - Topic subscription receiver via create_receiver_for_subscription(topic, subscription, None).await? - - name: Message Operations - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Messages created via Message::from(text) or Message::from(bytes) - Send via sender.send_message(message, None).await? - Receive via receiver.receive_messages(count, None).await? returning Vec - Extract body via message.body_as_string()? or equivalent - - name: Message Settlement - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - After processing, call receiver.complete_message(&message, None).await? - For retriable errors, use receiver.abandon_message(&message, None).await? - No message is processed without explicit settlement - Proper Result-based error handling for settlement operations - - name: Queue and Topic Patterns - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Queues demonstrated via create_sender/create_receiver on queue names - Topics demonstrated via create_sender on topic names - Subscriptions accessed via create_receiver_for_subscription(topic_name, subscription_name, None).await? - Both patterns use same Message type and settlement semantics diff --git a/tests/scenarios/azure-servicebus-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-servicebus-rust/vally/skill_effectiveness_eval.yaml deleted file mode 100644 index 6d37af5b..00000000 --- a/tests/scenarios/azure-servicebus-rust/vally/skill_effectiveness_eval.yaml +++ /dev/null @@ -1,199 +0,0 @@ -name: azure-servicebus-rust-eval -description: Evaluation suite for azure-messaging-servicebus-rust -version: "1.0" -type: capability -scoring: - weights: {} - threshold: 0.75 -defaults: - runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: - - name: rust-servicebus-messaging - prompt: | - Write a Rust program named rust-servicebus-ops that demonstrates Azure Service Bus - messaging patterns including queues, topics, and subscriptions: - - Queue operations: - 1. Create a ServiceBusClient with namespace credentials - 2. Create a sender for a queue named "orders" - 3. Send messages to the queue via sender.send_message() - 4. Create a receiver for the queue - 5. Receive messages (up to 5) via receiver.receive_messages() - 6. Complete messages after processing via receiver.complete_message() - 7. Handle cases where messages must be abandoned for retry - - Topic and subscription operations: - 1. Create a sender for a topic named "notifications" - 2. Send messages to the topic - 3. Create a receiver for a subscription "alerts" on the topic - 4. Receive messages from the subscription - 5. Complete messages after processing - - Demonstrate: - - ServiceBusClient::builder().open(namespace, credential).await pattern - - Message creation via Message::from() - - Sender/receiver creation for both queues and topics - - Message body access via body_as_string() - - Message settlement: complete_message() for success, abandon for retry - - TokenCredential authentication (DeveloperToolsCredential/ManagedIdentityCredential) - - Async/await with Tokio runtime - - Proper Result-based error propagation - - The program should compile and run successfully with proper async message handling, - demonstrating the complete Service Bus workflow for enterprise messaging. - - rubric: - - "Uses official azure_messaging_servicebus SDK with proper client creation." - - "Implements ServiceBusClient with TokenCredential-based authentication." - - "Creates senders and receivers for both queues and topics." - - "Sends messages to queues and topics via sender.send_message()." - - "Receives messages with receiver.receive_messages(count, options)." - - "Completes messages after processing with receiver.complete_message()." - - "Handles message bodies with body_as_string() or equivalent extraction." - - "Demonstrates both queue (point-to-point) and topic (pub-sub) patterns." - - "Async/await with Tokio runtime and proper error handling." - - "Code compiles without warnings and follows Rust idioms." - workspace_files: - include: - - "**/Cargo.toml" - - "**/Cargo.lock" - - "**/src/**/*.rs" - exclude: - - "**/target/**" - - "**/*.rlib" - - "**/*.rmeta" - environment: - files: - - src: workspace/cargo.toml - dest: Cargo.toml - - src: ../../_shared/vally/tools/check-azure-crates.rs - dest: .vally/tools/check-azure-crates.rs - - src: ../../_shared/vally/tools/check-token-credential.rs - dest: .vally/tools/check-token-credential.rs - - src: ../../_shared/vally/tools/check-async-runtime.rs - dest: .vally/tools/check-async-runtime.rs - - src: ../../_shared/vally/tools/check-no-secrets.rs - dest: .vally/tools/check-no-secrets.rs - constraints: - expect_skills: - - azure-servicebus-rust - graders: - - type: prompt - name: "Service Bus messaging completeness" - config: - model: gpt-5.3-codex - prompt: | - - ServiceBusClient initialization with builder and open() - - Queue sender/receiver creation - - Topic sender and subscription receiver creation - - Message sending via send_message(message, None).await - - Message receiving via receive_messages(count, None).await - - Message body extraction via body_as_string() - - Message settlement with complete_message() - - Abandon message pattern for retries - - TokenCredential authentication pattern - - Async/await with Tokio #[tokio::main] - scoring: scale_1_5 - - - name: Official Azure SDK Crate Selection - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" - - name: TokenCredential Authentication - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" - - name: Async-First with Tokio Runtime - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" - - name: No Hardcoded Secrets - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" - - - type: run-command - name: "Cargo Check" - config: - command: cargo - args: - - "check" - timeout: 90s - - name: Rust Clippy Lints - type: run-command - weight: 1.0 - config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" - config: - command: cargo - args: - - "build" - timeout: 120s - - - name: ServiceBusClient Builder Pattern - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - ServiceBusClient created via ServiceBusClient::builder() - .open(namespace, credential).await? - Sender created via client.create_sender(queue_or_topic_name, None).await? - Receiver created via client.create_receiver(queue_name, None).await? - Topic subscription receiver via create_receiver_for_subscription(topic, subscription, None).await? - - name: Message Operations - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Messages created via Message::from(text) or Message::from(bytes) - Send via sender.send_message(message, None).await? - Receive via receiver.receive_messages(count, None).await? returning Vec - Extract body via message.body_as_string()? or equivalent - - name: Message Settlement - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - After processing, call receiver.complete_message(&message, None).await? - For retriable errors, use receiver.abandon_message(&message, None).await? - No message is processed without explicit settlement - Proper Result-based error handling for settlement operations - - name: Queue and Topic Patterns - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Queues demonstrated via create_sender/create_receiver on queue names - Topics demonstrated via create_sender on topic names - Subscriptions accessed via create_receiver_for_subscription(topic_name, subscription_name, None).await? - Both patterns use same Message type and settlement semantics diff --git a/tests/scenarios/azure-servicebus-rust/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-servicebus-rust/vally/skill_effectiveness_experiment.yaml deleted file mode 100644 index bb5e04c9..00000000 --- a/tests/scenarios/azure-servicebus-rust/vally/skill_effectiveness_experiment.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. -name: azure-servicebus-rust-skill-experiment -evals: - - skill_effectiveness_eval.yaml - -vary: - - /defaults/model - - /environment/skills - -baseline: sonnet_baseline - -variants: - sonnet_skill: - overrides: - model: claude-sonnet-4.6 - environment: - skills: - - ../../../../.github/plugins/azure-sdk-rust/skills/azure-servicebus-rust - sonnet_baseline: - overrides: - model: claude-sonnet-4.6 - environment: - skills: [] diff --git a/tests/scenarios/azure-servicebus-rust/vally/workspace/cargo.toml b/tests/scenarios/azure-servicebus-rust/vally/workspace/cargo.toml deleted file mode 100644 index daa8c449..00000000 --- a/tests/scenarios/azure-servicebus-rust/vally/workspace/cargo.toml +++ /dev/null @@ -1,2 +0,0 @@ -[workspace] -resolver = "3" diff --git a/tests/scenarios/azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml b/tests/scenarios/azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml index b0248616..c1da674a 100644 --- a/tests/scenarios/azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml +++ b/tests/scenarios/azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml @@ -7,7 +7,7 @@ scoring: threshold: 1.0 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 stimuli: - name: rust-storage-keyvault-encryption-library @@ -39,7 +39,7 @@ stimuli: - "Uses async Rust with Tokio runtime and idiomatic Result-based error propagation." - "Code compiles and passes cargo check for the generated library project." - "Avoids hardcoded secrets in source code or committed configuration." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -100,44 +100,36 @@ stimuli: - Reasonable error handling and no embedded secrets scoring: scale_1_5 - name: Official Azure SDK Crate Selection - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs + emit_in_metadata: true timeout: 2m - name: TokenCredential Authentication - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs + emit_in_metadata: true timeout: 2m - name: Async-First with Tokio Runtime - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs + emit_in_metadata: true timeout: 2m - name: No Hardcoded Secrets - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs + emit_in_metadata: true timeout: 2m - name: Rust Cargo Trajectory Build Failure type: rust-cargo-build-failure-check diff --git a/tests/scenarios/azure-storage-blob-rust/vally/eval-nobuild-experiment.yaml b/tests/scenarios/azure-storage-blob-rust/vally/eval-nobuild-experiment.yaml deleted file mode 100644 index 0045654b..00000000 --- a/tests/scenarios/azure-storage-blob-rust/vally/eval-nobuild-experiment.yaml +++ /dev/null @@ -1,268 +0,0 @@ -name: azure-storage-blob-rust-nobuild-eval -description: Evaluation suite for azure-storage-blob-rust -version: "1.0" -type: capability -scoring: - weights: {} - threshold: 1.0 -defaults: - runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: - - name: rust-crud-blobs-nobuild - prompt: | - Write a Rust program named rust-blob-nobuild that performs the following blob storage operations: - 1. Create a blob container called "my-container" if it does not already exist - 2. Upload a local file "example.txt" as a block blob named "uploads/example.txt" - 3. List all blobs in the container and print their names and sizes - 4. Download the blob back to a local file "downloaded-example.txt" - 5. Delete the blob and then delete the container - - Include proper error handling for cases like container already existing - or blob not found. - rubric: - - "Implements the full blob CRUD workflow end-to-end: create container, upload, list, download, and delete blob/container." - - "Uses official Azure Rust SDK crate patterns and a TokenCredential-based authentication flow." - - "Uses async Rust with Tokio runtime and idiomatic Result-based error propagation." - - "Code compiles and passes cargo check for the rust-blob project." - - "Uses correct Azure client construction, response handling, and pager usage where applicable." - - "Avoids hardcoded secrets in source code or configuration committed to the workspace." - - "rustfmt compliance is bonus quality signal only and should not block a successful evaluation when all substantive correctness and SDK-usage criteria pass." - workspace_files: - include: - - "**/Cargo.toml" - - "**/Cargo.lock" - - "**/build.rs" - - "**/src/**/*.rs" - - "**/examples/**/*.rs" - - "**/tests/**/*.rs" - exclude: - - "**/target/**" - - "**/*.rlib" - - "**/*.rmeta" - - "**/*.d" - - "**/*.o" - - "**/*.obj" - - "**/*.pdb" - - "**/*.exe" - - "**/*.dll" - - "**/*.so" - - "**/*.dylib" - - "**/*.a" - - "**/*.lib" - - "**/*.wasm" - environment: - skills: - - ../../../../.github/plugins/azure-sdk-rust/skills/azure-storage-blob-rust - files: - - src: ../../_shared/vally/tools/find-cargo-root.js - dest: .vally/tools/find-cargo-root.js - - src: ../../_shared/vally/tools/check-azure-crates.rs - dest: .vally/tools/check-azure-crates.rs - - src: ../../_shared/vally/tools/check-token-credential.rs - dest: .vally/tools/check-token-credential.rs - - src: ../../_shared/vally/tools/check-async-runtime.rs - dest: .vally/tools/check-async-runtime.rs - - src: ../../_shared/vally/tools/check-no-secrets.rs - dest: .vally/tools/check-no-secrets.rs - - src: workspace/cargo.toml - dest: Cargo.toml - constraints: - expect_skills: - - azure-storage-blob-rust - graders: - - type: prompt - name: "Prompt correctness and completeness of blob storage operations" - config: - model: gpt-5.3-codex - prompt: | - - The code should demonstrate an azure blob storage client with a storage account url and credential - - Creating a container for container creation - - Uploading from a file path - - Enumerate blobs in a container - - Download from a blob to a local file - - Deleting blob files and containers for cleanup - - Error handling - scoring: scale_1_5 - - - type: run-command - name: "Build Application" - config: - command: cargo - args: - - "build" - stderr_contains: "Finished" - timeout: 120s - - type: run-command - name: "Check Application" - config: - command: cargo - args: - - "check" - timeout: 90s - - name: Rust Clippy Lints - type: run-command - weight: 0.5 - config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - name: Official Azure SDK Crate Selection - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" - - name: TokenCredential Authentication - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" - - name: Async-First with Tokio Runtime - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" - - name: Idiomatic Error Handling - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Error handling uses Result and the ? operator. Calls to - .unwrap() are avoided or used only where failure - is impossible. Calls to expect() with a reasonable error message are allowed - only if termination of the application is acceptable. Azure service errors are - matched via azure_core::Error with ErrorKind (e.g., - ErrorKind::HttpResponse { status, error_code, .. } with - StatusCode comparison). The main function returns - Result<(), Box> or azure_core::Result<()>. - - name: Correct Client Construction - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Service clients are constructed using the ::new() constructor - with an endpoint string, credential, and Option parameter. - If a ClientOptions parameter is specified, structs use ..Default::default() - for forward compatibility. If all the fields in the ClientOptions struct are provided, the - #[allow(clippy::needless_update)] annotation is used. - - name: Pager and Stream Usage - type: prompt - model: gpt-5.3-codex - weight: 1.0 - prompt: > - List operations return and consume azure_core::Pager using - futures::TryStreamExt (try_next/try_collect). Pagination is - handled correctly by iterating all items. In most if not all - cases, individual items should be iterated over directly rather - than page-by-page, demonstrating the SDK's ability to handle - pagination internally. Page-by-page iteration via into_pages() - is only necessary when there is an explicit requirement for page-level - access. - - name: Response Handling - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Calls to service clients correctly handle Response. When service - methods return a model type, code calls into_model() to deserialize - the response body into the expected model type (not manually parsing - JSON). When raw HTTP details are needed, deconstruct() is used to - access status, headers, and body separately. - - name: Long-Running Operations - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Long-running operations use azure_core::Poller correctly. - If a client calls into an API returning a Poller, code awaits - the Poller directly (via IntoFuture) for the final result. - The only time that a client should iterate over status updates via - futures::TryStreamExt is when there is an explicit requirement for - status-level access. - - name: Struct Initialization with Default - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - If a client initializes a structure for SDK types (client options, method - options, model types) the code uses ..Default::default() to protect - against future field additions. - - name: Ownership and Borrowing - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Code uses references (&str, &T) for borrowed data and avoids - unnecessary .clone() calls. Credentials are shared via Arc or - clone(). No unnecessary Box or Rc where Arc is appropriate. - Parameters that are only used for URL construction borrow - rather than own the data. - - name: No Hardcoded Secrets - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" - - name: Rust Cargo Trajectory Build Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - collect_trajectory_compiler_errors: true - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail - - name: Post prompt Rust Cargo Build Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - commands: - - command: cargo build - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail - - name: Post prompt Rust Cargo Check Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - commands: - - command: cargo check - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail - - name: Post prompt Rust Cargo Clippy Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - commands: - - command: cargo clippy - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail diff --git a/tests/scenarios/azure-storage-blob-rust/vally/eval-nobuild.yaml b/tests/scenarios/azure-storage-blob-rust/vally/eval-nobuild.yaml deleted file mode 100644 index 803bcd3f..00000000 --- a/tests/scenarios/azure-storage-blob-rust/vally/eval-nobuild.yaml +++ /dev/null @@ -1,268 +0,0 @@ -name: azure-storage-blob-rust-nobuild-eval -description: Evaluation suite for azure-storage-blob-rust -version: "1.0" -type: capability -scoring: - weights: {} - threshold: 1.0 -defaults: - runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: - - name: rust-crud-blobs-nobuild - prompt: | - Write a Rust program named rust-blob-nobuild that performs the following blob storage operations: - 1. Create a blob container called "my-container" if it does not already exist - 2. Upload a local file "example.txt" as a block blob named "uploads/example.txt" - 3. List all blobs in the container and print their names and sizes - 4. Download the blob back to a local file "downloaded-example.txt" - 5. Delete the blob and then delete the container - - Include proper error handling for cases like container already existing - or blob not found. - rubric: - - "Implements the full blob CRUD workflow end-to-end: create container, upload, list, download, and delete blob/container." - - "Uses official Azure Rust SDK crate patterns and a TokenCredential-based authentication flow." - - "Uses async Rust with Tokio runtime and idiomatic Result-based error propagation." - - "Code compiles and passes cargo check for the rust-blob project." - - "Uses correct Azure client construction, response handling, and pager usage where applicable." - - "Avoids hardcoded secrets in source code or configuration committed to the workspace." - - "rustfmt compliance is bonus quality signal only and should not block a successful evaluation when all substantive correctness and SDK-usage criteria pass." - workspace_files: - include: - - "**/Cargo.toml" - - "**/Cargo.lock" - - "**/build.rs" - - "**/src/**/*.rs" - - "**/examples/**/*.rs" - - "**/tests/**/*.rs" - exclude: - - "**/target/**" - - "**/*.rlib" - - "**/*.rmeta" - - "**/*.d" - - "**/*.o" - - "**/*.obj" - - "**/*.pdb" - - "**/*.exe" - - "**/*.dll" - - "**/*.so" - - "**/*.dylib" - - "**/*.a" - - "**/*.lib" - - "**/*.wasm" - environment: - skills: - - ../../../../.github/plugins/azure-sdk-rust/skills/azure-storage-blob-rust - files: - - src: ../../_shared/vally/tools/find-cargo-root.js - dest: .vally/tools/find-cargo-root.js - - src: ../../_shared/vally/tools/check-azure-crates.rs - dest: .vally/tools/check-azure-crates.rs - - src: ../../_shared/vally/tools/check-token-credential.rs - dest: .vally/tools/check-token-credential.rs - - src: ../../_shared/vally/tools/check-async-runtime.rs - dest: .vally/tools/check-async-runtime.rs - - src: ../../_shared/vally/tools/check-no-secrets.rs - dest: .vally/tools/check-no-secrets.rs - - src: workspace/cargo.toml - dest: Cargo.toml - constraints: - expect_skills: - - azure-storage-blob-rust - graders: - - type: prompt - name: "Prompt correctness and completeness of blob storage operations" - config: - model: gpt-5.3-codex - prompt: | - - The code should demonstrate an azure blob storage client with a storage account url and credential - - Creating a container for container creation - - Uploading from a file path - - Enumerate blobs in a container - - Download from a blob to a local file - - Deleting blob files and containers for cleanup - - Error handling - scoring: scale_1_5 - - - type: run-command - name: "Build Application" - config: - command: cargo - args: - - "build" - stderr_contains: "Finished" - timeout: 3m - - type: run-command - name: "Check Application" - config: - command: cargo - args: - - "check" - timeout: 90s - - name: Rust Clippy Lints - type: run-command - weight: 0.5 - config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - name: Official Azure SDK Crate Selection - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" - - name: TokenCredential Authentication - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" - - name: Async-First with Tokio Runtime - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" - - name: Idiomatic Error Handling - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Error handling uses Result and the ? operator. Calls to - .unwrap() are avoided or used only where failure - is impossible. Calls to expect() with a reasonable error message are allowed - only if termination of the application is acceptable. Azure service errors are - matched via azure_core::Error with ErrorKind (e.g., - ErrorKind::HttpResponse { status, error_code, .. } with - StatusCode comparison). The main function returns - Result<(), Box> or azure_core::Result<()>. - - name: Correct Client Construction - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Service clients are constructed using the ::new() constructor - with an endpoint string, credential, and Option parameter. - If a ClientOptions parameter is specified, structs use ..Default::default() - for forward compatibility. If all the fields in the ClientOptions struct are provided, the - #[allow(clippy::needless_update)] annotation is used. - - name: Pager and Stream Usage - type: prompt - model: gpt-5.3-codex - weight: 1.0 - prompt: > - List operations return and consume azure_core::Pager using - futures::TryStreamExt (try_next/try_collect). Pagination is - handled correctly by iterating all items. In most if not all - cases, individual items should be iterated over directly rather - than page-by-page, demonstrating the SDK's ability to handle - pagination internally. Page-by-page iteration via into_pages() - is only necessary when there is an explicit requirement for page-level - access. - - name: Response Handling - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Calls to service clients correctly handle Response. When service - methods return a model type, code calls into_model() to deserialize - the response body into the expected model type (not manually parsing - JSON). When raw HTTP details are needed, deconstruct() is used to - access status, headers, and body separately. - - name: Long-Running Operations - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Long-running operations use azure_core::Poller correctly. - If a client calls into an API returning a Poller, code awaits - the Poller directly (via IntoFuture) for the final result. - The only time that a client should iterate over status updates via - futures::TryStreamExt is when there is an explicit requirement for - status-level access. - - name: Struct Initialization with Default - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - If a client initializes a structure for SDK types (client options, method - options, model types) the code uses ..Default::default() to protect - against future field additions. - - name: Ownership and Borrowing - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Code uses references (&str, &T) for borrowed data and avoids - unnecessary .clone() calls. Credentials are shared via Arc or - clone(). No unnecessary Box or Rc where Arc is appropriate. - Parameters that are only used for URL construction borrow - rather than own the data. - - name: No Hardcoded Secrets - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" - - name: Rust Cargo Trajectory Build Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - collect_trajectory_compiler_errors: true - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail - - name: Post prompt Rust Cargo Build Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - commands: - - command: cargo build - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail - - name: Post prompt Rust Cargo Check Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - commands: - - command: cargo check - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail - - name: Post prompt Rust Cargo Clippy Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - commands: - - command: cargo clippy - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail diff --git a/tests/scenarios/azure-storage-blob-rust/vally/eval.yaml b/tests/scenarios/azure-storage-blob-rust/vally/eval.yaml index 7064a05d..c9eca445 100644 --- a/tests/scenarios/azure-storage-blob-rust/vally/eval.yaml +++ b/tests/scenarios/azure-storage-blob-rust/vally/eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-crud-blobs prompt: | @@ -37,7 +38,7 @@ stimuli: - "Uses correct Azure client construction, response handling, and pager usage where applicable." - "Avoids hardcoded secrets in source code or configuration committed to the workspace." - "rustfmt compliance is bonus quality signal only and should not block a successful evaluation when all substantive correctness and SDK-usage criteria pass." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -83,7 +84,6 @@ stimuli: - type: prompt name: "Prompt correctness and completeness of blob storage operations" config: - model: gpt-5.3-codex prompt: | - The code should demonstrate an azure blob storage client with a storage account url and credential - Creating a container for container creation @@ -95,36 +95,29 @@ stimuli: scoring: scale_1_5 - name: Official Azure SDK Crate Selection - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs + emit_in_metadata: true - name: TokenCredential Authentication - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs + emit_in_metadata: true - name: Async-First with Tokio Runtime - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs + emit_in_metadata: true - name: Idiomatic Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Error handling uses Result and the ? operator. Calls to .unwrap() are avoided or used only where failure @@ -137,7 +130,6 @@ stimuli: - name: Correct Client Construction type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Service clients are constructed using the ::new() constructor with an endpoint string, credential, and Option parameter. @@ -146,7 +138,6 @@ stimuli: #[allow(clippy::needless_update)] annotation is used. - name: Pager and Stream Usage type: prompt - model: gpt-5.3-codex weight: 1.0 prompt: > List operations return and consume azure_core::Pager using @@ -160,7 +151,6 @@ stimuli: - name: Response Handling type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Calls to service clients correctly handle Response. When service methods return a model type, code calls into_model() to deserialize @@ -170,7 +160,6 @@ stimuli: - name: Long-Running Operations type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Long-running operations use azure_core::Poller correctly. If a client calls into an API returning a Poller, code awaits @@ -181,7 +170,6 @@ stimuli: - name: Struct Initialization with Default type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > If a client initializes a structure for SDK types (client options, method options, model types) the code uses ..Default::default() to protect @@ -189,7 +177,6 @@ stimuli: - name: Ownership and Borrowing type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Code uses references (&str, &T) for borrowed data and avoids unnecessary .clone() calls. Credentials are shared via Arc or @@ -197,14 +184,12 @@ stimuli: Parameters that are only used for URL construction borrow rather than own the data. - name: No Hardcoded Secrets - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs + emit_in_metadata: true - name: Rust Cargo Trajectory Build Failure type: rust-cargo-build-failure-check weight: 1.0 diff --git a/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_eval.yaml index e04f29f1..08818e29 100644 --- a/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-crud-blobs prompt: | @@ -37,7 +38,7 @@ stimuli: - "Uses correct Azure client construction, response handling, and pager usage where applicable." - "Avoids hardcoded secrets in source code or configuration committed to the workspace." - "rustfmt compliance is bonus quality signal only and should not block a successful evaluation when all substantive correctness and SDK-usage criteria pass." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -61,8 +62,6 @@ stimuli: - "**/*.lib" - "**/*.wasm" environment: - skills: - - ../../../../.github/plugins/azure-sdk-rust/skills/azure-storage-blob-rust files: - src: ../../_shared/vally/tools/find-cargo-root.js dest: .vally/tools/find-cargo-root.js @@ -83,7 +82,7 @@ stimuli: - type: prompt name: "Prompt correctness and completeness of blob storage operations" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - The code should demonstrate an azure blob storage client with a storage account url and credential - Creating a container for container creation @@ -100,7 +99,7 @@ stimuli: command: cargo args: - "build" - timeout: 120s + timeout: 3m stderr_contains: "Finished" - type: run-command name: "Check Application" @@ -108,7 +107,7 @@ stimuli: command: cargo args: - "check" - timeout: 90s + timeout: 3m - name: Rust Clippy Lints type: run-command weight: 0.5 @@ -150,7 +149,7 @@ stimuli: - name: Idiomatic Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Error handling uses Result and the ? operator. Calls to .unwrap() are avoided or used only where failure @@ -163,7 +162,7 @@ stimuli: - name: Correct Client Construction type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Service clients are constructed using the ::new() constructor with an endpoint string, credential, and Option parameter. @@ -172,7 +171,7 @@ stimuli: #[allow(clippy::needless_update)] annotation is used. - name: Pager and Stream Usage type: prompt - model: gpt-5.3-codex + model: claude-sonnet-4.6 weight: 1.0 prompt: > List operations return and consume azure_core::Pager using @@ -186,7 +185,7 @@ stimuli: - name: Response Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Calls to service clients correctly handle Response. When service methods return a model type, code calls into_model() to deserialize @@ -196,7 +195,7 @@ stimuli: - name: Long-Running Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Long-running operations use azure_core::Poller correctly. If a client calls into an API returning a Poller, code awaits @@ -207,7 +206,7 @@ stimuli: - name: Struct Initialization with Default type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > If a client initializes a structure for SDK types (client options, method options, model types) the code uses ..Default::default() to protect @@ -215,7 +214,7 @@ stimuli: - name: Ownership and Borrowing type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Code uses references (&str, &T) for borrowed data and avoids unnecessary .clone() calls. Credentials are shared via Arc or @@ -231,46 +230,3 @@ stimuli: - "+nightly" - "-Zscript" - ".vally/tools/check-no-secrets.rs" - - name: Rust Cargo Trajectory Build Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - collect_trajectory_compiler_errors: true - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail - - name: Post prompt Rust Cargo Build Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - commands: - - command: cargo build - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail - - name: Post prompt Rust Cargo Check Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - commands: - - command: cargo check - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail - - name: Post prompt Rust Cargo Clippy Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - commands: - - command: cargo clippy - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail diff --git a/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_experiment.yaml index 859e46e3..42bebb73 100644 --- a/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_experiment.yaml +++ b/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_experiment.yaml @@ -2,8 +2,6 @@ name: azure-storage-blob-rust-skill-experiment evals: - skill_effectiveness_eval.yaml - - eval-nobuild-experiment.yaml - - eval-keyvault-encryption.yaml vary: - /defaults/model diff --git a/tests/scenarios/azure-storage-queue-rust/vally/eval-experiment.yaml b/tests/scenarios/azure-storage-queue-rust/vally/eval-experiment.yaml deleted file mode 100644 index a48e09f8..00000000 --- a/tests/scenarios/azure-storage-queue-rust/vally/eval-experiment.yaml +++ /dev/null @@ -1,165 +0,0 @@ -name: azure-storage-queue-rust-eval -description: Evaluation suite for azure-storage-queue-rust -version: "1.0" -type: capability -scoring: - weights: {} - threshold: 1.0 -defaults: - runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: - - name: rust-queue-operations - prompt: | - Write a Rust program named rust-queue-ops that performs the following queue storage operations: - 1. Create a QueueServiceClient from an endpoint URL with TokenCredential authentication - 2. Create or get a queue client for a queue named "work-queue" - 3. Send 3 messages to the queue with different message text - 4. Receive messages from the queue (up to 5 messages) - 5. Peek at messages in the queue without removing them - 6. For each received message, extract the message ID, pop receipt, and message text - 7. Delete messages from the queue using message ID and pop receipt - 8. Demonstrate proper error handling for queue operations - - The program should compile and run successfully with the official Azure Storage Queue SDK, - demonstrating full CRUD operations on queue messages with proper TokenCredential-based authentication. - - Ensure the following SDK patterns are demonstrated: - - QueueServiceClient initialization with endpoint and credential - - QueueClient derivation from QueueServiceClient - - Message send/receive/peek/delete workflows - - Proper use of try_into() for message models - - Error handling with Result-based patterns - rubric: - - "Implements complete queue message lifecycle: send, receive, peek, and delete operations end-to-end." - - "Uses official azure_storage_queue SDK with TokenCredential-based authentication." - - "Uses async Rust with Tokio runtime and idiomatic Result-based error propagation." - - "Code compiles and passes cargo check." - - "Correctly derives QueueClient from QueueServiceClient and uses queue_client() method." - - "Message receive operations properly extract message_id, pop_receipt, and message_text fields." - - "Message delete operations use both message_id and pop_receipt from received messages." - - "Peek operations demonstrate reading messages without consuming them from the queue." - - "Avoids hardcoded credentials in source code." - workspace_files: - include: - - "**/Cargo.toml" - - "**/Cargo.lock" - - "**/build.rs" - - "**/src/**/*.rs" - - "**/examples/**/*.rs" - - "**/tests/**/*.rs" - exclude: - - "**/target/**" - - "**/*.rlib" - - "**/*.rmeta" - - "**/*.d" - - "**/*.o" - - "**/*.obj" - - "**/*.pdb" - - "**/*.exe" - - "**/*.dll" - - "**/*.so" - - "**/*.dylib" - - "**/*.a" - - "**/*.lib" - - "**/*.wasm" - environment: - files: - - src: workspace/cargo.toml - dest: Cargo.toml - constraints: - expect_skills: - - azure-storage-queue-rust - graders: - - type: prompt - name: "Queue operations completeness" - config: - model: gpt-5.3-codex - prompt: | - - QueueServiceClient initialization with endpoint URL and credential - - QueueClient derived from service client - - Send message operations with message_text - - Receive messages with proper message_id, pop_receipt extraction - - Peek messages without removal - - Delete messages using message_id and pop_receipt pair - - Error handling for queue operations - scoring: scale_1_5 - - - type: run-command - name: "Build Application" - config: - command: cargo - args: - - "build" - timeout: 120s - stderr_contains: "Finished" - - type: run-command - name: "Check Application" - config: - command: cargo - args: - - "check" - timeout: 90s - - name: Rust Clippy Lints - type: run-command - weight: 1.0 - config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - name: Official Azure SDK Crate Selection - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" - - name: TokenCredential Authentication - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Authentication uses TokenCredential from azure_identity, created via - DeveloperToolsCredential::new(None) or ManagedIdentityCredential::new(None). - The credential is passed to QueueServiceClient::new() as Some(credential). - For production, uses ManagedIdentityCredential; for local dev, uses DeveloperToolsCredential. - - name: QueueServiceClient Construction - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - QueueServiceClient is created via QueueServiceClient::new() with three parameters: - service_url (Url type), Some(credential), and None for options. - QueueClient is derived via service_client.queue_client("")? - - name: Queue Message Operations - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Send messages via queue_client.send_message(message.try_into()?, None).await?. - Receive messages via queue_client.receive_messages(None).await? returning a response - that converts to model with items containing message_id, pop_receipt, and message_text fields. - Delete messages via queue_client.delete_message(id, pop_receipt, None).await?. - Peek via queue_client.peek_messages(None).await?. - - name: Async Runtime - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Code uses #[tokio::main] and async/await. Queue operations are properly awaited. - Result> main signature for error propagation. - - name: Error Handling - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Queue operations are wrapped in Result-based error handling using ? operator. - No .unwrap() calls except where failure is impossible. - Main function returns Result for proper error propagation. diff --git a/tests/scenarios/azure-storage-queue-rust/vally/eval.yaml b/tests/scenarios/azure-storage-queue-rust/vally/eval.yaml index 131b8112..4d5c5aa7 100644 --- a/tests/scenarios/azure-storage-queue-rust/vally/eval.yaml +++ b/tests/scenarios/azure-storage-queue-rust/vally/eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-queue-operations prompt: | @@ -41,7 +42,7 @@ stimuli: - "Message delete operations use both message_id and pop_receipt from received messages." - "Peek operations demonstrate reading messages without consuming them from the queue." - "Avoids hardcoded credentials in source code." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -68,6 +69,16 @@ stimuli: skills: - ../../../../.github/plugins/azure-sdk-rust/skills/azure-storage-queue-rust files: + - src: ../../_shared/vally/tools/find-cargo-root.js + dest: .vally/tools/find-cargo-root.js + - src: ../../_shared/vally/tools/check-azure-crates.rs + dest: .vally/tools/check-azure-crates.rs + - src: ../../_shared/vally/tools/check-token-credential.rs + dest: .vally/tools/check-token-credential.rs + - src: ../../_shared/vally/tools/check-async-runtime.rs + dest: .vally/tools/check-async-runtime.rs + - src: ../../_shared/vally/tools/check-no-secrets.rs + dest: .vally/tools/check-no-secrets.rs - src: workspace/cargo.toml dest: Cargo.toml constraints: @@ -77,7 +88,6 @@ stimuli: - type: prompt name: "Queue operations completeness" config: - model: gpt-5.3-codex prompt: | - QueueServiceClient initialization with endpoint URL and credential - QueueClient derived from service client @@ -88,73 +98,67 @@ stimuli: - Error handling for queue operations scoring: scale_1_5 - - type: run-command - name: "Build Application" + - name: "Build Application" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "build" - timeout: 120s - stderr_contains: "Finished" - - type: run-command - name: "Check Application" + commands: + - command: cargo build + emit_in_metadata: true + - name: "Check Application" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "check" - timeout: 90s + commands: + - command: cargo check + emit_in_metadata: true - name: Rust Clippy Lints - type: run-command + type: rust-cargo-build-failure-check weight: 1.0 config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s + commands: + - command: cargo clippy -- -D warnings + emit_in_metadata: true - name: Official Azure SDK Crate Selection - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs + emit_in_metadata: true - name: TokenCredential Authentication - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs + emit_in_metadata: true - name: Async-First with Tokio Runtime - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs + emit_in_metadata: true - name: No Hardcoded Secrets - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs + emit_in_metadata: true + - name: Rust Cargo Trajectory Build Failure + type: rust-cargo-build-failure-check + weight: 1.0 + config: + collect_trajectory_compiler_errors: true + emit_in_metadata: true + error_exceptions: + - error_code: E0277 + message_pattern: "Option" + action: ignore_for_fail - name: QueueServiceClient Construction type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > QueueServiceClient is created via QueueServiceClient::new() with three parameters: service_url (Url type), Some(credential), and None for options. @@ -162,7 +166,6 @@ stimuli: - name: Queue Message Operations type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Send messages via queue_client.send_message(message.try_into()?, None).await?. Receive messages via queue_client.receive_messages(None).await? returning a response @@ -172,7 +175,6 @@ stimuli: - name: Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Queue operations are wrapped in Result-based error handling using ? operator. No .unwrap() calls except where failure is impossible. diff --git a/tests/scenarios/azure-storage-queue-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-storage-queue-rust/vally/skill_effectiveness_eval.yaml index f4ae4478..78ad4708 100644 --- a/tests/scenarios/azure-storage-queue-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-storage-queue-rust/vally/skill_effectiveness_eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-queue-operations prompt: | @@ -41,7 +42,7 @@ stimuli: - "Message delete operations use both message_id and pop_receipt from received messages." - "Peek operations demonstrate reading messages without consuming them from the queue." - "Avoids hardcoded credentials in source code." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -83,7 +84,7 @@ stimuli: - type: prompt name: "Queue operations completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - QueueServiceClient initialization with endpoint URL and credential - QueueClient derived from service client @@ -159,7 +160,7 @@ stimuli: - name: QueueServiceClient Construction type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > QueueServiceClient is created via QueueServiceClient::new() with three parameters: service_url (Url type), Some(credential), and None for options. @@ -167,7 +168,7 @@ stimuli: - name: Queue Message Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Send messages via queue_client.send_message(message.try_into()?, None).await?. Receive messages via queue_client.receive_messages(None).await? returning a response @@ -177,7 +178,7 @@ stimuli: - name: Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Queue operations are wrapped in Result-based error handling using ? operator. No .unwrap() calls except where failure is impossible. From b5ede07682443b7966cd819c672bcdd11337a353 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 16 Jul 2026 22:16:40 +0000 Subject: [PATCH 02/10] Fix Rust Vally review feedback in PR #381 --- tests/package.json | 4 +- .../rust-cargo-build-failure/package.json | 2 +- .../azure-cosmos-rust/vally/eval.yaml | 40 +++++++++++-------- .../azure-eventhub-rust/vally/eval.yaml | 40 +++++++++++-------- .../azure-identity-rust/vally/eval.yaml | 40 +++++++++++-------- .../vally/eval.yaml | 40 +++++++++++-------- .../azure-keyvault-keys-rust/vally/eval.yaml | 40 +++++++++++-------- .../vally/eval.yaml | 40 +++++++++++-------- .../vally/eval-keyvault-encryption.yaml | 40 +++++++++++-------- .../azure-storage-blob-rust/vally/eval.yaml | 40 +++++++++++-------- .../azure-storage-queue-rust/vally/eval.yaml | 40 +++++++++++-------- 11 files changed, 219 insertions(+), 147 deletions(-) diff --git a/tests/package.json b/tests/package.json index 8f9b7b74..fefc9b18 100644 --- a/tests/package.json +++ b/tests/package.json @@ -21,7 +21,7 @@ "zod": "^3.23.8" }, "devDependencies": { - "@microsoft/vally": "^0.6.0", + "@microsoft/vally": "^0.9.0", "@types/node": "^20.14.0", "@vitest/coverage-v8": "^4.1.9", "eslint": "^9.5.0", @@ -33,4 +33,4 @@ "engines": { "node": ">=20.0.0" } -} \ No newline at end of file +} diff --git a/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/package.json b/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/package.json index 839e049e..9c5c5d6a 100644 --- a/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/package.json +++ b/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/package.json @@ -10,7 +10,7 @@ "typecheck": "tsc --noEmit" }, "devDependencies": { - "@microsoft/vally": "^0.7.0", + "@microsoft/vally": "^0.9.0", "@types/node": "^20.0.0", "typescript": "^5.4.5" }, diff --git a/tests/scenarios/azure-cosmos-rust/vally/eval.yaml b/tests/scenarios/azure-cosmos-rust/vally/eval.yaml index 039aac5a..f70e9a5d 100644 --- a/tests/scenarios/azure-cosmos-rust/vally/eval.yaml +++ b/tests/scenarios/azure-cosmos-rust/vally/eval.yaml @@ -111,33 +111,41 @@ stimuli: scoring: scale_1_5 - name: Official Azure SDK Crate Selection - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-azure-crates.rs" - name: TokenCredential Authentication - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-token-credential.rs" - name: Async-First with Tokio Runtime - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-async-runtime.rs" - name: No Hardcoded Secrets - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-no-secrets.rs" - name: Rust Cargo Trajectory Build Failure type: rust-cargo-build-failure-check weight: 1.0 diff --git a/tests/scenarios/azure-eventhub-rust/vally/eval.yaml b/tests/scenarios/azure-eventhub-rust/vally/eval.yaml index 5ba5449c..e9a12102 100644 --- a/tests/scenarios/azure-eventhub-rust/vally/eval.yaml +++ b/tests/scenarios/azure-eventhub-rust/vally/eval.yaml @@ -103,33 +103,41 @@ stimuli: scoring: scale_1_5 - name: Official Azure SDK Crate Selection - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-azure-crates.rs" - name: TokenCredential Authentication - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-token-credential.rs" - name: Async-First with Tokio Runtime - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-async-runtime.rs" - name: No Hardcoded Secrets - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-no-secrets.rs" - name: Rust Cargo Trajectory Build Failure type: rust-cargo-build-failure-check weight: 1.0 diff --git a/tests/scenarios/azure-identity-rust/vally/eval.yaml b/tests/scenarios/azure-identity-rust/vally/eval.yaml index eaf46317..11958622 100644 --- a/tests/scenarios/azure-identity-rust/vally/eval.yaml +++ b/tests/scenarios/azure-identity-rust/vally/eval.yaml @@ -130,33 +130,41 @@ stimuli: emit_in_metadata: true - name: Official Azure SDK Crate Selection - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-azure-crates.rs" - name: TokenCredential Authentication - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-token-credential.rs" - name: Async-First with Tokio Runtime - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-async-runtime.rs" - name: No Hardcoded Secrets - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-no-secrets.rs" - name: Rust Cargo Trajectory Build Failure type: rust-cargo-build-failure-check weight: 1.0 diff --git a/tests/scenarios/azure-keyvault-certificates-rust/vally/eval.yaml b/tests/scenarios/azure-keyvault-certificates-rust/vally/eval.yaml index e2a28c6a..7577937d 100644 --- a/tests/scenarios/azure-keyvault-certificates-rust/vally/eval.yaml +++ b/tests/scenarios/azure-keyvault-certificates-rust/vally/eval.yaml @@ -122,33 +122,41 @@ stimuli: scoring: scale_1_5 - name: Official Azure SDK Crate Selection - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-azure-crates.rs" - name: TokenCredential Authentication - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-token-credential.rs" - name: Async-First with Tokio Runtime - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-async-runtime.rs" - name: No Hardcoded Secrets - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-no-secrets.rs" - name: Rust Cargo Trajectory Build Failure type: rust-cargo-build-failure-check weight: 1.0 diff --git a/tests/scenarios/azure-keyvault-keys-rust/vally/eval.yaml b/tests/scenarios/azure-keyvault-keys-rust/vally/eval.yaml index 26ad503e..97dfce74 100644 --- a/tests/scenarios/azure-keyvault-keys-rust/vally/eval.yaml +++ b/tests/scenarios/azure-keyvault-keys-rust/vally/eval.yaml @@ -120,33 +120,41 @@ stimuli: scoring: scale_1_5 - name: Official Azure SDK Crate Selection - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-azure-crates.rs" - name: TokenCredential Authentication - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-token-credential.rs" - name: Async-First with Tokio Runtime - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-async-runtime.rs" - name: No Hardcoded Secrets - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-no-secrets.rs" - name: Rust Cargo Trajectory Build Failure type: rust-cargo-build-failure-check weight: 1.0 diff --git a/tests/scenarios/azure-keyvault-secrets-rust/vally/eval.yaml b/tests/scenarios/azure-keyvault-secrets-rust/vally/eval.yaml index 866150c1..cd3f875a 100644 --- a/tests/scenarios/azure-keyvault-secrets-rust/vally/eval.yaml +++ b/tests/scenarios/azure-keyvault-secrets-rust/vally/eval.yaml @@ -110,33 +110,41 @@ stimuli: - Async/await throughout scoring: scale_1_5 - name: Official Azure SDK Crate Selection - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-azure-crates.rs" - name: TokenCredential Authentication - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-token-credential.rs" - name: Async-First with Tokio Runtime - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-async-runtime.rs" - name: No Hardcoded Secrets - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-no-secrets.rs" - name: Rust Cargo Trajectory Build Failure type: rust-cargo-build-failure-check weight: 1.0 diff --git a/tests/scenarios/azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml b/tests/scenarios/azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml index c1da674a..2c73994a 100644 --- a/tests/scenarios/azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml +++ b/tests/scenarios/azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml @@ -100,36 +100,44 @@ stimuli: - Reasonable error handling and no embedded secrets scoring: scale_1_5 - name: Official Azure SDK Crate Selection - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-azure-crates.rs" timeout: 2m - name: TokenCredential Authentication - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-token-credential.rs" timeout: 2m - name: Async-First with Tokio Runtime - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-async-runtime.rs" timeout: 2m - name: No Hardcoded Secrets - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-no-secrets.rs" timeout: 2m - name: Rust Cargo Trajectory Build Failure type: rust-cargo-build-failure-check diff --git a/tests/scenarios/azure-storage-blob-rust/vally/eval.yaml b/tests/scenarios/azure-storage-blob-rust/vally/eval.yaml index c9eca445..2e6db220 100644 --- a/tests/scenarios/azure-storage-blob-rust/vally/eval.yaml +++ b/tests/scenarios/azure-storage-blob-rust/vally/eval.yaml @@ -95,26 +95,32 @@ stimuli: scoring: scale_1_5 - name: Official Azure SDK Crate Selection - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-azure-crates.rs" - name: TokenCredential Authentication - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-token-credential.rs" - name: Async-First with Tokio Runtime - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-async-runtime.rs" - name: Idiomatic Error Handling type: prompt weight: 1.0 @@ -184,12 +190,14 @@ stimuli: Parameters that are only used for URL construction borrow rather than own the data. - name: No Hardcoded Secrets - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-no-secrets.rs" - name: Rust Cargo Trajectory Build Failure type: rust-cargo-build-failure-check weight: 1.0 diff --git a/tests/scenarios/azure-storage-queue-rust/vally/eval.yaml b/tests/scenarios/azure-storage-queue-rust/vally/eval.yaml index 4d5c5aa7..007e4d67 100644 --- a/tests/scenarios/azure-storage-queue-rust/vally/eval.yaml +++ b/tests/scenarios/azure-storage-queue-rust/vally/eval.yaml @@ -118,33 +118,41 @@ stimuli: - command: cargo clippy -- -D warnings emit_in_metadata: true - name: Official Azure SDK Crate Selection - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-azure-crates.rs" - name: TokenCredential Authentication - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-token-credential.rs" - name: Async-First with Tokio Runtime - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-async-runtime.rs" - name: No Hardcoded Secrets - type: rust-cargo-build-failure-check + type: program weight: 1.0 config: - commands: - - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs - emit_in_metadata: true + program: cargo + args: + - "+nightly" + - "-Zscript" + - ".vally/tools/check-no-secrets.rs" - name: Rust Cargo Trajectory Build Failure type: rust-cargo-build-failure-check weight: 1.0 From 0d4babed7c3166467af8d98011652a91bc9228e3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:09:30 +0000 Subject: [PATCH 03/10] Fix CI: update pnpm-lock.yaml and add pnpm-workspace.yaml for @microsoft/vally ^0.9.0 --- tests/pnpm-lock.yaml | 143 +++++++++++++++++++++++++++++++++++--- tests/pnpm-workspace.yaml | 2 + 2 files changed, 136 insertions(+), 9 deletions(-) create mode 100644 tests/pnpm-workspace.yaml diff --git a/tests/pnpm-lock.yaml b/tests/pnpm-lock.yaml index bc23d1e8..87acd923 100644 --- a/tests/pnpm-lock.yaml +++ b/tests/pnpm-lock.yaml @@ -25,8 +25,8 @@ importers: version: 3.25.76 devDependencies: '@microsoft/vally': - specifier: ^0.6.0 - version: 0.6.0 + specifier: ^0.9.0 + version: 0.9.0 '@types/node': specifier: ^20.14.0 version: 20.19.30 @@ -47,7 +47,7 @@ importers: version: 8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2) vitest: specifier: ^4.1.9 - version: 4.1.9(@types/node@20.19.30)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2)) + version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@20.19.30)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2)) packages: @@ -281,56 +281,120 @@ packages: os: [darwin] hasBin: true + '@github/copilot-darwin-arm64@1.0.70': + resolution: {integrity: sha512-NewReqBmTxtAzApZNmUsY4Xqy8N086MiQm7k35i9i+WrGyRiHxdZ/n/lMLCkqWoelr7pZhcZ7gQ7fHbEq1KdoQ==} + cpu: [arm64] + os: [darwin] + hasBin: true + '@github/copilot-darwin-x64@1.0.64': resolution: {integrity: sha512-mwjZ0/HZ7loXnahkqhy7LXvNVzq12eghoawd+M2d7kiHMR5yAHK6XA++6kKFj3vZnDanjau+wyvTx0z++mQ3LA==} cpu: [x64] os: [darwin] hasBin: true + '@github/copilot-darwin-x64@1.0.70': + resolution: {integrity: sha512-ydUEYI3udNAjdMsLPFQLH/JG9YFKcxuAUxYIyOK+F/m/Of9nODKdYa2hw2mlLanP4cNyuxGLflu+rtIqIb+cPw==} + cpu: [x64] + os: [darwin] + hasBin: true + '@github/copilot-linux-arm64@1.0.64': resolution: {integrity: sha512-Krg/3ZWxXB7Dw4VOLZEZrYmqc39Yvz9M1K9SPOfjpEy2SFnF/KVLaFt/6E1uYdjgvJ7BmocfVcFYp0hUmy5Axw==} cpu: [arm64] os: [linux] + libc: [glibc] + hasBin: true + + '@github/copilot-linux-arm64@1.0.70': + resolution: {integrity: sha512-EpR3VEEqMy5M9t3cs+6FtjX/AhyfoefzmcMAjBls+RGv1fILjaAby+rhKHzX5YVSxOu9OyQDlQVHK3kxznTU5Q==} + cpu: [arm64] + os: [linux] + libc: [glibc] hasBin: true '@github/copilot-linux-x64@1.0.64': resolution: {integrity: sha512-2k9FGppYnxHLwVH+TVCf13JfjSlvS15wsZM5xYxEFlqL/CIBvTK7yn5p7M+P1dWoTUpCmavKG3601BtHPenrRg==} cpu: [x64] os: [linux] + libc: [glibc] + hasBin: true + + '@github/copilot-linux-x64@1.0.70': + resolution: {integrity: sha512-4pyNKunm7GEzQZ+09Dwr4BwixJVNcQGBeqiZPKWBGxcSipwj90t8tidLYdNjgmXJoLerANhXZcY52wJW/HubEA==} + cpu: [x64] + os: [linux] + libc: [glibc] hasBin: true '@github/copilot-linuxmusl-arm64@1.0.64': resolution: {integrity: sha512-C+EYoMvmlUxR0YYxLkD3nwn940y0zId8z+pPl9rFO6f9heMGXYCwCZL2i2c3GW6CvGgZF6Wbzw1Kk0Gvw46F2w==} cpu: [arm64] os: [linux] + libc: [musl] + hasBin: true + + '@github/copilot-linuxmusl-arm64@1.0.70': + resolution: {integrity: sha512-Xy3tDjIMmMkZZzJjCrK5aDCLLV5+pyOfIcT1lWLmqVWJG0erpHXL6KU82nGW+0nc0TPqGZFHvOyQeLuXl+s3ZA==} + cpu: [arm64] + os: [linux] + libc: [musl] hasBin: true '@github/copilot-linuxmusl-x64@1.0.64': resolution: {integrity: sha512-VSTGl7dGQDhH2ACeyYq1hYhAsMUbumpqeVe+CGo/u7fKOMrTNMTR5SB3s+zsHCskKZkJkZ2gsUWROCiDBsThEg==} cpu: [x64] os: [linux] + libc: [musl] + hasBin: true + + '@github/copilot-linuxmusl-x64@1.0.70': + resolution: {integrity: sha512-AHWayI1lVTzxYkNkKrCBOo3XPG+Fb67zcqFivRjGaXG5tr9Bt870LIjIAWoJqQ1Clc3K2PsxyYZ1d8T6vjpWnA==} + cpu: [x64] + os: [linux] + libc: [musl] hasBin: true '@github/copilot-sdk@1.0.3': resolution: {integrity: sha512-ujnH2QVw3+xvjgo9cbpY0wik4fNxAmdMDSFnxGScDSvRuK2vUCL2xWW4V2ANc9pWwRHPBpEpMuNJMtmydmLCIQ==} engines: {node: ^20.19.0 || >=22.12.0} + '@github/copilot-sdk@1.0.6': + resolution: {integrity: sha512-24eI5UNY3dWjGBNTLhNJf8ki3dNIE2o2jlEZv2F79N5MHzVpnDWbpM8vUI8L1furBXqc9dWRhLPJG/4Z7Me8Ow==} + engines: {node: ^20.19.0 || >=22.12.0} + '@github/copilot-win32-arm64@1.0.64': resolution: {integrity: sha512-16gb2T0rQ3QI/rs7mJa3xSks0/ZFrFKoMro3tvgM/c89J/5PTpykpLIuD2Zhl1cC3FqaFzcSYzdupxu82Aj2tQ==} cpu: [arm64] os: [win32] hasBin: true + '@github/copilot-win32-arm64@1.0.70': + resolution: {integrity: sha512-oWsWCTlUyIv4S3FdYoBNCscndLrUv+C3qgbfJ9mf+5igPqbIYL8alkc8FBHWPB/cornDNj65yd4s8dZrSkFPpg==} + cpu: [arm64] + os: [win32] + hasBin: true + '@github/copilot-win32-x64@1.0.64': resolution: {integrity: sha512-Ki+St5eggcATHdPLc58RGrOuCX+igblD3/TmlQ8WTHmZxcyuKJl9RNzui3ioVi7ntLmKWriW5r/VH/DTHVKT8g==} cpu: [x64] os: [win32] hasBin: true + '@github/copilot-win32-x64@1.0.70': + resolution: {integrity: sha512-KFDZ750CwhjKF6uATQ1pv1XrYHI+/qTQta954gYZaa+YJEXRstGd284uJ6NI3YXfshZwhTs64JdjN4xLodYwrA==} + cpu: [x64] + os: [win32] + hasBin: true + '@github/copilot@1.0.64': resolution: {integrity: sha512-Dch34NNBWjWlkEUxrC4CkDFunQs5cRmkEEO/PR6Lqj23zzwDwzJUxWA/6K28EokfoPhc9hLyPsCbS9rQ9v2TFA==} hasBin: true + '@github/copilot@1.0.70': + resolution: {integrity: sha512-onEwx5tod9t31Lc2rT5ns6s/fY6jtdwVWS3Fzs8hKUjCv7LFIMGf71MLcikl4GBXn6cq44+HVdNzCMWnLjYl3g==} + hasBin: true + '@humanfs/core@0.19.1': resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==} engines: {node: '>=18.18.0'} @@ -357,8 +421,8 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} - '@microsoft/vally@0.6.0': - resolution: {integrity: sha512-b283YRDFZXUkKNKY3+1EfMBVbHrBLIs5jfUi7lIQ8N0Y10lVsNnNGkRbtTbd7tLYZajr6AhtnpIroc4RFzo1cQ==} + '@microsoft/vally@0.9.0': + resolution: {integrity: sha512-6OYrhfwwssauk5OxxKvLFxsxOfhJ2vln5hoqMNvp1AxxVAW9mh6wyKydkW+UFaRm81kEqXUxU9WBNbmM0goy8Q==} '@napi-rs/wasm-runtime@1.1.5': resolution: {integrity: sha512-AWPoBRJ9tsnVhor4sjO7rkni+7p+2IAEFj6cx06UgP10jkQHqay/36uRV/bFkgrh18D9vb4cr8Q0Pthskgzy+Q==} @@ -366,6 +430,10 @@ packages: '@emnapi/core': ^1.7.1 '@emnapi/runtime': ^1.7.1 + '@opentelemetry/api@1.9.1': + resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==} + engines: {node: '>=8.0.0'} + '@oxc-project/types@0.137.0': resolution: {integrity: sha512-WT+Gb24i8hmvo85AIv2oEYouEXkRlKAlT9WaCa3TfLgNCN+GhrJOGZuIlMouAh38Qe4QOx26eUOVsq70qXrywA==} @@ -404,36 +472,42 @@ packages: engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] + libc: [glibc] '@rolldown/binding-linux-arm64-musl@1.1.2': resolution: {integrity: sha512-LjQP/iZLBu8o8PjIfk4x3At0/mT6h282pvz8Z5LAyhGbu/kDezyO7ea62rF5uoqmgnIYqbN/MqJ3Si3Aymi7xQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] + libc: [musl] '@rolldown/binding-linux-ppc64-gnu@1.1.2': resolution: {integrity: sha512-X/7bVLWelEsbyWDUSXt7zVsTniLLPIY2n1rH58qr78l9i7MNbbxBWD8gI2vRfBWf4NUXJCUuQnfZDsp32LqsfQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [ppc64] os: [linux] + libc: [glibc] '@rolldown/binding-linux-s390x-gnu@1.1.2': resolution: {integrity: sha512-gb6dYKW/1KDorGXyy48glEBJs/sxVSC5pcVrox/pFGV4mvwSFeg2sK5L2tRkVsVlh7kueqOgg4GEcuipJcGuKg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [s390x] os: [linux] + libc: [glibc] '@rolldown/binding-linux-x64-gnu@1.1.2': resolution: {integrity: sha512-JY4w85pU3iAiJVMh5nuk4/Mh9GjMsupe8MrIN53rwxAZW64GKrWeJBuN6SxQg9QTU5uB1cxyhDzW8jqRn1EABw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] + libc: [glibc] '@rolldown/binding-linux-x64-musl@1.1.2': resolution: {integrity: sha512-xvpA7o5KCYLB0Rwscmuylb1/zHHSUx4g4xilm4prC5jP76pEUlzBmMbgpbh7bVDbId4NcfT96gN5i6mE6UDaiw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] + libc: [musl] '@rolldown/binding-openharmony-arm64@1.1.2': resolution: {integrity: sha512-p/ts6KBLjuk49Bp21XH77poQGt02iNz7ChgHep7tudPOaLinR/De/RHdxF8w8Yj4r/bF/bqXwH6PZrB2sA+Nvw==} @@ -822,24 +896,28 @@ packages: engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] + libc: [glibc] lightningcss-linux-arm64-musl@1.32.0: resolution: {integrity: sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==} engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] + libc: [musl] lightningcss-linux-x64-gnu@1.32.0: resolution: {integrity: sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] + libc: [glibc] lightningcss-linux-x64-musl@1.32.0: resolution: {integrity: sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] + libc: [musl] lightningcss-win32-arm64-msvc@1.32.0: resolution: {integrity: sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==} @@ -1304,33 +1382,63 @@ snapshots: '@github/copilot-darwin-arm64@1.0.64': optional: true + '@github/copilot-darwin-arm64@1.0.70': + optional: true + '@github/copilot-darwin-x64@1.0.64': optional: true + '@github/copilot-darwin-x64@1.0.70': + optional: true + '@github/copilot-linux-arm64@1.0.64': optional: true + '@github/copilot-linux-arm64@1.0.70': + optional: true + '@github/copilot-linux-x64@1.0.64': optional: true + '@github/copilot-linux-x64@1.0.70': + optional: true + '@github/copilot-linuxmusl-arm64@1.0.64': optional: true + '@github/copilot-linuxmusl-arm64@1.0.70': + optional: true + '@github/copilot-linuxmusl-x64@1.0.64': optional: true + '@github/copilot-linuxmusl-x64@1.0.70': + optional: true + '@github/copilot-sdk@1.0.3': dependencies: '@github/copilot': 1.0.64 vscode-jsonrpc: 8.2.1 zod: 4.4.3 + '@github/copilot-sdk@1.0.6': + dependencies: + '@github/copilot': 1.0.70 + vscode-jsonrpc: 8.2.1 + zod: 4.4.3 + '@github/copilot-win32-arm64@1.0.64': optional: true + '@github/copilot-win32-arm64@1.0.70': + optional: true + '@github/copilot-win32-x64@1.0.64': optional: true + '@github/copilot-win32-x64@1.0.70': + optional: true + '@github/copilot@1.0.64': dependencies: detect-libc: 2.1.2 @@ -1344,6 +1452,19 @@ snapshots: '@github/copilot-win32-arm64': 1.0.64 '@github/copilot-win32-x64': 1.0.64 + '@github/copilot@1.0.70': + dependencies: + detect-libc: 2.1.2 + optionalDependencies: + '@github/copilot-darwin-arm64': 1.0.70 + '@github/copilot-darwin-x64': 1.0.70 + '@github/copilot-linux-arm64': 1.0.70 + '@github/copilot-linux-x64': 1.0.70 + '@github/copilot-linuxmusl-arm64': 1.0.70 + '@github/copilot-linuxmusl-x64': 1.0.70 + '@github/copilot-win32-arm64': 1.0.70 + '@github/copilot-win32-x64': 1.0.70 + '@humanfs/core@0.19.1': {} '@humanfs/node@0.16.7': @@ -1364,9 +1485,10 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 - '@microsoft/vally@0.6.0': + '@microsoft/vally@0.9.0': dependencies: - '@github/copilot-sdk': 1.0.3 + '@github/copilot-sdk': 1.0.6 + '@opentelemetry/api': 1.9.1 js-tiktoken: 1.0.21 picomatch: 4.0.4 yaml: 2.9.0 @@ -1379,6 +1501,8 @@ snapshots: '@tybys/wasm-util': 0.10.3 optional: true + '@opentelemetry/api@1.9.1': {} + '@oxc-project/types@0.137.0': {} '@rolldown/binding-android-arm64@1.1.2': @@ -1466,7 +1590,7 @@ snapshots: obug: 2.1.3 std-env: 4.1.0 tinyrainbow: 3.1.0 - vitest: 4.1.9(@types/node@20.19.30)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2)) + vitest: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@20.19.30)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2)) '@vitest/expect@4.1.9': dependencies: @@ -2001,7 +2125,7 @@ snapshots: tsx: 4.21.0 yaml: 2.8.2 - vitest@4.1.9(@types/node@20.19.30)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2)): + vitest@4.1.9(@opentelemetry/api@1.9.1)(@types/node@20.19.30)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2)): dependencies: '@vitest/expect': 4.1.9 '@vitest/mocker': 4.1.9(vite@8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2)) @@ -2024,6 +2148,7 @@ snapshots: vite: 8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2) why-is-node-running: 2.3.0 optionalDependencies: + '@opentelemetry/api': 1.9.1 '@types/node': 20.19.30 '@vitest/coverage-v8': 4.1.9(vitest@4.1.9) transitivePeerDependencies: diff --git a/tests/pnpm-workspace.yaml b/tests/pnpm-workspace.yaml new file mode 100644 index 00000000..5ed0b5af --- /dev/null +++ b/tests/pnpm-workspace.yaml @@ -0,0 +1,2 @@ +allowBuilds: + esbuild: true From ed141a759e49177ac703bf5493fa364f075ab007 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:33:53 +0000 Subject: [PATCH 04/10] fix: resolve nested plugin sub-skill directories in harness --- tests/harness/copilot-client.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/harness/copilot-client.ts b/tests/harness/copilot-client.ts index 1e05bb6a..e6a12464 100644 --- a/tests/harness/copilot-client.ts +++ b/tests/harness/copilot-client.ts @@ -128,6 +128,24 @@ export class SkillCopilotClient implements CopilotClient { if (existsSync(candidate)) { return candidate; } + + const pluginSkillsDir = join(this.pluginsDir, entry.name, "skills"); + const skillEntries = existsSync(pluginSkillsDir) + ? readdirSync(pluginSkillsDir, { withFileTypes: true }) + : []; + for (const skillEntry of skillEntries) { + if (!skillEntry.isDirectory()) { + continue; + } + const nestedCandidate = join( + pluginSkillsDir, + skillEntry.name, + skillName, + ); + if (existsSync(join(nestedCandidate, "SKILL.md"))) { + return nestedCandidate; + } + } } } From cf5b071986a44a74ec2d7100f8e93173af0b7af9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 17 Jul 2026 20:28:47 +0000 Subject: [PATCH 05/10] fix: add packages entry to tests pnpm workspace --- tests/pnpm-workspace.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/pnpm-workspace.yaml b/tests/pnpm-workspace.yaml index 5ed0b5af..03cbbe52 100644 --- a/tests/pnpm-workspace.yaml +++ b/tests/pnpm-workspace.yaml @@ -1,2 +1,5 @@ +packages: + - . + allowBuilds: esbuild: true From b78c67dab79518e251d680e349774011cb8e37c1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 17 Jul 2026 21:02:52 +0000 Subject: [PATCH 06/10] fix: align vally workflow CLI with 0.9 metadata --- .github/workflows/vally-evaluation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vally-evaluation.yml b/.github/workflows/vally-evaluation.yml index b5975531..dabe4923 100644 --- a/.github/workflows/vally-evaluation.yml +++ b/.github/workflows/vally-evaluation.yml @@ -36,7 +36,7 @@ jobs: node-version: "20" - name: Install Vally CLI - run: npm install -g @microsoft/vally-cli@0.6.0 + run: npm install -g @microsoft/vally-cli@0.9.0 - name: Verify Vally CLI run: vally --version From c0d30ac821998107a42ab588fe5daced507e8a1d Mon Sep 17 00:00:00 2001 From: Larry Osterman Date: Tue, 21 Jul 2026 14:47:25 -0700 Subject: [PATCH 07/10] Use vally version 0.10.0 --- .github/workflows/run-vally-evaluations.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/run-vally-evaluations.yml b/.github/workflows/run-vally-evaluations.yml index 657e54d9..a3afcdad 100644 --- a/.github/workflows/run-vally-evaluations.yml +++ b/.github/workflows/run-vally-evaluations.yml @@ -44,17 +44,13 @@ jobs: node-version: "24" - name: Install Vally CLI -<<<<<<< HEAD:.github/workflows/vally-evaluation.yml - run: npm install -g @microsoft/vally-cli@0.9.0 -======= - run: npm install -g @microsoft/vally-cli@0.7.0 + run: npm install -g @microsoft/vally-cli@0.10.0 - name: Install Rust toolchain uses: actions-rs/toolchain@v1 with: toolchain: stable override: true ->>>>>>> main:.github/workflows/run-vally-evaluations.yml - name: Verify Vally CLI run: vally --version From 3f7eb205d0032c8f06adb392daa81688f650d828 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 21 Jul 2026 21:52:49 +0000 Subject: [PATCH 08/10] Fix review comments for workflow and nested skill tests --- .github/workflows/run-vally-evaluations.yml | 2 +- tests/harness/copilot-client.test.ts | 56 +++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-vally-evaluations.yml b/.github/workflows/run-vally-evaluations.yml index a3afcdad..e49244c4 100644 --- a/.github/workflows/run-vally-evaluations.yml +++ b/.github/workflows/run-vally-evaluations.yml @@ -44,7 +44,7 @@ jobs: node-version: "24" - name: Install Vally CLI - run: npm install -g @microsoft/vally-cli@0.10.0 + run: npm install -g @microsoft/vally-cli@0.9.0 - name: Install Rust toolchain uses: actions-rs/toolchain@v1 diff --git a/tests/harness/copilot-client.test.ts b/tests/harness/copilot-client.test.ts index 53100cdc..9abad66c 100644 --- a/tests/harness/copilot-client.test.ts +++ b/tests/harness/copilot-client.test.ts @@ -1,4 +1,12 @@ import { describe, expect, it } from "vitest"; +import { + mkdirSync, + mkdtempSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; import { CopilotGenerationError, @@ -49,6 +57,54 @@ describe("SkillCopilotClient.extractCode", () => { }); }); +describe("SkillCopilotClient.loadSkillContext", () => { + it("loads skill context from nested plugins//skills//", () => { + const basePath = mkdtempSync(join(tmpdir(), "copilot-client-")); + const nestedSkillDir = join( + basePath, + ".github/plugins/test-plugin/skills/rust/azure-cosmos-rust", + ); + mkdirSync(nestedSkillDir, { recursive: true }); + writeFileSync( + join(nestedSkillDir, "SKILL.md"), + "---\nname: azure-cosmos-rust\ndescription: test\n---\n\ncontent", + "utf-8", + ); + + try { + const client = new SkillCopilotClient(basePath, true); + const context = client.loadSkillContext("azure-cosmos-rust"); + expect(context).toContain("# Skill: azure-cosmos-rust"); + expect(context).toContain("content"); + } finally { + rmSync(basePath, { recursive: true, force: true }); + } + }); + + it("does not match non-skill nested directories", () => { + const basePath = mkdtempSync(join(tmpdir(), "copilot-client-")); + const nonMatchingDir = join( + basePath, + ".github/plugins/test-plugin/skills/rust/not-the-skill", + ); + mkdirSync(nonMatchingDir, { recursive: true }); + writeFileSync( + join(nonMatchingDir, "SKILL.md"), + "---\nname: not-the-skill\ndescription: test\n---\n\ncontent", + "utf-8", + ); + + try { + const client = new SkillCopilotClient(basePath, true); + expect(() => client.loadSkillContext("azure-cosmos-rust")).toThrow( + "Skill not found: azure-cosmos-rust", + ); + } finally { + rmSync(basePath, { recursive: true, force: true }); + } + }); +}); + describe("classifyCopilotError", () => { it("classifies timeout and marks retryable", () => { const classified = classifyCopilotError( From bd1d8a8496ffe2b7f2aa42cdaf5a6ffd01f903bd Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:49:38 +0000 Subject: [PATCH 09/10] fix review comments from review 4749328063 --- .../vally/grader-plugins/rust-cargo-build-failure/index.ts | 2 +- .../azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/index.ts b/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/index.ts index 786fbb03..db1d6452 100644 --- a/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/index.ts +++ b/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/index.ts @@ -424,7 +424,7 @@ class CargoBuildTrajectoryGrader implements Grader { name: "rust-cargo-build-failure-check", description: "Rust-specific grader: checks trajectory for failed cargo build tool calls, extracts Rust compiler errors (E0XXX), and scales score by failure ratio", - behavior: { requiresWorkspace: false }, + behavior: { requiresWorkspace: true }, determinism: "static", reference: "reference-free", temporalScope: "trajectory-level", diff --git a/tests/scenarios/azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml b/tests/scenarios/azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml index 2c73994a..64b668f8 100644 --- a/tests/scenarios/azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml +++ b/tests/scenarios/azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml @@ -9,6 +9,7 @@ defaults: runs: 1 timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-storage-keyvault-encryption-library prompt: | @@ -89,7 +90,6 @@ stimuli: - type: prompt name: "Key Vault-backed encryption-at-rest design completeness" config: - model: gpt-5.3-codex prompt: | Verify the generated Rust solution is a reusable library and includes: - Blob Storage client setup with account endpoint + TokenCredential From 1475e00303293865ed4dbabf510d597452fdbfce Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 21 Jul 2026 23:02:14 +0000 Subject: [PATCH 10/10] Rename grader_model to judge_model in Vally eval defaults for Rust scenarios --- tests/scenarios/azure-eventhub-rust/vally/eval.yaml | 2 +- .../azure-eventhub-rust/vally/skill_effectiveness_eval.yaml | 2 +- tests/scenarios/azure-identity-rust/vally/eval.yaml | 2 +- .../azure-identity-rust/vally/skill_effectiveness_eval.yaml | 2 +- .../scenarios/azure-keyvault-certificates-rust/vally/eval.yaml | 2 +- .../vally/skill_effectiveness_eval.yaml | 2 +- tests/scenarios/azure-keyvault-keys-rust/vally/eval.yaml | 2 +- .../vally/skill_effectiveness_eval.yaml | 2 +- tests/scenarios/azure-keyvault-secrets-rust/vally/eval.yaml | 2 +- .../vally/skill_effectiveness_eval.yaml | 2 +- .../azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml | 2 +- tests/scenarios/azure-storage-blob-rust/vally/eval.yaml | 2 +- .../azure-storage-blob-rust/vally/skill_effectiveness_eval.yaml | 2 +- tests/scenarios/azure-storage-queue-rust/vally/eval.yaml | 2 +- .../vally/skill_effectiveness_eval.yaml | 2 +- 15 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/scenarios/azure-eventhub-rust/vally/eval.yaml b/tests/scenarios/azure-eventhub-rust/vally/eval.yaml index e9a12102..350a0187 100644 --- a/tests/scenarios/azure-eventhub-rust/vally/eval.yaml +++ b/tests/scenarios/azure-eventhub-rust/vally/eval.yaml @@ -9,7 +9,7 @@ defaults: runs: 1 timeout: 60m model: claude-sonnet-4.6 - grader_model: gpt-5.5 + judge_model: gpt-5.5 stimuli: - name: rust-eventhub-producer-consumer prompt: | diff --git a/tests/scenarios/azure-eventhub-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-eventhub-rust/vally/skill_effectiveness_eval.yaml index 04589dd5..c3834eb8 100644 --- a/tests/scenarios/azure-eventhub-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-eventhub-rust/vally/skill_effectiveness_eval.yaml @@ -9,7 +9,7 @@ defaults: runs: 1 timeout: 60m model: claude-sonnet-4.6 - grader_model: gpt-5.5 + judge_model: gpt-5.5 stimuli: - name: rust-eventhub-producer-consumer prompt: | diff --git a/tests/scenarios/azure-identity-rust/vally/eval.yaml b/tests/scenarios/azure-identity-rust/vally/eval.yaml index 11958622..ce7a0b47 100644 --- a/tests/scenarios/azure-identity-rust/vally/eval.yaml +++ b/tests/scenarios/azure-identity-rust/vally/eval.yaml @@ -9,7 +9,7 @@ defaults: runs: 1 timeout: 60m model: claude-sonnet-4.6 - grader_model: gpt-5.5 + judge_model: gpt-5.5 stimuli: - name: rust-identity-credentials prompt: | diff --git a/tests/scenarios/azure-identity-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-identity-rust/vally/skill_effectiveness_eval.yaml index 4195da52..882fbf57 100644 --- a/tests/scenarios/azure-identity-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-identity-rust/vally/skill_effectiveness_eval.yaml @@ -9,7 +9,7 @@ defaults: runs: 1 timeout: 60m model: claude-sonnet-4.6 - grader_model: gpt-5.5 + judge_model: gpt-5.5 stimuli: - name: rust-identity-credentials prompt: | diff --git a/tests/scenarios/azure-keyvault-certificates-rust/vally/eval.yaml b/tests/scenarios/azure-keyvault-certificates-rust/vally/eval.yaml index 7577937d..579af3b4 100644 --- a/tests/scenarios/azure-keyvault-certificates-rust/vally/eval.yaml +++ b/tests/scenarios/azure-keyvault-certificates-rust/vally/eval.yaml @@ -9,7 +9,7 @@ defaults: runs: 1 timeout: 60m model: claude-sonnet-4.6 - grader_model: gpt-5.5 + judge_model: gpt-5.5 stimuli: - name: rust-keyvault-certificates-operations prompt: | diff --git a/tests/scenarios/azure-keyvault-certificates-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-keyvault-certificates-rust/vally/skill_effectiveness_eval.yaml index 6b1de8c7..e40678c6 100644 --- a/tests/scenarios/azure-keyvault-certificates-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-keyvault-certificates-rust/vally/skill_effectiveness_eval.yaml @@ -9,7 +9,7 @@ defaults: runs: 1 timeout: 60m model: claude-sonnet-4.6 - grader_model: gpt-5.5 + judge_model: gpt-5.5 stimuli: - name: rust-keyvault-certificates-operations prompt: | diff --git a/tests/scenarios/azure-keyvault-keys-rust/vally/eval.yaml b/tests/scenarios/azure-keyvault-keys-rust/vally/eval.yaml index 97dfce74..07ecb25c 100644 --- a/tests/scenarios/azure-keyvault-keys-rust/vally/eval.yaml +++ b/tests/scenarios/azure-keyvault-keys-rust/vally/eval.yaml @@ -9,7 +9,7 @@ defaults: runs: 1 timeout: 60m model: claude-sonnet-4.6 - grader_model: gpt-5.5 + judge_model: gpt-5.5 stimuli: - name: rust-keyvault-keys-operations prompt: | diff --git a/tests/scenarios/azure-keyvault-keys-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-keyvault-keys-rust/vally/skill_effectiveness_eval.yaml index 3b2fd9b8..b1c8cce1 100644 --- a/tests/scenarios/azure-keyvault-keys-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-keyvault-keys-rust/vally/skill_effectiveness_eval.yaml @@ -9,7 +9,7 @@ defaults: runs: 1 timeout: 60m model: claude-sonnet-4.6 - grader_model: gpt-5.5 + judge_model: gpt-5.5 stimuli: - name: rust-keyvault-keys-operations prompt: | diff --git a/tests/scenarios/azure-keyvault-secrets-rust/vally/eval.yaml b/tests/scenarios/azure-keyvault-secrets-rust/vally/eval.yaml index cd3f875a..80ca4ecb 100644 --- a/tests/scenarios/azure-keyvault-secrets-rust/vally/eval.yaml +++ b/tests/scenarios/azure-keyvault-secrets-rust/vally/eval.yaml @@ -9,7 +9,7 @@ defaults: runs: 1 timeout: 60m model: claude-sonnet-4.6 - grader_model: gpt-5.5 + judge_model: gpt-5.5 stimuli: - name: rust-keyvault-secrets-operations prompt: | diff --git a/tests/scenarios/azure-keyvault-secrets-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-keyvault-secrets-rust/vally/skill_effectiveness_eval.yaml index 1278b75a..bfa17bb0 100644 --- a/tests/scenarios/azure-keyvault-secrets-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-keyvault-secrets-rust/vally/skill_effectiveness_eval.yaml @@ -9,7 +9,7 @@ defaults: runs: 1 timeout: 60m model: claude-sonnet-4.6 - grader_model: gpt-5.5 + judge_model: gpt-5.5 stimuli: - name: rust-keyvault-secrets-operations prompt: | diff --git a/tests/scenarios/azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml b/tests/scenarios/azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml index 64b668f8..dd7d981e 100644 --- a/tests/scenarios/azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml +++ b/tests/scenarios/azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml @@ -9,7 +9,7 @@ defaults: runs: 1 timeout: 60m model: claude-sonnet-4.6 - grader_model: gpt-5.5 + judge_model: gpt-5.5 stimuli: - name: rust-storage-keyvault-encryption-library prompt: | diff --git a/tests/scenarios/azure-storage-blob-rust/vally/eval.yaml b/tests/scenarios/azure-storage-blob-rust/vally/eval.yaml index 2e6db220..ee58b032 100644 --- a/tests/scenarios/azure-storage-blob-rust/vally/eval.yaml +++ b/tests/scenarios/azure-storage-blob-rust/vally/eval.yaml @@ -9,7 +9,7 @@ defaults: runs: 1 timeout: 60m model: claude-sonnet-4.6 - grader_model: gpt-5.5 + judge_model: gpt-5.5 stimuli: - name: rust-crud-blobs prompt: | diff --git a/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_eval.yaml index 08818e29..67974afa 100644 --- a/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_eval.yaml @@ -9,7 +9,7 @@ defaults: runs: 1 timeout: 60m model: claude-sonnet-4.6 - grader_model: gpt-5.5 + judge_model: gpt-5.5 stimuli: - name: rust-crud-blobs prompt: | diff --git a/tests/scenarios/azure-storage-queue-rust/vally/eval.yaml b/tests/scenarios/azure-storage-queue-rust/vally/eval.yaml index 007e4d67..e750afe6 100644 --- a/tests/scenarios/azure-storage-queue-rust/vally/eval.yaml +++ b/tests/scenarios/azure-storage-queue-rust/vally/eval.yaml @@ -9,7 +9,7 @@ defaults: runs: 1 timeout: 60m model: claude-sonnet-4.6 - grader_model: gpt-5.5 + judge_model: gpt-5.5 stimuli: - name: rust-queue-operations prompt: | diff --git a/tests/scenarios/azure-storage-queue-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-storage-queue-rust/vally/skill_effectiveness_eval.yaml index 78ad4708..1ca91b02 100644 --- a/tests/scenarios/azure-storage-queue-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-storage-queue-rust/vally/skill_effectiveness_eval.yaml @@ -9,7 +9,7 @@ defaults: runs: 1 timeout: 60m model: claude-sonnet-4.6 - grader_model: gpt-5.5 + judge_model: gpt-5.5 stimuli: - name: rust-queue-operations prompt: |