Standardize Rust Vally scenarios and remove azure-servicebus-rust

.github/workflows/run-vally-evaluations.yml

@@ -44,7 +44,7 @@ jobs:
           node-version: "24"
 
       - name: Install Vally CLI
-        run: npm install -g @microsoft/vally-cli@0.7.0
+        run: npm install -g @microsoft/vally-cli@0.9.0
 
       - name: Install Rust toolchain
         uses: actions-rs/toolchain@v1

tests/harness/copilot-client.test.ts

@@ -1,4 +1,12 @@
 import { describe, expect, it } from "vitest";
+import {
+  mkdirSync,
+  mkdtempSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
 
 import {
   CopilotGenerationError,
@@ -49,6 +57,54 @@ describe("SkillCopilotClient.extractCode", () => {
   });
 });
 
+describe("SkillCopilotClient.loadSkillContext", () => {
+  it("loads skill context from nested plugins/<plugin>/skills/<group>/<skill>", () => {
+    const basePath = mkdtempSync(join(tmpdir(), "copilot-client-"));
+    const nestedSkillDir = join(
+      basePath,
+      ".github/plugins/test-plugin/skills/rust/azure-cosmos-rust",
+    );
+    mkdirSync(nestedSkillDir, { recursive: true });
+    writeFileSync(
+      join(nestedSkillDir, "SKILL.md"),
+      "---\nname: azure-cosmos-rust\ndescription: test\n---\n\ncontent",
+      "utf-8",
+    );
+
+    try {
+      const client = new SkillCopilotClient(basePath, true);
+      const context = client.loadSkillContext("azure-cosmos-rust");
+      expect(context).toContain("# Skill: azure-cosmos-rust");
+      expect(context).toContain("content");
+    } finally {
+      rmSync(basePath, { recursive: true, force: true });
+    }
+  });
+
+  it("does not match non-skill nested directories", () => {
+    const basePath = mkdtempSync(join(tmpdir(), "copilot-client-"));
+    const nonMatchingDir = join(
+      basePath,
+      ".github/plugins/test-plugin/skills/rust/not-the-skill",
+    );
+    mkdirSync(nonMatchingDir, { recursive: true });
+    writeFileSync(
+      join(nonMatchingDir, "SKILL.md"),
+      "---\nname: not-the-skill\ndescription: test\n---\n\ncontent",
+      "utf-8",
+    );
+
+    try {
+      const client = new SkillCopilotClient(basePath, true);
+      expect(() => client.loadSkillContext("azure-cosmos-rust")).toThrow(
+        "Skill not found: azure-cosmos-rust",
+      );
+    } finally {
+      rmSync(basePath, { recursive: true, force: true });
+    }
+  });
+});
+
 describe("classifyCopilotError", () => {
   it("classifies timeout and marks retryable", () => {
     const classified = classifyCopilotError(

tests/harness/copilot-client.ts

@@ -197,6 +197,24 @@ export class SkillCopilotClient implements CopilotClient {
         if (existsSync(candidate)) {
           return candidate;
         }
+
+        const pluginSkillsDir = join(this.pluginsDir, entry.name, "skills");
+        const skillEntries = existsSync(pluginSkillsDir)
+          ? readdirSync(pluginSkillsDir, { withFileTypes: true })
+          : [];
+        for (const skillEntry of skillEntries) {
+          if (!skillEntry.isDirectory()) {
+            continue;
+          }
+          const nestedCandidate = join(
+            pluginSkillsDir,
+            skillEntry.name,
+            skillName,
+          );
+          if (existsSync(join(nestedCandidate, "SKILL.md"))) {
+            return nestedCandidate;
+          }
+        }
       }
     }
 

tests/package.json

@@ -21,7 +21,7 @@
     "zod": "^3.23.8"
   },
   "devDependencies": {
-    "@microsoft/vally": "^0.6.0",
+    "@microsoft/vally": "^0.9.0",
     "@types/node": "^20.14.0",
     "@vitest/coverage-v8": "^4.1.9",
     "eslint": "^9.5.0",
@@ -33,4 +33,4 @@
   "engines": {
     "node": ">=20.0.0"
   }
-}
+}