feat: Add evally test runner and implementation for vscode-evals

Author: dcbartlett

.gitignore

@@ -51,4 +51,6 @@ logs
 qdrant_storage/
 
 # Architect plans
-plans/
+plans/
+
+.results

apps/vscode-evals/.env.local.sample

@@ -0,0 +1 @@
+OPENROUTER_API_KEY=sk-or-v1-...

apps/vscode-evals/.vscode-test.mjs

@@ -0,0 +1,16 @@
+/**
+ * See: https://code.visualstudio.com/api/working-with-extensions/testing-extension
+ */
+
+import { defineConfig } from "@vscode/test-cli"
+
+export default defineConfig({
+	label: "integrationTest",
+	files: "out/suite/**/*.test.js",
+	workspaceFolder: ".",
+	mocha: {
+		ui: "tdd",
+		timeout: 60000,
+	},
+	launchArgs: ["--enable-proposed-api=RooVeterinaryInc.roo-cline", "--disable-extensions"],
+})

apps/vscode-evals/eslint.config.mjs

@@ -0,0 +1,4 @@
+import { config } from "@roo-code/config-eslint/base"
+
+/** @type {import("eslint").Linter.Config} */
+export default [...config]

apps/vscode-evals/package.json

@@ -0,0 +1,26 @@
+{
+	"name": "@roo-code/vscode-evals",
+	"private": true,
+	"scripts": {
+		"lint": "eslint src --ext=ts --max-warnings=0",
+		"check-types": "tsc -p tsconfig.esm.json --noEmit",
+		"format": "prettier --write src",
+		"test:ci": "pnpm -w bundle && pnpm --filter @roo-code/vscode-webview build && pnpm test:run",
+		"test:run": "rimraf out && tsc -p tsconfig.json && npx dotenvx run -f .env.local -- node ./out/runTest.js",
+		"clean": "rimraf out .turbo"
+	},
+	"devDependencies": {
+		"@roo-code/evally": "workspace:^",
+		"@roo-code/config-eslint": "workspace:^",
+		"@roo-code/config-typescript": "workspace:^",
+		"@roo-code/types": "workspace:^",
+		"@types/vscode": "^1.95.0",
+		"@vscode/test-cli": "^0.0.11",
+		"@vscode/test-electron": "^2.4.0",
+		"rimraf": "^6.0.1",
+		"typescript": "5.8.3"
+	},
+	"dependencies": {
+		"vscode": "^1.1.37"
+	}
+}

apps/vscode-evals/src/runTest.ts

@@ -0,0 +1,33 @@
+import * as path from "path"
+import * as os from "os"
+import * as fs from "fs/promises"
+
+import { runTests } from "@vscode/test-electron"
+
+async function main() {
+	try {
+		const extensionDevelopmentPath = path.resolve(__dirname, "../../../src")
+		const extensionTestsPath = path.resolve(__dirname, "./suite/index")
+		const testWorkspace = await fs.mkdtemp(path.join(os.tmpdir(), "roo-evals-workspace-"))
+		const testGrep = process.argv.find((arg, i) => process.argv[i - 1] === "--grep") || process.env.TEST_GREP
+		const testFile = process.argv.find((arg, i) => process.argv[i - 1] === "--file") || process.env.TEST_FILE
+		const extensionTestsEnv = {
+			...process.env,
+			...(testGrep && { TEST_GREP: testGrep }),
+			...(testFile && { TEST_FILE: testFile }),
+		}
+		await runTests({
+			extensionDevelopmentPath,
+			extensionTestsPath,
+			launchArgs: [testWorkspace],
+			extensionTestsEnv,
+			version: process.env.VSCODE_VERSION || "1.101.2",
+		})
+		await fs.rm(testWorkspace, { recursive: true, force: true })
+	} catch (error) {
+		console.error("Failed to run vscode evals", error)
+		process.exit(1)
+	}
+}
+
+main()