From 6fcebacb92018b9e0b223843ec7b042221bda5c0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Nov 2025 04:02:57 +0000 Subject: [PATCH 1/4] Add Metrics Agent for frontmatter validation and reporting Implements a comprehensive frontmatter metrics system to track coverage, validate schema compliance, detect broken references, and identify version skew across the repository. The system provides weekly automated reporting via GitHub Actions and creates/updates tracking issues for visibility. - Add metrics configuration (metrics/metrics.config.json) - Add frontmatter validation script (metrics/frontmatter-metrics.js) - Add weekly GitHub Actions workflow (.github/workflows/frontmatter-metrics.yml) - Update package.json with metrics scripts and required dependencies - Support for AJV schema validation, broken reference detection, and version comparison - Configurable thresholds for coverage, unknown keys, broken refs, and version skew --- .github/workflows/frontmatter-metrics.yml | 78 +++++++ metrics/frontmatter-metrics.js | 251 ++++++++++++++++++++++ metrics/metrics.config.json | 35 +++ package.json | 8 + 4 files changed, 372 insertions(+) create mode 100644 .github/workflows/frontmatter-metrics.yml create mode 100644 metrics/frontmatter-metrics.js create mode 100644 metrics/metrics.config.json diff --git a/.github/workflows/frontmatter-metrics.yml b/.github/workflows/frontmatter-metrics.yml new file mode 100644 index 000000000..088edfd9d --- /dev/null +++ b/.github/workflows/frontmatter-metrics.yml @@ -0,0 +1,78 @@ +name: Frontmatter Metrics + +on: + workflow_dispatch: + schedule: + - cron: "0 6 * * 1" # Mondays 06:00 UTC + +permissions: + contents: read + actions: read + issues: write + +jobs: + run: + runs-on: ubuntu-latest + steps: + - name: Checkout (develop) + uses: actions/checkout@v4 + with: + ref: develop + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Install deps + run: npm ci || npm i + + - name: Run metrics + run: npm run metrics:ci + + - name: Upload JSON artifact + uses: actions/upload-artifact@v4 + with: + name: frontmatter-metrics-json + path: metrics/out/frontmatter-metrics.json + if-no-files-found: warn + + - name: Upload Markdown report + uses: actions/upload-artifact@v4 + with: + name: frontmatter-metrics-md + path: metrics/out/frontmatter-metrics.md + if-no-files-found: error + + - name: Create or update tracking issue + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const body = fs.readFileSync('metrics/out/frontmatter-metrics.md','utf8'); + const title = 'Weekly Frontmatter Metrics'; + // Find existing issue + const { data: issues } = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: '' + }); + const existing = issues.find(i => i.title === title); + if (existing) { + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + body + }); + core.info(`Updated issue #${existing.number}`); + } else { + const res = await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title, + body + }); + core.info(`Created issue #${res.data.number}`); + } diff --git a/metrics/frontmatter-metrics.js b/metrics/frontmatter-metrics.js new file mode 100644 index 000000000..a5fc877bf --- /dev/null +++ b/metrics/frontmatter-metrics.js @@ -0,0 +1,251 @@ +#!/usr/bin/env node +/* eslint-disable no-console */ +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import glob from "glob"; +import matter from "gray-matter"; +import micromatch from "micromatch"; +import YAML from "js-yaml"; +import Ajv from "ajv"; +import addFormats from "ajv-formats"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +async function readJSON(p) { + return JSON.parse(await fs.readFile(p, "utf8")); +} +async function readText(p) { + return await fs.readFile(p, "utf8"); +} + +function loadConfig() { + const cfgPath = path.join(process.cwd(), "metrics/metrics.config.json"); + return readJSON(cfgPath); +} + +async function listFiles(includeGlobs, excludeGlobs) { + const matches = await new Promise((resolve, reject) => + glob("**/*", { dot: true, nodir: true }, (e, files) => + e ? reject(e) : resolve(files) + ) + ); + const included = micromatch(matches, includeGlobs); + return micromatch.not(included, excludeGlobs); +} + +function getCategoryForPath(fp) { + if (fp.startsWith(".github/ISSUE_TEMPLATE/")) return "issue_template"; + if (fp.startsWith(".github/PULL_REQUEST_TEMPLATE/")) return "pr_template"; + if (fp.startsWith(".github/DISCUSSION_TEMPLATE/")) return "discussion_template"; + return "md"; +} + +function extractFrontmatterMD(text) { + // gray-matter handles '---' fenced YAML + const fm = matter(text); + if (!fm.data || Object.keys(fm.data).length === 0) return { data: null }; + return { data: fm.data }; +} + +function extractFrontmatterYAML(text) { + // For *.yml templates that are entirely YAML (no fences) + try { + const data = YAML.load(text); + if (data && typeof data === "object") return { data }; + } catch (e) { /* noop */ } + return { data: null }; +} + +async function loadSchemaAndValidator() { + const ajv = new Ajv({ allErrors: true, strict: false }); + addFormats(ajv); + const schemaPath = path.join(process.cwd(), "schemas/frontmatter.schema.json"); + const schema = await readJSON(schemaPath); + const validate = ajv.compile(schema); + return { validate, schema }; +} + +async function readRepoVersion(versionPath) { + try { + const txt = await readText(path.join(process.cwd(), versionPath)); + return txt.trim(); + } catch { + return null; + } +} + +function semverCompare(a, b) { + const pa = a.split(".").map(Number); + const pb = b.split(".").map(Number); + for (let i = 0; i < 3; i++) { + if ((pa[i] ?? 0) > (pb[i] ?? 0)) return 1; + if ((pa[i] ?? 0) < (pb[i] ?? 0)) return -1; + } + return 0; +} + +function collectReferences(obj) { + const refs = obj?.references; + if (!Array.isArray(refs)) return []; + return refs.filter(Boolean); +} + +function isInternalDevelop(url) { + return typeof url === "string" && + url.includes("github.com/lightspeedwp/.github/blob/develop/"); +} + +async function urlLooksResolvable(u) { + // For internal blob links we rely on path existence in the repo (fast, offline). + if (!isInternalDevelop(u)) return true; // ignore external for now + const rel = u.split("blob/develop/")[1]; + try { + await fs.access(path.join(process.cwd(), rel)); + return true; + } catch { + return false; + } +} + +async function main() { + const cfg = await loadConfig(); + const repoVersion = await readRepoVersion(cfg.version.repoVersionFile); + + const files = await listFiles(cfg.includeGlobs, cfg.excludeGlobs); + const { validate } = await loadSchemaAndValidator(); + + let eligible = 0, withValid = 0; + const unknownKeys = []; + const brokenRefs = []; + const versionSkews = []; + const perFile = []; + + for (const fp of files) { + const category = getCategoryForPath(fp); + const eligibleByType = Boolean(cfg.frontmatterEligible[category]); + const text = await readText(fp); + + let data = null; + if (fp.endsWith(".md")) data = extractFrontmatterMD(text).data; + else if (fp.endsWith(".yml") || fp.endsWith(".yaml")) data = extractFrontmatterYAML(text).data; + + if (!eligibleByType) { + perFile.push({ fp, category, eligible: false, hasFrontmatter: Boolean(data) }); + continue; + } + + eligible++; + if (!data) { + perFile.push({ fp, category, eligible: true, hasFrontmatter: false, errors: ["missing frontmatter"] }); + continue; + } + + // Attach category so schema can branch if it uses if/then/else on `category` + data.category = data.category ?? category; + + const ok = validate(data); + if (ok) withValid++; + else { + const errs = (validate.errors || []).map(e => `${e.instancePath || "."} ${e.message}`); + perFile.push({ fp, category, eligible: true, hasFrontmatter: true, errors: errs }); + } + + // Unknown keys (if schema uses additionalProperties: false) + if (validate.errors) { + const extras = validate.errors.filter(e => e.keyword === "additionalProperties"); + extras.forEach(e => unknownKeys.push({ fp, detail: e.message })); + } + + // references audit + const refs = collectReferences(data); + for (const r of refs) { + const okRef = await urlLooksResolvable(r); + if (!okRef) brokenRefs.push({ fp, r }); + } + + // version skew (file ahead of repo) + if (cfg.version.enforceFileNotAboveRepo && repoVersion && data.version) { + if (semverCompare(String(data.version), String(repoVersion)) === 1) { + versionSkews.push({ fp, fileVersion: data.version, repoVersion }); + } + } + + if (ok) { + perFile.push({ fp, category, eligible: true, hasFrontmatter: true, errors: [] }); + } + } + + const coveragePct = eligible === 0 ? 100 : Math.round((withValid / eligible) * 100); + + const out = { + summary: { + eligible, + withValid, + coveragePct, + unknownKeys: unknownKeys.length, + brokenRefs: brokenRefs.length, + versionSkews: versionSkews.length, + thresholds: cfg.thresholds + }, + unknownKeys, + brokenRefs, + versionSkews, + files: perFile + }; + + await fs.mkdir(path.join(process.cwd(), "metrics/out"), { recursive: true }); + if (cfg.report.storeJsonArtifact) { + await fs.writeFile(cfg.report.artifactPath, JSON.stringify(out, null, 2)); + } + + // Markdown report + const md = [ + "# Weekly Frontmatter Metrics", + "", + `**Coverage:** ${coveragePct}% (${withValid}/${eligible})`, + `**Unknown keys:** ${unknownKeys.length}`, + `**Broken references:** ${brokenRefs.length}`, + `**Version skews:** ${versionSkews.length}`, + "", + "## Notes", + "- Eligible = files expected to contain valid frontmatter.", + "- Version skew = file version greater than repo VERSION.", + "", + "## Broken references", + ...(brokenRefs.length ? brokenRefs.map(b => `- \`${b.fp}\` → ${b.r}`) : ["- None"]), + "", + "## Unknown keys", + ...(unknownKeys.length ? unknownKeys.map(u => `- \`${u.fp}\`: ${u.detail}`) : ["- None"]), + "", + "## Version skews", + ...(versionSkews.length ? versionSkews.map(v => `- \`${v.fp}\`: file=${v.fileVersion} > repo=${v.repoVersion}`) : ["- None"]), + "", + "## File results (errors only)", + ...perFile + .filter(f => Array.isArray(f.errors) && f.errors.length) + .map(f => `- \`${f.fp}\` → ${f.errors.join("; ")}`) + ].join("\n"); + + await fs.writeFile(cfg.report.reportPath, md, "utf8"); + + // Set non-zero exit if thresholds fail + const t = cfg.thresholds; + const fails = []; + if (coveragePct < t.coveragePctMin) fails.push(`coverage < ${t.coveragePctMin}%`); + if (unknownKeys.length > t.unknownKeysMax) fails.push(`unknownKeys > ${t.unknownKeysMax}`); + if (brokenRefs.length > t.brokenRefsMax) fails.push(`brokenRefs > ${t.brokenRefsMax}`); + if (versionSkews.length > t.versionSkewMax) fails.push(`versionSkews > ${t.versionSkewMax}`); + + if (fails.length) { + console.error("Thresholds failed:", fails.join(", ")); + // Do not fail the job by default; uncomment the next line once calibrated: + // process.exit(1); + } +} + +main().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/metrics/metrics.config.json b/metrics/metrics.config.json new file mode 100644 index 000000000..fb3928ac5 --- /dev/null +++ b/metrics/metrics.config.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://json.schemastore.org/json-schema.json", + "includeGlobs": [ + "**/*.md", + ".github/ISSUE_TEMPLATE/*.yml", + ".github/ISSUE_TEMPLATE/*.yaml" + ], + "excludeGlobs": [ + "**/node_modules/**", + "**/.git/**", + "**/CHANGELOG.md" + ], + "frontmatterEligible": { + "md": true, + "issue_template": true, + "pr_template": false, + "discussion_template": true + }, + "thresholds": { + "coveragePctMin": 90, + "unknownKeysMax": 0, + "brokenRefsMax": 0, + "versionSkewMax": 0 + }, + "report": { + "issueTitle": "Weekly Frontmatter Metrics", + "storeJsonArtifact": true, + "artifactPath": "metrics/out/frontmatter-metrics.json", + "reportPath": "metrics/out/frontmatter-metrics.md" + }, + "version": { + "repoVersionFile": "VERSION", + "enforceFileNotAboveRepo": true + } +} diff --git a/package.json b/package.json index 18f6f39fc..4b9334250 100644 --- a/package.json +++ b/package.json @@ -83,6 +83,8 @@ "@types/node": "^20.0.0", "@typescript-eslint/eslint-plugin": "^6.0.0", "@typescript-eslint/parser": "^6.0.0", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", "all-contributors-cli": "6.26.1", "babel-jest": "^29.7.0", "bats": "^1.8.0", @@ -90,11 +92,15 @@ "eslint": "^8.0.0", "eslint-config-prettier": "^9.0.0", "eslint-plugin-prettier": "^5.0.0", + "glob": "^10.3.12", + "gray-matter": "^4.0.3", "jest": "^30.0.1", "jest-environment-jsdom": "^30.0.1", "js-yaml": "^4.1.0", "markdownlint": "^0.28.1", "mermaid": "^10.9.0", + "micromatch": "^4.0.7", + "node-fetch": "^3.3.2", "npm-package-json-lint": "^7.0.0", "playwright": "^1.40.0", "prettier": "^3.0.0", @@ -121,6 +127,8 @@ "format:md": "prettier '**/*.md' --write && markdownlint '**/*.md' --fix", "format": "npm run format:js && npm run format:css", "sync-version": "node scripts/sync-version.js", + "metrics:run": "node metrics/frontmatter-metrics.js", + "metrics:ci": "node metrics/frontmatter-metrics.js", "test:js": "jest --coverage --forceExit --detectOpenHandles", "test": "npm run test:js", "contributors:add": "all-contributors add", From d045177575860657784f1c7484b93ff2219fc9ec Mon Sep 17 00:00:00 2001 From: Ash Shaw Date: Wed, 12 Nov 2025 11:29:13 +0700 Subject: [PATCH 2/4] Update metrics/frontmatter-metrics.js Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Ash Shaw --- metrics/frontmatter-metrics.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metrics/frontmatter-metrics.js b/metrics/frontmatter-metrics.js index a5fc877bf..a7ef62cf3 100644 --- a/metrics/frontmatter-metrics.js +++ b/metrics/frontmatter-metrics.js @@ -54,7 +54,10 @@ function extractFrontmatterYAML(text) { try { const data = YAML.load(text); if (data && typeof data === "object") return { data }; - } catch (e) { /* noop */ } + } catch (e) { + // Log YAML parsing errors for debugging, but continue gracefully. + console.error("YAML parsing error in extractFrontmatterYAML:", e); + } return { data: null }; } From aafebbfef56b695af95cb0b7a34dc146a32660a2 Mon Sep 17 00:00:00 2001 From: Ash Shaw Date: Wed, 12 Nov 2025 11:29:24 +0700 Subject: [PATCH 3/4] Update metrics/frontmatter-metrics.js Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Ash Shaw --- metrics/frontmatter-metrics.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/metrics/frontmatter-metrics.js b/metrics/frontmatter-metrics.js index a7ef62cf3..81d9081a8 100644 --- a/metrics/frontmatter-metrics.js +++ b/metrics/frontmatter-metrics.js @@ -243,8 +243,10 @@ async function main() { if (fails.length) { console.error("Thresholds failed:", fails.join(", ")); - // Do not fail the job by default; uncomment the next line once calibrated: - // process.exit(1); + // Fail the job if configured to do so (set thresholds.failOnError: true in metrics.config.json) + if (t.failOnError) { + process.exit(1); + } } } From e180caf4cbf095707f338c2523831f5a38dc0a9e Mon Sep 17 00:00:00 2001 From: Ash Shaw Date: Wed, 12 Nov 2025 11:29:30 +0700 Subject: [PATCH 4/4] Update .github/workflows/frontmatter-metrics.yml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Ash Shaw --- .github/workflows/frontmatter-metrics.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/frontmatter-metrics.yml b/.github/workflows/frontmatter-metrics.yml index 088edfd9d..828867891 100644 --- a/.github/workflows/frontmatter-metrics.yml +++ b/.github/workflows/frontmatter-metrics.yml @@ -25,7 +25,7 @@ jobs: node-version: "20" - name: Install deps - run: npm ci || npm i + run: npm ci - name: Run metrics run: npm run metrics:ci