From 9c03b7fe0895eb251bf8f228b7ecbb74ff6adb98 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Thu, 2 Jul 2026 17:26:00 +0800 Subject: [PATCH] chore: add dependency matrix sync and CI gates Co-Authored-By: Codex --- .github/workflows/validate.yml | 8 +++ docs/internal_dependency_pin_policy.md | 7 +++ docs/internal_dependency_pin_policy.zh-CN.md | 7 +++ python/README.md | 3 + python/pyproject.toml | 1 + .../check_internal_dependency_matrix.py | 55 +++++++++++++++++ .../tests/test_internal_dependency_matrix.py | 61 +++++++++++++++++++ 7 files changed, 142 insertions(+) create mode 100644 python/README.md diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 29980fa..a58c757 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -8,6 +8,14 @@ permissions: contents: read jobs: + actionlint: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6 + - name: Actionlint + uses: docker://rhysd/actionlint:1.7.12 + python: runs-on: ubuntu-latest timeout-minutes: 15 diff --git a/docs/internal_dependency_pin_policy.md b/docs/internal_dependency_pin_policy.md index d7324c9..5371662 100644 --- a/docs/internal_dependency_pin_policy.md +++ b/docs/internal_dependency_pin_policy.md @@ -13,6 +13,13 @@ QuantStrategyLab shares Python packages across platforms, strategies, and pipeli python3 scripts/check_internal_dependency_matrix.py --projects-root .. --strict ``` +When drift is found repeatedly, you can regenerate the matrix from local consumer files in a single step: + +```bash +python3 scripts/check_internal_dependency_matrix.py --projects-root .. --generate --json > /tmp/internal_dependency_matrix.json +python3 scripts/check_internal_dependency_matrix.py --projects-root .. --sync +``` + The checker compares matrix entries against consumer `requirements.txt`, `requirements-lock.txt`, and `pyproject.toml` files in sibling repositories. With `--strict`, ref mismatches fail CI even when sibling repos are not checked out locally. ## Pin formats diff --git a/docs/internal_dependency_pin_policy.zh-CN.md b/docs/internal_dependency_pin_policy.zh-CN.md index b1b12f3..1e7968a 100644 --- a/docs/internal_dependency_pin_policy.zh-CN.md +++ b/docs/internal_dependency_pin_policy.zh-CN.md @@ -13,6 +13,13 @@ QuantStrategyLab 通过 git URL pin 在平台、策略与 pipeline 之间共享 python3 scripts/check_internal_dependency_matrix.py --projects-root .. --strict ``` +若本地 consumer 文件与 matrix 漂移，可先从本地消费方依赖文件重建并同步： + +```bash +python3 scripts/check_internal_dependency_matrix.py --projects-root .. --generate --json > /tmp/internal_dependency_matrix.json +python3 scripts/check_internal_dependency_matrix.py --projects-root .. --sync +``` + 该脚本会将 matrix 条目与各 consumer 仓库中的 `requirements.txt`、`requirements-lock.txt`、`pyproject.toml` 对比。启用 `--strict` 时，即使本地未 checkout sibling 仓库，ref 不一致也会导致 CI 失败。 ## Pin 格式 diff --git a/python/README.md b/python/README.md new file mode 100644 index 0000000..f2a5f6d --- /dev/null +++ b/python/README.md @@ -0,0 +1,3 @@ +# quant-runtime-settings + +Python helper package for QuantStrategyLab runtime target settings and validation scripts. diff --git a/python/pyproject.toml b/python/pyproject.toml index f1b20b0..44ddd30 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -2,6 +2,7 @@ name = "quant-runtime-settings" version = "0.1.0" description = "Declarative runtime target settings for QuantStrategyLab deployments" +readme = "README.md" requires-python = ">=3.11" [tool.ruff] diff --git a/python/scripts/check_internal_dependency_matrix.py b/python/scripts/check_internal_dependency_matrix.py index 5c84be6..ed06150 100644 --- a/python/scripts/check_internal_dependency_matrix.py +++ b/python/scripts/check_internal_dependency_matrix.py @@ -19,6 +19,14 @@ r"git\+https://github\.com/QuantStrategyLab/" r"(?P[A-Za-z0-9_.-]+)\.git@(?P[A-Za-z0-9_.-]+)" ) +TRACKED_DEPENDENCY_PATHS = ("requirements.txt", "requirements-lock.txt", "pyproject.toml") + + +def _sort_dependency_pins(pins: list[DependencyPin]) -> list[DependencyPin]: + return sorted( + set(pins), + key=lambda pin: (pin.consumer_repo, pin.path, pin.package, pin.source_repo, pin.ref), + ) @dataclass(frozen=True) @@ -48,6 +56,33 @@ def ok(self) -> bool: return not self.issues +def matrix_payload(pins: list[DependencyPin]) -> dict[str, Any]: + return { + "schema_version": 1, + "dependencies": [ + { + "consumer_repo": pin.consumer_repo, + "path": pin.path, + "package": pin.package, + "source_repo": pin.source_repo, + "ref": pin.ref, + } + for pin in _sort_dependency_pins(pins) + ], + } + + +def collect_dependency_pins_from_projects(projects_root: Path) -> list[DependencyPin]: + pins: list[DependencyPin] = [] + for project_dir in sorted(p for p in projects_root.iterdir() if p.is_dir() and not p.name.startswith(".")): + for relative_path in TRACKED_DEPENDENCY_PATHS: + path = project_dir / relative_path + if not path.is_file(): + continue + pins.extend(parse_dependency_pins(project_dir.name, relative_path, path.read_text(encoding="utf-8"))) + return _sort_dependency_pins(pins) + + def load_matrix(path: Path) -> list[DependencyPin]: payload = json.loads(path.read_text(encoding="utf-8")) if payload.get("schema_version") != 1: @@ -127,6 +162,16 @@ def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="Report QuantStrategyLab internal dependency pin drift.") parser.add_argument("--matrix", type=Path, default=DEFAULT_MATRIX_PATH) parser.add_argument("--projects-root", type=Path, default=DEFAULT_PROJECTS_ROOT) + parser.add_argument( + "--generate", + action="store_true", + help="Generate internal dependency matrix payload from local consumer dependency files.", + ) + parser.add_argument( + "--sync", + action="store_true", + help="Overwrite --matrix with generated internal dependency payload.", + ) parser.add_argument("--json", action="store_true", help="Print machine-readable report.") parser.add_argument("--strict", action="store_true", help="Exit non-zero when drift is detected.") parser.add_argument( @@ -139,6 +184,16 @@ def build_parser() -> argparse.ArgumentParser: def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) + if args.generate or args.sync: + generated_payload = matrix_payload(collect_dependency_pins_from_projects(projects_root=args.projects_root)) + rendered_payload = json.dumps(generated_payload, ensure_ascii=False, indent=2) + print(rendered_payload) + if args.sync: + args.matrix.write_text(rendered_payload + "\n", encoding="utf-8") + if not args.json: + print(f"synced matrix -> {args.matrix}") + return 0 + report = check_matrix(matrix_pins=load_matrix(args.matrix), projects_root=args.projects_root) issues = list(report.issues) if args.require_consumer_files and report.missing_files: diff --git a/python/tests/test_internal_dependency_matrix.py b/python/tests/test_internal_dependency_matrix.py index 83bfccf..8b54410 100644 --- a/python/tests/test_internal_dependency_matrix.py +++ b/python/tests/test_internal_dependency_matrix.py @@ -1,6 +1,7 @@ from __future__ import annotations import importlib.util +import json import sys import unittest from pathlib import Path @@ -100,6 +101,66 @@ def test_require_consumer_files_treats_missing_paths_as_issues(self): self.assertEqual(report.missing_files, ["ExamplePlatform/requirements.txt"]) self.assertEqual(report.issues, []) + def test_collect_dependency_pins_from_projects(self): + projects_root = self._make_projects_root( + { + "ExampleA/pyproject.toml": "quant-platform-kit @ git+https://github.com/QuantStrategyLab/QuantPlatformKit.git@a11", + "ExampleB/requirements.txt": "us-equity-strategies @ git+https://github.com/QuantStrategyLab/UsEquityStrategies.git@b22", + "ExampleB/requirements-lock.txt": "crypto-strategies @ git+https://github.com/QuantStrategyLab/CryptoStrategies.git@c33", + } + ) + + pins = check_internal_dependency_matrix.collect_dependency_pins_from_projects(projects_root) + rows = [(pin.consumer_repo, pin.path, pin.package, pin.source_repo, pin.ref) for pin in pins] + + self.assertEqual(rows, [ + ("ExampleA", "pyproject.toml", "quant-platform-kit", "QuantPlatformKit", "a11"), + ("ExampleB", "requirements-lock.txt", "crypto-strategies", "CryptoStrategies", "c33"), + ("ExampleB", "requirements.txt", "us-equity-strategies", "UsEquityStrategies", "b22"), + ]) + + def test_sync_rewrites_matrix_with_stable_order(self): + projects_root = self._make_projects_root( + { + "ExampleB/requirements.txt": "us-equity-strategies @ git+https://github.com/QuantStrategyLab/UsEquityStrategies.git@b22", + "ExampleA/pyproject.toml": "quant-platform-kit @ git+https://github.com/QuantStrategyLab/QuantPlatformKit.git@a11", + } + ) + matrix_path = projects_root / "internal_dependency_matrix.json" + matrix_path.write_text( + ( + "{\n" + ' "schema_version": 1,\n' + ' "dependencies": [\n' + ' {\n' + ' "consumer_repo": "ExampleB",\n' + ' "path": "requirements.txt",\n' + ' "package": "us-equity-strategies",\n' + ' "source_repo": "UsEquityStrategies",\n' + ' "ref": "b22"\n' + " }\n" + " ]\n" + "}\n" + ), + encoding="utf-8", + ) + + projects_payload = check_internal_dependency_matrix.matrix_payload( + check_internal_dependency_matrix.collect_dependency_pins_from_projects(projects_root) + ) + exit_code = check_internal_dependency_matrix.main( + ["--sync", "--projects-root", str(projects_root), "--matrix", str(matrix_path)] + ) + synced_payload = json.loads(matrix_path.read_text(encoding="utf-8")) + + self.assertEqual(exit_code, 0) + self.assertEqual(synced_payload["schema_version"], 1) + self.assertEqual(synced_payload["dependencies"], projects_payload["dependencies"]) + self.assertLess( + synced_payload["dependencies"][0]["path"], + synced_payload["dependencies"][-1]["path"], + ) + def _make_projects_root(self, files: dict[str, str]) -> Path: import tempfile