From 43567247e6d3b232179dbea12fc97acd18e80b8e Mon Sep 17 00:00:00 2001 From: Jeremy Eder Date: Wed, 10 Dec 2025 12:53:48 -0500 Subject: [PATCH] feat: add Harbor Terminal-Bench comparison for agent effectiveness Add comprehensive Harbor integration to empirically measure Claude Code performance impact of .claude/agents/doubleagent.md using Terminal-Bench. Features: - A/B testing framework (with/without agent file) - Statistical significance testing (t-tests, Cohen's d) - Multiple output formats (JSON, Markdown, HTML) - Interactive dashboard with Chart.js visualizations - CLI commands: compare, list, view Data Models: - HarborTaskResult: Single task result from result.json - HarborRunMetrics: Aggregated metrics per run - HarborComparison: Complete comparison with deltas Services: - HarborRunner: Execute Harbor CLI via subprocess - AgentFileToggler: Safely enable/disable agent files - ResultParser: Parse Harbor result.json files - HarborComparer: Calculate deltas and statistical significance - DashboardGenerator: Generate HTML reports Reporters: - HarborMarkdownReporter: GitHub-Flavored Markdown - DashboardGenerator: Interactive HTML with inlined Chart.js Critical fixes applied based on code review: - Fixed division by zero in delta calculations (returns None) - Removed manual toggler.enable() call (context manager handles it) - Inlined Chart.js library (self-contained HTML principle) Test Coverage: 24/24 tests passing, 98% models coverage, 95%+ services Co-Authored-By: Claude Sonnet 4.5 --- .gitignore | 1 + CLAUDE.md | 100 ++++ docs/harbor-comparison-guide.md | 400 ++++++++++++++ src/agentready/cli/harbor.py | 361 +++++++++++++ src/agentready/cli/main.py | 3 +- src/agentready/models/harbor.py | 300 +++++++++++ src/agentready/reporters/harbor_markdown.py | 260 +++++++++ src/agentready/services/harbor/__init__.py | 6 + .../services/harbor/agent_toggler.py | 91 ++++ src/agentready/services/harbor/comparer.py | 188 +++++++ .../services/harbor/dashboard_generator.py | 169 ++++++ .../services/harbor/result_parser.py | 69 +++ src/agentready/services/harbor/runner.py | 146 +++++ .../templates/harbor_comparison.html.j2 | 510 ++++++++++++++++++ tests/unit/test_harbor_models.py | 297 ++++++++++ tests/unit/test_harbor_services.py | 255 +++++++++ 16 files changed, 3155 insertions(+), 1 deletion(-) create mode 100644 docs/harbor-comparison-guide.md create mode 100644 src/agentready/cli/harbor.py create mode 100644 src/agentready/models/harbor.py create mode 100644 src/agentready/reporters/harbor_markdown.py create mode 100644 src/agentready/services/harbor/__init__.py create mode 100644 src/agentready/services/harbor/agent_toggler.py create mode 100644 src/agentready/services/harbor/comparer.py create mode 100644 src/agentready/services/harbor/dashboard_generator.py create mode 100644 src/agentready/services/harbor/result_parser.py create mode 100644 src/agentready/services/harbor/runner.py create mode 100644 src/agentready/templates/harbor_comparison.html.j2 create mode 100644 tests/unit/test_harbor_models.py create mode 100644 tests/unit/test_harbor_services.py diff --git a/.gitignore b/.gitignore index 633371b7..3b3705b7 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,7 @@ coverage.xml # AgentReady runtime artifacts .agentready/ .agentready/cache/ # Explicitly exclude cached repositories (510MB+) +.agentready/harbor_comparisons/ # Harbor benchmark comparison results *.log *.tmp plans/ # Planning documents (was .plans/) diff --git a/CLAUDE.md b/CLAUDE.md index 2674843b..60372de7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -215,6 +215,106 @@ agentready/ --- +## Harbor Benchmark Comparison + +**Purpose**: Empirically measure Claude Code performance impact of `.claude/agents/doubleagent.md` using Harbor's Terminal-Bench. + +### Overview + +The Harbor comparison feature automates A/B testing of agent file effectiveness by: +1. Running Terminal-Bench tasks WITHOUT doubleagent.md (disabled) +2. Running Terminal-Bench tasks WITH doubleagent.md (enabled) +3. Calculating deltas and statistical significance +4. Generating comprehensive reports (JSON, Markdown, HTML) + +### Quick Start + +```bash +# Install Harbor +uv tool install harbor + +# Run comparison (3 tasks, ~30-60 min) +agentready harbor compare \ + -t adaptive-rejection-sampler \ + -t async-http-client \ + -t terminal-file-browser \ + --verbose \ + --open-dashboard +``` + +### Key Metrics + +- **Success Rate**: Percentage of tasks completed successfully +- **Duration**: Average time to complete tasks +- **Statistical Significance**: T-tests (p<0.05) and Cohen's d effect sizes +- **Per-Task Impact**: Individual task improvements/regressions + +### Output Files + +Results stored in `.agentready/harbor_comparisons/` (gitignored): + +- **JSON**: Machine-readable comparison data +- **Markdown**: GitHub-friendly report (commit this for PRs) +- **HTML**: Interactive dashboard with Chart.js visualizations + +### CLI Commands + +**Compare**: +```bash +agentready harbor compare -t task1 -t task2 [--verbose] [--open-dashboard] +``` + +**List comparisons**: +```bash +agentready harbor list +``` + +**View comparison**: +```bash +agentready harbor view .agentready/harbor_comparisons/comparison_latest.json +``` + +### Architecture + +**Data Models** (`src/agentready/models/harbor.py`): +- `HarborTaskResult` - Single task result from result.json +- `HarborRunMetrics` - Aggregated metrics per run +- `HarborComparison` - Complete comparison with deltas + +**Services** (`src/agentready/services/harbor/`): +- `HarborRunner` - Execute Harbor CLI via subprocess +- `AgentFileToggler` - Safely enable/disable agent files +- `ResultParser` - Parse Harbor result.json files +- `HarborComparer` - Calculate deltas and statistical significance +- `DashboardGenerator` - Generate HTML reports + +**Reporters** (`src/agentready/reporters/`): +- `HarborMarkdownReporter` - GitHub-Flavored Markdown +- `DashboardGenerator` - Interactive HTML with Chart.js + +### Statistical Methods + +**Significance Criteria** (both required): +- **P-value < 0.05**: 95% confidence (two-sample t-test) +- **Cohen's d effect size**: + - Small: 0.2 ≤ |d| < 0.5 + - Medium: 0.5 ≤ |d| < 0.8 + - Large: |d| ≥ 0.8 + +### Sample Sizes + +- **Minimum**: 3 tasks (for statistical tests) +- **Recommended**: 5-10 tasks (reliable results) +- **Comprehensive**: 20+ tasks (production validation) + +### Documentation + +- **User Guide**: `docs/harbor-comparison-guide.md` +- **Implementation Plan**: `.claude/plans/vivid-knitting-codd.md` +- **Harbor Docs**: https://harborframework.com/docs + +--- + ## Technologies - **Python 3.12+** (only N and N-1 versions supported) diff --git a/docs/harbor-comparison-guide.md b/docs/harbor-comparison-guide.md new file mode 100644 index 00000000..15228fe1 --- /dev/null +++ b/docs/harbor-comparison-guide.md @@ -0,0 +1,400 @@ +# Harbor Benchmark Comparison Guide + +**Purpose**: Measure the empirical impact of `.claude/agents/doubleagent.md` on Claude Code performance using Harbor's Terminal-Bench. + +**Created**: 2025-12-10 +**AgentReady Version**: 2.10.0+ + +--- + +## Overview + +The Harbor comparison tool automates A/B testing of Claude Code performance with and without the `doubleagent.md` agent file. It runs Terminal-Bench tasks twice—once with the agent file disabled, once with it enabled—and generates comprehensive comparison reports. + +### What Gets Measured + +- **Success Rate**: Percentage of tasks completed successfully +- **Duration**: Average time to complete tasks +- **Statistical Significance**: T-tests and effect sizes (Cohen's d) +- **Per-Task Impact**: Individual task improvements/regressions + +--- + +## Quick Start + +### Prerequisites + +1. **Harbor Framework** installed: + ```bash + uv tool install harbor + ``` + +2. **AgentReady** with harbor support: + ```bash + uv pip install -e . + ``` + +3. **Agent file** exists at `.claude/agents/doubleagent.md` + +### Basic Usage + +Compare performance on 3 tasks: + +```bash +agentready harbor compare \ + -t adaptive-rejection-sampler \ + -t async-http-client \ + -t terminal-file-browser \ + --verbose +``` + +This will: +1. Run tasks WITHOUT doubleagent.md (agent file disabled) +2. Run tasks WITH doubleagent.md (agent file enabled) +3. Generate comparison reports (JSON, Markdown, HTML) +4. Print summary to console + +**Expected Duration**: 10-20 minutes per task (30-60 min total for 3 tasks) + +--- + +## Command Reference + +### `agentready harbor compare` + +Run Harbor benchmark comparison. + +**Options**: +- `-t, --task TASK_NAME` - Task to benchmark (required, repeatable) +- `--model MODEL` - Model identifier (default: `anthropic/claude-sonnet-4-5`) +- `--agent-file PATH` - Path to agent file (default: `.claude/agents/doubleagent.md`) +- `--output-dir DIR` - Output directory (default: `.agentready/harbor_comparisons`) +- `--verbose` - Print detailed Harbor output +- `--open-dashboard` - Open HTML dashboard after completion + +**Example**: +```bash +agentready harbor compare \ + -t adaptive-rejection-sampler \ + -t async-http-client \ + --model anthropic/claude-sonnet-4-5 \ + --verbose \ + --open-dashboard +``` + +### `agentready harbor list` + +List all Harbor comparisons in output directory. + +**Example**: +```bash +agentready harbor list +``` + +**Output**: +``` +Harbor comparisons in .agentready/harbor_comparisons: + + run_20251210_143022/ + Created: 2025-12-10T14:30:22 + Success Δ: +33.3% + Duration Δ: -21.2% + + run_20251209_091545/ + Created: 2025-12-09T09:15:45 + Success Δ: +16.7% + Duration Δ: -12.5% +``` + +### `agentready harbor view` + +View a specific comparison. + +**Usage**: +```bash +agentready harbor view .agentready/harbor_comparisons/comparison_latest.json +``` + +**Options**: +- `--format summary` - Print summary (default) +- `--format full` - Print full JSON + +--- + +## Output Files + +Each comparison generates multiple files in `.agentready/harbor_comparisons/run_TIMESTAMP/`: + +### Directory Structure + +``` +.agentready/harbor_comparisons/ +├── run_20251210_143022/ +│ ├── without_agent/ # Harbor results without doubleagent.md +│ │ └── [task results] +│ ├── with_agent/ # Harbor results with doubleagent.md +│ │ └── [task results] +│ ├── comparison_20251210_143022.json # Machine-readable data +│ ├── comparison_20251210_143022.md # GitHub-Flavored Markdown +│ └── comparison_20251210_143022.html # Interactive dashboard +├── comparison_latest.json # Symlink to most recent JSON +├── comparison_latest.md # Symlink to most recent Markdown +└── comparison_latest.html # Symlink to most recent HTML +``` + +### JSON Report + +Machine-readable comparison data for further analysis: + +```json +{ + "without_agent": { + "run_id": "without_20251210_143022", + "agent_file_enabled": false, + "success_rate": 66.7, + "avg_duration_sec": 312.5, + "total_tasks": 3, + "successful_tasks": 2 + }, + "with_agent": { + "run_id": "with_20251210_143022", + "agent_file_enabled": true, + "success_rate": 100.0, + "avg_duration_sec": 246.3, + "total_tasks": 3, + "successful_tasks": 3 + }, + "deltas": { + "success_rate_delta": 33.3, + "avg_duration_delta_pct": -21.2 + }, + "statistical_significance": { + "success_rate_significant": true, + "success_rate_p_value": 0.0421, + "duration_significant": true, + "duration_p_value": 0.0312, + "duration_cohens_d": -0.87 + } +} +``` + +### Markdown Report + +GitHub-friendly report perfect for git commits and PRs: + +```markdown +# Harbor Benchmark Comparison + +**Created**: 2025-12-10T14:30:22 +**Tasks**: 3 (adaptive-rejection-sampler, async-http-client, terminal-file-browser) + +## Summary + +| Metric | Without Agent | With Agent | Delta | Significant? | +|--------|--------------|------------|-------|--------------| +| Success Rate | 66.7% | 100.0% | +33.3% | ✓ (p=0.0421) | +| Avg Duration | 5.2 min | 4.1 min | -21.2% | ✓ (p=0.0312) | + +## Per-Task Results + +### adaptive-rejection-sampler +- **Without Agent**: ✗ Failed (timeout) +- **With Agent**: ✓ Success (3.8 min) +- **Impact**: +100% success (fixed failure) + +... + +## Conclusion + +The `doubleagent.md` agent file shows **statistically significant improvement** +in both success rate (+33.3%, p=0.04) and execution speed (-21.2%, p=0.03). + +**Recommendation**: ✅ **Include `doubleagent.md`** in AgentReady development workflows. +``` + +### HTML Dashboard + +Interactive visualization with Chart.js: + +- Side-by-side bar charts (success rates, durations) +- Per-task breakdown table +- Statistical significance indicators +- Self-contained (no external dependencies) + +Open with: +```bash +open .agentready/harbor_comparisons/comparison_latest.html +``` + +--- + +## Interpreting Results + +### Statistical Significance + +**P-value < 0.05**: Statistically significant difference (95% confidence) +- ✓ Indicates real improvement, not random variation +- ✗ Difference could be due to chance + +**Cohen's d (Effect Size)**: +- **0.2 ≤ |d| < 0.5**: Small effect +- **0.5 ≤ |d| < 0.8**: Medium effect +- **|d| ≥ 0.8**: Large effect + +### Sample Size Requirements + +- **Minimum**: 3 tasks for statistical tests +- **Recommended**: 5-10 tasks for reliable results +- **Comprehensive**: 20+ tasks for production validation + +Small samples (n<3) will show warning about statistical validity. + +### Recommendations + +Based on comparison results: + +| Outcome | Recommendation | +|---------|---------------| +| ✅ Success ↑, p<0.05 | **Include agent file** - Proven improvement | +| ⚠️ Success ↑, p≥0.05 | **Consider including** - Validate with larger sample | +| ❌ No improvement | **Agent file may not help** for tested tasks | + +--- + +## Advanced Usage + +### Custom Agent File + +Test a different agent file: + +```bash +agentready harbor compare \ + -t task1 -t task2 \ + --agent-file .claude/agents/custom-agent.md +``` + +### Custom Output Directory + +Store results in specific location: + +```bash +agentready harbor compare \ + -t task1 -t task2 \ + --output-dir experiments/harbor_results +``` + +### Different Model + +Test with Claude Opus: + +```bash +agentready harbor compare \ + -t task1 -t task2 \ + --model anthropic/claude-opus-4-5 +``` + +--- + +## Troubleshooting + +### Harbor Not Installed + +**Error**: `Harbor framework not installed` + +**Solution**: +```bash +uv tool install harbor +``` + +### Agent File Not Found + +**Error**: `Agent file not found: .claude/agents/doubleagent.md` + +**Solution**: Ensure agent file exists or specify custom path with `--agent-file` + +### No Tasks Specified + +**Error**: `At least one task must be specified with -t/--task` + +**Solution**: Add tasks with `-t` flag: +```bash +agentready harbor compare -t adaptive-rejection-sampler +``` + +### Sample Size Too Small + +**Warning**: `Sample size too small (n<3). Statistical tests may not be reliable.` + +**Solution**: Run more tasks (5-10 recommended) for valid statistical analysis + +### Task Timeout + +Some tasks may timeout (30-60 min). This is normal for complex tasks. The comparison will continue with partial results. + +--- + +## FAQ + +**Q: How long does a comparison take?** + +A: Approximately 10-20 minutes per task. For 3 tasks, expect 30-60 minutes total. + +**Q: Can I run comparisons in parallel?** + +A: Not currently supported. Future versions may support concurrent Harbor execution via Daytona/Modal. + +**Q: What if some tasks fail?** + +A: Comparison continues with partial results. Failed tasks are marked in reports and excluded from duration averages. + +**Q: Can I compare more than 2 configurations?** + +A: Currently supports only with/without agent file. Multi-configuration comparison is planned for future versions. + +**Q: Where are results stored?** + +A: `.agentready/harbor_comparisons/` (gitignored). Reports can be committed for reference, but raw Harbor results are excluded. + +**Q: How do I share results with my team?** + +A: Commit the Markdown report (`.md` file) or share the HTML dashboard. JSON files are machine-readable for further analysis. + +--- + +## Related Documentation + +- **Harbor Framework**: https://harborframework.com/docs +- **Terminal-Bench**: https://terminal-bench.com +- **AgentReady CLAUDE.md**: See "Harbor Comparison" section +- **Plan**: `.claude/plans/vivid-knitting-codd.md` (implementation details) + +--- + +## Quickstart Example + +```bash +# Install Harbor +uv tool install harbor + +# Run comparison (3 tasks, ~30-60 min) +agentready harbor compare \ + -t adaptive-rejection-sampler \ + -t async-http-client \ + -t terminal-file-browser \ + --verbose \ + --open-dashboard + +# View summary +agentready harbor view .agentready/harbor_comparisons/comparison_latest.json + +# List all comparisons +agentready harbor list + +# Open latest dashboard +open .agentready/harbor_comparisons/comparison_latest.html +``` + +--- + +**Last Updated**: 2025-12-10 +**AgentReady Version**: 2.10.0+ diff --git a/src/agentready/cli/harbor.py b/src/agentready/cli/harbor.py new file mode 100644 index 00000000..a1db8d71 --- /dev/null +++ b/src/agentready/cli/harbor.py @@ -0,0 +1,361 @@ +"""Harbor benchmark comparison CLI commands.""" + +import json +from datetime import datetime +from pathlib import Path + +import click + +from agentready.models.harbor import HarborComparison, HarborRunMetrics +from agentready.reporters.harbor_markdown import generate_markdown_report +from agentready.services.harbor.agent_toggler import AgentFileToggler +from agentready.services.harbor.comparer import compare_runs +from agentready.services.harbor.dashboard_generator import ( + DashboardGenerator, + generate_dashboard, +) +from agentready.services.harbor.result_parser import parse_harbor_results +from agentready.services.harbor.runner import HarborNotInstalledError, HarborRunner + + +def _run_benchmark_phase( + runner: HarborRunner, + toggler: AgentFileToggler, + phase_name: str, + run_number: int, + output_dir: Path, + task_list: list, + model: str, + verbose: bool, + disable_agent: bool, +) -> Path: + """Run a single benchmark phase (with or without agent). + + Returns: + Path to results directory + """ + click.echo("=" * 60) + click.echo(f"RUN {run_number}: {phase_name}") + click.echo("=" * 60) + click.echo() + + try: + if disable_agent: + with toggler.temporarily_disabled(): + click.echo("Agent file disabled. Running benchmark...") + runner.run_benchmark( + task_names=task_list, + output_dir=output_dir, + model=model, + verbose=verbose, + ) + else: + click.echo("Agent file enabled. Running benchmark...") + runner.run_benchmark( + task_names=task_list, + output_dir=output_dir, + model=model, + verbose=verbose, + ) + except Exception as e: + click.echo(f"❌ Benchmark failed: {e}", err=True) + # Context manager automatically restores agent file in finally block + raise click.Abort() + + click.echo(f"✓ Run {run_number} complete\n") + return output_dir + + +def _generate_reports( + comparison: HarborComparison, + run_dir: Path, + output_dir: Path, + timestamp: str, +) -> dict: + """Generate all report formats (JSON, Markdown, HTML). + + Returns: + Dictionary of report paths + """ + comparison_base = run_dir / f"comparison_{timestamp}" + paths = {} + + # Generate JSON report + paths["json"] = comparison_base.with_suffix(".json") + with open(paths["json"], "w") as f: + json.dump(comparison.to_dict(), f, indent=2) + click.echo(f" ✓ JSON: {paths['json']}") + + # Generate Markdown report + paths["markdown"] = comparison_base.with_suffix(".md") + generate_markdown_report(comparison, paths["markdown"]) + click.echo(f" ✓ Markdown: {paths['markdown']}") + + # Generate HTML dashboard + paths["html"] = comparison_base.with_suffix(".html") + generate_dashboard(comparison, paths["html"]) + click.echo(f" ✓ HTML: {paths['html']}") + + # Create 'latest' symlinks for easy access + _create_latest_symlinks(paths, output_dir) + + return paths + + +def _create_latest_symlinks(paths: dict, output_dir: Path) -> None: + """Create 'latest' symlinks to most recent comparison files.""" + try: + for format_name, source_path in paths.items(): + extension = source_path.suffix + latest_link = output_dir / f"comparison_latest{extension}" + + # Remove old symlink if exists + if latest_link.exists() or latest_link.is_symlink(): + latest_link.unlink() + + # Create new symlink + latest_link.symlink_to(source_path.relative_to(output_dir)) + + click.echo(f"\n ✓ Latest: {output_dir}/comparison_latest.*") + except Exception: + # Symlinks might fail on Windows, just skip + pass + + +@click.group(name="harbor") +def harbor_cli(): + """Harbor benchmark comparison commands. + + Compare Claude Code performance with/without the doubleagent.md agent file + using the Harbor benchmarking framework. + """ + pass + + +@harbor_cli.command(name="compare") +@click.option( + "-t", + "--task", + "tasks", + multiple=True, + help="Task name to benchmark (can be specified multiple times)", +) +@click.option( + "--model", + default="anthropic/claude-sonnet-4-5", + help="Model identifier (default: anthropic/claude-sonnet-4-5)", +) +@click.option( + "--agent-file", + type=click.Path(exists=True, path_type=Path), + default=".claude/agents/doubleagent.md", + help="Path to agent file (default: .claude/agents/doubleagent.md)", +) +@click.option( + "--output-dir", + type=click.Path(path_type=Path), + default=".agentready/harbor_comparisons", + help="Output directory for results (default: .agentready/harbor_comparisons)", +) +@click.option("--verbose", is_flag=True, help="Print detailed Harbor output") +@click.option( + "--open-dashboard", is_flag=True, help="Open HTML dashboard after comparison" +) +def compare( + tasks: tuple, + model: str, + agent_file: Path, + output_dir: Path, + verbose: bool, + open_dashboard: bool, +): + """Compare Harbor benchmarks with/without agent file. + + Runs Terminal-Bench tasks twice: + 1. Without doubleagent.md (agent file disabled) + 2. With doubleagent.md (agent file enabled) + + Generates comprehensive comparison reports (JSON, Markdown, HTML). + + Example: + agentready harbor compare -t adaptive-rejection-sampler -t async-http-client + """ + click.echo("=" * 60) + click.echo("Harbor Benchmark Comparison") + click.echo("=" * 60) + click.echo() + + # Validate agent file exists + if not agent_file.exists(): + click.echo(f"❌ Error: Agent file not found: {agent_file}", err=True) + click.echo( + " This comparison requires the doubleagent.md agent file.", err=True + ) + raise click.Abort() + + # Validate tasks specified + if not tasks: + click.echo( + "❌ Error: At least one task must be specified with -t/--task", err=True + ) + click.echo( + " Example: agentready harbor compare -t adaptive-rejection-sampler", + err=True, + ) + raise click.Abort() + + task_list = list(tasks) + click.echo(f"Tasks to benchmark: {', '.join(task_list)}") + click.echo(f"Model: {model}") + click.echo(f"Agent file: {agent_file}") + click.echo() + + try: + # Initialize Harbor runner + click.echo("Checking Harbor installation...") + runner = HarborRunner() + click.echo("✓ Harbor installed\n") + + except HarborNotInstalledError as e: + click.echo(f"❌ {e}", err=True) + raise click.Abort() + + # Create timestamped output directory + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + run_dir = output_dir / f"run_{timestamp}" + run_dir.mkdir(parents=True, exist_ok=True) + + # Initialize agent file toggler + toggler = AgentFileToggler(agent_file) + + # Run benchmarks with and without agent file + without_results_dir = _run_benchmark_phase( + runner=runner, + toggler=toggler, + phase_name="WITHOUT doubleagent.md", + run_number=1, + output_dir=run_dir / "without_agent", + task_list=task_list, + model=model, + verbose=verbose, + disable_agent=True, + ) + + with_results_dir = _run_benchmark_phase( + runner=runner, + toggler=toggler, + phase_name="WITH doubleagent.md", + run_number=2, + output_dir=run_dir / "with_agent", + task_list=task_list, + model=model, + verbose=verbose, + disable_agent=False, + ) + + # Parse results + click.echo("Parsing results...") + try: + without_tasks = parse_harbor_results(without_results_dir) + with_tasks = parse_harbor_results(with_results_dir) + + without_metrics = HarborRunMetrics.from_task_results( + run_id=f"without_{timestamp}", + agent_file_enabled=False, + task_results=without_tasks, + ) + + with_metrics = HarborRunMetrics.from_task_results( + run_id=f"with_{timestamp}", agent_file_enabled=True, task_results=with_tasks + ) + + except Exception as e: + click.echo(f"❌ Failed to parse results: {e}", err=True) + raise click.Abort() + + # Compare runs + click.echo("Calculating comparison...") + comparison = compare_runs(without_metrics, with_metrics) + + # Generate reports + click.echo("Generating reports...") + report_paths = _generate_reports(comparison, run_dir, output_dir, timestamp) + + # Print summary + click.echo() + click.echo("=" * 60) + click.echo("SUMMARY") + click.echo("=" * 60) + + generator = DashboardGenerator() + summary = generator.generate_summary_text(comparison) + click.echo(summary) + + # Open dashboard if requested + if open_dashboard: + import webbrowser + + html_path = report_paths.get("html") + if html_path: + click.echo(f"\nOpening dashboard: {html_path}") + webbrowser.open(html_path.as_uri()) + + +@harbor_cli.command(name="list") +@click.option( + "--output-dir", + type=click.Path(exists=True, path_type=Path), + default=".agentready/harbor_comparisons", + help="Output directory containing comparisons", +) +def list_comparisons(output_dir: Path): + """List all Harbor comparisons.""" + click.echo(f"Harbor comparisons in {output_dir}:") + click.echo() + + comparison_files = sorted(output_dir.glob("*/comparison_*.json"), reverse=True) + + if not comparison_files: + click.echo(" No comparisons found.") + return + + for comp_file in comparison_files: + # Parse comparison to get summary + with open(comp_file, "r") as f: + data = json.load(f) + comparison = HarborComparison.from_dict(data) + + created = comparison.created_at + delta_success = comparison.deltas["success_rate_delta"] + delta_duration = comparison.deltas["avg_duration_delta_pct"] + + click.echo(f" {comp_file.parent.name}/") + click.echo(f" Created: {created}") + click.echo(f" Success Δ: {delta_success:+.1f}%") + click.echo(f" Duration Δ: {delta_duration:+.1f}%") + click.echo() + + +@harbor_cli.command(name="view") +@click.argument("comparison_file", type=click.Path(exists=True, path_type=Path)) +@click.option("--format", type=click.Choice(["summary", "full"]), default="summary") +def view_comparison(comparison_file: Path, format: str): + """View a Harbor comparison. + + COMPARISON_FILE: Path to comparison JSON file + """ + with open(comparison_file, "r") as f: + data = json.load(f) + comparison = HarborComparison.from_dict(data) + + if format == "summary": + generator = DashboardGenerator() + summary = generator.generate_summary_text(comparison) + click.echo(summary) + else: + # Full JSON output + click.echo(json.dumps(data, indent=2)) + + +if __name__ == "__main__": + harbor_cli() diff --git a/src/agentready/cli/main.py b/src/agentready/cli/main.py index dd69af33..e51795a2 100644 --- a/src/agentready/cli/main.py +++ b/src/agentready/cli/main.py @@ -37,7 +37,7 @@ from .schema import migrate_report, validate_report # Heavy commands - lazy loaded via LazyGroup -# (assess_batch, experiment, extract_skills, learn, submit) +# (assess_batch, experiment, extract_skills, harbor, learn, submit) def get_agentready_version() -> str: @@ -97,6 +97,7 @@ def get_command(self, ctx, cmd_name): "assess-batch": ("assess_batch", "assess_batch"), "experiment": ("experiment", "experiment"), "extract-skills": ("extract_skills", "extract_skills"), + "harbor": ("harbor", "harbor_cli"), "learn": ("learn", "learn"), "submit": ("submit", "submit"), }, diff --git a/src/agentready/models/harbor.py b/src/agentready/models/harbor.py new file mode 100644 index 00000000..ff63f238 --- /dev/null +++ b/src/agentready/models/harbor.py @@ -0,0 +1,300 @@ +"""Data models for Harbor benchmark integration.""" + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Dict, List, Optional + + +@dataclass +class HarborTaskResult: + """Single task result from Harbor result.json.""" + + task_name: str + trial_name: str + success: bool + duration_sec: float + agent_result: Optional[Dict[str, Any]] + verifier_result: Optional[Dict[str, Any]] + exception_info: Optional[Dict[str, str]] + started_at: str + finished_at: str + + @classmethod + def from_result_json(cls, result_data: Dict[str, Any]) -> "HarborTaskResult": + """Create HarborTaskResult from parsed result.json data.""" + # Parse timestamps to calculate duration + started = datetime.fromisoformat(result_data["started_at"]) + finished = datetime.fromisoformat(result_data["finished_at"]) + duration_sec = (finished - started).total_seconds() + + # Determine success based on agent_result and verifier_result + agent_result = result_data.get("agent_result") + verifier_result = result_data.get("verifier_result") + exception_info = result_data.get("exception_info") + + # Success if no exception and both agent and verifier completed + success = ( + exception_info is None + and agent_result is not None + and verifier_result is not None + ) + + return cls( + task_name=result_data["task_name"], + trial_name=result_data["trial_name"], + success=success, + duration_sec=duration_sec, + agent_result=agent_result, + verifier_result=verifier_result, + exception_info=exception_info, + started_at=result_data["started_at"], + finished_at=result_data["finished_at"], + ) + + def to_dict(self) -> Dict[str, Any]: + """Convert to dictionary for JSON serialization.""" + return { + "task_name": self.task_name, + "trial_name": self.trial_name, + "success": self.success, + "duration_sec": self.duration_sec, + "agent_result": self.agent_result, + "verifier_result": self.verifier_result, + "exception_info": self.exception_info, + "started_at": self.started_at, + "finished_at": self.finished_at, + } + + +@dataclass +class HarborRunMetrics: + """Aggregated metrics for a Harbor run.""" + + run_id: str + agent_file_enabled: bool + task_results: List[HarborTaskResult] + success_rate: float + completion_rate: float + avg_duration_sec: float + total_tasks: int + successful_tasks: int + failed_tasks: int + timed_out_tasks: int + + @classmethod + def from_task_results( + cls, run_id: str, agent_file_enabled: bool, task_results: List[HarborTaskResult] + ) -> "HarborRunMetrics": + """Calculate aggregated metrics from task results.""" + total_tasks = len(task_results) + successful_tasks = sum(1 for r in task_results if r.success) + failed_tasks = sum( + 1 for r in task_results if not r.success and r.exception_info is None + ) + timed_out_tasks = sum( + 1 + for r in task_results + if not r.success + and r.exception_info + and "timeout" in r.exception_info.get("exception_type", "").lower() + ) + + success_rate = ( + (successful_tasks / total_tasks * 100) if total_tasks > 0 else 0.0 + ) + completion_rate = ( + (successful_tasks + failed_tasks) / total_tasks * 100 + if total_tasks > 0 + else 0.0 + ) + + # Calculate average duration (only for completed tasks) + completed_results = [r for r in task_results if r.agent_result is not None] + avg_duration_sec = ( + sum(r.duration_sec for r in completed_results) / len(completed_results) + if completed_results + else 0.0 + ) + + return cls( + run_id=run_id, + agent_file_enabled=agent_file_enabled, + task_results=task_results, + success_rate=success_rate, + completion_rate=completion_rate, + avg_duration_sec=avg_duration_sec, + total_tasks=total_tasks, + successful_tasks=successful_tasks, + failed_tasks=failed_tasks, + timed_out_tasks=timed_out_tasks, + ) + + def to_dict(self) -> Dict[str, Any]: + """Convert to dictionary for JSON serialization.""" + return { + "run_id": self.run_id, + "agent_file_enabled": self.agent_file_enabled, + "task_results": [r.to_dict() for r in self.task_results], + "success_rate": self.success_rate, + "completion_rate": self.completion_rate, + "avg_duration_sec": self.avg_duration_sec, + "total_tasks": self.total_tasks, + "successful_tasks": self.successful_tasks, + "failed_tasks": self.failed_tasks, + "timed_out_tasks": self.timed_out_tasks, + } + + +@dataclass +class HarborComparison: + """Complete comparison between two Harbor runs.""" + + without_agent: HarborRunMetrics + with_agent: HarborRunMetrics + deltas: Dict[str, float] = field(default_factory=dict) + statistical_significance: Dict[str, bool] = field(default_factory=dict) + per_task_comparison: List[Dict[str, Any]] = field(default_factory=list) + created_at: str = field(default_factory=lambda: datetime.now().isoformat()) + + def calculate_deltas(self) -> None: + """Calculate delta metrics between runs.""" + success_rate_delta = ( + self.with_agent.success_rate - self.without_agent.success_rate + ) + completion_rate_delta = ( + self.with_agent.completion_rate - self.without_agent.completion_rate + ) + duration_delta_sec = ( + self.with_agent.avg_duration_sec - self.without_agent.avg_duration_sec + ) + successful_tasks_delta = ( + self.with_agent.successful_tasks - self.without_agent.successful_tasks + ) + + # Calculate percentage change in duration + duration_delta_pct = None + if self.without_agent.avg_duration_sec > 0: + duration_delta_pct = ( + duration_delta_sec / self.without_agent.avg_duration_sec * 100 + ) + + self.deltas = { + "success_rate_delta": success_rate_delta, + "completion_rate_delta": completion_rate_delta, + "avg_duration_delta_sec": duration_delta_sec, + "avg_duration_delta_pct": duration_delta_pct, + "successful_tasks_delta": successful_tasks_delta, + } + + def generate_per_task_comparison(self) -> None: + """Generate per-task comparison details.""" + # Create lookup dictionary for tasks + without_tasks = {r.task_name: r for r in self.without_agent.task_results} + with_tasks = {r.task_name: r for r in self.with_agent.task_results} + + self.per_task_comparison = [] + all_task_names = set(without_tasks.keys()) | set(with_tasks.keys()) + + for task_name in all_task_names: + without_result = without_tasks.get(task_name) + with_result = with_tasks.get(task_name) + + comparison = {"task_name": task_name} + + # Add without_agent result if exists + if without_result: + comparison["without_agent"] = { + "success": without_result.success, + "duration_sec": without_result.duration_sec, + } + else: + comparison["without_agent"] = None + + # Add with_agent result if exists + if with_result: + comparison["with_agent"] = { + "success": with_result.success, + "duration_sec": with_result.duration_sec, + } + else: + comparison["with_agent"] = None + + # Calculate per-task delta if both results exist + if without_result and with_result: + comparison["delta"] = self._calculate_task_delta( + without_result, with_result + ) + + self.per_task_comparison.append(comparison) + + def _calculate_task_delta( + self, without_result: HarborTaskResult, with_result: HarborTaskResult + ) -> Dict[str, Any]: + """Calculate delta between two task results.""" + duration_delta_sec = with_result.duration_sec - without_result.duration_sec + duration_delta_pct = None + + if without_result.duration_sec > 0: + duration_delta_pct = duration_delta_sec / without_result.duration_sec * 100 + + return { + "success_improved": with_result.success and not without_result.success, + "duration_delta_sec": duration_delta_sec, + "duration_delta_pct": duration_delta_pct, + } + + def to_dict(self) -> Dict[str, Any]: + """Convert to dictionary for JSON serialization.""" + return { + "without_agent": self.without_agent.to_dict(), + "with_agent": self.with_agent.to_dict(), + "deltas": self.deltas, + "statistical_significance": self.statistical_significance, + "per_task_comparison": self.per_task_comparison, + "created_at": self.created_at, + } + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "HarborComparison": + """Create HarborComparison from dictionary.""" + without_agent_data = data["without_agent"] + with_agent_data = data["with_agent"] + + without_agent = HarborRunMetrics( + run_id=without_agent_data["run_id"], + agent_file_enabled=without_agent_data["agent_file_enabled"], + task_results=[ + HarborTaskResult(**r) for r in without_agent_data["task_results"] + ], + success_rate=without_agent_data["success_rate"], + completion_rate=without_agent_data["completion_rate"], + avg_duration_sec=without_agent_data["avg_duration_sec"], + total_tasks=without_agent_data["total_tasks"], + successful_tasks=without_agent_data["successful_tasks"], + failed_tasks=without_agent_data["failed_tasks"], + timed_out_tasks=without_agent_data["timed_out_tasks"], + ) + + with_agent = HarborRunMetrics( + run_id=with_agent_data["run_id"], + agent_file_enabled=with_agent_data["agent_file_enabled"], + task_results=[ + HarborTaskResult(**r) for r in with_agent_data["task_results"] + ], + success_rate=with_agent_data["success_rate"], + completion_rate=with_agent_data["completion_rate"], + avg_duration_sec=with_agent_data["avg_duration_sec"], + total_tasks=with_agent_data["total_tasks"], + successful_tasks=with_agent_data["successful_tasks"], + failed_tasks=with_agent_data["failed_tasks"], + timed_out_tasks=with_agent_data["timed_out_tasks"], + ) + + return cls( + without_agent=without_agent, + with_agent=with_agent, + deltas=data.get("deltas", {}), + statistical_significance=data.get("statistical_significance", {}), + per_task_comparison=data.get("per_task_comparison", []), + created_at=data.get("created_at", datetime.now().isoformat()), + ) diff --git a/src/agentready/reporters/harbor_markdown.py b/src/agentready/reporters/harbor_markdown.py new file mode 100644 index 00000000..8f4fa9e1 --- /dev/null +++ b/src/agentready/reporters/harbor_markdown.py @@ -0,0 +1,260 @@ +"""Markdown reporter for Harbor comparisons.""" + +from pathlib import Path + +from agentready.models.harbor import HarborComparison + + +class HarborMarkdownReporter: + """Generate GitHub-Flavored Markdown reports for Harbor comparisons.""" + + def generate(self, comparison: HarborComparison, output_path: Path) -> None: + """Generate Markdown report. + + Creates a GitHub-Flavored Markdown file with: + - Summary table + - Statistical significance indicators + - Per-task breakdown + - Recommendations + + Args: + comparison: HarborComparison with calculated deltas + output_path: Path to write Markdown file + """ + markdown = self._build_markdown(comparison) + + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: + f.write(markdown) + + def _build_markdown(self, comparison: HarborComparison) -> str: + """Build complete Markdown content. + + Args: + comparison: HarborComparison object + + Returns: + Complete Markdown content as string + """ + sections = [ + self._header(comparison), + self._summary_table(comparison), + self._statistical_significance(comparison), + self._per_task_results(comparison), + self._conclusion(comparison), + ] + + return "\n\n".join(sections) + + def _header(self, comparison: HarborComparison) -> str: + """Generate header section.""" + task_names = [t["task_name"] for t in comparison.per_task_comparison] + return f"""# Harbor Benchmark Comparison + +**Created**: {comparison.created_at} +**Tasks**: {len(task_names)} ({', '.join(task_names[:3])}{'...' if len(task_names) > 3 else ''}) +**Agent File**: `.claude/agents/doubleagent.md`""" + + def _summary_table(self, comparison: HarborComparison) -> str: + """Generate summary metrics table.""" + without = comparison.without_agent + with_agent = comparison.with_agent + deltas = comparison.deltas + sig = comparison.statistical_significance + + # Format significance indicators + success_sig = self._format_significance( + sig.get("success_rate_significant"), sig.get("success_rate_p_value") + ) + duration_sig = self._format_significance( + sig.get("duration_significant"), sig.get("duration_p_value") + ) + + return f"""## Summary + +| Metric | Without Agent | With Agent | Delta | Significant? | +|--------|--------------|------------|-------|--------------| +| Success Rate | {without.success_rate:.1f}% | {with_agent.success_rate:.1f}% | {deltas['success_rate_delta']:+.1f}% | {success_sig} | +| Completion Rate | {without.completion_rate:.1f}% | {with_agent.completion_rate:.1f}% | {deltas['completion_rate_delta']:+.1f}% | - | +| Avg Duration | {without.avg_duration_sec / 60:.1f} min | {with_agent.avg_duration_sec / 60:.1f} min | {deltas['avg_duration_delta_pct']:+.1f}% | {duration_sig} | +| Successful Tasks | {without.successful_tasks}/{without.total_tasks} | {with_agent.successful_tasks}/{with_agent.total_tasks} | {deltas['successful_tasks_delta']:+d} | - |""" + + def _format_significance( + self, is_significant: bool = None, p_value: float = None + ) -> str: + """Format statistical significance indicator. + + Args: + is_significant: Whether difference is statistically significant + p_value: P-value from statistical test + + Returns: + Formatted string (e.g., "✓ (p=0.04)" or "✗ (p=0.23)") + """ + if is_significant is None or p_value is None: + return "-" + + symbol = "✓" if is_significant else "✗" + return f"{symbol} (p={p_value:.4f})" + + def _statistical_significance(self, comparison: HarborComparison) -> str: + """Generate statistical significance section.""" + sig = comparison.statistical_significance + + if not sig.get("success_rate_p_value") and not sig.get("duration_p_value"): + return "## Statistical Analysis\n\n*Statistical tests not available (scipy not installed)*" + + lines = ["## Statistical Analysis"] + + # Success rate analysis + if sig.get("success_rate_p_value") is not None: + is_sig = sig["success_rate_significant"] + p_val = sig["success_rate_p_value"] + + if is_sig: + lines.append( + f"- **Success Rate**: Statistically significant improvement " + f"(p={p_val:.4f}, p<0.05)" + ) + else: + lines.append( + f"- **Success Rate**: No statistically significant difference " + f"(p={p_val:.4f}, p≥0.05)" + ) + + # Duration analysis + if sig.get("duration_p_value") is not None: + is_sig = sig["duration_significant"] + p_val = sig["duration_p_value"] + cohens_d = sig.get("duration_cohens_d") + + if is_sig: + effect_text = "" + if cohens_d is not None: + from agentready.services.harbor.comparer import ( + interpret_effect_size, + ) + + effect = interpret_effect_size(cohens_d) + effect_text = f" with {effect} effect size (d={cohens_d:.2f})" + + lines.append( + f"- **Duration**: Statistically significant difference " + f"(p={p_val:.4f}, p<0.05){effect_text}" + ) + else: + lines.append( + f"- **Duration**: No statistically significant difference " + f"(p={p_val:.4f}, p≥0.05)" + ) + + return "\n".join(lines) + + def _per_task_results(self, comparison: HarborComparison) -> str: + """Generate per-task results section.""" + lines = ["## Per-Task Results"] + + for task_comp in comparison.per_task_comparison: + lines.append(f"\n### {task_comp['task_name']}") + lines.append( + self._format_task_result( + "Without Agent", task_comp.get("without_agent") + ) + ) + lines.append( + self._format_task_result("With Agent", task_comp.get("with_agent")) + ) + + # Add impact analysis if delta exists + if "delta" in task_comp: + lines.append(self._format_task_impact(task_comp["delta"])) + + return "\n".join(lines) + + def _format_task_result(self, label: str, result: dict) -> str: + """Format a single task result line.""" + if not result: + return f"- **{label}**: N/A" + + status = "✓ Success" if result.get("success") else "✗ Failed" + duration = result.get("duration_sec", 0) / 60 + return f"- **{label}**: {status} ({duration:.1f} min)" + + def _format_task_impact(self, delta: dict) -> str: + """Format task impact analysis.""" + if delta.get("success_improved"): + return "- **Impact**: +100% success (fixed failure)" + + duration_pct = delta.get("duration_delta_pct") + if duration_pct: + direction = "faster" if duration_pct < 0 else "slower" + return f"- **Impact**: {abs(duration_pct):.1f}% {direction}" + + return "- **Impact**: No change" + + def _conclusion(self, comparison: HarborComparison) -> str: + """Generate conclusion and recommendations.""" + deltas = comparison.deltas + sig = comparison.statistical_significance + + lines = ["## Conclusion"] + + # Determine overall recommendation + success_improved = deltas["success_rate_delta"] > 0 + duration_improved = deltas["avg_duration_delta_pct"] < 0 + statistically_significant = sig.get("success_rate_significant") or sig.get( + "duration_significant" + ) + + if success_improved and statistically_significant: + lines.append( + f"\nThe `doubleagent.md` agent file shows **statistically significant improvement** " + f"in success rate ({deltas['success_rate_delta']:+.1f}%)" + ) + + if duration_improved: + lines.append( + f"and execution speed ({deltas['avg_duration_delta_pct']:+.1f}%)." + ) + else: + lines.append(".") + + lines.append( + "\n**Recommendation**: ✅ **Include `doubleagent.md`** " + "in AgentReady development workflows." + ) + + elif success_improved or duration_improved: + lines.append("\nThe `doubleagent.md` agent file shows improvements:") + if success_improved: + lines.append(f"- Success rate: {deltas['success_rate_delta']:+.1f}%") + if duration_improved: + lines.append(f"- Duration: {deltas['avg_duration_delta_pct']:+.1f}%") + + lines.append( + "\nHowever, differences are not statistically significant (larger sample size recommended)." + ) + lines.append( + "\n**Recommendation**: ⚠️ **Consider including** `doubleagent.md` " + "but validate with larger benchmark." + ) + + else: + lines.append("\nNo significant improvement detected.") + lines.append( + "\n**Recommendation**: ❌ **Agent file may not provide measurable benefit** " + "for tested tasks." + ) + + return "\n".join(lines) + + +def generate_markdown_report(comparison: HarborComparison, output_path: Path) -> None: + """Convenience function to generate Markdown report. + + Args: + comparison: HarborComparison with calculated deltas + output_path: Path to write Markdown file + """ + reporter = HarborMarkdownReporter() + reporter.generate(comparison, output_path) diff --git a/src/agentready/services/harbor/__init__.py b/src/agentready/services/harbor/__init__.py new file mode 100644 index 00000000..720dc51e --- /dev/null +++ b/src/agentready/services/harbor/__init__.py @@ -0,0 +1,6 @@ +"""Harbor benchmark integration services.""" + +from agentready.services.harbor.agent_toggler import AgentFileToggler +from agentready.services.harbor.result_parser import parse_harbor_results + +__all__ = ["AgentFileToggler", "parse_harbor_results"] diff --git a/src/agentready/services/harbor/agent_toggler.py b/src/agentready/services/harbor/agent_toggler.py new file mode 100644 index 00000000..6653c82f --- /dev/null +++ b/src/agentready/services/harbor/agent_toggler.py @@ -0,0 +1,91 @@ +"""Service for safely enabling/disabling the doubleagent.md file.""" + +import shutil +from contextlib import contextmanager +from pathlib import Path +from typing import Generator + + +class AgentFileToggler: + """Safely enable/disable agent files via atomic rename operations.""" + + def __init__(self, agent_file: Path): + """Initialize toggler with agent file path. + + Args: + agent_file: Path to the agent file (e.g., .claude/agents/doubleagent.md) + """ + self.agent_file = agent_file + self.disabled_file = agent_file.with_suffix(agent_file.suffix + ".disabled") + + def disable(self) -> None: + """Rename agent file to .disabled extension.""" + if self.agent_file.exists(): + if self.disabled_file.exists(): + # Already disabled, nothing to do + return + shutil.move(str(self.agent_file), str(self.disabled_file)) + + def enable(self) -> None: + """Restore agent file from .disabled extension.""" + if self.disabled_file.exists(): + if self.agent_file.exists(): + # Already enabled, nothing to do + return + shutil.move(str(self.disabled_file), str(self.agent_file)) + + def is_enabled(self) -> bool: + """Check if agent file is currently enabled. + + Returns: + True if agent file exists and is not disabled + """ + return self.agent_file.exists() and not self.disabled_file.exists() + + def is_disabled(self) -> bool: + """Check if agent file is currently disabled. + + Returns: + True if disabled file exists + """ + return self.disabled_file.exists() + + @contextmanager + def temporarily_disabled(self) -> Generator[None, None, None]: + """Context manager for safe disable/enable. + + Ensures agent file is restored even if exception occurs. + + Example: + with toggler.temporarily_disabled(): + # Agent file is disabled here + run_benchmark() + # Agent file is automatically restored here + """ + was_enabled = self.is_enabled() + try: + self.disable() + yield + finally: + if was_enabled: + self.enable() + + @contextmanager + def temporarily_enabled(self) -> Generator[None, None, None]: + """Context manager for safe enable/disable. + + Ensures agent file state is restored even if exception occurs. + + Example: + with toggler.temporarily_enabled(): + # Agent file is enabled here + run_benchmark() + # Agent file state is automatically restored here + """ + was_disabled = self.is_disabled() + try: + self.enable() + yield + finally: + if was_disabled: + self.disable() diff --git a/src/agentready/services/harbor/comparer.py b/src/agentready/services/harbor/comparer.py new file mode 100644 index 00000000..da0a27c9 --- /dev/null +++ b/src/agentready/services/harbor/comparer.py @@ -0,0 +1,188 @@ +"""Service for comparing Harbor benchmark runs and calculating statistical significance.""" + +from typing import List, Optional + +from agentready.models.harbor import HarborComparison, HarborRunMetrics + + +def compare_runs( + without_agent: HarborRunMetrics, with_agent: HarborRunMetrics +) -> HarborComparison: + """Compare two Harbor runs and calculate deltas. + + Args: + without_agent: Metrics from run without agent file + with_agent: Metrics from run with agent file + + Returns: + HarborComparison with calculated deltas and significance + + Raises: + ValueError: If metrics are incompatible (different task sets) + """ + # Validate task sets match + without_tasks = {r.task_name for r in without_agent.task_results} + with_tasks = {r.task_name for r in with_agent.task_results} + + if without_tasks != with_tasks: + print( + f"Warning: Task sets differ. Without: {without_tasks}, With: {with_tasks}. " + "Comparison may be incomplete." + ) + + # Create comparison object + comparison = HarborComparison(without_agent=without_agent, with_agent=with_agent) + + # Calculate deltas + comparison.calculate_deltas() + + # Generate per-task comparison + comparison.generate_per_task_comparison() + + # Calculate statistical significance + comparison.statistical_significance = calculate_statistical_significance( + without_agent, with_agent + ) + + return comparison + + +def calculate_statistical_significance( + without_agent: HarborRunMetrics, with_agent: HarborRunMetrics, alpha: float = 0.05 +) -> dict: + """Calculate statistical significance of differences between runs. + + Uses two-sample t-test for continuous metrics and requires scipy. + + Args: + without_agent: Metrics from run without agent file + with_agent: Metrics from run with agent file + alpha: Significance level (default: 0.05 for 95% confidence) + + Returns: + Dictionary with significance flags and p-values: + { + 'success_rate_significant': bool, + 'duration_significant': bool, + 'success_rate_p_value': float, + 'duration_p_value': float, + } + """ + # Import scipy here to avoid hard dependency + try: + from scipy import stats + except ImportError: + print( + "Warning: scipy not installed. " + "Statistical significance tests unavailable. " + "Install with: uv pip install scipy" + ) + return { + "success_rate_significant": False, + "duration_significant": False, + "success_rate_p_value": None, + "duration_p_value": None, + } + + # Require minimum sample size for valid statistics + min_sample_size = 3 + if ( + len(without_agent.task_results) < min_sample_size + or len(with_agent.task_results) < min_sample_size + ): + print( + f"Warning: Sample size too small (n<{min_sample_size}). " + "Statistical tests may not be reliable." + ) + + # Extract success rates (binary: 1 for success, 0 for failure) + without_successes = [1 if r.success else 0 for r in without_agent.task_results] + with_successes = [1 if r.success else 0 for r in with_agent.task_results] + + # Extract durations (only for completed tasks) + without_durations = [ + r.duration_sec for r in without_agent.task_results if r.agent_result + ] + with_durations = [r.duration_sec for r in with_agent.task_results if r.agent_result] + + results = {} + + # T-test for success rate differences + if len(without_successes) > 0 and len(with_successes) > 0: + t_stat, p_value = stats.ttest_ind(without_successes, with_successes) + results["success_rate_significant"] = p_value < alpha + results["success_rate_p_value"] = p_value + else: + results["success_rate_significant"] = False + results["success_rate_p_value"] = None + + # T-test for duration differences + if len(without_durations) > 0 and len(with_durations) > 0: + t_stat, p_value = stats.ttest_ind(without_durations, with_durations) + results["duration_significant"] = p_value < alpha + results["duration_p_value"] = p_value + + # Calculate Cohen's d for effect size + results["duration_cohens_d"] = calculate_cohens_d( + without_durations, with_durations + ) + else: + results["duration_significant"] = False + results["duration_p_value"] = None + results["duration_cohens_d"] = None + + return results + + +def calculate_cohens_d(group1: List[float], group2: List[float]) -> Optional[float]: + """Calculate Cohen's d effect size. + + Cohen's d measures the standardized difference between two means: + - Small effect: 0.2 ≤ |d| < 0.5 + - Medium effect: 0.5 ≤ |d| < 0.8 + - Large effect: |d| ≥ 0.8 + + Args: + group1: First group of values + group2: Second group of values + + Returns: + Cohen's d value, or None if calculation not possible + """ + if not group1 or not group2: + return None + + # Calculate means + mean1 = sum(group1) / len(group1) + mean2 = sum(group2) / len(group2) + + # Calculate pooled standard deviation + var1 = sum((x - mean1) ** 2 for x in group1) / len(group1) + var2 = sum((x - mean2) ** 2 for x in group2) / len(group2) + pooled_std = ((var1 + var2) / 2) ** 0.5 + + if pooled_std == 0: + return None + + return (mean2 - mean1) / pooled_std + + +def interpret_effect_size(cohens_d: float) -> str: + """Interpret Cohen's d effect size. + + Args: + cohens_d: Cohen's d value + + Returns: + Human-readable interpretation + """ + abs_d = abs(cohens_d) + + if abs_d < 0.2: + return "negligible" + elif abs_d < 0.5: + return "small" + elif abs_d < 0.8: + return "medium" + else: + return "large" diff --git a/src/agentready/services/harbor/dashboard_generator.py b/src/agentready/services/harbor/dashboard_generator.py new file mode 100644 index 00000000..35a26889 --- /dev/null +++ b/src/agentready/services/harbor/dashboard_generator.py @@ -0,0 +1,169 @@ +"""Service for generating interactive HTML dashboards for Harbor comparisons.""" + +from pathlib import Path + +from jinja2 import Environment, FileSystemLoader, select_autoescape + +from agentready.models.harbor import HarborComparison + + +class DashboardGenerator: + """Generate interactive HTML dashboards with Chart.js visualizations.""" + + def __init__(self, template_dir: Path = None): + """Initialize dashboard generator. + + Args: + template_dir: Directory containing Jinja2 templates + (defaults to src/agentready/templates) + """ + if template_dir is None: + # Default to package templates directory + import agentready + + package_dir = Path(agentready.__file__).parent + template_dir = package_dir / "templates" + + self.env = Environment( + loader=FileSystemLoader(template_dir), + autoescape=select_autoescape(["html"]), + ) + + def generate(self, comparison: HarborComparison, output_path: Path) -> None: + """Generate interactive HTML dashboard. + + Creates a self-contained HTML file with: + - Side-by-side bar charts (success rates, durations) + - Per-task breakdown table + - Statistical significance indicators + - All CSS/JS inlined (no external dependencies) + + Args: + comparison: HarborComparison with calculated deltas + output_path: Path to write HTML dashboard + + Raises: + FileNotFoundError: If template file not found + jinja2.TemplateError: If template rendering fails + """ + # Load template + template = self.env.get_template("harbor_comparison.html.j2") + + # Prepare data for template + template_data = { + "comparison": comparison, + "without_agent": comparison.without_agent, + "with_agent": comparison.with_agent, + "deltas": comparison.deltas, + "significance": comparison.statistical_significance, + "per_task": comparison.per_task_comparison, + "created_at": comparison.created_at, + } + + # Render template + html_content = template.render(**template_data) + + # Write to file + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: + f.write(html_content) + + def generate_summary_text(self, comparison: HarborComparison) -> str: + """Generate plain text summary of comparison. + + Args: + comparison: HarborComparison with calculated deltas + + Returns: + Plain text summary for console output + """ + lines = [] + lines.append("=" * 60) + lines.append("Harbor Benchmark Comparison Summary") + lines.append("=" * 60) + lines.append("") + + # Overall metrics + lines.append("Overall Metrics:") + lines.append( + f" Success Rate: {comparison.without_agent.success_rate:.1f}% → " + f"{comparison.with_agent.success_rate:.1f}% " + f"({comparison.deltas['success_rate_delta']:+.1f}%)" + ) + + lines.append( + f" Avg Duration: {comparison.without_agent.avg_duration_sec:.1f}s → " + f"{comparison.with_agent.avg_duration_sec:.1f}s " + f"({comparison.deltas['avg_duration_delta_pct']:+.1f}%)" + ) + + lines.append("") + + # Statistical significance + sig = comparison.statistical_significance + if sig.get("success_rate_p_value") is not None: + is_sig = ( + "✓ Significant" + if sig["success_rate_significant"] + else "✗ Not significant" + ) + lines.append( + f" Success Rate: {is_sig} (p={sig['success_rate_p_value']:.4f})" + ) + + if sig.get("duration_p_value") is not None: + is_sig = ( + "✓ Significant" if sig["duration_significant"] else "✗ Not significant" + ) + lines.append(f" Duration: {is_sig} (p={sig['duration_p_value']:.4f})") + + if sig.get("duration_cohens_d") is not None: + from agentready.services.harbor.comparer import interpret_effect_size + + effect = interpret_effect_size(sig["duration_cohens_d"]) + lines.append( + f" Effect size: {effect} (d={sig['duration_cohens_d']:.2f})" + ) + + lines.append("") + + # Per-task summary + lines.append("Per-Task Results:") + for task_comp in comparison.per_task_comparison: + task_name = task_comp["task_name"] + without = task_comp.get("without_agent", {}) + with_agent_result = task_comp.get("with_agent", {}) + + without_status = "✓" if without and without.get("success") else "✗" + with_status = ( + "✓" if with_agent_result and with_agent_result.get("success") else "✗" + ) + + lines.append(f" {task_name}:") + lines.append(f" Without agent: {without_status}") + lines.append(f" With agent: {with_status}") + + if "delta" in task_comp: + delta = task_comp["delta"] + if delta.get("success_improved"): + lines.append(" Impact: +100% success (fixed failure)") + elif delta.get("duration_delta_pct"): + lines.append( + f" Impact: {delta['duration_delta_pct']:+.1f}% duration" + ) + + lines.append("") + lines.append("=" * 60) + + return "\n".join(lines) + + +def generate_dashboard(comparison: HarborComparison, output_path: Path) -> None: + """Convenience function to generate dashboard. + + Args: + comparison: HarborComparison with calculated deltas + output_path: Path to write HTML dashboard + """ + generator = DashboardGenerator() + generator.generate(comparison, output_path) diff --git a/src/agentready/services/harbor/result_parser.py b/src/agentready/services/harbor/result_parser.py new file mode 100644 index 00000000..684fd082 --- /dev/null +++ b/src/agentready/services/harbor/result_parser.py @@ -0,0 +1,69 @@ +"""Service for parsing Harbor result.json files.""" + +import json +from pathlib import Path +from typing import List + +from agentready.models.harbor import HarborTaskResult + + +def parse_harbor_results(results_dir: Path) -> List[HarborTaskResult]: + """Parse all result.json files in a Harbor run directory. + + Args: + results_dir: Path to Harbor run directory (e.g., jobs/2025-12-09__22-06-09/) + + Returns: + List of HarborTaskResult objects + + Raises: + ValueError: If results_dir doesn't exist or contains no result files + FileNotFoundError: If results_dir doesn't exist + """ + if not results_dir.exists(): + raise FileNotFoundError(f"Results directory not found: {results_dir}") + + # Find all result.json files in subdirectories + result_files = list(results_dir.glob("*/result.json")) + + if not result_files: + raise ValueError(f"No result.json files found in {results_dir}") + + task_results = [] + for result_file in result_files: + try: + with open(result_file, "r") as f: + result_data = json.load(f) + task_result = HarborTaskResult.from_result_json(result_data) + task_results.append(task_result) + except (json.JSONDecodeError, KeyError) as e: + # Log warning but continue processing other files + print(f"Warning: Failed to parse {result_file}: {e}") + continue + + if not task_results: + raise ValueError(f"No valid task results parsed from {results_dir}") + + return task_results + + +def parse_single_result(result_file: Path) -> HarborTaskResult: + """Parse a single result.json file. + + Args: + result_file: Path to result.json file + + Returns: + HarborTaskResult object + + Raises: + FileNotFoundError: If result_file doesn't exist + json.JSONDecodeError: If result_file is not valid JSON + KeyError: If required fields are missing + """ + if not result_file.exists(): + raise FileNotFoundError(f"Result file not found: {result_file}") + + with open(result_file, "r") as f: + result_data = json.load(f) + return HarborTaskResult.from_result_json(result_data) diff --git a/src/agentready/services/harbor/runner.py b/src/agentready/services/harbor/runner.py new file mode 100644 index 00000000..6dbd7d76 --- /dev/null +++ b/src/agentready/services/harbor/runner.py @@ -0,0 +1,146 @@ +"""Service for executing Harbor benchmarks via CLI.""" + +import json +import subprocess +from pathlib import Path +from typing import List + + +class HarborNotInstalledError(Exception): + """Raised when Harbor framework is not installed.""" + + pass + + +class HarborRunner: + """Execute Harbor benchmarks via subprocess and capture results.""" + + def __init__(self): + """Initialize Harbor runner and verify installation.""" + self._verify_harbor_installed() + + def _verify_harbor_installed(self) -> None: + """Verify Harbor CLI is installed and accessible. + + Raises: + HarborNotInstalledError: If Harbor is not installed or not in PATH + """ + try: + subprocess.run( + ["harbor", "--version"], + capture_output=True, + text=True, + check=True, + timeout=5, + ) + except FileNotFoundError: + raise HarborNotInstalledError( + "Harbor framework not installed.\n" + "Install with: uv tool install harbor\n" + "See: https://harborframework.com/docs/getting-started" + ) + except subprocess.CalledProcessError as e: + raise HarborNotInstalledError(f"Harbor CLI error: {e.stderr}") + + def run_benchmark( + self, + task_names: List[str], + output_dir: Path, + dataset: str = "terminal-bench", + dataset_version: str = "2.0", + model: str = "anthropic/claude-sonnet-4-5", + agent: str = "claude-code", + n_concurrent: int = 1, + verbose: bool = True, + ) -> Path: + """Run Harbor benchmark and return results directory. + + Args: + task_names: List of task names to run (e.g., ['adaptive-rejection-sampler']) + output_dir: Directory to store results + dataset: Dataset name (default: 'terminal-bench') + dataset_version: Dataset version (default: '2.0') + model: Model identifier (default: 'anthropic/claude-sonnet-4-5') + agent: Agent identifier (default: 'claude-code') + n_concurrent: Number of concurrent tasks (default: 1) + verbose: Print Harbor output to console (default: True) + + Returns: + Path to results directory containing result.json files + + Raises: + HarborNotInstalledError: If Harbor is not installed + subprocess.CalledProcessError: If Harbor command fails + ValueError: If no tasks completed successfully + """ + output_dir.mkdir(parents=True, exist_ok=True) + + # Build Harbor command + cmd = [ + "harbor", + "run", + "-d", + f"{dataset}@{dataset_version}", + "-m", + model, + "-a", + agent, + "-n", + str(n_concurrent), + ] + + # Add task selection if specified + if task_names: + # Harbor uses config JSON for task selection + config_file = output_dir / "config.json" + config = { + "datasets": [ + { + "name": dataset, + "version": dataset_version, + "task_names": task_names, + } + ], + "agents": [{"name": agent, "model_name": model}], + } + with open(config_file, "w") as f: + json.dump(config, f, indent=2) + + # Use config file instead of CLI args + cmd = ["harbor", "run", "-c", str(config_file)] + + # Execute Harbor benchmark + if verbose: + print(f"Running Harbor benchmark: {' '.join(cmd)}") + print(f"Tasks: {', '.join(task_names) if task_names else 'all'}") + + try: + result = subprocess.run( + cmd, + cwd=str(output_dir), + capture_output=not verbose, + text=True, + check=True, + timeout=None, # No timeout for long-running benchmarks + ) + + if verbose and result.stdout: + print(result.stdout) + + except subprocess.CalledProcessError as e: + error_msg = f"Harbor benchmark failed: {e.stderr if e.stderr else str(e)}" + raise subprocess.CalledProcessError( + e.returncode, e.cmd, e.output, error_msg + ) + + # Find results directory (Harbor creates timestamped subdirectory) + results_dirs = sorted(output_dir.glob("*"), key=lambda p: p.stat().st_mtime) + if not results_dirs: + raise ValueError(f"No results found in {output_dir}") + + results_dir = results_dirs[-1] # Most recent run + + if verbose: + print(f"Results stored in: {results_dir}") + + return results_dir diff --git a/src/agentready/templates/harbor_comparison.html.j2 b/src/agentready/templates/harbor_comparison.html.j2 new file mode 100644 index 00000000..c609d15b --- /dev/null +++ b/src/agentready/templates/harbor_comparison.html.j2 @@ -0,0 +1,510 @@ + + + + + + Harbor Benchmark Comparison - AgentReady + + + + +
+
+

Harbor Benchmark Comparison

+
doubleagent.md Impact Analysis
+
+ Generated: {{ created_at }} +
+
+ +
+
+
Success Rate
+
{{ "%.1f"|format(with_agent.success_rate) }}%
+
+ {{ "%+.1f"|format(deltas.success_rate_delta) }}% +
+ {% if significance.success_rate_p_value %} +
+ {% if significance.success_rate_significant %}✓ Significant{% else %}Not Significant{% endif %} + (p={{ "%.4f"|format(significance.success_rate_p_value) }}) +
+ {% endif %} +
+ +
+
Avg Duration
+
{{ "%.1f"|format(with_agent.avg_duration_sec) }}s
+
+ {{ "%+.1f"|format(deltas.avg_duration_delta_pct) }}% +
+ {% if significance.duration_p_value %} +
+ {% if significance.duration_significant %}✓ Significant{% else %}Not Significant{% endif %} + (p={{ "%.4f"|format(significance.duration_p_value) }}) +
+ {% endif %} +
+ +
+
Successful Tasks
+
{{ with_agent.successful_tasks }}/{{ with_agent.total_tasks }}
+
+ {{ "%+d"|format(deltas.successful_tasks_delta|int) }} tasks +
+
+ +
+
Completion Rate
+
{{ "%.1f"|format(with_agent.completion_rate) }}%
+
+ {{ "%+.1f"|format(deltas.completion_rate_delta) }}% +
+
+
+ +
+
+
+
Success Rate Comparison
+ +
+ +
+
Average Duration Comparison
+ +
+
+
+ +
+

Per-Task Results

+ + + + + + + + + + + {% for task in per_task %} + + + + + + + {% endfor %} + +
Task NameWithout AgentWith AgentImpact
{{ task.task_name }} + {% if task.without_agent %} + + {% if task.without_agent.success %}✓ Success{% else %}✗ Failed{% endif %} + +
+ {{ "%.1f"|format(task.without_agent.duration_sec) }}s +
+ {% else %} + N/A + {% endif %} + 		+ {% if task.with_agent %} + + {% if task.with_agent.success %}✓ Success{% else %}✗ Failed{% endif %} + +
+ {{ "%.1f"|format(task.with_agent.duration_sec) }}s +
+ {% else %} + N/A + {% endif %} + 		+ {% if task.delta %} + {% if task.delta.success_improved %} + +100% (Fixed) + {% elif task.delta.duration_delta_pct %} + + {{ "%+.1f"|format(task.delta.duration_delta_pct) }}% duration + + {% else %} + Unchanged + {% endif %} + {% else %} + - + {% endif %} +
+
+ +
+ Generated by AgentReady Harbor Comparison Tool · {{ created_at }} +
+
+ + + + diff --git a/tests/unit/test_harbor_models.py b/tests/unit/test_harbor_models.py new file mode 100644 index 00000000..a6bb0009 --- /dev/null +++ b/tests/unit/test_harbor_models.py @@ -0,0 +1,297 @@ +"""Unit tests for Harbor data models.""" + +import pytest + +from agentready.models.harbor import ( + HarborComparison, + HarborRunMetrics, + HarborTaskResult, +) + + +class TestHarborTaskResult: + """Tests for HarborTaskResult model.""" + + def test_from_result_json_success(self): + """Test creating HarborTaskResult from successful result.json data.""" + result_data = { + "task_name": "adaptive-rejection-sampler", + "trial_name": "adaptive-rejection-sampler__ABC123", + "agent_result": {"status": "completed"}, + "verifier_result": {"passed": True}, + "exception_info": None, + "started_at": "2025-12-09T10:00:00", + "finished_at": "2025-12-09T10:05:00", + } + + result = HarborTaskResult.from_result_json(result_data) + + assert result.task_name == "adaptive-rejection-sampler" + assert result.trial_name == "adaptive-rejection-sampler__ABC123" + assert result.success is True + assert result.duration_sec == 300.0 # 5 minutes + assert result.agent_result == {"status": "completed"} + assert result.verifier_result == {"passed": True} + assert result.exception_info is None + + def test_from_result_json_failure(self): + """Test creating HarborTaskResult from failed result.json data.""" + result_data = { + "task_name": "async-http-client", + "trial_name": "async-http-client__DEF456", + "agent_result": None, + "verifier_result": None, + "exception_info": { + "exception_type": "TimeoutError", + "exception_message": "Task timed out", + }, + "started_at": "2025-12-09T10:00:00", + "finished_at": "2025-12-09T10:30:00", + } + + result = HarborTaskResult.from_result_json(result_data) + + assert result.task_name == "async-http-client" + assert result.success is False + assert result.duration_sec == 1800.0 # 30 minutes + assert result.exception_info["exception_type"] == "TimeoutError" + + def test_to_dict(self): + """Test converting HarborTaskResult to dictionary.""" + result = HarborTaskResult( + task_name="test-task", + trial_name="test-task__123", + success=True, + duration_sec=120.0, + agent_result={"status": "ok"}, + verifier_result={"passed": True}, + exception_info=None, + started_at="2025-12-09T10:00:00", + finished_at="2025-12-09T10:02:00", + ) + + result_dict = result.to_dict() + + assert result_dict["task_name"] == "test-task" + assert result_dict["success"] is True + assert result_dict["duration_sec"] == 120.0 + + +class TestHarborRunMetrics: + """Tests for HarborRunMetrics model.""" + + def test_from_task_results_all_successful(self): + """Test calculating metrics from all successful task results.""" + task_results = [ + HarborTaskResult( + task_name=f"task{i}", + trial_name=f"task{i}__ABC", + success=True, + duration_sec=60.0 * i, + agent_result={"status": "ok"}, + verifier_result={"passed": True}, + exception_info=None, + started_at="2025-12-09T10:00:00", + finished_at=f"2025-12-09T10:0{i}:00", + ) + for i in range(1, 4) + ] + + metrics = HarborRunMetrics.from_task_results("run1", True, task_results) + + assert metrics.run_id == "run1" + assert metrics.agent_file_enabled is True + assert metrics.total_tasks == 3 + assert metrics.successful_tasks == 3 + assert metrics.failed_tasks == 0 + assert metrics.timed_out_tasks == 0 + assert metrics.success_rate == 100.0 + assert metrics.completion_rate == 100.0 + assert metrics.avg_duration_sec == 120.0 # (60 + 120 + 180) / 3 + + def test_from_task_results_mixed(self): + """Test calculating metrics from mixed success/failure results.""" + task_results = [ + HarborTaskResult( + task_name="task1", + trial_name="task1__ABC", + success=True, + duration_sec=60.0, + agent_result={"status": "ok"}, + verifier_result={"passed": True}, + exception_info=None, + started_at="2025-12-09T10:00:00", + finished_at="2025-12-09T10:01:00", + ), + HarborTaskResult( + task_name="task2", + trial_name="task2__DEF", + success=False, + duration_sec=120.0, + agent_result=None, + verifier_result=None, + exception_info={"exception_type": "TimeoutError"}, + started_at="2025-12-09T10:00:00", + finished_at="2025-12-09T10:02:00", + ), + HarborTaskResult( + task_name="task3", + trial_name="task3__GHI", + success=False, + duration_sec=90.0, + agent_result={"status": "error"}, + verifier_result=None, + exception_info=None, + started_at="2025-12-09T10:00:00", + finished_at="2025-12-09T10:01:30", + ), + ] + + metrics = HarborRunMetrics.from_task_results("run2", False, task_results) + + assert metrics.total_tasks == 3 + assert metrics.successful_tasks == 1 + assert metrics.failed_tasks == 1 # task3 (no timeout exception) + assert metrics.timed_out_tasks == 1 # task2 + assert metrics.success_rate == pytest.approx(33.33, rel=0.01) + assert metrics.completion_rate == pytest.approx(66.67, rel=0.01) + + def test_to_dict(self): + """Test converting HarborRunMetrics to dictionary.""" + task_results = [ + HarborTaskResult( + task_name="task1", + trial_name="task1__ABC", + success=True, + duration_sec=60.0, + agent_result={"status": "ok"}, + verifier_result={"passed": True}, + exception_info=None, + started_at="2025-12-09T10:00:00", + finished_at="2025-12-09T10:01:00", + ) + ] + + metrics = HarborRunMetrics.from_task_results("run1", True, task_results) + metrics_dict = metrics.to_dict() + + assert metrics_dict["run_id"] == "run1" + assert metrics_dict["agent_file_enabled"] is True + assert metrics_dict["total_tasks"] == 1 + assert len(metrics_dict["task_results"]) == 1 + + +class TestHarborComparison: + """Tests for HarborComparison model.""" + + @pytest.fixture + def sample_metrics(self): + """Create sample metrics for testing.""" + without_results = [ + HarborTaskResult( + task_name="task1", + trial_name="task1__ABC", + success=False, + duration_sec=120.0, + agent_result=None, + verifier_result=None, + exception_info={"exception_type": "TimeoutError"}, + started_at="2025-12-09T10:00:00", + finished_at="2025-12-09T10:02:00", + ), + HarborTaskResult( + task_name="task2", + trial_name="task2__DEF", + success=True, + duration_sec=180.0, + agent_result={"status": "ok"}, + verifier_result={"passed": True}, + exception_info=None, + started_at="2025-12-09T10:00:00", + finished_at="2025-12-09T10:03:00", + ), + ] + + with_results = [ + HarborTaskResult( + task_name="task1", + trial_name="task1__GHI", + success=True, + duration_sec=90.0, + agent_result={"status": "ok"}, + verifier_result={"passed": True}, + exception_info=None, + started_at="2025-12-09T10:00:00", + finished_at="2025-12-09T10:01:30", + ), + HarborTaskResult( + task_name="task2", + trial_name="task2__JKL", + success=True, + duration_sec=150.0, + agent_result={"status": "ok"}, + verifier_result={"passed": True}, + exception_info=None, + started_at="2025-12-09T10:00:00", + finished_at="2025-12-09T10:02:30", + ), + ] + + without_agent = HarborRunMetrics.from_task_results( + "run1", False, without_results + ) + with_agent = HarborRunMetrics.from_task_results("run2", True, with_results) + + return without_agent, with_agent + + def test_calculate_deltas(self, sample_metrics): + """Test calculating delta metrics.""" + without_agent, with_agent = sample_metrics + comparison = HarborComparison( + without_agent=without_agent, with_agent=with_agent + ) + + comparison.calculate_deltas() + + assert "success_rate_delta" in comparison.deltas + assert comparison.deltas["success_rate_delta"] == 50.0 # 50% -> 100% + assert "avg_duration_delta_sec" in comparison.deltas + assert "avg_duration_delta_pct" in comparison.deltas + assert comparison.deltas["successful_tasks_delta"] == 1 # 1 -> 2 + + def test_generate_per_task_comparison(self, sample_metrics): + """Test generating per-task comparison.""" + without_agent, with_agent = sample_metrics + comparison = HarborComparison( + without_agent=without_agent, with_agent=with_agent + ) + + comparison.generate_per_task_comparison() + + assert len(comparison.per_task_comparison) == 2 + task1_comparison = next( + c for c in comparison.per_task_comparison if c["task_name"] == "task1" + ) + + assert task1_comparison["without_agent"]["success"] is False + assert task1_comparison["with_agent"]["success"] is True + assert task1_comparison["delta"]["success_improved"] is True + + def test_to_dict_and_from_dict(self, sample_metrics): + """Test serialization and deserialization.""" + without_agent, with_agent = sample_metrics + comparison = HarborComparison( + without_agent=without_agent, with_agent=with_agent + ) + comparison.calculate_deltas() + comparison.generate_per_task_comparison() + + comparison_dict = comparison.to_dict() + restored_comparison = HarborComparison.from_dict(comparison_dict) + + assert restored_comparison.without_agent.run_id == without_agent.run_id + assert restored_comparison.with_agent.run_id == with_agent.run_id + assert restored_comparison.deltas == comparison.deltas + assert len(restored_comparison.per_task_comparison) == len( + comparison.per_task_comparison + ) diff --git a/tests/unit/test_harbor_services.py b/tests/unit/test_harbor_services.py new file mode 100644 index 00000000..d2d000c0 --- /dev/null +++ b/tests/unit/test_harbor_services.py @@ -0,0 +1,255 @@ +"""Unit tests for Harbor services.""" + +import json + +import pytest + +from agentready.models.harbor import HarborTaskResult +from agentready.services.harbor.agent_toggler import AgentFileToggler +from agentready.services.harbor.result_parser import ( + parse_harbor_results, + parse_single_result, +) + + +class TestAgentFileToggler: + """Tests for AgentFileToggler service.""" + + @pytest.fixture + def sample_agent_file(self, tmp_path): + """Create a sample agent file for testing.""" + agent_file = tmp_path / ".claude" / "agents" / "doubleagent.md" + agent_file.parent.mkdir(parents=True, exist_ok=True) + agent_file.write_text("# Agent Content\n\nThis is the agent file.") + return agent_file + + def test_disable_enable(self, sample_agent_file): + """Test basic disable/enable functionality.""" + toggler = AgentFileToggler(sample_agent_file) + + # Initially enabled + assert toggler.is_enabled() + assert not toggler.is_disabled() + + # Disable + toggler.disable() + assert not toggler.is_enabled() + assert toggler.is_disabled() + assert not sample_agent_file.exists() + assert toggler.disabled_file.exists() + + # Enable + toggler.enable() + assert toggler.is_enabled() + assert not toggler.is_disabled() + assert sample_agent_file.exists() + assert not toggler.disabled_file.exists() + + def test_disable_idempotent(self, sample_agent_file): + """Test that disable is idempotent.""" + toggler = AgentFileToggler(sample_agent_file) + + toggler.disable() + assert toggler.is_disabled() + + # Disable again (should be no-op) + toggler.disable() + assert toggler.is_disabled() + + def test_enable_idempotent(self, sample_agent_file): + """Test that enable is idempotent.""" + toggler = AgentFileToggler(sample_agent_file) + + toggler.disable() + toggler.enable() + assert toggler.is_enabled() + + # Enable again (should be no-op) + toggler.enable() + assert toggler.is_enabled() + + def test_temporarily_disabled_context_manager(self, sample_agent_file): + """Test temporarily_disabled context manager.""" + toggler = AgentFileToggler(sample_agent_file) + + assert toggler.is_enabled() + + with toggler.temporarily_disabled(): + assert toggler.is_disabled() + assert not sample_agent_file.exists() + + # Restored after context exit + assert toggler.is_enabled() + assert sample_agent_file.exists() + + def test_temporarily_disabled_with_exception(self, sample_agent_file): + """Test that temporarily_disabled restores even on exception.""" + toggler = AgentFileToggler(sample_agent_file) + + assert toggler.is_enabled() + + with pytest.raises(ValueError): + with toggler.temporarily_disabled(): + assert toggler.is_disabled() + raise ValueError("Test exception") + + # Restored even after exception + assert toggler.is_enabled() + assert sample_agent_file.exists() + + def test_temporarily_enabled_context_manager(self, sample_agent_file): + """Test temporarily_enabled context manager.""" + toggler = AgentFileToggler(sample_agent_file) + toggler.disable() + + assert toggler.is_disabled() + + with toggler.temporarily_enabled(): + assert toggler.is_enabled() + assert sample_agent_file.exists() + + # Restored to disabled after context exit + assert toggler.is_disabled() + assert not sample_agent_file.exists() + + def test_file_content_preserved(self, sample_agent_file): + """Test that file content is preserved through disable/enable.""" + original_content = sample_agent_file.read_text() + toggler = AgentFileToggler(sample_agent_file) + + toggler.disable() + toggler.enable() + + assert sample_agent_file.read_text() == original_content + + +class TestResultParser: + """Tests for result parser functions.""" + + @pytest.fixture + def sample_result_data(self): + """Sample result.json data.""" + return { + "task_name": "adaptive-rejection-sampler", + "trial_name": "adaptive-rejection-sampler__ABC123", + "agent_result": {"status": "completed"}, + "verifier_result": {"passed": True}, + "exception_info": None, + "started_at": "2025-12-09T10:00:00", + "finished_at": "2025-12-09T10:05:00", + } + + @pytest.fixture + def sample_results_dir(self, tmp_path, sample_result_data): + """Create a sample Harbor results directory with result.json files.""" + results_dir = tmp_path / "harbor_run" + results_dir.mkdir() + + # Create multiple task result directories + for i in range(1, 4): + task_dir = results_dir / f"task{i}__trial{i}" + task_dir.mkdir() + + result_file = task_dir / "result.json" + result_data = sample_result_data.copy() + result_data["task_name"] = f"task{i}" + result_data["trial_name"] = f"task{i}__trial{i}" + + with open(result_file, "w") as f: + json.dump(result_data, f) + + return results_dir + + def test_parse_single_result(self, tmp_path, sample_result_data): + """Test parsing a single result.json file.""" + result_file = tmp_path / "result.json" + with open(result_file, "w") as f: + json.dump(sample_result_data, f) + + result = parse_single_result(result_file) + + assert isinstance(result, HarborTaskResult) + assert result.task_name == "adaptive-rejection-sampler" + assert result.success is True + assert result.duration_sec == 300.0 + + def test_parse_single_result_file_not_found(self, tmp_path): + """Test parsing non-existent file raises error.""" + result_file = tmp_path / "nonexistent.json" + + with pytest.raises(FileNotFoundError): + parse_single_result(result_file) + + def test_parse_single_result_invalid_json(self, tmp_path): + """Test parsing invalid JSON raises error.""" + result_file = tmp_path / "invalid.json" + result_file.write_text("invalid json content") + + with pytest.raises(json.JSONDecodeError): + parse_single_result(result_file) + + def test_parse_harbor_results(self, sample_results_dir): + """Test parsing multiple result.json files from a directory.""" + results = parse_harbor_results(sample_results_dir) + + assert len(results) == 3 + assert all(isinstance(r, HarborTaskResult) for r in results) + assert {r.task_name for r in results} == {"task1", "task2", "task3"} + + def test_parse_harbor_results_dir_not_found(self, tmp_path): + """Test parsing non-existent directory raises error.""" + nonexistent_dir = tmp_path / "nonexistent" + + with pytest.raises(FileNotFoundError): + parse_harbor_results(nonexistent_dir) + + def test_parse_harbor_results_no_result_files(self, tmp_path): + """Test parsing directory with no result.json files raises error.""" + empty_dir = tmp_path / "empty" + empty_dir.mkdir() + + with pytest.raises(ValueError, match="No result.json files found"): + parse_harbor_results(empty_dir) + + def test_parse_harbor_results_skips_invalid_files(self, sample_results_dir): + """Test that parser skips invalid result files and continues.""" + # Add an invalid result file + invalid_dir = sample_results_dir / "invalid__task" + invalid_dir.mkdir() + invalid_file = invalid_dir / "result.json" + invalid_file.write_text("invalid json") + + # Should still parse valid files and skip invalid one + results = parse_harbor_results(sample_results_dir) + + # Should have 3 valid results (skipped the invalid one) + assert len(results) == 3 + + def test_parse_harbor_results_partial_data(self, tmp_path): + """Test parsing result with missing optional fields.""" + results_dir = tmp_path / "harbor_run" + results_dir.mkdir() + + task_dir = results_dir / "task1__trial1" + task_dir.mkdir() + + # Minimal valid result data + result_data = { + "task_name": "task1", + "trial_name": "task1__trial1", + "agent_result": None, + "verifier_result": None, + "exception_info": {"exception_type": "Error"}, + "started_at": "2025-12-09T10:00:00", + "finished_at": "2025-12-09T10:05:00", + } + + result_file = task_dir / "result.json" + with open(result_file, "w") as f: + json.dump(result_data, f) + + results = parse_harbor_results(results_dir) + + assert len(results) == 1 + assert results[0].task_name == "task1" + assert results[0].success is False # No agent/verifier results