diff --git a/src/agentready/cli/learn.py b/src/agentready/cli/learn.py index 1f9e10d8..0909c4c6 100644 --- a/src/agentready/cli/learn.py +++ b/src/agentready/cli/learn.py @@ -58,6 +58,12 @@ is_flag=True, help="Bypass LLM response cache (always call API)", ) +@click.option( + "--llm-max-retries", + type=click.IntRange(min=0, max=10), + default=3, + help="Maximum retry attempts for LLM rate limits (default: 3)", +) def learn( repository, output_format, @@ -68,6 +74,7 @@ def learn( enable_llm, llm_budget, llm_no_cache, + llm_max_retries, ): """Extract reusable patterns and generate Claude Code skills. @@ -172,6 +179,7 @@ def learn( attribute_ids=list(attribute) if attribute else None, enable_llm=enable_llm, llm_budget=llm_budget, + llm_max_retries=llm_max_retries, ) except Exception as e: click.echo(f"\nError during learning: {str(e)}", err=True) diff --git a/src/agentready/learners/llm_enricher.py b/src/agentready/learners/llm_enricher.py index 7b3475d5..3012410b 100644 --- a/src/agentready/learners/llm_enricher.py +++ b/src/agentready/learners/llm_enricher.py @@ -3,6 +3,7 @@ import hashlib import json import logging +import random from pathlib import Path from time import sleep @@ -44,6 +45,8 @@ def enrich_skill( repository: Repository, finding: Finding, use_cache: bool = True, + max_retries: int = 3, + _retry_count: int = 0, ) -> DiscoveredSkill: """Enrich skill with LLM-generated content. @@ -52,9 +55,12 @@ def enrich_skill( repository: Repository being assessed finding: Finding that generated this skill use_cache: Whether to use cached responses + max_retries: Maximum retry attempts for rate limits (default: 3) + _retry_count: Internal retry counter (do not set manually) Returns: - Enriched DiscoveredSkill with LLM-generated content + Enriched DiscoveredSkill with LLM-generated content, or original + skill if enrichment fails after max retries """ # Generate cache key evidence_str = "".join(finding.evidence) if finding.evidence else "" @@ -91,12 +97,31 @@ def enrich_skill( return enriched_skill except RateLimitError as e: - logger.warning(f"Rate limit hit for {skill.skill_id}: {e}") - # Exponential backoff + # Check if max retries exceeded + if _retry_count >= max_retries: + logger.error( + f"Max retries ({max_retries}) exceeded for {skill.skill_id}. " + f"Falling back to heuristic skill. " + f"Check API quota: https://console.anthropic.com/settings/limits" + ) + return skill # Graceful fallback to heuristic skill + + # Calculate backoff with jitter to prevent thundering herd retry_after = int(getattr(e, "retry_after", 60)) - logger.info(f"Retrying after {retry_after} seconds...") - sleep(retry_after) - return self.enrich_skill(skill, repository, finding, use_cache) + jitter = random.uniform(0, min(retry_after * 0.1, 5)) + total_wait = retry_after + jitter + + logger.warning( + f"Rate limit hit for {skill.skill_id} " + f"(retry {_retry_count + 1}/{max_retries}): {e}" + ) + logger.info(f"Retrying after {total_wait:.1f} seconds...") + + sleep(total_wait) + + return self.enrich_skill( + skill, repository, finding, use_cache, max_retries, _retry_count + 1 + ) except APIError as e: # Security: Sanitize error message to prevent API key exposure diff --git a/src/agentready/services/learning_service.py b/src/agentready/services/learning_service.py index 306a4a21..007bf3f9 100644 --- a/src/agentready/services/learning_service.py +++ b/src/agentready/services/learning_service.py @@ -66,6 +66,7 @@ def extract_patterns_from_file( attribute_ids: list[str] | None = None, enable_llm: bool = False, llm_budget: int = 5, + llm_max_retries: int = 3, ) -> list[DiscoveredSkill]: """Extract patterns from an assessment file. @@ -74,6 +75,7 @@ def extract_patterns_from_file( attribute_ids: Optional list of specific attributes to extract enable_llm: Enable LLM enrichment llm_budget: Max number of skills to enrich with LLM + llm_max_retries: Maximum retry attempts for LLM rate limits Returns: List of discovered skills meeting confidence threshold @@ -168,7 +170,7 @@ def extract_patterns_from_file( # Optionally enrich with LLM if enable_llm and discovered_skills: discovered_skills = self._enrich_with_llm( - discovered_skills, assessment, llm_budget + discovered_skills, assessment, llm_budget, llm_max_retries ) return discovered_skills @@ -243,7 +245,11 @@ def _generate_json(self, skills: list[DiscoveredSkill]) -> Path: return json_file def _enrich_with_llm( - self, skills: list[DiscoveredSkill], assessment: Assessment, budget: int + self, + skills: list[DiscoveredSkill], + assessment: Assessment, + budget: int, + max_retries: int = 3, ) -> list[DiscoveredSkill]: """Enrich top N skills with LLM analysis. @@ -251,6 +257,7 @@ def _enrich_with_llm( skills: List of discovered skills assessment: Full assessment with findings budget: Max skills to enrich + max_retries: Maximum retry attempts for LLM rate limits Returns: List with top skills enriched @@ -288,7 +295,10 @@ def _enrich_with_llm( if finding: try: enriched = enricher.enrich_skill( - skill, assessment.repository, finding + skill, + assessment.repository, + finding, + max_retries=max_retries, ) enriched_skills.append(enriched) except Exception as e: @@ -318,6 +328,7 @@ def run_full_workflow( attribute_ids: list[str] | None = None, enable_llm: bool = False, llm_budget: int = 5, + llm_max_retries: int = 3, ) -> dict: """Run complete learning workflow: extract + generate. @@ -327,6 +338,7 @@ def run_full_workflow( attribute_ids: Optional specific attributes to extract enable_llm: Enable LLM enrichment llm_budget: Max skills to enrich with LLM + llm_max_retries: Maximum retry attempts for LLM rate limits Returns: Dictionary with workflow results @@ -337,6 +349,7 @@ def run_full_workflow( attribute_ids, enable_llm=enable_llm, llm_budget=llm_budget, + llm_max_retries=llm_max_retries, ) # Generate output files diff --git a/tests/e2e/test_critical_paths.py b/tests/e2e/test_critical_paths.py index 960ad7b1..ad49a76d 100644 --- a/tests/e2e/test_critical_paths.py +++ b/tests/e2e/test_critical_paths.py @@ -11,10 +11,15 @@ """ import json +import os import subprocess import tempfile from pathlib import Path +# Configurable timeout for subprocess tests (default: 90s) +# Can be overridden via AGENTREADY_TEST_TIMEOUT environment variable +DEFAULT_TIMEOUT = int(os.getenv("AGENTREADY_TEST_TIMEOUT", "90")) + class TestCriticalAssessmentFlow: """Test the primary assessment workflow end-to-end.""" @@ -34,7 +39,7 @@ def test_assess_current_repository(self): ["agentready", "assess", ".", "--output-dir", str(output_dir)], capture_output=True, text=True, - timeout=60, + timeout=DEFAULT_TIMEOUT, ) # Verify success @@ -56,7 +61,7 @@ def test_assess_generates_all_output_files(self): ["agentready", "assess", ".", "--output-dir", str(output_dir)], capture_output=True, text=True, - timeout=60, + timeout=DEFAULT_TIMEOUT, ) assert result.returncode == 0 @@ -85,7 +90,7 @@ def test_assess_json_output_is_valid(self): ["agentready", "assess", ".", "--output-dir", str(output_dir)], capture_output=True, text=True, - timeout=60, + timeout=DEFAULT_TIMEOUT, ) assert result.returncode == 0 @@ -147,7 +152,7 @@ def test_assess_html_report_generated(self): ["agentready", "assess", ".", "--output-dir", str(output_dir)], capture_output=True, text=True, - timeout=60, + timeout=DEFAULT_TIMEOUT, ) assert result.returncode == 0 @@ -171,7 +176,7 @@ def test_assess_markdown_report_generated(self): ["agentready", "assess", ".", "--output-dir", str(output_dir)], capture_output=True, text=True, - timeout=60, + timeout=DEFAULT_TIMEOUT, ) assert result.returncode == 0 @@ -294,7 +299,7 @@ def test_assess_with_valid_config(self): ], capture_output=True, text=True, - timeout=60, + timeout=DEFAULT_TIMEOUT, ) assert result.returncode == 0 @@ -308,3 +313,31 @@ def test_assess_with_valid_config(self): # Check that repomix_config is not in findings finding_ids = [f["attribute"]["id"] for f in data["findings"]] assert "repomix_config" not in finding_ids + + +class TestCriticalSecurityFeatures: + """Test critical security features work end-to-end.""" + + def test_assess_blocks_sensitive_directories(self): + """E2E: Verify sensitive directory scanning is blocked. + + Critical security feature: AgentReady should warn users before + scanning sensitive system directories and allow them to decline. + """ + # Test with /etc directory (common sensitive directory) + result = subprocess.run( + ["agentready", "assess", "/etc"], + capture_output=True, + text=True, + timeout=10, + input="n\n", # Decline to continue + ) + + # Should fail when user declines + assert result.returncode != 0, "Should fail when user declines to scan /etc" + + # Should show warning message about sensitive directory + output = result.stdout + result.stderr + assert ( + "sensitive" in output.lower() or "warning" in output.lower() + ), f"No warning about sensitive directory in output: {output[:500]}"