diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 7577634..bf7a19a 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -62,13 +62,16 @@ jobs:
run: uv run mypy src/promptfoo/
test:
- name: Test Python ${{ matrix.python-version }}
+ name: Test (py${{ matrix.python-version }}, ${{ matrix.os }})
runs-on: ${{ matrix.os }}
timeout-minutes: 15
strategy:
matrix:
- os: [ubuntu-latest, macos-latest, windows-latest]
- python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+ # Temporarily excluding macos-latest due to GitHub Actions runner resource constraints
+ # causing BlockingIOError [Errno 35] when spawning subprocess
+ os: [ubuntu-latest, windows-latest]
+ # Test only min and max supported Python versions for efficiency
+ python-version: ["3.9", "3.13"]
steps:
- uses: actions/checkout@v6
@@ -76,6 +79,46 @@ jobs:
with:
node-version: "24"
+ - name: Configure npm on Windows
+ if: matrix.os == 'windows-latest'
+ shell: pwsh
+ run: |
+ # Configure cache location (applies immediately to this step)
+ $cacheDir = Join-Path $env:RUNNER_TEMP "npm-cache"
+ New-Item -ItemType Directory -Force -Path $cacheDir | Out-Null
+ npm config set cache $cacheDir --location=user
+
+ # Configure prefix location (applies immediately to this step)
+ $globalPrefix = npm config get prefix
+ if (-not $globalPrefix -or $globalPrefix -eq "undefined") {
+ $globalPrefix = Join-Path $env:APPDATA "npm"
+ }
+ $globalPrefix = $globalPrefix.Trim()
+ npm config set prefix $globalPrefix --location=user
+
+ # NOW clean and verify cache (cleans the correctly-configured cache)
+ npm cache clean --force
+ npm cache verify
+
+ # Export settings for future steps
+ "NPM_CONFIG_CACHE=$cacheDir" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+ "NPM_CONFIG_PREFIX=$globalPrefix" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+ "npm_config_prefix=$globalPrefix" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+
+ # Add global bin directories to PATH
+ $binPaths = @($globalPrefix, (Join-Path $globalPrefix "bin")) | Where-Object { Test-Path $_ }
+ foreach ($binPath in $binPaths) {
+ $binPath | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+ }
+
+ Write-Host "npm cache: $cacheDir"
+ Write-Host "npm prefix: $globalPrefix"
+
+ - name: Install promptfoo globally
+ run: npm install -g promptfoo@latest
+ env:
+ NODE_OPTIONS: --max-old-space-size=4096
+
- uses: astral-sh/setup-uv@v7
with:
enable-cache: true
@@ -92,6 +135,72 @@ jobs:
- name: Test Node.js detection
run: uv run python -c "from promptfoo.cli import check_node_installed, check_npx_installed; assert check_node_installed(); assert check_npx_installed()"
+ test-npx-fallback:
+ name: Test npx fallback (py${{ matrix.python-version }}, ${{ matrix.os }})
+ runs-on: ${{ matrix.os }}
+ timeout-minutes: 15
+ strategy:
+ matrix:
+ # Test npx fallback (without global install)
+ # Temporarily excluding macos-latest due to GitHub Actions runner resource constraints
+ os: [ubuntu-latest, windows-latest]
+ # Use middle-version Python for this test
+ python-version: ["3.12"]
+ steps:
+ - uses: actions/checkout@v6
+
+ - uses: actions/setup-node@v6
+ with:
+ node-version: "24"
+
+ - name: Configure npm on Windows
+ if: matrix.os == 'windows-latest'
+ shell: pwsh
+ run: |
+ # Configure cache location (applies immediately to this step)
+ $cacheDir = Join-Path $env:RUNNER_TEMP "npm-cache"
+ New-Item -ItemType Directory -Force -Path $cacheDir | Out-Null
+ npm config set cache $cacheDir --location=user
+
+ # Configure prefix location (applies immediately to this step)
+ $globalPrefix = npm config get prefix
+ if (-not $globalPrefix -or $globalPrefix -eq "undefined") {
+ $globalPrefix = Join-Path $env:APPDATA "npm"
+ }
+ $globalPrefix = $globalPrefix.Trim()
+ npm config set prefix $globalPrefix --location=user
+
+ # NOW clean and verify cache (cleans the correctly-configured cache)
+ npm cache clean --force
+ npm cache verify
+
+ # Export settings for future steps
+ "NPM_CONFIG_CACHE=$cacheDir" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+ "NPM_CONFIG_PREFIX=$globalPrefix" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+ "npm_config_prefix=$globalPrefix" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+
+ Write-Host "npm cache: $cacheDir"
+ Write-Host "npm prefix: $globalPrefix"
+
+ # Intentionally skip installing promptfoo globally
+ # This tests the npx fallback path
+
+ - uses: astral-sh/setup-uv@v7
+ with:
+ enable-cache: true
+
+ - name: Pin Python version
+ run: uv python pin ${{ matrix.python-version }}
+
+ - name: Install package
+ run: uv sync
+
+ - name: Test CLI fallback to npx (no global install)
+ run: uv run promptfoo --version
+
+ - name: Test Node.js detection
+ run: uv run python -c "from promptfoo.cli import check_node_installed, check_npx_installed; assert check_node_installed(); assert check_npx_installed()"
+
build:
name: Build Package
runs-on: ubuntu-latest
@@ -117,7 +226,7 @@ jobs:
ci-success:
name: CI Success
- needs: [lint, type-check, test, build]
+ needs: [lint, type-check, test, test-npx-fallback, build]
if: always()
runs-on: ubuntu-latest
steps:
@@ -126,17 +235,20 @@ jobs:
LINT_RESULT="${{ needs.lint.result }}"
TYPE_CHECK_RESULT="${{ needs.type-check.result }}"
TEST_RESULT="${{ needs.test.result }}"
+ TEST_NPX_FALLBACK_RESULT="${{ needs.test-npx-fallback.result }}"
BUILD_RESULT="${{ needs.build.result }}"
echo "Job results:"
echo " lint: $LINT_RESULT"
echo " type-check: $TYPE_CHECK_RESULT"
echo " test: $TEST_RESULT"
+ echo " test-npx-fallback: $TEST_NPX_FALLBACK_RESULT"
echo " build: $BUILD_RESULT"
if [[ "$LINT_RESULT" == "failure" || "$LINT_RESULT" == "cancelled" ||
"$TYPE_CHECK_RESULT" == "failure" || "$TYPE_CHECK_RESULT" == "cancelled" ||
"$TEST_RESULT" == "failure" || "$TEST_RESULT" == "cancelled" ||
+ "$TEST_NPX_FALLBACK_RESULT" == "failure" || "$TEST_NPX_FALLBACK_RESULT" == "cancelled" ||
"$BUILD_RESULT" == "failure" || "$BUILD_RESULT" == "cancelled" ]]; then
echo "Some CI checks failed!"
exit 1
diff --git a/README.md b/README.md
index cab65f1..143c077 100644
--- a/README.md
+++ b/README.md
@@ -1,26 +1,54 @@
-# promptfoo - Python wrapper
-
-[](https://pypi.org/project/promptfoo/)
-[](https://pypi.org/project/promptfoo/)
-[](https://opensource.org/licenses/MIT)
-
-Python wrapper for [promptfoo](https://www.promptfoo.dev) - the LLM testing, red teaming, and security evaluation framework.
-
-## What is promptfoo?
-
-Promptfoo is a TypeScript/Node.js tool for:
-
-- **LLM Testing & Evaluation** - Compare prompts, models, and RAG systems
-- **Red Teaming** - Automated vulnerability testing and adversarial attacks
-- **Security Scanning** - Detect prompt injection, jailbreaks, and data leaks
-- **CI/CD Integration** - Add automated AI security checks to your pipeline
+# Promptfoo: LLM evals & red teaming
+
+
+ 
+ 
+ 
+ 
+ 
+
+
+---
+
+> **๐ฆ About this Python package**
+>
+> This is a lightweight wrapper that installs promptfoo via `pip`. It requires **Node.js 20+** and executes `npx promptfoo@latest` under the hood.
+>
+> **๐ก If you have Node.js installed**, we recommend using `npx promptfoo@latest` directly for better performance:
+>
+> ```bash
+> npx promptfoo@latest init
+> npx promptfoo@latest eval
+> ```
+>
+> See the [main project](https://github.com/promptfoo/promptfoo) for the official npm package.
+>
+> **๐ Use this pip wrapper when you:**
+>
+> - Need to install via `pip` for Python-only CI/CD environments
+> - Want to manage promptfoo with poetry/pipenv/pip alongside Python dependencies
+> - Work in environments where pip packages are easier to approve than npm
+
+---
+
+
+ promptfoo is a developer-friendly local tool for testing LLM applications. Stop the trial-and-error approach - start shipping secure, reliable AI apps.
+
+
+
+ Website ยท
+ Getting Started ยท
+ Red Teaming ยท
+ Documentation ยท
+ Discord
+
## Installation
### Requirements
- **Python 3.9+** (for this wrapper)
-- **Node.js 18+** (to run the actual promptfoo CLI)
+- **Node.js 20+** (required to run promptfoo)
### Install from PyPI
@@ -28,83 +56,121 @@ Promptfoo is a TypeScript/Node.js tool for:
pip install promptfoo
```
-This Python package is a lightweight wrapper that calls the official promptfoo CLI via `npx`.
+### Alternative: Use npx (Recommended)
-### Verify Installation
+If you have Node.js installed, you can skip the wrapper and use npx directly:
```bash
-# Check that Node.js is installed
-node --version
-
-# Run promptfoo
-promptfoo --version
+npx promptfoo@latest init
+npx promptfoo@latest eval
```
+This is faster and gives you direct access to the latest version.
+
## Quick Start
```bash
-# Initialize a new project
+# Install
+pip install promptfoo
+
+# Initialize project
promptfoo init
-# Run an evaluation
+# Run your first evaluation
promptfoo eval
+```
-# Start red teaming
-promptfoo redteam run
+See [Getting Started](https://www.promptfoo.dev/docs/getting-started/) (evals) or [Red Teaming](https://www.promptfoo.dev/docs/red-team/) (vulnerability scanning) for more.
-# View results in the web UI
-promptfoo view
-```
+## What can you do with Promptfoo?
-## Usage
+- **Test your prompts and models** with [automated evaluations](https://www.promptfoo.dev/docs/getting-started/)
+- **Secure your LLM apps** with [red teaming](https://www.promptfoo.dev/docs/red-team/) and vulnerability scanning
+- **Compare models** side-by-side (OpenAI, Anthropic, Azure, Bedrock, Ollama, and [more](https://www.promptfoo.dev/docs/providers/))
+- **Automate checks** in [CI/CD](https://www.promptfoo.dev/docs/integrations/ci-cd/)
+- **Review pull requests** for LLM-related security and compliance issues with [code scanning](https://www.promptfoo.dev/docs/code-scanning/)
+- **Share results** with your team
-The `promptfoo` command behaves identically to the official Node.js CLI. All arguments are passed through:
+Here's what it looks like in action:
-```bash
-# Get help
-promptfoo --help
+
-# Run tests
-promptfoo eval
+It works on the command line too:
-# Generate red team attacks
-promptfoo redteam generate
+
-# Run vulnerability scans
-promptfoo redteam run
+It also can generate [security vulnerability reports](https://www.promptfoo.dev/docs/red-team/):
-# View results
-promptfoo view
+
-# Export results
-promptfoo export --format json --output results.json
-```
+## Why Promptfoo?
-## How It Works
+- ๐ **Developer-first**: Fast, with features like live reload and caching
+- ๐ **Private**: LLM evals run 100% locally - your prompts never leave your machine
+- ๐ง **Flexible**: Works with any LLM API or programming language
+- ๐ช **Battle-tested**: Powers LLM apps serving 10M+ users in production
+- ๐ **Data-driven**: Make decisions based on metrics, not gut feel
+- ๐ค **Open source**: MIT licensed, with an active community
+
+## How This Wrapper Works
This Python package is a thin wrapper that:
-1. Checks if Node.js and npx are installed
-2. Executes `npx promptfoo@latest `
+1. Checks if Node.js is installed
+2. Executes `npx promptfoo@latest ` (or uses globally installed promptfoo if available)
3. Passes through all arguments and environment variables
4. Returns the same exit code
-The actual promptfoo logic runs via the TypeScript package from npm.
+The actual promptfoo logic runs via the official TypeScript package from npm. All features and commands work identically.
+
+## Python-Specific Usage
+
+### With pip
+
+```bash
+pip install promptfoo
+promptfoo eval
+```
+
+### With poetry
+
+```bash
+poetry add --group dev promptfoo
+poetry run promptfoo eval
+```
+
+### With requirements.txt
+
+```bash
+echo "promptfoo>=0.2.0" >> requirements.txt
+pip install -r requirements.txt
+promptfoo eval
+```
-## Why a Python Wrapper?
+### In CI/CD (GitHub Actions example)
-Many Python developers prefer `pip install` over `npm install` for tools in their workflow. This wrapper allows you to:
+```yaml
+- name: Setup Node.js
+ uses: actions/setup-node@v4
+ with:
+ node-version: "20"
-- Install promptfoo alongside your Python dependencies
-- Use it in Python-based CI/CD pipelines
-- Manage it with standard Python tooling (pip, poetry, pipenv, etc.)
+- name: Install promptfoo
+ run: pip install promptfoo
-## Documentation
+- name: Run red team tests
+ run: promptfoo redteam run
+```
+
+## Learn More
-- **Website**: https://www.promptfoo.dev
-- **Docs**: https://www.promptfoo.dev/docs
-- **GitHub**: https://github.com/promptfoo/promptfoo
-- **Discord**: https://discord.gg/promptfoo
+- ๐ [Full Documentation](https://www.promptfoo.dev/docs/intro/)
+- ๐ [Red Teaming Guide](https://www.promptfoo.dev/docs/red-team/)
+- ๐ฏ [Getting Started](https://www.promptfoo.dev/docs/getting-started/)
+- ๐ป [CLI Usage](https://www.promptfoo.dev/docs/usage/command-line/)
+- ๐ฆ [Main Project (npm)](https://github.com/promptfoo/promptfoo)
+- ๐ค [Supported Models](https://www.promptfoo.dev/docs/providers/)
+- ๐ฌ [Code Scanning Guide](https://www.promptfoo.dev/docs/code-scanning/)
## Troubleshooting
@@ -119,20 +185,43 @@ The wrapper needs Node.js to run. Install it:
### Slow First Run
-The first time you run `promptfoo`, npx will download the latest version from npm. Subsequent runs are fast.
+The first time you run `promptfoo`, npx downloads the latest version from npm (typically ~50MB). Subsequent runs use the cached version and are fast.
+
+To speed this up, install promptfoo globally:
+
+```bash
+npm install -g promptfoo
+```
+
+The Python wrapper will automatically use the global installation when available.
### Version Pinning
-By default, this wrapper uses `npx promptfoo@latest`. To pin a specific version, set the `PROMPTFOO_VERSION` environment variable:
+By default, this wrapper uses `npx promptfoo@latest`. To pin a specific version:
```bash
export PROMPTFOO_VERSION=0.95.0
promptfoo --version
```
-## Development
+Or install a specific version globally:
+
+```bash
+npm install -g promptfoo@0.95.0
+```
+
+## Contributing
+
+We welcome contributions! Check out our [contributing guide](https://www.promptfoo.dev/docs/contributing/) to get started.
+
+Join our [Discord community](https://discord.gg/promptfoo) for help and discussion.
+
+**For wrapper-specific issues**: Report them in this repository
+**For promptfoo features/bugs**: Report in the [main project](https://github.com/promptfoo/promptfoo)
-This is a minimal wrapper - the actual promptfoo source code lives in the main TypeScript repository.
+
+ 
+
## License
diff --git a/src/promptfoo/cli.py b/src/promptfoo/cli.py
index 261783a..a75927a 100644
--- a/src/promptfoo/cli.py
+++ b/src/promptfoo/cli.py
@@ -2,14 +2,17 @@
CLI wrapper for promptfoo
This module provides a thin wrapper around the promptfoo Node.js CLI tool.
-It executes the npx promptfoo command and passes through all arguments.
+It executes a global promptfoo binary when available, falling back to npx.
"""
import os
import shutil
import subprocess
import sys
-from typing import NoReturn
+from typing import NoReturn, Optional
+
+_WRAPPER_ENV = "PROMPTFOO_PY_WRAPPER"
+_WINDOWS_SHELL_EXTENSIONS = (".bat", ".cmd")
def check_node_installed() -> bool:
@@ -35,41 +38,128 @@ def print_installation_help() -> None:
print(" https://github.com/nvm-sh/nvm", file=sys.stderr)
+def _normalize_path(path: str) -> str:
+ return os.path.normcase(os.path.abspath(path))
+
+
+def _strip_quotes(path: str) -> str:
+ if len(path) >= 2 and path[0] == path[-1] and path[0] in ('"', "'"):
+ return path[1:-1]
+ return path
+
+
+def _split_path(path_value: str) -> list[str]:
+ entries = []
+ for entry in path_value.split(os.pathsep):
+ entry = _strip_quotes(entry.strip())
+ if entry:
+ entries.append(entry)
+ return entries
+
+
+def _resolve_argv0() -> Optional[str]:
+ if not sys.argv:
+ return None
+ argv0 = sys.argv[0]
+ if not argv0:
+ return None
+ if os.path.sep in argv0 or (os.path.altsep and os.path.altsep in argv0):
+ return _normalize_path(argv0)
+ resolved = shutil.which(argv0)
+ if resolved:
+ return _normalize_path(resolved)
+ return None
+
+
+def _find_windows_promptfoo() -> Optional[str]:
+ candidates = []
+ for key in ("NPM_CONFIG_PREFIX", "npm_config_prefix"):
+ prefix = os.environ.get(key)
+ if prefix:
+ candidates.append(prefix)
+ appdata = os.environ.get("APPDATA")
+ if appdata:
+ candidates.append(os.path.join(appdata, "npm"))
+ localappdata = os.environ.get("LOCALAPPDATA")
+ if localappdata:
+ candidates.append(os.path.join(localappdata, "npm"))
+ for env_key in ("ProgramFiles", "ProgramFiles(x86)"):
+ program_files = os.environ.get(env_key)
+ if program_files:
+ candidates.append(os.path.join(program_files, "nodejs"))
+ for base in candidates:
+ for name in ("promptfoo.cmd", "promptfoo.exe"):
+ candidate = os.path.join(base, name)
+ if os.path.isfile(candidate):
+ return candidate
+ return None
+
+
+def _find_external_promptfoo() -> Optional[str]:
+ promptfoo_path = shutil.which("promptfoo")
+ if not promptfoo_path:
+ if os.name == "nt":
+ return _find_windows_promptfoo()
+ return None
+ argv0_path = _resolve_argv0()
+ if argv0_path and _normalize_path(promptfoo_path) == argv0_path:
+ wrapper_dir = _normalize_path(os.path.dirname(promptfoo_path))
+ path_entries = [
+ entry for entry in _split_path(os.environ.get("PATH", "")) if _normalize_path(entry) != wrapper_dir
+ ]
+ if path_entries:
+ candidate = shutil.which("promptfoo", path=os.pathsep.join(path_entries))
+ if candidate:
+ return candidate
+ if os.name == "nt":
+ return _find_windows_promptfoo()
+ return None
+ return promptfoo_path
+
+
+def _requires_shell(executable: str) -> bool:
+ if os.name != "nt":
+ return False
+ _, ext = os.path.splitext(executable)
+ return ext.lower() in _WINDOWS_SHELL_EXTENSIONS
+
+
+def _run_command(cmd: list[str], env: Optional[dict[str, str]] = None) -> subprocess.CompletedProcess:
+ if _requires_shell(cmd[0]):
+ return subprocess.run(subprocess.list2cmdline(cmd), shell=True, env=env)
+ return subprocess.run(cmd, env=env)
+
+
def main() -> NoReturn:
"""
Main entry point for the promptfoo CLI wrapper.
- Executes `npx promptfoo@latest ` and passes through all arguments.
- Exits with the same exit code as the underlying promptfoo command.
+ Executes promptfoo using subprocess.run() with minimal configuration.
"""
# Check for Node.js installation
if not check_node_installed():
print_installation_help()
sys.exit(1)
- if not check_npx_installed():
- print("ERROR: npx is not available. Please ensure Node.js is properly installed.", file=sys.stderr)
- sys.exit(1)
-
- # Build the command: npx promptfoo@latest
- # Use @latest to always get the most recent version
- cmd = ["npx", "--yes", "promptfoo@latest"] + sys.argv[1:]
-
- try:
- # Execute the command and inherit stdio
- result = subprocess.run(
- cmd,
- env=os.environ.copy(),
- check=False, # Don't raise exception on non-zero exit
- )
- sys.exit(result.returncode)
- except KeyboardInterrupt:
- # Handle Ctrl+C gracefully
- print("\nInterrupted by user", file=sys.stderr)
- sys.exit(130)
- except Exception as e:
- print(f"ERROR: Failed to execute promptfoo: {e}", file=sys.stderr)
- sys.exit(1)
+ # Build command: try external promptfoo first, fall back to npx
+ promptfoo_path = None if os.environ.get(_WRAPPER_ENV) else _find_external_promptfoo()
+ if promptfoo_path:
+ cmd = [promptfoo_path] + sys.argv[1:]
+ env = os.environ.copy()
+ env[_WRAPPER_ENV] = "1"
+ result = _run_command(cmd, env=env)
+ else:
+ npx_path = shutil.which("npx")
+ if npx_path:
+ cmd = [npx_path, "-y", "promptfoo@latest"] + sys.argv[1:]
+ result = _run_command(cmd)
+ else:
+ print("ERROR: Neither promptfoo nor npx is available.", file=sys.stderr)
+ print("Please install promptfoo: npm install -g promptfoo", file=sys.stderr)
+ print("Or ensure Node.js is properly installed.", file=sys.stderr)
+ sys.exit(1)
+
+ sys.exit(result.returncode)
if __name__ == "__main__":
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 0000000..3302a01
--- /dev/null
+++ b/tests/__init__.py
@@ -0,0 +1 @@
+"""Tests for the promptfoo Python wrapper."""
diff --git a/tests/test_cli.py b/tests/test_cli.py
new file mode 100644
index 0000000..1acf1c7
--- /dev/null
+++ b/tests/test_cli.py
@@ -0,0 +1,499 @@
+"""
+Tests for the promptfoo CLI wrapper.
+
+This module tests all functionality of the CLI wrapper including:
+- Dependency detection (Node.js, npx)
+- External promptfoo detection and recursion prevention
+- Command execution with proper shell handling
+- Error handling and exit codes
+- Platform-specific behavior (Windows vs Unix)
+"""
+
+import os
+import subprocess
+import sys
+from typing import Any, Optional
+from unittest.mock import MagicMock
+
+import pytest
+
+from promptfoo.cli import (
+ _WINDOWS_SHELL_EXTENSIONS,
+ _WRAPPER_ENV,
+ _find_external_promptfoo,
+ _find_windows_promptfoo,
+ _normalize_path,
+ _requires_shell,
+ _resolve_argv0,
+ _run_command,
+ _split_path,
+ _strip_quotes,
+ check_node_installed,
+ check_npx_installed,
+ main,
+ print_installation_help,
+)
+
+
+# =============================================================================
+# Unit Tests for Helper Functions
+# =============================================================================
+
+
+class TestNodeDetection:
+ """Test Node.js and npx detection functions."""
+
+ def test_check_node_installed_when_available(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Node.js detection returns True when node is in PATH."""
+ monkeypatch.setattr("shutil.which", lambda cmd: "/usr/bin/node" if cmd == "node" else None)
+ assert check_node_installed() is True
+
+ def test_check_node_installed_when_not_available(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Node.js detection returns False when node is not in PATH."""
+ monkeypatch.setattr("shutil.which", lambda cmd: None)
+ assert check_node_installed() is False
+
+ def test_check_npx_installed_when_available(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """npx detection returns True when npx is in PATH."""
+ monkeypatch.setattr("shutil.which", lambda cmd: "/usr/bin/npx" if cmd == "npx" else None)
+ assert check_npx_installed() is True
+
+ def test_check_npx_installed_when_not_available(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """npx detection returns False when npx is not in PATH."""
+ monkeypatch.setattr("shutil.which", lambda cmd: None)
+ assert check_npx_installed() is False
+
+
+class TestInstallationHelp:
+ """Test installation help message output."""
+
+ def test_print_installation_help_outputs_to_stderr(self, capsys: pytest.CaptureFixture) -> None:
+ """Installation help is printed to stderr with expected content."""
+ print_installation_help()
+ captured = capsys.readouterr()
+ assert captured.out == "" # Nothing to stdout
+ assert "ERROR: promptfoo requires Node.js" in captured.err
+ assert "brew install node" in captured.err
+ assert "apt install nodejs npm" in captured.err
+ assert "nodejs.org" in captured.err
+ assert "nvm" in captured.err
+
+
+class TestPathUtilities:
+ """Test path normalization and manipulation functions."""
+
+ def test_normalize_path(self) -> None:
+ """Path normalization converts to absolute normalized case."""
+ result = _normalize_path(".")
+ assert os.path.isabs(result)
+ assert result == os.path.normcase(os.path.abspath("."))
+
+ @pytest.mark.parametrize(
+ "input_path,expected",
+ [
+ ('"/usr/bin"', "/usr/bin"),
+ ("'/usr/bin'", "/usr/bin"),
+ ("/usr/bin", "/usr/bin"),
+ ('""', ""),
+ ("''", ""),
+ ('"incomplete', '"incomplete'),
+ ("'incomplete", "'incomplete"),
+ ],
+ )
+ def test_strip_quotes(self, input_path: str, expected: str) -> None:
+ """Quote stripping handles various quote patterns correctly."""
+ assert _strip_quotes(input_path) == expected
+
+ @pytest.mark.parametrize(
+ "path_value,expected",
+ [
+ ("/usr/bin:/usr/local/bin", ["/usr/bin", "/usr/local/bin"]),
+ ('"/usr/bin":/usr/local/bin', ["/usr/bin", "/usr/local/bin"]),
+ ("/usr/bin::/usr/local/bin", ["/usr/bin", "/usr/local/bin"]), # Empty entry removed
+ (" /usr/bin : /usr/local/bin ", ["/usr/bin", "/usr/local/bin"]), # Whitespace
+ ("", []),
+ (":::", []), # Only separators
+ ],
+ )
+ def test_split_path(self, path_value: str, expected: list[str]) -> None:
+ """PATH splitting handles quotes, empty entries, and whitespace."""
+ assert _split_path(path_value) == expected
+
+
+class TestArgvResolution:
+ """Test sys.argv[0] resolution logic."""
+
+ def test_resolve_argv0_with_empty_argv(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Returns None when sys.argv is empty."""
+ monkeypatch.setattr(sys, "argv", [])
+ assert _resolve_argv0() is None
+
+ def test_resolve_argv0_with_empty_string(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Returns None when argv[0] is empty string."""
+ monkeypatch.setattr(sys, "argv", [""])
+ assert _resolve_argv0() is None
+
+ def test_resolve_argv0_with_absolute_path(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Returns normalized path when argv[0] contains path separator."""
+ test_path = "/usr/bin/promptfoo"
+ monkeypatch.setattr(sys, "argv", [test_path])
+ result = _resolve_argv0()
+ assert result == _normalize_path(test_path)
+
+ def test_resolve_argv0_with_command_name(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Resolves command name via which() when no path separator."""
+ monkeypatch.setattr(sys, "argv", ["promptfoo"])
+ monkeypatch.setattr("shutil.which", lambda cmd: "/usr/bin/promptfoo" if cmd == "promptfoo" else None)
+ result = _resolve_argv0()
+ assert result == _normalize_path("/usr/bin/promptfoo")
+
+ def test_resolve_argv0_with_unresolvable_command(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Returns None when command cannot be resolved via which()."""
+ monkeypatch.setattr(sys, "argv", ["promptfoo"])
+ monkeypatch.setattr("shutil.which", lambda cmd: None)
+ assert _resolve_argv0() is None
+
+
+class TestWindowsPromptfooDiscovery:
+ """Test Windows-specific promptfoo discovery."""
+
+ def test_find_windows_promptfoo_in_npm_prefix(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Finds promptfoo.cmd in npm prefix directory."""
+ monkeypatch.setenv("NPM_CONFIG_PREFIX", "C:\\npm")
+
+ def mock_isfile(p: str) -> bool:
+ return p == os.path.join("C:\\npm", "promptfoo.cmd")
+
+ monkeypatch.setattr(os.path, "isfile", mock_isfile)
+
+ # Only test on Windows or mock the function call
+ if os.name == "nt":
+ result = _find_windows_promptfoo()
+ assert result == os.path.join("C:\\npm", "promptfoo.cmd")
+ else:
+ # On non-Windows, test the logic by directly calling with mocked env
+ # This is testing the Windows code path even on Unix
+ pytest.skip("Windows-specific test, skipping on non-Windows platform")
+
+ def test_find_windows_promptfoo_in_appdata(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Finds promptfoo.cmd in APPDATA npm directory."""
+ monkeypatch.setenv("APPDATA", "C:\\Users\\test\\AppData\\Roaming")
+
+ expected_path = os.path.join("C:\\Users\\test\\AppData\\Roaming", "npm", "promptfoo.cmd")
+
+ def mock_isfile(p: str) -> bool:
+ return p == expected_path
+
+ monkeypatch.setattr(os.path, "isfile", mock_isfile)
+
+ # Only test on Windows
+ if os.name == "nt":
+ result = _find_windows_promptfoo()
+ assert result == expected_path
+ else:
+ pytest.skip("Windows-specific test, skipping on non-Windows platform")
+
+ def test_find_windows_promptfoo_not_found(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Returns None when no promptfoo found in Windows locations."""
+ monkeypatch.setattr(os.path, "isfile", lambda p: False)
+
+ # Only test on Windows
+ if os.name == "nt":
+ assert _find_windows_promptfoo() is None
+ else:
+ pytest.skip("Windows-specific test, skipping on non-Windows platform")
+
+
+class TestExternalPromptfooDiscovery:
+ """Test external promptfoo detection and recursion prevention."""
+
+ def test_find_external_promptfoo_when_not_in_path(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Returns None when no promptfoo in PATH."""
+ monkeypatch.setattr("shutil.which", lambda cmd, path=None: None)
+ monkeypatch.setattr(os, "name", "posix")
+ assert _find_external_promptfoo() is None
+
+ def test_find_external_promptfoo_when_found(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Returns path when promptfoo found and not this wrapper."""
+ promptfoo_path = "/usr/local/bin/promptfoo"
+ monkeypatch.setattr(
+ "shutil.which",
+ lambda cmd, path=None: promptfoo_path if cmd == "promptfoo" else None
+ )
+ monkeypatch.setattr(sys, "argv", ["different-script"])
+ result = _find_external_promptfoo()
+ assert result == promptfoo_path
+
+ def test_find_external_promptfoo_prevents_recursion(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Filters out wrapper directory from PATH to prevent recursion."""
+ wrapper_path = "/home/user/.local/bin/promptfoo"
+ real_promptfoo = "/usr/local/bin/promptfoo"
+
+ monkeypatch.setattr(sys, "argv", [wrapper_path])
+ monkeypatch.setenv("PATH", "/home/user/.local/bin:/usr/local/bin")
+
+ def mock_which(cmd: str, path: Optional[str] = None) -> Optional[str]:
+ if cmd != "promptfoo":
+ return None
+ if path is None:
+ return wrapper_path
+ # When called with filtered PATH, return the real one
+ if "/home/user/.local/bin" not in path:
+ return real_promptfoo
+ return None
+
+ monkeypatch.setattr("shutil.which", mock_which)
+ result = _find_external_promptfoo()
+ assert result == real_promptfoo
+
+
+class TestShellRequirement:
+ """Test Windows shell requirement detection for .bat/.cmd files."""
+
+ def test_requires_shell_on_windows_with_bat(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Returns True for .bat files on Windows."""
+ monkeypatch.setattr(os, "name", "nt")
+ assert _requires_shell("promptfoo.bat") is True
+
+ def test_requires_shell_on_windows_with_cmd(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Returns True for .cmd files on Windows."""
+ monkeypatch.setattr(os, "name", "nt")
+ assert _requires_shell("promptfoo.cmd") is True
+
+ def test_requires_shell_on_windows_with_exe(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Returns False for .exe files on Windows."""
+ monkeypatch.setattr(os, "name", "nt")
+ assert _requires_shell("promptfoo.exe") is False
+
+ def test_requires_shell_on_unix(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Returns False for all files on Unix."""
+ monkeypatch.setattr(os, "name", "posix")
+ assert _requires_shell("promptfoo.bat") is False
+ assert _requires_shell("promptfoo.cmd") is False
+ assert _requires_shell("promptfoo") is False
+
+
+class TestCommandExecution:
+ """Test command execution with proper shell handling."""
+
+ def test_run_command_with_shell_for_bat_file(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Uses shell=True for .bat files on Windows."""
+ monkeypatch.setattr(os, "name", "nt")
+ mock_run = MagicMock(return_value=subprocess.CompletedProcess([], 0))
+ monkeypatch.setattr(subprocess, "run", mock_run)
+
+ cmd = ["promptfoo.bat", "eval"]
+ _run_command(cmd)
+
+ # Should be called with shell=True
+ assert mock_run.call_count == 1
+ call_args = mock_run.call_args
+ assert call_args.kwargs.get("shell") is True
+
+ def test_run_command_without_shell_for_regular_executable(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Uses shell=False for regular executables."""
+ monkeypatch.setattr(os, "name", "posix")
+ mock_run = MagicMock(return_value=subprocess.CompletedProcess([], 0))
+ monkeypatch.setattr(subprocess, "run", mock_run)
+
+ cmd = ["/usr/bin/promptfoo", "eval"]
+ _run_command(cmd)
+
+ # Should be called with the list directly, no shell
+ assert mock_run.call_count == 1
+ call_args = mock_run.call_args
+ assert call_args.args[0] == cmd
+ assert call_args.kwargs.get("shell") is not True
+
+ def test_run_command_passes_environment(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Passes environment variables to subprocess."""
+ monkeypatch.setattr(os, "name", "posix")
+ mock_run = MagicMock(return_value=subprocess.CompletedProcess([], 0))
+ monkeypatch.setattr(subprocess, "run", mock_run)
+
+ cmd = ["promptfoo", "eval"]
+ env = {"TEST": "value"}
+ _run_command(cmd, env=env)
+
+ call_args = mock_run.call_args
+ assert call_args.kwargs.get("env") == env
+
+
+# =============================================================================
+# Integration Tests for main()
+# =============================================================================
+
+
+class TestMainFunction:
+ """Test the main CLI entry point with various scenarios."""
+
+ def test_main_exits_when_node_not_installed(
+ self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture
+ ) -> None:
+ """Exits with code 1 and prints help when Node.js not found."""
+ monkeypatch.setattr("shutil.which", lambda cmd: None)
+
+ with pytest.raises(SystemExit) as exc_info:
+ main()
+
+ assert exc_info.value.code == 1
+ captured = capsys.readouterr()
+ assert "ERROR: promptfoo requires Node.js" in captured.err
+
+ def test_main_uses_external_promptfoo_when_available(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Uses external promptfoo when found and sets wrapper env var."""
+ monkeypatch.setattr(sys, "argv", ["promptfoo", "eval"])
+ monkeypatch.setattr("shutil.which", lambda cmd, path=None: {
+ "node": "/usr/bin/node",
+ "promptfoo": "/usr/local/bin/promptfoo"
+ }.get(cmd))
+
+ mock_result = subprocess.CompletedProcess([], 0)
+ mock_run = MagicMock(return_value=mock_result)
+ monkeypatch.setattr(subprocess, "run", mock_run)
+
+ with pytest.raises(SystemExit) as exc_info:
+ main()
+
+ assert exc_info.value.code == 0
+ assert mock_run.call_count == 1
+
+ # Check command and environment
+ call_args = mock_run.call_args
+ if call_args.kwargs.get("shell"):
+ # Shell mode - check environment
+ assert call_args.kwargs["env"][_WRAPPER_ENV] == "1"
+ else:
+ # Non-shell mode
+ cmd = call_args.args[0]
+ assert cmd[0] == "/usr/local/bin/promptfoo"
+ assert cmd[1] == "eval"
+ assert call_args.kwargs["env"][_WRAPPER_ENV] == "1"
+
+ def test_main_skips_external_when_wrapper_env_set(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Skips external promptfoo search when wrapper env var is set."""
+ monkeypatch.setattr(sys, "argv", ["promptfoo", "eval"])
+ monkeypatch.setenv(_WRAPPER_ENV, "1")
+ monkeypatch.setattr("shutil.which", lambda cmd, path=None: {
+ "node": "/usr/bin/node",
+ "npx": "/usr/bin/npx",
+ "promptfoo": "/usr/local/bin/promptfoo"
+ }.get(cmd))
+
+ mock_result = subprocess.CompletedProcess([], 0)
+ mock_run = MagicMock(return_value=mock_result)
+ monkeypatch.setattr(subprocess, "run", mock_run)
+
+ with pytest.raises(SystemExit) as exc_info:
+ main()
+
+ assert exc_info.value.code == 0
+
+ # Should use npx, not external promptfoo
+ call_args = mock_run.call_args
+ if not call_args.kwargs.get("shell"):
+ cmd = call_args.args[0]
+ assert "npx" in cmd[0]
+ assert "promptfoo@latest" in cmd
+
+ def test_main_falls_back_to_npx(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Falls back to npx when no external promptfoo found."""
+ monkeypatch.setattr(sys, "argv", ["promptfoo", "eval"])
+ monkeypatch.setattr("shutil.which", lambda cmd, path=None: {
+ "node": "/usr/bin/node",
+ "npx": "/usr/bin/npx"
+ }.get(cmd))
+
+ mock_result = subprocess.CompletedProcess([], 0)
+ mock_run = MagicMock(return_value=mock_result)
+ monkeypatch.setattr(subprocess, "run", mock_run)
+
+ with pytest.raises(SystemExit) as exc_info:
+ main()
+
+ assert exc_info.value.code == 0
+ assert mock_run.call_count == 1
+
+ # Check that npx was used
+ call_args = mock_run.call_args
+ if not call_args.kwargs.get("shell"):
+ cmd = call_args.args[0]
+ assert cmd[0] == "/usr/bin/npx"
+ assert "-y" in cmd
+ assert "promptfoo@latest" in cmd
+ assert "eval" in cmd
+
+ def test_main_exits_when_neither_external_nor_npx_available(
+ self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture
+ ) -> None:
+ """Exits with error when neither external promptfoo nor npx found."""
+ monkeypatch.setattr(sys, "argv", ["promptfoo", "eval"])
+ monkeypatch.setattr("shutil.which", lambda cmd, path=None: {
+ "node": "/usr/bin/node"
+ }.get(cmd))
+
+ with pytest.raises(SystemExit) as exc_info:
+ main()
+
+ assert exc_info.value.code == 1
+ captured = capsys.readouterr()
+ assert "ERROR: Neither promptfoo nor npx is available" in captured.err
+
+ def test_main_passes_arguments_correctly(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Passes command-line arguments to the subprocess."""
+ monkeypatch.setattr(sys, "argv", ["promptfoo", "redteam", "run", "--config", "test.yaml"])
+ monkeypatch.setattr("shutil.which", lambda cmd, path=None: {
+ "node": "/usr/bin/node",
+ "npx": "/usr/bin/npx"
+ }.get(cmd))
+
+ mock_result = subprocess.CompletedProcess([], 0)
+ mock_run = MagicMock(return_value=mock_result)
+ monkeypatch.setattr(subprocess, "run", mock_run)
+
+ with pytest.raises(SystemExit):
+ main()
+
+ call_args = mock_run.call_args
+ if not call_args.kwargs.get("shell"):
+ cmd = call_args.args[0]
+ assert "redteam" in cmd
+ assert "run" in cmd
+ assert "--config" in cmd
+ assert "test.yaml" in cmd
+
+ def test_main_returns_subprocess_exit_code(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Returns the exit code from the subprocess."""
+ monkeypatch.setattr(sys, "argv", ["promptfoo", "eval"])
+ monkeypatch.setattr("shutil.which", lambda cmd, path=None: {
+ "node": "/usr/bin/node",
+ "npx": "/usr/bin/npx"
+ }.get(cmd))
+
+ # Test non-zero exit code
+ mock_result = subprocess.CompletedProcess([], 42)
+ mock_run = MagicMock(return_value=mock_result)
+ monkeypatch.setattr(subprocess, "run", mock_run)
+
+ with pytest.raises(SystemExit) as exc_info:
+ main()
+
+ assert exc_info.value.code == 42
+
+
+# =============================================================================
+# Platform-Specific Tests
+# =============================================================================
+
+
+class TestPlatformSpecificBehavior:
+ """Test platform-specific code paths."""
+
+ def test_windows_shell_extensions_constant(self) -> None:
+ """Windows shell extensions constant contains expected values."""
+ assert ".bat" in _WINDOWS_SHELL_EXTENSIONS
+ assert ".cmd" in _WINDOWS_SHELL_EXTENSIONS
+
+ def test_wrapper_env_constant(self) -> None:
+ """Wrapper environment variable constant has expected value."""
+ assert _WRAPPER_ENV == "PROMPTFOO_PY_WRAPPER"