diff --git a/README.md b/README.md index d92e8697..a3320309 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,9 @@ SHARPI_MODEL=models/SmolLM2-1.7B-Instruct-Q4_K_M.gguf \ dotnet run --project src/SharpInference.Server.Host -c Release ``` +GPU flags mirror llama.cpp: `-g`/`--ngl`/`--n-gpu-layers` are interchangeable, and `--device <0|CUDA0|none>` +pins a single GPU (no multi-GPU split). + ## Image generation Two pipelines, auto-detected from model filename. Benchmarked on AMD Zen 4 + RTX 4070 Ti (CUDA, 4 steps, diff --git a/SharpInference.slnx b/SharpInference.slnx index 9679de3f..c688bf87 100644 --- a/SharpInference.slnx +++ b/SharpInference.slnx @@ -23,6 +23,7 @@ + diff --git a/src/SharpInference.Cli/GpuDevice.cs b/src/SharpInference.Cli/GpuDevice.cs new file mode 100644 index 00000000..541849d0 --- /dev/null +++ b/src/SharpInference.Cli/GpuDevice.cs @@ -0,0 +1,67 @@ +namespace SharpInference.Cli; + +/// +/// Parses the llama.cpp-style -dev/--device option into a single GPU selection and +/// applies it. Unlike llama.cpp's comma-separated multi-GPU list, SharpInference targets a +/// single device (it has no tensor/row split), so only one device may be named. +/// +/// Accepted values: +/// +/// null / empty / auto — auto-select (returns index -1, no change) +/// none / cpu — don't offload (sets = true) +/// a bare index — 0, 1, … +/// a named device with a trailing index — CUDA0, Vulkan1, GPU2 +/// +/// +/// When a concrete index is selected it pins the CUDA runtime to that physical GPU via +/// CUDA_VISIBLE_DEVICES (honored process-wide by every CUDA host thread, unlike a +/// per-thread cudaSetDevice) and returns the same index for the Vulkan physical-device +/// selector. The CUDA env var is only set when the caller has not already constrained the +/// visible set themselves. +/// +internal static class GpuDevice +{ + public static int Resolve(string? device, out bool none) + { + none = false; + if (string.IsNullOrWhiteSpace(device)) + return -1; + + var value = device.Trim(); + if (value.Equals("auto", StringComparison.OrdinalIgnoreCase)) + return -1; + if (value.Equals("none", StringComparison.OrdinalIgnoreCase) || + value.Equals("cpu", StringComparison.OrdinalIgnoreCase)) + { + none = true; + return -1; + } + + if (value.Contains(',')) + throw new InvalidOperationException( + $"--device '{device}': multi-device split is not supported; specify a single device " + + "(e.g. 0, CUDA0, Vulkan1, or 'none' for CPU)."); + + // Strip an optional leading backend name (CUDA/Vulkan/GPU/…) and read the trailing index. + int i = value.Length; + while (i > 0 && char.IsDigit(value[i - 1])) i--; + var digits = value[i..]; + // The part before the index must be empty or a plain backend name (letters only) — + // this rejects things like "-1" (which would otherwise parse as device 1). + bool prefixOk = true; + for (int j = 0; j < i; j++) + if (!char.IsLetter(value[j])) { prefixOk = false; break; } + if (!prefixOk || digits.Length == 0 || !int.TryParse(digits, out int index) || index < 0) + throw new InvalidOperationException( + $"--device '{device}': expected a device index (0, 1, …), a named device " + + "(CUDA0, Vulkan1), 'auto', or 'none'."); + + // Pin CUDA to this physical device process-wide. Harmless on the Vulkan path (Vulkan + // enumerates all devices regardless and uses the returned index). Don't override an + // explicit CUDA_VISIBLE_DEVICES the user already set in the environment. + if (Environment.GetEnvironmentVariable("CUDA_VISIBLE_DEVICES") is null) + Environment.SetEnvironmentVariable("CUDA_VISIBLE_DEVICES", index.ToString()); + + return index; + } +} diff --git a/src/SharpInference.Cli/ImageCommand.cs b/src/SharpInference.Cli/ImageCommand.cs index 4401f602..ed6c606d 100644 --- a/src/SharpInference.Cli/ImageCommand.cs +++ b/src/SharpInference.Cli/ImageCommand.cs @@ -67,11 +67,15 @@ public sealed class Settings : CommandSettings [Description("(Z-Image) Path to Qwen3 tokenizer.json")] public string? QwenTokenizerPath { get; init; } - [CommandOption("-g|--n-gpu-layers")] + [CommandOption("--ngl|--n-gpu-layers|--gpu-layers|-g")] [Description("(Z-Image) GPU acceleration: -1 = auto (CUDA→Vulkan→CPU, default), 0 = CPU only")] [DefaultValue(-1)] public int NGpuLayers { get; init; } + [CommandOption("--device")] + [Description("(Z-Image) GPU to use: auto (default), none/cpu, an index (0, 1), or a named device (CUDA0, Vulkan1)")] + public string? Device { get; init; } + [CommandOption("--backend")] [Description("(Z-Image) Force compute backend: auto (default), cuda, vulkan, cpu")] public string? Backend { get; init; } @@ -181,10 +185,21 @@ private static int RunNative(Settings s) return 1; } - return IsZImage(modelPath) ? RunZImage(s, modelPath) : RunFlux(s, modelPath); + int deviceIndex; + bool deviceNone; + try { deviceIndex = GpuDevice.Resolve(s.Device, out deviceNone); } + catch (InvalidOperationException ex) + { + AnsiConsole.MarkupLine($"[red]Error:[/] {Markup.Escape(ex.Message)}"); + return 1; + } + + return IsZImage(modelPath) + ? RunZImage(s, modelPath, deviceIndex, deviceNone) + : RunFlux(s, modelPath, deviceIndex, deviceNone); } - private static int RunZImage(Settings s, string modelPath) + private static int RunZImage(Settings s, string modelPath, int deviceIndex, bool deviceNone) { // Auto-discover component paths from models/ directory if not explicitly provided string? vaePath = s.VaePath ?? ResolveZImageVae(); @@ -229,7 +244,9 @@ private static int RunZImage(Settings s, string modelPath) // -g 0 → CPU only // default (-1) → CUDA → Vulkan → CPU fallback string backendChoice = (s.Backend ?? "auto").ToLowerInvariant(); - bool forceCpu = s.NGpuLayers == 0 || backendChoice == "cpu"; + if (deviceNone && (backendChoice is "cuda" or "vulkan")) + AnsiConsole.MarkupLine("[yellow]Note:[/] --device none overrides --backend; running on CPU."); + bool forceCpu = s.NGpuLayers == 0 || backendChoice == "cpu" || deviceNone; bool forceCuda = backendChoice == "cuda"; bool forceVulkan = backendChoice == "vulkan"; @@ -244,12 +261,14 @@ private static int RunZImage(Settings s, string modelPath) { try { - var vulkan = new VulkanBackend(); + var vulkan = new VulkanBackend(deviceIndex); gpu = vulkan; vulkan.PrintDeviceInfo(); AnsiConsole.MarkupLine("[dim]Backend:[/] GPU (Vulkan SGEMM)"); } - catch + // Only auto-select (deviceIndex < 0) silently falls back to CPU. + // An explicit --device must surface its error, not vanish into a CPU run. + catch when (deviceIndex < 0) { AnsiConsole.MarkupLine("[dim]Backend:[/] CPU (no GPU detected)"); } @@ -275,15 +294,19 @@ private static int RunZImage(Settings s, string modelPath) { try { - upscalerGpu = new VulkanBackend(); + upscalerGpu = new VulkanBackend(deviceIndex); upscalerBackend = upscalerGpu; AnsiConsole.MarkupLine("[dim]Upscaler backend:[/] GPU (Vulkan, native kernels)"); } - catch + catch when (deviceIndex < 0) { - upscalerBackend = gpu; // fallback: SGEMM on main backend + upscalerBackend = gpu; // auto only: fall back to SGEMM on main backend } } + if (upscalerBackend is null) + AnsiConsole.MarkupLine(deviceNone + ? "[dim]Upscaler backend:[/] CPU (--device none)" + : "[dim]Upscaler backend:[/] CPU"); upscaler = RRDBNet.Load(s.UpscalerPath, upscalerBackend); AnsiConsole.MarkupLine($"[dim]Upscaler:[/] RRDBNet ×{upscaler.Scale} loaded"); } @@ -330,7 +353,7 @@ private static int RunZImage(Settings s, string modelPath) return 0; } - private static int RunFlux(Settings s, string modelPath) + private static int RunFlux(Settings s, string modelPath, int deviceIndex, bool deviceNone) { if (!RequireFile(s.VaePath, "--vae", "ae.safetensors")) return 1; if (!RequireFile(s.ClipLPath, "--clip-l", "clip_l.safetensors")) return 1; @@ -376,9 +399,14 @@ private static int RunFlux(Settings s, string modelPath) { // Prefer CudaBackend directly when NVRTC image kernels are available. // Fall back to VulkanBackend, then CudaBackend SGEMM path. + // --device none/cpu (deviceNone) skips the GPU upscaler entirely. try { - if (CudaBackend.IsAvailable()) + if (deviceNone) + { + AnsiConsole.MarkupLine("[dim]Upscaler backend:[/] CPU (--device none)"); + } + else if (CudaBackend.IsAvailable()) { var cudaForUpscaler = CudaBackend.Create(); if (cudaForUpscaler.ImageKernelsAvailable) @@ -401,10 +429,12 @@ private static int RunFlux(Settings s, string modelPath) { try { - upscalerGpu = new VulkanBackend(); + upscalerGpu = new VulkanBackend(deviceIndex); AnsiConsole.MarkupLine("[dim]Upscaler backend:[/] GPU (Vulkan, native kernels)"); } - catch + // An explicit --device error propagates to the outer handler (surfaced, + // not swallowed into a wrong-device or silent-CPU fallback). + catch when (deviceIndex < 0) { try { diff --git a/src/SharpInference.Cli/RunCommand.cs b/src/SharpInference.Cli/RunCommand.cs index a2cfc0e9..e88666c2 100644 --- a/src/SharpInference.Cli/RunCommand.cs +++ b/src/SharpInference.Cli/RunCommand.cs @@ -79,11 +79,16 @@ public sealed class Settings : CommandSettings [DefaultValue(false)] public bool VerbosePrompt { get; init; } - [CommandOption("--n-gpu-layers|-g")] - [Description("Layers on GPU (0=CPU only, -1=all, default: 0)")] + [CommandOption("--ngl|--n-gpu-layers|--gpu-layers|-g")] + [Description("Layers on GPU (0=CPU only, -1=all, default: 0). Mirrors llama.cpp's --n-gpu-layers/--ngl.")] [DefaultValue(0)] public int NGpuLayers { get; init; } + [CommandOption("--device")] + [Description("GPU device to offload to: index (0,1,…), name (CUDA0, Vulkan1), or 'none' for CPU. " + + "Default: auto. Single-device only (no multi-GPU split). Mirrors llama.cpp's --device.")] + public string? Device { get; init; } + [CommandOption("-c|--ctx-size")] [Description("Context size / max sequence length (0 = model default)")] [DefaultValue(0)] @@ -98,8 +103,8 @@ public sealed class Settings : CommandSettings [Description("KV-cache element type for the CUDA backend: fp32 (default), bf16 (half the KV VRAM → ~2x context), or q8_0 (quarter → ~4x). Like llama.cpp --cache-type-k/v. Env: SHARPI_KV_DTYPE.")] public string? KvType { get; init; } - [CommandOption("--draft-model")] - [Description("Path to a smaller draft model for speculative decoding (greedy only, requires --temp 0)")] + [CommandOption("--model-draft|--draft-model")] + [Description("Path to a smaller draft model for speculative decoding (greedy only, requires --temp 0). Mirrors llama.cpp's --model-draft.")] public string? DraftModelPath { get; init; } [CommandOption("--spec-lookahead|--draft-tokens")] @@ -142,8 +147,8 @@ public sealed class Settings : CommandSettings [DefaultValue(long.MinValue)] public long PrefillDequantCacheMb { get; init; } - [CommandOption("--rep-penalty")] - [Description("Repetition penalty (1.0 = disabled, >1.0 penalizes repeated tokens, default: 1.1)")] + [CommandOption("--repeat-penalty|--rep-penalty")] + [Description("Repetition penalty (1.0 = disabled, >1.0 penalizes repeated tokens, default: 1.1). Mirrors llama.cpp's --repeat-penalty.")] [DefaultValue(1.1f)] public float RepPenalty { get; init; } @@ -218,6 +223,24 @@ protected override int Execute(CommandContext context, Settings settings, Cancel if (settings.MinBatchBlas > 0) SimdKernels.MinBatchForBlas = settings.MinBatchBlas; + // Resolve --device before any GPU call (it may set CUDA_VISIBLE_DEVICES, which the CUDA + // driver only reads at first init; Vulkan takes the index explicitly below). `--device none` + // forces the CPU path, overriding --n-gpu-layers. + int gpuDeviceIndex; + bool deviceNone; + try + { + gpuDeviceIndex = GpuDevice.Resolve(settings.Device, out deviceNone); + } + catch (InvalidOperationException ex) + { + AnsiConsole.MarkupLine($"[red]Error:[/] {Markup.Escape(ex.Message)}"); + return 1; + } + if (deviceNone && settings.NGpuLayers != 0) + AnsiConsole.MarkupLine("[yellow]Note:[/] --device none overrides --ngl/-g; running on CPU."); + int effNGpuLayers = deviceNone ? 0 : settings.NGpuLayers; + // MoE expert-cache knobs are read from the environment inside the engine // (WarmPinConfig / HybridForwardPass / slot-manager dispose). Surface them as // CLI flags by setting the env var here — before any forward pass is built — @@ -322,7 +345,7 @@ protected override int Execute(CommandContext context, Settings settings, Cancel // chosen forward pass ships an MTP head. The actual MTP gating happens later in // RunSinglePrompt / RunInteractive based on sp.SpecType. IForwardPass? mtpFwd = null; - if (hp.IsHybridSsm && settings.NGpuLayers == 0) + if (hp.IsHybridSsm && effNGpuLayers == 0) { hybridFwd = new HybridGdnForwardPass(model, cpuBackend, hp); if (hybridFwd.HasMtpHead) mtpFwd = hybridFwd; @@ -331,7 +354,7 @@ protected override int Execute(CommandContext context, Settings settings, Cancel { // #189 dequant cache: only the pure-CPU path (no GPU offload) runs the batched // CPU prefill that consults it; under -g it would be a wasted F32 model copy. - long dequantBytes = settings.NGpuLayers != 0 + long dequantBytes = effNGpuLayers != 0 ? 0 : settings.PrefillDequantCacheMb == long.MinValue ? long.MinValue // auto / SHARPI_PREFILL_DEQUANT_MB @@ -358,7 +381,7 @@ protected override int Execute(CommandContext context, Settings settings, Cancel } } - int nGpuLayers = settings.NGpuLayers; + int nGpuLayers = effNGpuLayers; // Issue #2 (MoE on hybrid GPU+CPU produced NaN/garbled output) was resolved by // fixing the descriptor-set reuse hazard in ComputePipeline.RecordWith. @@ -591,7 +614,7 @@ protected override int Execute(CommandContext context, Settings settings, Cancel return 1; } - var gpu = new VulkanBackend(); + var gpu = new VulkanBackend(gpuDeviceIndex); gpuBackend = gpu; try { @@ -640,8 +663,8 @@ protected override int Execute(CommandContext context, Settings settings, Cancel // Hybrid: N layers GPU, rest CPU var placement = TierPlanner.Plan(model, hp, hwProfile, settings.TurboQuant, requestedCtxSize: ctxSize); - // Override with explicit -g N if user specified it - if (settings.NGpuLayers > 0) + // Override with explicit -g/--ngl N if user specified it + if (effNGpuLayers > 0) placement = placement with { GpuLayers = nGpuLayers, CpuLayers = hp.NumLayers - nGpuLayers }; var hfwd = new HybridForwardPass(model, gpu, hp, placement, settings.TurboQuant); diff --git a/src/SharpInference.Cli/SharpInference.Cli.csproj b/src/SharpInference.Cli/SharpInference.Cli.csproj index 4b9f6055..915f9e3b 100644 --- a/src/SharpInference.Cli/SharpInference.Cli.csproj +++ b/src/SharpInference.Cli/SharpInference.Cli.csproj @@ -52,4 +52,8 @@ + + + + diff --git a/src/SharpInference.Vulkan/VulkanBackend.cs b/src/SharpInference.Vulkan/VulkanBackend.cs index 2c30569a..f583b49f 100644 --- a/src/SharpInference.Vulkan/VulkanBackend.cs +++ b/src/SharpInference.Vulkan/VulkanBackend.cs @@ -251,7 +251,11 @@ private void SubmitAndWaitAsync() _vkd.vkWaitForFences(1, &fence, true, ulong.MaxValue).CheckResult(); } - public VulkanBackend() + /// + /// Physical-device index to select (from --device), or -1 to auto-select + /// (prefer a discrete GPU, fall back to any compute-capable device). + /// + public VulkanBackend(int deviceIndex = -1) { // 1. Initialize Vulkan loader vkInitialize().CheckResult(); @@ -269,7 +273,7 @@ public VulkanBackend() _vki = new VkInstanceApi(in _instance); // 3. Select physical device (prefer discrete GPU) - _physicalDevice = SelectPhysicalDevice(); + _physicalDevice = SelectPhysicalDevice(deviceIndex); _vki.vkGetPhysicalDeviceProperties(_physicalDevice, out _deviceProperties); VkPhysicalDeviceMemoryProperties memProps; _vki.vkGetPhysicalDeviceMemoryProperties(_physicalDevice, &memProps); @@ -528,7 +532,7 @@ public uint FindMemoryType(uint typeFilter, VkMemoryPropertyFlags properties) // Physical device selection // ================================================================ - private VkPhysicalDevice SelectPhysicalDevice() + private VkPhysicalDevice SelectPhysicalDevice(int deviceIndex) { uint count = 0; _vki.vkEnumeratePhysicalDevices(&count, null); @@ -538,6 +542,19 @@ private VkPhysicalDevice SelectPhysicalDevice() fixed (VkPhysicalDevice* p = devices) _vki.vkEnumeratePhysicalDevices(&count, p); + // Explicit device requested via --device: honor it exactly (no discrete-GPU fallback). + if (deviceIndex >= 0) + { + if (deviceIndex >= (int)count) + throw new InvalidOperationException( + $"--device {deviceIndex}: only {count} Vulkan device(s) present (valid indices 0..{count - 1})."); + var chosen = devices[deviceIndex]; + if (!HasComputeQueue(chosen)) + throw new InvalidOperationException( + $"--device {deviceIndex}: the selected Vulkan device has no compute queue."); + return chosen; + } + // Prefer discrete GPU, fall back to any compute-capable device VkPhysicalDevice fallback = default; foreach (var gpu in devices) diff --git a/tests/SharpInference.Tests.Cli/GpuDeviceTests.cs b/tests/SharpInference.Tests.Cli/GpuDeviceTests.cs new file mode 100644 index 00000000..78c1576a --- /dev/null +++ b/tests/SharpInference.Tests.Cli/GpuDeviceTests.cs @@ -0,0 +1,99 @@ +using SharpInference.Cli; + +namespace SharpInference.Tests.Cli; + +/// +/// Unit tests for — the llama.cpp-style --device parser. +/// Each test starts from a cleared CUDA_VISIBLE_DEVICES (restored on dispose) so the +/// env-var side effect is deterministic. xunit.runner.json disables collection parallelism, so +/// these env-mutating tests never run concurrently. +/// +public sealed class GpuDeviceTests : IDisposable +{ + private const string CvdVar = "CUDA_VISIBLE_DEVICES"; + private readonly string? _savedCvd; + + public GpuDeviceTests() + { + _savedCvd = Environment.GetEnvironmentVariable(CvdVar); + Environment.SetEnvironmentVariable(CvdVar, null); + } + + public void Dispose() => Environment.SetEnvironmentVariable(CvdVar, _savedCvd); + + [Theory] + [InlineData("0", 0)] + [InlineData("1", 1)] + [InlineData("CUDA0", 0)] + [InlineData("Vulkan1", 1)] + [InlineData("GPU2", 2)] + [InlineData("cuda3", 3)] // case-insensitive backend prefix + public void Resolve_ConcreteIndex_ReturnsIndex(string input, int expected) + { + int index = GpuDevice.Resolve(input, out bool none); + Assert.Equal(expected, index); + Assert.False(none); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + [InlineData("auto")] + [InlineData("AUTO")] + public void Resolve_Auto_ReturnsMinusOne(string? input) + { + int index = GpuDevice.Resolve(input, out bool none); + Assert.Equal(-1, index); + Assert.False(none); + } + + [Theory] + [InlineData("none")] + [InlineData("None")] + [InlineData("cpu")] + [InlineData("CPU")] + public void Resolve_None_SetsNoneFlag(string input) + { + int index = GpuDevice.Resolve(input, out bool none); + Assert.Equal(-1, index); + Assert.True(none); + } + + [Theory] + [InlineData("-1")] // must NOT mis-parse as device 1 + [InlineData("+1")] + [InlineData("0,1")] // multi-device split unsupported + [InlineData("foo")] + [InlineData("CUDA")] // named device with no index + [InlineData("0x1F")] + [InlineData("99999999999999")] // int overflow + public void Resolve_Invalid_Throws(string input) + { + Assert.Throws(() => GpuDevice.Resolve(input, out _)); + } + + [Fact] + public void Resolve_ConcreteIndex_PinsCudaVisibleDevicesWhenUnset() + { + int index = GpuDevice.Resolve("CUDA2", out _); + Assert.Equal(2, index); + Assert.Equal("2", Environment.GetEnvironmentVariable(CvdVar)); + } + + [Fact] + public void Resolve_DoesNotOverrideUserSetCudaVisibleDevices() + { + Environment.SetEnvironmentVariable(CvdVar, "7"); + int index = GpuDevice.Resolve("0", out _); + Assert.Equal(0, index); + Assert.Equal("7", Environment.GetEnvironmentVariable(CvdVar)); // preserved, not clobbered + } + + [Fact] + public void Resolve_Auto_DoesNotTouchCudaVisibleDevices() + { + GpuDevice.Resolve("auto", out _); + Assert.Null(Environment.GetEnvironmentVariable(CvdVar)); + } +} diff --git a/tests/SharpInference.Tests.Cli/SharpInference.Tests.Cli.csproj b/tests/SharpInference.Tests.Cli/SharpInference.Tests.Cli.csproj new file mode 100644 index 00000000..114e5146 --- /dev/null +++ b/tests/SharpInference.Tests.Cli/SharpInference.Tests.Cli.csproj @@ -0,0 +1,25 @@ + + + false + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/SharpInference.Tests.Cli/xunit.runner.json b/tests/SharpInference.Tests.Cli/xunit.runner.json new file mode 100644 index 00000000..dd80f43a --- /dev/null +++ b/tests/SharpInference.Tests.Cli/xunit.runner.json @@ -0,0 +1,5 @@ +{ + "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", + "parallelizeAssembly": false, + "parallelizeTestCollections": false +}