diff --git a/core/config/backend_capabilities.go b/core/config/backend_capabilities.go index a4067d4623b2..479a80cda238 100644 --- a/core/config/backend_capabilities.go +++ b/core/config/backend_capabilities.go @@ -621,6 +621,40 @@ func NormalizeBackendName(backend string) string { return strings.ReplaceAll(backend, ".", "-") } +// llamaCppChannelSuffixes are the release-channel suffixes appended to a +// llama.cpp backend name in the gallery ("llama-cpp" vs +// "llama-cpp-development"). They carry no engine information, so they are +// stripped before the family check below. +var llamaCppChannelSuffixes = []string{"-development", "-quantization"} + +// IsLlamaCppBackend reports whether a backend name refers to a build of the +// llama.cpp gRPC server. The gallery ships one concrete backend per hardware +// capability ("vulkan-llama-cpp", "cuda12-llama-cpp", "metal-llama-cpp", ...) +// behind the "llama-cpp" meta name, and an operator may pin any of them in a +// model config. They all run the same server, so anything gated on "is this +// llama.cpp" must accept the whole family: an exact match against "llama-cpp" +// silently skips every pinned variant (see #10945, where skipping the media +// marker probe broke all vision requests). +// +// The empty name matches too: it is the GGUF auto-detect path, which resolves +// to llama.cpp. +// +// ik-llama.cpp is deliberately excluded. It is a separate engine with its own +// gRPC server that happens to share the "-llama-cpp" suffix. +func IsLlamaCppBackend(backend string) bool { + name := NormalizeBackendName(backend) + if name == "" { + return true + } + for _, suffix := range llamaCppChannelSuffixes { + name = strings.TrimSuffix(name, suffix) + } + if strings.HasSuffix(name, "ik-llama-cpp") { + return false + } + return name == "llama-cpp" || strings.HasSuffix(name, "-llama-cpp") +} + // nonLlamaSamplerBackends lists backends whose native sampler defaults differ // from llama.cpp's, so LocalAI must NOT inject llama.cpp's top_k=40 default for // them (issue #6632). mlx_lm's intended default is top_k=0 (disabled) and mlx diff --git a/core/config/backend_capabilities_test.go b/core/config/backend_capabilities_test.go index fe983c21de44..1a5e1d7db3ac 100644 --- a/core/config/backend_capabilities_test.go +++ b/core/config/backend_capabilities_test.go @@ -109,6 +109,39 @@ var _ = Describe("IsValidUsecaseForBackend", func() { }) }) +var _ = Describe("IsLlamaCppBackend", func() { + DescribeTable("classifies a backend name", + func(backend string, expected bool) { + Expect(IsLlamaCppBackend(backend)).To(Equal(expected)) + }, + Entry("meta name", "llama-cpp", true), + Entry("dotted spelling", "llama.cpp", true), + Entry("auto-detect (empty)", "", true), + Entry("development channel", "llama-cpp-development", true), + Entry("quantization channel", "llama-cpp-quantization", true), + Entry("vulkan variant", "vulkan-llama-cpp", true), + Entry("cuda 12 variant", "cuda12-llama-cpp", true), + Entry("cuda 13 variant", "cuda13-llama-cpp", true), + Entry("jetson variant", "cuda13-nvidia-l4t-arm64-llama-cpp", true), + Entry("rocm variant", "rocm-llama-cpp", true), + Entry("metal variant", "metal-llama-cpp", true), + Entry("intel sycl f16 variant", "intel-sycl-f16-llama-cpp", true), + Entry("intel sycl f32 variant", "intel-sycl-f32-llama-cpp", true), + Entry("cpu variant", "cpu-llama-cpp", true), + Entry("variant on the development channel", "rocm-llama-cpp-development", true), + Entry("darwin quantization variant", "metal-darwin-arm64-llama-cpp-quantization", true), + // ik-llama.cpp is a distinct engine that merely shares the suffix. + Entry("ik-llama-cpp", "ik-llama-cpp", false), + Entry("ik-llama-cpp development", "ik-llama-cpp-development", false), + Entry("cpu ik-llama-cpp", "cpu-ik-llama-cpp", false), + Entry("cpu ik-llama-cpp development", "cpu-ik-llama-cpp-development", false), + Entry("vllm", "vllm", false), + Entry("mlx", "mlx", false), + Entry("whisper", "whisper", false), + Entry("bark-cpp", "bark-cpp", false), + ) +}) + var _ = Describe("AllBackendNames", func() { It("returns 30+ backends in sorted order", func() { names := AllBackendNames() diff --git a/core/config/gguf.go b/core/config/gguf.go index 186c9438f106..7935d98d1c3f 100644 --- a/core/config/gguf.go +++ b/core/config/gguf.go @@ -142,7 +142,10 @@ func DetectThinkingSupportFromBackend(ctx context.Context, cfg *ModelConfig, bac // Only llama-cpp exposes ModelMetadata today. Other backends will either error // or return an empty response — both are fine, we just bail before calling. - if cfg.Backend != "llama-cpp" { + // The check must cover every llama.cpp build, not just the "llama-cpp" meta + // name: a config pinning a concrete variant ("vulkan-llama-cpp", ...) runs the + // same server and needs the same probe (#10945). + if !IsLlamaCppBackend(cfg.Backend) { xlog.Debug("[gguf] DetectThinkingSupportFromBackend: skipping detection", "backend", cfg.Backend) return } diff --git a/core/config/gguf_media_marker_test.go b/core/config/gguf_media_marker_test.go new file mode 100644 index 000000000000..5bbca00e9fc5 --- /dev/null +++ b/core/config/gguf_media_marker_test.go @@ -0,0 +1,82 @@ +package config + +import ( + "context" + + "github.com/mudler/LocalAI/pkg/grpc" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + grpclib "google.golang.org/grpc" +) + +// markerBackend answers ModelMetadata with a fixed marker and records whether it +// was called at all. Embedding grpc.Backend keeps the rest of the (large) +// interface unimplemented on purpose: DetectThinkingSupportFromBackend must only +// reach for ModelMetadata, and any other call would panic loudly. +type markerBackend struct { + grpc.Backend + marker string + called bool +} + +func (m *markerBackend) ModelMetadata(ctx context.Context, in *pb.ModelOptions, opts ...grpclib.CallOption) (*pb.ModelMetadataResponse, error) { + m.called = true + return &pb.ModelMetadataResponse{MediaMarker: m.marker}, nil +} + +var _ = Describe("media marker probing across llama.cpp backend variants", func() { + // llama.cpp picks a random per-process media marker (ggml-org/llama.cpp#21962), + // so the marker MUST come from the backend. A config that pins a concrete + // variant name still runs the same llama.cpp gRPC server and must be probed, + // or the rendered prompt keeps LocalAI's "<__media__>" sentinel and + // mtmd_tokenize reports zero markers against one bitmap (#10945). + const backendMarker = "<__media_9fJqQpVb2hZK3nT7__>" + + probe := func(backend string) *ModelConfig { + cfg := &ModelConfig{} + cfg.Backend = backend + client := &markerBackend{marker: backendMarker} + DetectThinkingSupportFromBackend(context.Background(), cfg, client, &pb.ModelOptions{}) + return cfg + } + + DescribeTable("captures the backend-reported marker", + func(backend string) { + Expect(probe(backend).MediaMarker).To(Equal(backendMarker)) + }, + Entry("meta backend", "llama-cpp"), + Entry("auto-detected (empty) backend", ""), + Entry("development channel", "llama-cpp-development"), + Entry("vulkan variant", "vulkan-llama-cpp"), + Entry("vulkan development variant", "vulkan-llama-cpp-development"), + Entry("cuda 12 variant", "cuda12-llama-cpp"), + Entry("cuda 13 jetson variant", "cuda13-nvidia-l4t-arm64-llama-cpp"), + Entry("rocm variant", "rocm-llama-cpp"), + Entry("metal variant", "metal-llama-cpp"), + Entry("intel sycl variant", "intel-sycl-f16-llama-cpp"), + Entry("cpu variant", "cpu-llama-cpp"), + Entry("quantization variant", "llama-cpp-quantization"), + ) + + DescribeTable("leaves the marker untouched for backends that are not llama.cpp", + func(backend string) { + cfg := &ModelConfig{} + cfg.Backend = backend + client := &markerBackend{marker: backendMarker} + DetectThinkingSupportFromBackend(context.Background(), cfg, client, &pb.ModelOptions{}) + Expect(client.called).To(BeFalse(), "backend %q must not be probed", backend) + Expect(cfg.MediaMarker).To(BeEmpty()) + }, + // ik-llama.cpp is a separate engine with its own gRPC server that still + // uses mtmd_default_marker(), so the sentinel already matches and there + // is nothing to probe. + Entry("ik-llama-cpp", "ik-llama-cpp"), + Entry("ik-llama-cpp development", "ik-llama-cpp-development"), + Entry("cpu ik-llama-cpp", "cpu-ik-llama-cpp"), + Entry("vllm", "vllm"), + Entry("mlx", "mlx"), + Entry("whisper", "whisper"), + ) +})