From c6f073c4e085e6a33f08d720c1a1058d83987f52 Mon Sep 17 00:00:00 2001 From: stefanwalcz Date: Mon, 20 Jul 2026 13:20:35 +0200 Subject: [PATCH 1/5] chore(rocm): bump ROCm base image 7.2.1 -> 7.14.0-full (native gfx1151 / Strix Halo) ROCm 7.14 adds native gfx1151 (Strix Halo / RDNA 3.5) runtime support. The prebuilt ROCm images pinned rocm/dev-ubuntu-24.04:7.2.1, which predates gfx1151 support, so the published -gpu-rocm-hipblas-* images do not run on that hardware even though gfx1151 is already in AMDGPU_TARGETS. Bump the pin to rocm/dev-ubuntu-24.04:7.14.0-full everywhere it is referenced: - .github/backend-matrix.yml (all hipblas backend build entries) - .github/workflows/base-images.yml (base-grpc-rocm-amd64 gRPC cache is rebuilt on 7.14 so it stays ABI-compatible with the new runtime base) - .github/workflows/image.yml + image-pr.yml (core hipblas image) - .agents/*.md examples + backend/Dockerfile.base-grpc-builder comment Verified on gfx1151 / Strix Halo: a full ROCm 7.14 stack (LocalAI + llama.cpp hipblas, vLLM, embeddings/reranker) runs natively, no HSA_OVERRIDE_GFX_VERSION needed. Other AMD architectures (gfx908/90a/942/1030/1100/1200/1201) build on the same base and are covered by CI; I do not have that hardware to test at runtime. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: stefanwalcz --- .agents/adding-backends.md | 2 +- .agents/building-and-testing.md | 2 +- .agents/ci-caching.md | 2 +- .github/backend-matrix.yml | 60 ++++++++++++++-------------- .github/workflows/base-images.yml | 2 +- .github/workflows/image-pr.yml | 2 +- .github/workflows/image.yml | 2 +- backend/Dockerfile.base-grpc-builder | 2 +- 8 files changed, 37 insertions(+), 37 deletions(-) diff --git a/.agents/adding-backends.md b/.agents/adding-backends.md index 969684a7d4e3..9c11b1e7ee68 100644 --- a/.agents/adding-backends.md +++ b/.agents/adding-backends.md @@ -92,7 +92,7 @@ If you have a `prepare.sh` doing the clone, delete it — the recipe belongs in - CUDA 13 builds: Add after other CUDA 13 builds (e.g., after `gpu-nvidia-cuda-13-chatterbox`) **Additional build types you may need:** -- ROCm/HIP: Use `build-type: 'hipblas'` with `base-image: "rocm/dev-ubuntu-24.04:7.2.1"` +- ROCm/HIP: Use `build-type: 'hipblas'` with `base-image: "rocm/dev-ubuntu-24.04:7.14.0-full"` - Intel/SYCL: Use `build-type: 'intel'` or `build-type: 'sycl_f16'`/`sycl_f32` with `base-image: "intel/oneapi-basekit:2025.3.2-0-devel-ubuntu24.04"` - L4T (ARM): Use `build-type: 'l4t'` with `platforms: 'linux/arm64'` and `runs-on: 'ubuntu-24.04-arm'` diff --git a/.agents/building-and-testing.md b/.agents/building-and-testing.md index 3cf85c0dc283..6672545a4aad 100644 --- a/.agents/building-and-testing.md +++ b/.agents/building-and-testing.md @@ -11,7 +11,7 @@ Let's say the user wants to build a particular backend for a given platform. For - Use `.github/backend-matrix.yml` as a reference — it's the data-only YAML that lists every backend variant's `build-type`, `base-image`, `platforms`, etc. (`backend.yml` and `backend_pr.yml` consume it via `scripts/changed-backends.js`). - l4t and cublas also require the CUDA major and minor version. - For llama-cpp / ik-llama-cpp / turboquant the matrix also sets `builder-base-image` pointing at a prebuilt `quay.io/go-skynet/ci-cache:base-grpc-*` tag. Local `make backends/` defaults to `BUILDER_TARGET=builder-fromsource` and doesn't need it — the Dockerfile's from-source stage installs everything itself. -- You can pretty print a command like `DOCKER_MAKEFLAGS=-j$(nproc --ignore=1) BUILD_TYPE=hipblas BASE_IMAGE=rocm/dev-ubuntu-24.04:7.2.1 make docker-build-coqui` +- You can pretty print a command like `DOCKER_MAKEFLAGS=-j$(nproc --ignore=1) BUILD_TYPE=hipblas BASE_IMAGE=rocm/dev-ubuntu-24.04:7.14.0-full make docker-build-coqui` - Unless the user specifies that they want you to run the command, then just print it because not all agent frontends handle long running jobs well and the output may overflow your context - The user may say they want to build AMD or ROCM instead of hipblas, or Intel instead of SYCL or NVIDIA insted of l4t or cublas. Ask for confirmation if there is ambiguity. - Sometimes the user may need extra parameters to be added to `docker build` (e.g. `--platform` for cross-platform builds or `--progress` to view the full logs), in which case you can generate the `docker build` command directly. diff --git a/.agents/ci-caching.md b/.agents/ci-caching.md index 17cc6001d908..03799eb29305 100644 --- a/.agents/ci-caching.md +++ b/.agents/ci-caching.md @@ -51,7 +51,7 @@ The C++ backend Dockerfiles (`Dockerfile.{llama-cpp,ik-llama-cpp,turboquant}`) c | `base-grpc-cuda-13-amd64` | the above + CUDA 13.0 toolkit (Ubuntu 22.04 base) | | `base-grpc-cuda-13-arm64` | the above + CUDA 13.0 sbsa toolkit (Ubuntu 24.04 base) | | `base-grpc-l4t-cuda-12-arm64` | JetPack r36.4.0 base (CUDA preinstalled, `SKIP_DRIVERS=true`) + gRPC | -| `base-grpc-rocm-amd64` | rocm/dev-ubuntu-24.04:7.2.1 base + hipblas/hipblaslt/rocblas + gRPC | +| `base-grpc-rocm-amd64` | rocm/dev-ubuntu-24.04:7.14.0-full base + hipblas/hipblaslt/rocblas + gRPC | | `base-grpc-vulkan-amd64` / `base-grpc-vulkan-arm64` | Ubuntu 24.04 + Vulkan SDK 1.4.335 + gRPC | | `base-grpc-intel-amd64` | intel/oneapi-basekit:2025.3.2 base + gRPC | diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index 5d4163f602fc..962cb457eb3e 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -2047,7 +2047,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-rerankers' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "rerankers" dockerfile: "./backend/Dockerfile.python" @@ -2061,7 +2061,7 @@ include: tag-suffix: '-gpu-rocm-hipblas-llama-cpp' builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-rocm-amd64' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "llama-cpp" dockerfile: "./backend/Dockerfile.llama-cpp" @@ -2075,7 +2075,7 @@ include: tag-suffix: '-gpu-rocm-hipblas-bonsai' builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-rocm-amd64' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "bonsai" dockerfile: "./backend/Dockerfile.bonsai" @@ -2088,7 +2088,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-vllm' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "vllm" dockerfile: "./backend/Dockerfile.python" @@ -2101,7 +2101,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-vllm-omni' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "vllm-omni" dockerfile: "./backend/Dockerfile.python" @@ -2114,7 +2114,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-sglang' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "sglang" dockerfile: "./backend/Dockerfile.python" @@ -2127,7 +2127,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-transformers' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "transformers" dockerfile: "./backend/Dockerfile.python" @@ -2140,7 +2140,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-diffusers' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "diffusers" dockerfile: "./backend/Dockerfile.python" @@ -2153,7 +2153,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-ace-step' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "ace-step" dockerfile: "./backend/Dockerfile.python" @@ -2167,7 +2167,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-kokoro' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "kokoro" dockerfile: "./backend/Dockerfile.python" @@ -2180,7 +2180,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-vibevoice' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "vibevoice" dockerfile: "./backend/Dockerfile.python" @@ -2193,7 +2193,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-liquid-audio' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "liquid-audio" dockerfile: "./backend/Dockerfile.python" @@ -2206,7 +2206,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-qwen-asr' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "qwen-asr" dockerfile: "./backend/Dockerfile.python" @@ -2219,7 +2219,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-nemo' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "nemo" dockerfile: "./backend/Dockerfile.python" @@ -2232,7 +2232,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-qwen-tts' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "qwen-tts" dockerfile: "./backend/Dockerfile.python" @@ -2245,7 +2245,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-fish-speech' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "fish-speech" dockerfile: "./backend/Dockerfile.python" @@ -2258,7 +2258,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-voxcpm' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "voxcpm" dockerfile: "./backend/Dockerfile.python" @@ -2271,7 +2271,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-pocket-tts' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "pocket-tts" dockerfile: "./backend/Dockerfile.python" @@ -2284,7 +2284,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-faster-whisper' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "faster-whisper" dockerfile: "./backend/Dockerfile.python" @@ -2297,7 +2297,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-coqui' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "coqui" dockerfile: "./backend/Dockerfile.python" @@ -3748,7 +3748,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-whisper' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "whisper" @@ -3761,7 +3761,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-crispasr' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "crispasr" @@ -3870,7 +3870,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-parakeet-cpp' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "parakeet-cpp" @@ -3979,7 +3979,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-moss-transcribe-cpp' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "moss-transcribe-cpp" @@ -4127,7 +4127,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-ced' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "ced" @@ -4275,7 +4275,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-voice-detect' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "voice-detect" @@ -4423,7 +4423,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-face-detect' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "face-detect" @@ -4532,7 +4532,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-acestep-cpp' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "acestep-cpp" @@ -5328,7 +5328,7 @@ include: # platforms: 'linux/amd64' # tag-latest: 'auto' # tag-suffix: '-gpu-hipblas-rfdetr' - # base-image: "rocm/dev-ubuntu-24.04:7.2.1" + # base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" # runs-on: 'ubuntu-latest' # skip-drivers: 'false' # backend: "rfdetr" @@ -5400,7 +5400,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-neutts' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "neutts" dockerfile: "./backend/Dockerfile.python" diff --git a/.github/workflows/base-images.yml b/.github/workflows/base-images.yml index 637b603a56fb..87ce9c3fded7 100644 --- a/.github/workflows/base-images.yml +++ b/.github/workflows/base-images.yml @@ -83,7 +83,7 @@ jobs: ubuntu-version: '2404' - tag: 'base-grpc-rocm-amd64' runs-on: 'ubuntu-latest' - base-image: 'rocm/dev-ubuntu-24.04:7.2.1' + base-image: 'rocm/dev-ubuntu-24.04:7.14.0-full' build-type: 'hipblas' cuda-major-version: '' cuda-minor-version: '' diff --git a/.github/workflows/image-pr.yml b/.github/workflows/image-pr.yml index cfc7c47f423b..c6ff8e4c0aa9 100644 --- a/.github/workflows/image-pr.yml +++ b/.github/workflows/image-pr.yml @@ -59,7 +59,7 @@ platforms: 'linux/amd64' tag-latest: 'false' tag-suffix: '-hipblas' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' makeflags: "--jobs=3 --output-sync=target" ubuntu-version: '2404' diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index 716faaaee102..4506648acd1f 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -40,7 +40,7 @@ platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-hipblas' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' makeflags: "--jobs=3 --output-sync=target" ubuntu-version: '2404' diff --git a/backend/Dockerfile.base-grpc-builder b/backend/Dockerfile.base-grpc-builder index 8843a0a72c29..c7c681dde82b 100644 --- a/backend/Dockerfile.base-grpc-builder +++ b/backend/Dockerfile.base-grpc-builder @@ -26,7 +26,7 @@ # base-grpc-cuda-13-amd64 ubuntu:22.04 + CUDA 13.0 # base-grpc-cuda-13-arm64 ubuntu:24.04 + CUDA 13.0 (sbsa) # base-grpc-l4t-cuda-12-arm64 ubuntu:22.04 + CUDA 12.x (legacy JetPack) -# base-grpc-rocm-amd64 rocm/dev-ubuntu-24.04:7.2.1 + hipblas +# base-grpc-rocm-amd64 rocm/dev-ubuntu-24.04:7.14.0-full + hipblas # base-grpc-vulkan-amd64 ubuntu:24.04 + Vulkan SDK 1.4.335 # base-grpc-vulkan-arm64 ubuntu:24.04 + Vulkan SDK ARM 1.4.335 # base-grpc-intel-amd64 intel/oneapi-basekit:2025.3.2 (sycl) From 1b874f5144cec1637e90fc672f46690d84b9cf77 Mon Sep 17 00:00:00 2001 From: stefanwalcz Date: Mon, 20 Jul 2026 13:58:32 +0200 Subject: [PATCH 2/5] fix(rocm): ROCm 7.x / TheRock build compat (drop legacy -dev pkgs, register lib paths) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ROCm 7.x moved to 'TheRock' packaging. Bumping the base to rocm/dev-ubuntu-24.04:7.14.0-full surfaces two build breakages, fixed here in ALL THREE spots that installed the legacy ROCm dev metapackages: 1) Legacy hipblas-dev / hipblaslt-dev / rocblas-dev metapackages were removed (consolidated + arch-split: amdrocm-blas-gfx*, amdrocm-blas-dev, ...). 'apt-get install' of the old names fails. The -full base already ships the BLAS dev libs+headers, so the install is dropped in: - Dockerfile (requirements-drivers) -> core image - .docker/install-base-deps.sh (section 6) -> C++ backend builder - backend/Dockerfile.python (hipblas block) -> Python backends (vllm, sglang, transformers, diffusers, kokoro, ...) 2) TheRock scatters the ROCm libs across /opt/rocm/lib, /opt/rocm/lib/rocm_sysdeps/lib and /opt/rocm/llvm/lib with no ld.so.conf.d entry, so the dynamic linker can't resolve them and the built backends fail at RUNTIME ('... cannot open shared object file') — a failure CI never sees because it only surfaces when a backend loads on an AMD GPU. Fixed by registering all three lib dirs in /etc/ld.so.conf.d/rocm.conf before ldconfig in all three blocks. Validated end-to-end on gfx1151 / Strix Halo (build + GPU inference, llama-cpp). Ref: https://rocm.docs.amd.com/en/latest/about/transition-guide-TheRock.html Assisted-by: Claude:claude-opus-4-8 Signed-off-by: stefanwalcz --- .docker/install-base-deps.sh | 22 ++++++++++++---------- Dockerfile | 23 ++++++++++++++--------- backend/Dockerfile.python | 17 ++++++++--------- 3 files changed, 34 insertions(+), 28 deletions(-) diff --git a/.docker/install-base-deps.sh b/.docker/install-base-deps.sh index 2b0e7e0c6a95..f8a5396d20cc 100755 --- a/.docker/install-base-deps.sh +++ b/.docker/install-base-deps.sh @@ -178,16 +178,18 @@ fi # --- 6. ROCm / HIP build deps (BUILD_TYPE=hipblas) --- if [ "${BUILD_TYPE:-}" = "hipblas" ] && [ "${SKIP_DRIVERS:-false}" = "false" ]; then - apt-get update - apt-get install -y --no-install-recommends \ - hipblas-dev \ - hipblaslt-dev \ - rocblas-dev - apt-get clean - rm -rf /var/lib/apt/lists/* - # I have no idea why, but the ROCM lib packages don't trigger ldconfig after they install, - # which results in local-ai and others not being able to locate the libraries. - # We run ldconfig ourselves to work around this packaging deficiency. + # ROCm 7.x ("TheRock" packaging): the legacy hipblas-dev / hipblaslt-dev / + # rocblas-dev metapackages were removed — BLAS is consolidated and split per + # GPU arch (amdrocm-blas-gfx*, amdrocm-blas-dev, amdrocm-blas-host). The + # rocm/dev-ubuntu-*:*-full base image already ships those BLAS dev libs and + # headers, so no apt install is needed here. + # TheRock ships the ROCm libs under /opt/rocm/core*/lib (reached via the + # /opt/rocm/lib alternatives symlink) but registers no ld.so.conf.d entry, so + # the dynamic linker can't resolve them. Without this the ldd-based GPU-lib + # packaging (scripts/build/package-gpu-libs.sh) silently skips librocm_kpack, + # rocBLAS, hipBLASLt, the bundled rocm_sysdeps (zlib/zstd/elf) and the LLVM + # runtime, and the built backend fails to load at runtime. + printf '/opt/rocm/lib\n/opt/rocm/lib/rocm_sysdeps/lib\n/opt/rocm/llvm/lib\n' > /etc/ld.so.conf.d/rocm.conf ldconfig # Log which GPU architectures have rocBLAS kernel support echo "rocBLAS library data architectures:" diff --git a/Dockerfile b/Dockerfile index 7e1e2f4c3ed4..08e12ee65c2c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -158,16 +158,21 @@ RUN if [ "${BUILD_TYPE}" = "clblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ ; fi RUN if [ "${BUILD_TYPE}" = "hipblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - hipblas-dev \ - hipblaslt-dev \ - rocblas-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ + # ROCm 7.x ("TheRock" packaging): the legacy hipblas-dev / hipblaslt-dev / + # rocblas-dev metapackages were removed — BLAS is consolidated and split + # per GPU arch (amdrocm-blas-gfx*, amdrocm-blas-dev, amdrocm-blas-host). + # The rocm/dev-ubuntu-*:*-full base image already ships those dev libraries + # and headers, so no apt install is needed here anymore. We still set the + # AMD capability marker. echo "amd" > /run/localai/capability && \ - # I have no idea why, but the ROCM lib packages don't trigger ldconfig after they install, which results in local-ai and others not being able - # to locate the libraries. We run ldconfig ourselves to work around this packaging deficiency + # TheRock ships the ROCm libs under /opt/rocm/core*/lib (reached via the + # /opt/rocm/lib alternatives symlink) but registers no ld.so.conf.d entry, + # so the dynamic linker can't find them. Register the path so ldconfig + # (and the ldd-based backend GPU-lib packaging) resolves librocm_kpack, + # rocBLAS, hipBLASLt, the bundled rocm_sysdeps (zlib/zstd/elf) and the + # LLVM runtime — otherwise the built backend fails at runtime with e.g. + # "librocm_kpack.so.0 / librocm_sysdeps_z.so.1: cannot open shared object". + printf '/opt/rocm/lib\n/opt/rocm/lib/rocm_sysdeps/lib\n/opt/rocm/llvm/lib\n' > /etc/ld.so.conf.d/rocm.conf && \ ldconfig \ ; fi diff --git a/backend/Dockerfile.python b/backend/Dockerfile.python index 2522a6f56be0..b075ed84f47c 100644 --- a/backend/Dockerfile.python +++ b/backend/Dockerfile.python @@ -171,15 +171,14 @@ RUN if [ "${BUILD_TYPE}" = "clblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ ; fi RUN if [ "${BUILD_TYPE}" = "hipblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - hipblas-dev \ - hipblaslt-dev \ - rocblas-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - # I have no idea why, but the ROCM lib packages don't trigger ldconfig after they install, which results in local-ai and others not being able - # to locate the libraries. We run ldconfig ourselves to work around this packaging deficiency + # ROCm 7.x ("TheRock" packaging): the legacy hipblas-dev / hipblaslt-dev / + # rocblas-dev metapackages were removed (consolidated + arch-split). The + # rocm/dev-ubuntu-*:*-full base image already ships the BLAS dev libs and + # headers, so no apt install is needed. TheRock also scatters the ROCm libs + # across /opt/rocm/lib, /opt/rocm/lib/rocm_sysdeps/lib and /opt/rocm/llvm/lib + # with no ld.so.conf.d entry — register them so ldconfig (and Python + # backends that dlopen ROCm libs at runtime) can resolve them. + printf '/opt/rocm/lib\n/opt/rocm/lib/rocm_sysdeps/lib\n/opt/rocm/llvm/lib\n' > /etc/ld.so.conf.d/rocm.conf && \ ldconfig \ ; fi From 7239e3bca6a7e40164f949552cfd9ee21cb2021c Mon Sep 17 00:00:00 2001 From: stefanwalcz Date: Mon, 20 Jul 2026 17:31:57 +0200 Subject: [PATCH 3/5] fix(rocm): update Go backend Dockerfile for TheRock packaging Apply the same ROCm 7.14 dependency + linker-path migration to backend/Dockerfile.golang: the removed legacy hipblas-dev / hipblaslt-dev / rocblas-dev metapackages are no longer installed (the rocm/dev-ubuntu-*:*-full base already ships them), and TheRock's scattered ROCm lib dirs are registered in /etc/ld.so.conf.d/rocm.conf before ldconfig so cgo-based Go backends resolve the ROCm runtime libraries at build/link and runtime. Identified by the LocalAI maintenance bot; re-authored under my identity per DCO. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: stefanwalcz --- backend/Dockerfile.golang | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/backend/Dockerfile.golang b/backend/Dockerfile.golang index 13032fa2209d..5dc1eada4cd7 100644 --- a/backend/Dockerfile.golang +++ b/backend/Dockerfile.golang @@ -169,15 +169,12 @@ RUN if [ "${BUILD_TYPE}" = "clblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ ; fi RUN if [ "${BUILD_TYPE}" = "hipblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - hipblas-dev \ - hipblaslt-dev \ - rocblas-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - # I have no idea why, but the ROCM lib packages don't trigger ldconfig after they install, which results in local-ai and others not being able - # to locate the libraries. We run ldconfig ourselves to work around this packaging deficiency + # ROCm 7.x ("TheRock" packaging): the legacy hipblas-dev / hipblaslt-dev / + # rocblas-dev metapackages were removed (consolidated + arch-split). The + # rocm/dev-ubuntu-*:*-full base image already ships the BLAS dev libs and + # headers, so no apt install is needed. Register TheRock's library paths + # so Go backends using cgo can resolve the ROCm runtime libraries. + printf '/opt/rocm/lib\n/opt/rocm/lib/rocm_sysdeps/lib\n/opt/rocm/llvm/lib\n' > /etc/ld.so.conf.d/rocm.conf && \ ldconfig \ ; fi From 21d2c924324e1b8c263f6d30482cea38c68736a6 Mon Sep 17 00:00:00 2001 From: stefanwalcz Date: Mon, 20 Jul 2026 22:41:10 +0200 Subject: [PATCH 4/5] feat(backend/python): AMD multi-arch ROCm torch for transformers/diffusers/kokoro The community whl/rocm7.0 torch wheel these backends install does not enumerate consumer/RDNA AMD GPUs (torch.cuda.device_count() == 0 on e.g. Strix Halo / gfx1151), so they silently fall back to CPU or fail to load. AMD publishes a stable ROCm 7.14 torch for essentially every AMD arch on its multi-arch index, selected per GPU via the torch[device-gfx] extra. Install torch from that index in an isolated step: the index returns 403 for packages it doesn't serve (accelerate/transformers/...), which uv treats as fatal, so only the torch family is pulled there and everything else resolves from PyPI. The arch comes from the build via a new AMDGPU_TARGETS ARG/ENV on backend/Dockerfile.python, defaulting to gfx1151. Only gfx1151 was validated on real hardware (transformers generates text, diffusers runs SD-1.5, kokoro runs TTS); the mechanism is arch-generic for any AMD GPU AMD ships a device wheel for. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: stefanwalcz --- backend/Dockerfile.python | 5 +++++ backend/python/diffusers/install.sh | 17 +++++++++++++++++ .../python/diffusers/requirements-hipblas.txt | 8 +++++--- backend/python/kokoro/install.sh | 17 +++++++++++++++++ backend/python/kokoro/requirements-hipblas.txt | 8 +++++--- backend/python/transformers/install.sh | 16 ++++++++++++++++ .../transformers/requirements-hipblas.txt | 6 ++++-- 7 files changed, 69 insertions(+), 8 deletions(-) diff --git a/backend/Dockerfile.python b/backend/Dockerfile.python index b075ed84f47c..ad363e320343 100644 --- a/backend/Dockerfile.python +++ b/backend/Dockerfile.python @@ -6,6 +6,11 @@ FROM ${BASE_IMAGE} AS builder ARG BACKEND=rerankers ARG BUILD_TYPE ENV BUILD_TYPE=${BUILD_TYPE} +# AMD GPU target arch(es) for ROCm builds (e.g. gfx1151, or a ';'-list). Exposed +# to install.sh so ROCm backends can select AMD's per-GPU torch and target the +# right arch when building from source, instead of hardcoding one. +ARG AMDGPU_TARGETS +ENV AMDGPU_TARGETS=${AMDGPU_TARGETS} ARG CUDA_MAJOR_VERSION ARG CUDA_MINOR_VERSION ARG SKIP_DRIVERS=false diff --git a/backend/python/diffusers/install.sh b/backend/python/diffusers/install.sh index 83703b1b2853..fc27a53c1a22 100755 --- a/backend/python/diffusers/install.sh +++ b/backend/python/diffusers/install.sh @@ -27,4 +27,21 @@ if [ "x${BUILD_PROFILE}" == "xl4t13" ]; then PY_STANDALONE_TAG="20251120" fi +# ROCm/gfx1151: the community whl/rocm7.0 torch wheels do not enumerate Strix +# Halo (device_count == 0). AMD ships stable ROCm 7.14 builds via its multi-arch +# index, selected per GPU with the torch[device-gfx] extra. Install torch + +# torchvision (the same pinned versions, from that index) in an isolated step: +# uv aborts on that index's 403-for-missing-package responses, so we must not let +# it resolve PyPI-only packages (diffusers, transformers, ...) there. Everything +# else then resolves from PyPI in installRequirements. +if [ "x${BUILD_PROFILE}" == "xhipblas" ]; then + # Arch from the build (AMDGPU_TARGETS); default gfx1151 -- the only arch we + # could validate on real hardware. The torch[device-...] extra takes one arch. + _gpu_arch="${AMDGPU_TARGETS:-gfx1151}"; _gpu_arch="${_gpu_arch%%;*}"; _gpu_arch="${_gpu_arch%% *}" + ensureVenv + uv pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ \ + "torch[device-${_gpu_arch}]==2.10.0+rocm7.14.0" \ + "torchvision==0.25.0+rocm7.14.0" +fi + installRequirements diff --git a/backend/python/diffusers/requirements-hipblas.txt b/backend/python/diffusers/requirements-hipblas.txt index f3666d5f504c..47846fc5c91c 100644 --- a/backend/python/diffusers/requirements-hipblas.txt +++ b/backend/python/diffusers/requirements-hipblas.txt @@ -1,6 +1,8 @@ ---extra-index-url https://download.pytorch.org/whl/rocm7.0 -torch==2.10.0+rocm7.0 -torchvision==0.25.0+rocm7.0 +# torch + torchvision are installed separately from AMD's multi-arch ROCm 7.14 +# index in install.sh (gfx1151 needs torch built against ROCm 7.13+; the +# community whl/rocm7.0 wheel does not enumerate Strix Halo). uv aborts on that +# index's 403-for-missing-pkg, so torch is pulled there; everything below resolves +# from PyPI. diffusers==0.38.0 opencv-python transformers==4.57.6 diff --git a/backend/python/kokoro/install.sh b/backend/python/kokoro/install.sh index 3e1ab8606480..c652ba6a8980 100755 --- a/backend/python/kokoro/install.sh +++ b/backend/python/kokoro/install.sh @@ -20,6 +20,23 @@ if [ "x${BUILD_PROFILE}" == "xl4t12" ]; then USE_PIP=true fi +# ROCm/gfx1151: the community whl/rocm7.0 torch wheels do not enumerate Strix +# Halo (device_count == 0). AMD ships stable ROCm 7.14 builds via its multi-arch +# index, selected per GPU with the torch[device-gfx] extra. Install torch + +# torchaudio (the same pinned versions, from that index) in an isolated step: uv +# aborts on that index's 403-for-missing-package responses, so we must not let it +# resolve PyPI-only packages (kokoro, transformers, ...) there. Everything else +# then resolves from PyPI in installRequirements. +if [ "x${BUILD_PROFILE}" == "xhipblas" ]; then + # Arch from the build (AMDGPU_TARGETS); default gfx1151 -- the only arch we + # could validate on real hardware. The torch[device-...] extra takes one arch. + _gpu_arch="${AMDGPU_TARGETS:-gfx1151}"; _gpu_arch="${_gpu_arch%%;*}"; _gpu_arch="${_gpu_arch%% *}" + ensureVenv + uv pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ \ + "torch[device-${_gpu_arch}]==2.10.0+rocm7.14.0" \ + "torchaudio==2.10.0+rocm7.14.0" +fi + installRequirements # spaCy is a dependency of misaki (used by kokoro for English phonemization). diff --git a/backend/python/kokoro/requirements-hipblas.txt b/backend/python/kokoro/requirements-hipblas.txt index 0ca1352282c9..9edeff371eeb 100644 --- a/backend/python/kokoro/requirements-hipblas.txt +++ b/backend/python/kokoro/requirements-hipblas.txt @@ -1,6 +1,8 @@ ---extra-index-url https://download.pytorch.org/whl/rocm7.0 -torch==2.10.0+rocm7.0 -torchaudio==2.10.0+rocm7.0 +# torch + torchaudio are installed separately from AMD's multi-arch ROCm 7.14 +# index in install.sh (gfx1151 needs torch built against ROCm 7.13+; the +# community whl/rocm7.0 wheel does not enumerate Strix Halo). uv aborts on that +# index's 403-for-missing-pkg, so torch is pulled there; everything below resolves +# from PyPI. transformers accelerate kokoro diff --git a/backend/python/transformers/install.sh b/backend/python/transformers/install.sh index 32befa8e6c03..e2084bb30744 100755 --- a/backend/python/transformers/install.sh +++ b/backend/python/transformers/install.sh @@ -16,4 +16,20 @@ if [ "x${BUILD_PROFILE}" == "xintel" ]; then EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match" fi +# ROCm/gfx1151: the community whl/rocm7.0 torch wheel does not enumerate Strix +# Halo (device_count == 0). AMD ships a stable ROCm 7.14 build via its multi-arch +# index, selected per GPU with the torch[device-gfx] extra. Install it (plus +# its rocm/triton deps, which all live on that same index) in an isolated step: +# uv aborts on that index's 403-for-missing-package responses, so we must not let +# it look up PyPI-only packages (accelerate, transformers, ...) there. Everything +# else then resolves from PyPI in installRequirements. +if [ "x${BUILD_PROFILE}" == "xhipblas" ]; then + # Arch from the build (AMDGPU_TARGETS); default gfx1151 -- the only arch we + # could validate on real hardware. The torch[device-...] extra takes one arch. + _gpu_arch="${AMDGPU_TARGETS:-gfx1151}"; _gpu_arch="${_gpu_arch%%;*}"; _gpu_arch="${_gpu_arch%% *}" + ensureVenv + uv pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ \ + "torch[device-${_gpu_arch}]==2.12.0+rocm7.14.0" +fi + installRequirements diff --git a/backend/python/transformers/requirements-hipblas.txt b/backend/python/transformers/requirements-hipblas.txt index 4c8ba84755a8..7ad4855faaf3 100644 --- a/backend/python/transformers/requirements-hipblas.txt +++ b/backend/python/transformers/requirements-hipblas.txt @@ -1,5 +1,7 @@ ---extra-index-url https://download.pytorch.org/whl/rocm7.0 -torch==2.10.0+rocm7.0 +# torch is installed separately from AMD's multi-arch ROCm index in install.sh +# (gfx1151 needs torch built against ROCm 7.13+; the community whl/rocm7.0 wheel +# does not enumerate Strix Halo). uv aborts on that index's 403-for-missing-pkg, +# so torch is pulled with pip there; everything below resolves from PyPI. accelerate transformers>=5.14.1 llvmlite==0.43.0 From 3f39cec73385453a53737e64b1ffaa0b15028719 Mon Sep 17 00:00:00 2001 From: stefanwalcz Date: Mon, 20 Jul 2026 22:41:56 +0200 Subject: [PATCH 5/5] feat(backend/python): build vllm from source for AMD consumer/RDNA GPUs vllm's ROCm install pulls a prebuilt wheel from wheels.vllm.ai/rocm, but that wheel exists only for CDNA data-center arches (gfx942/gfx950) and is pinned to a rocm7.2.3 torch that can't enumerate consumer/RDNA GPUs (device_count == 0 on e.g. gfx1151). No prebuilt vllm wheel exists for these GPUs anywhere, so build it from source against AMD's ROCm 7.14 torch -- reproducing the recipe AMD's own rocm/vllm:*_rdna_* image uses. - torch[device-gfx] + rocm-sdk-devel (hipcc + the HIP CMake packages the runtime SDK lacks) from AMD's multi-arch index; ROCM_PATH / CMAKE_PREFIX_PATH resolved from the SDK's own `rocm-sdk path` CLI -- self-contained, no system /opt/rocm needed. - vLLM detects ROCm at runtime via `import amdsmi`; register the SDK's in-place amd_smi with a .pth. A pip copy breaks its relative libamd_smi.so lookup, and forcing the lib via LD_LIBRARY_PATH shadows torch's own ROCm runtime and zeroes device_count. - run.sh exposes the bundled amdclang as CC/CXX: Triton JIT-compiles kernels at first inference and the runtime image ships hipcc but not cc/gcc, so without it inference dies with "Failed to find C compiler". Mirrors the CPU toolchain block just above it. - arch from AMDGPU_TARGETS (added in the multi-arch-torch change this stacks on), defaulting to gfx1151. Validated on gfx1151: builds from a single `make`, loads natively via LocalAI (no external gRPC) and generates. Only gfx1151 was hardware-tested; other AMD arches use the same mechanism but are unverified. Depends on #10978 (ROCm 7.14 base) and the multi-arch-torch change it stacks on. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: stefanwalcz --- backend/python/vllm/install.sh | 91 +++++++++++++++++--- backend/python/vllm/requirements-hipblas.txt | 7 +- backend/python/vllm/run.sh | 14 +++ 3 files changed, 97 insertions(+), 15 deletions(-) diff --git a/backend/python/vllm/install.sh b/backend/python/vllm/install.sh index 683b7635b21a..e7c4ad30cd79 100755 --- a/backend/python/vllm/install.sh +++ b/backend/python/vllm/install.sh @@ -215,22 +215,87 @@ elif [ "x${BUILD_TYPE}" == "xintel" ]; then export CMAKE_PREFIX_PATH="$(python -c 'import site; print(site.getsitepackages()[0])'):${CMAKE_PREFIX_PATH:-}" VLLM_TARGET_DEVICE=xpu uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --no-deps . popd -# AMD ROCm: install vllm from its dedicated ROCm wheel index instead of the -# CUDA-only PyPI wheel. installRequirements brings the base ROCm -# torch/transformers (requirements-hipblas.txt), then we pull vllm (plus the -# matching ROCm torch, via --upgrade) from wheels.vllm.ai/rocm. This is the -# method upstream prescribes for AMD; the Python-3.12 pin is set above. -# There is intentionally no requirements-hipblas-after.txt: a bare `vllm` -# there would resolve to the CUDA wheel, and installRequirements never loads -# a ${BUILD_TYPE}-after file for hipblas anyway (BUILD_TYPE == BUILD_PROFILE). -# https://docs.vllm.ai/en/latest/getting_started/installation/gpu.html?device=rocm +# AMD ROCm / gfx1151 (RDNA 3.5, Strix Halo): there is NO prebuilt vllm wheel for +# this GPU. wheels.vllm.ai/rocm ships only a gfx942/gfx950 (CDNA) build pinned to +# a rocm7.2.3 torch that cannot even enumerate Strix Halo (device_count == 0), and +# AMD's gfx1151 wheel index carries torch/triton but no vllm. So build vllm from +# source against AMD's ROCm 7.14 torch, targeting gfx1151 — mirrors the intel/cpu +# source-build branches above. This reproduces the exact stack AMD's own +# rocm/vllm:*_rdna_* image ships and runs on gfx1151 (torch 2.11.0+rocm7.14.0, +# vllm 0.23.1.dev1 @ commit 9ddef7117). The Python-3.12 pin is set above. elif [ "x${BUILD_TYPE}" == "xhipblas" ]; then + # vllm's HIP/C++ compile is large; MAX_JOBS=1 (set above for CUDA hosts) would + # take hours. 8 jobs balances throughput vs the per-job RAM of the ROCm kernels. + export MAX_JOBS=8 + + # rocm_smi's CMake package (pulled by vllm's ROCm build) probes system libdrm + # via pkg-config; these are OS-level build tools, not ROCm. No-op outside the + # Docker builder stage. The ROCm toolchain itself comes entirely from pip below. + if command -v apt-get >/dev/null 2>&1 && [ "$(id -u)" = "0" ]; then + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + pkg-config libdrm-dev + fi + + # GPU arch to build for. AMDGPU_TARGETS comes from the build (Dockerfile.python + # ARG/ENV); default to gfx1151 -- the only arch we could validate on real + # hardware. AMD ships a per-GPU torch build for essentially every AMD arch on + # the multi-arch index, selected via the torch[device-gfx] extra, which + # takes a single arch, so use the first target for that selection while the + # source build below targets the full AMDGPU_TARGETS list. + _gpu_targets="${AMDGPU_TARGETS:-gfx1151}" + _gpu_arch="${_gpu_targets%%;*}"; _gpu_arch="${_gpu_arch%% *}" + + ensureVenv + # Self-contained ROCm 7.14 build -- reproduces AMD's own rocm/vllm rdna recipe. + # From AMD's multi-arch index: the per-GPU torch (torch[device-gfx]) AND + # the full devel SDK (rocm-sdk-devel: hipcc + the HIP CMake packages that the + # runtime _rocm_sdk_core lacks). Everything lives in the venv, so the build + # needs NO system ROCm at /opt/rocm and reproduces on any base image with pip. + uv pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ \ + "torch[device-${_gpu_arch}]==2.11.0+rocm7.14.0" \ + "torchvision==0.26.0+rocm7.14.0" \ + "rocm-sdk-devel==7.14.0" installRequirements - # --upgrade reconciles the base ROCm torch to whatever the vllm ROCm wheel - # pins; --extra-index-url adds the ROCm wheel repository on top of PyPI. - uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} \ - --extra-index-url https://wheels.vllm.ai/rocm/ --upgrade vllm + # Expand the devel tree and link the installed rocm-sdk-device-* wheels into + # it (required before the SDK's hipcc/CMake are usable), then resolve every + # path from the SDK's own CLI — no hardcoded location, no host dependency. + rocm-sdk init + ROCM_PATH="$(rocm-sdk path --root)" + export ROCM_PATH ROCM_HOME="$ROCM_PATH" + export CMAKE_PREFIX_PATH="$(rocm-sdk path --cmake)${CMAKE_PREFIX_PATH:+:$CMAKE_PREFIX_PATH}" + export PATH="$(rocm-sdk path --bin):$PATH" + + # vLLM detects ROCm at runtime via `import amdsmi` (platforms/__init__.py: + # amdsmi_init() + get_processor_handles()); without it vLLM falls back to + # UnspecifiedPlatform and dies with "Device string must not be empty". + # rocm-sdk-core ships amd_smi in-place under _rocm_sdk_core/share/amd_smi, + # where it resolves its native libamd_smi.so RELATIVE to that directory. + # A pip copy into site-packages breaks that relative lookup, and forcing the + # lib via LD_LIBRARY_PATH shadows torch's own bundled ROCm runtime and drops + # device_count to 0. So register the in-place package with a .pth entry -- + # `import amdsmi` then resolves where its libs do, no LD override needed. + # (AMD's rocm/vllm image likewise imports amdsmi in-place from this path.) + _sp="$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')" + printf '_rocm_sdk_core/share/amd_smi\n' > "${_sp}/amdsmi.pth" + + # Build vllm from source against this ROCm 7.14 torch. Pin to the release tag + # closest to what AMD's rocm/vllm rdna image ships (0.23.1.dev1+g9ddef7117 + # sits between v0.23.0 and v0.23.1rc0). + VLLM_REF="${VLLM_REF:-v0.23.0}" + _vllm_src=$(mktemp -d) + trap 'rm -rf "${_vllm_src}"' EXIT + git clone --depth 1 --branch "${VLLM_REF}" \ + https://github.com/vllm-project/vllm "${_vllm_src}/vllm" + pushd "${_vllm_src}/vllm" + # Strip vllm's own torch/triton pins so it builds against our ROCm wheels. + python use_existing_torch.py || true + uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} -r requirements/rocm.txt + # No GPU in the build container -> pin the target arch(es) explicitly. + export PYTORCH_ROCM_ARCH="${_gpu_targets}" GPU_TARGETS="${_gpu_targets}" GPU_ARCHS="${_gpu_targets}" + VLLM_TARGET_DEVICE=rocm uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --no-deps . + popd # FROM_SOURCE=true on a CPU build skips the prebuilt vllm wheel in # requirements-cpu-after.txt and compiles vllm locally against the host's # actual CPU. Not used by default because it takes ~30-40 minutes, but diff --git a/backend/python/vllm/requirements-hipblas.txt b/backend/python/vllm/requirements-hipblas.txt index 1163f1b647c6..c26500ea1ed3 100644 --- a/backend/python/vllm/requirements-hipblas.txt +++ b/backend/python/vllm/requirements-hipblas.txt @@ -1,5 +1,8 @@ ---extra-index-url https://download.pytorch.org/whl/nightly/rocm7.0 +# torch is installed separately from AMD's multi-arch ROCm 7.14 index in +# install.sh (gfx1151 needs torch built against ROCm 7.13+; the community +# whl/rocm7.0 wheel does not enumerate Strix Halo). vllm itself is built from +# source there against that torch. uv aborts on the AMD index's 403-for-missing +# -pkg, so torch is pulled there; everything below resolves from PyPI. accelerate -torch transformers bitsandbytes \ No newline at end of file diff --git a/backend/python/vllm/run.sh b/backend/python/vllm/run.sh index 83607fb8d659..7c3668a5d096 100755 --- a/backend/python/vllm/run.sh +++ b/backend/python/vllm/run.sh @@ -47,6 +47,20 @@ if [ -d "${EDIR}/toolchain/usr/bin" ]; then export LD_LIBRARY_PATH="${_libpath}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" fi +# ROCm (hipblas) profile: Triton JIT-compiles its launcher/util C modules at first +# inference (e.g. the top-k/top-p sampling kernel), and torch._inductor's ISA probe +# shells out to a C compiler too. The LocalAI runtime image ships hipcc but not +# cc/gcc, and the FROM-scratch backend omits them as well -- so Triton dies with +# "Failed to find C compiler". rocm-sdk-devel (installed for the gfx1151 build) +# already bundles amdclang, so just put it on PATH and expose it as CC/CXX. Mirrors +# the CPU-toolchain block above; no-op for other profiles (the dir doesn't exist). +_rocm_sdk_bin="$(ls -d ${EDIR}/venv/lib/python*/site-packages/_rocm_sdk_devel/bin 2>/dev/null | head -1)" +if [ -n "${_rocm_sdk_bin}" ] && [ -x "${_rocm_sdk_bin}/amdclang" ]; then + export PATH="${_rocm_sdk_bin}:${PATH}" + export CC="${CC:-${_rocm_sdk_bin}/amdclang}" + export CXX="${CXX:-${_rocm_sdk_bin}/amdclang++}" +fi + # Multi-node DP follower mode: when the first arg is `serve`, exec into # vllm's own CLI instead of LocalAI's backend.py gRPC server. The # follower speaks ZMQ directly to the head node's vllm ranks — there