diff --git a/.agents/adding-backends.md b/.agents/adding-backends.md index a7880a17b400..f0f44b815e12 100644 --- a/.agents/adding-backends.md +++ b/.agents/adding-backends.md @@ -92,7 +92,7 @@ If you have a `prepare.sh` doing the clone, delete it — the recipe belongs in - CUDA 13 builds: Add after other CUDA 13 builds (e.g., after `gpu-nvidia-cuda-13-chatterbox`) **Additional build types you may need:** -- ROCm/HIP: Use `build-type: 'hipblas'` with `base-image: "rocm/dev-ubuntu-24.04:7.2.1"` +- ROCm/HIP: Use `build-type: 'hipblas'` with `base-image: "rocm/dev-ubuntu-24.04:7.14.0-full"` - Intel/SYCL: Use `build-type: 'intel'` or `build-type: 'sycl_f16'`/`sycl_f32` with `base-image: "intel/oneapi-basekit:2025.3.2-0-devel-ubuntu24.04"` - L4T (ARM): Use `build-type: 'l4t'` with `platforms: 'linux/arm64'` and `runs-on: 'ubuntu-24.04-arm'` diff --git a/.agents/building-and-testing.md b/.agents/building-and-testing.md index 021d555ec993..a98eeeb7a906 100644 --- a/.agents/building-and-testing.md +++ b/.agents/building-and-testing.md @@ -11,7 +11,7 @@ Let's say the user wants to build a particular backend for a given platform. For - Use `.github/backend-matrix.yml` as a reference — it's the data-only YAML that lists every backend variant's `build-type`, `base-image`, `platforms`, etc. (`backend.yml` and `backend_pr.yml` consume it via `scripts/changed-backends.js`). - l4t and cublas also require the CUDA major and minor version. - For llama-cpp / ik-llama-cpp / turboquant the matrix also sets `builder-base-image` pointing at a prebuilt `quay.io/go-skynet/ci-cache:base-grpc-*` tag. Local `make backends/` defaults to `BUILDER_TARGET=builder-fromsource` and doesn't need it — the Dockerfile's from-source stage installs everything itself. -- You can pretty print a command like `DOCKER_MAKEFLAGS=-j$(nproc --ignore=1) BUILD_TYPE=hipblas BASE_IMAGE=rocm/dev-ubuntu-24.04:7.2.1 make docker-build-coqui` +- You can pretty print a command like `DOCKER_MAKEFLAGS=-j$(nproc --ignore=1) BUILD_TYPE=hipblas BASE_IMAGE=rocm/dev-ubuntu-24.04:7.14.0-full make docker-build-coqui` - Unless the user specifies that they want you to run the command, then just print it because not all agent frontends handle long running jobs well and the output may overflow your context - The user may say they want to build AMD or ROCM instead of hipblas, or Intel instead of SYCL or NVIDIA insted of l4t or cublas. Ask for confirmation if there is ambiguity. - Sometimes the user may need extra parameters to be added to `docker build` (e.g. `--platform` for cross-platform builds or `--progress` to view the full logs), in which case you can generate the `docker build` command directly. diff --git a/.agents/ci-caching.md b/.agents/ci-caching.md index 17cc6001d908..03799eb29305 100644 --- a/.agents/ci-caching.md +++ b/.agents/ci-caching.md @@ -51,7 +51,7 @@ The C++ backend Dockerfiles (`Dockerfile.{llama-cpp,ik-llama-cpp,turboquant}`) c | `base-grpc-cuda-13-amd64` | the above + CUDA 13.0 toolkit (Ubuntu 22.04 base) | | `base-grpc-cuda-13-arm64` | the above + CUDA 13.0 sbsa toolkit (Ubuntu 24.04 base) | | `base-grpc-l4t-cuda-12-arm64` | JetPack r36.4.0 base (CUDA preinstalled, `SKIP_DRIVERS=true`) + gRPC | -| `base-grpc-rocm-amd64` | rocm/dev-ubuntu-24.04:7.2.1 base + hipblas/hipblaslt/rocblas + gRPC | +| `base-grpc-rocm-amd64` | rocm/dev-ubuntu-24.04:7.14.0-full base + hipblas/hipblaslt/rocblas + gRPC | | `base-grpc-vulkan-amd64` / `base-grpc-vulkan-arm64` | Ubuntu 24.04 + Vulkan SDK 1.4.335 + gRPC | | `base-grpc-intel-amd64` | intel/oneapi-basekit:2025.3.2 base + gRPC | diff --git a/.docker/install-base-deps.sh b/.docker/install-base-deps.sh index 2b0e7e0c6a95..f8a5396d20cc 100755 --- a/.docker/install-base-deps.sh +++ b/.docker/install-base-deps.sh @@ -178,16 +178,18 @@ fi # --- 6. ROCm / HIP build deps (BUILD_TYPE=hipblas) --- if [ "${BUILD_TYPE:-}" = "hipblas" ] && [ "${SKIP_DRIVERS:-false}" = "false" ]; then - apt-get update - apt-get install -y --no-install-recommends \ - hipblas-dev \ - hipblaslt-dev \ - rocblas-dev - apt-get clean - rm -rf /var/lib/apt/lists/* - # I have no idea why, but the ROCM lib packages don't trigger ldconfig after they install, - # which results in local-ai and others not being able to locate the libraries. - # We run ldconfig ourselves to work around this packaging deficiency. + # ROCm 7.x ("TheRock" packaging): the legacy hipblas-dev / hipblaslt-dev / + # rocblas-dev metapackages were removed — BLAS is consolidated and split per + # GPU arch (amdrocm-blas-gfx*, amdrocm-blas-dev, amdrocm-blas-host). The + # rocm/dev-ubuntu-*:*-full base image already ships those BLAS dev libs and + # headers, so no apt install is needed here. + # TheRock ships the ROCm libs under /opt/rocm/core*/lib (reached via the + # /opt/rocm/lib alternatives symlink) but registers no ld.so.conf.d entry, so + # the dynamic linker can't resolve them. Without this the ldd-based GPU-lib + # packaging (scripts/build/package-gpu-libs.sh) silently skips librocm_kpack, + # rocBLAS, hipBLASLt, the bundled rocm_sysdeps (zlib/zstd/elf) and the LLVM + # runtime, and the built backend fails to load at runtime. + printf '/opt/rocm/lib\n/opt/rocm/lib/rocm_sysdeps/lib\n/opt/rocm/llvm/lib\n' > /etc/ld.so.conf.d/rocm.conf ldconfig # Log which GPU architectures have rocBLAS kernel support echo "rocBLAS library data architectures:" diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index 2ae8a26af576..dde5d4de11cb 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -2140,7 +2140,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-rerankers' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "rerankers" dockerfile: "./backend/Dockerfile.python" @@ -2154,7 +2154,7 @@ include: tag-suffix: '-gpu-rocm-hipblas-llama-cpp' builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-rocm-amd64' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "llama-cpp" dockerfile: "./backend/Dockerfile.llama-cpp" @@ -2168,7 +2168,7 @@ include: tag-suffix: '-gpu-rocm-hipblas-bonsai' builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-rocm-amd64' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "bonsai" dockerfile: "./backend/Dockerfile.bonsai" @@ -2181,7 +2181,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-vllm' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "vllm" dockerfile: "./backend/Dockerfile.python" @@ -2194,7 +2194,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-vllm-omni' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "vllm-omni" dockerfile: "./backend/Dockerfile.python" @@ -2207,7 +2207,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-sglang' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "sglang" dockerfile: "./backend/Dockerfile.python" @@ -2220,7 +2220,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-transformers' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "transformers" dockerfile: "./backend/Dockerfile.python" @@ -2233,7 +2233,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-diffusers' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "diffusers" dockerfile: "./backend/Dockerfile.python" @@ -2246,7 +2246,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-ace-step' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "ace-step" dockerfile: "./backend/Dockerfile.python" @@ -2260,7 +2260,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-kokoro' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "kokoro" dockerfile: "./backend/Dockerfile.python" @@ -2273,7 +2273,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-vibevoice' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "vibevoice" dockerfile: "./backend/Dockerfile.python" @@ -2286,7 +2286,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-liquid-audio' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "liquid-audio" dockerfile: "./backend/Dockerfile.python" @@ -2299,7 +2299,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-qwen-asr' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "qwen-asr" dockerfile: "./backend/Dockerfile.python" @@ -2312,7 +2312,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-nemo' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "nemo" dockerfile: "./backend/Dockerfile.python" @@ -2325,7 +2325,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-qwen-tts' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "qwen-tts" dockerfile: "./backend/Dockerfile.python" @@ -2338,7 +2338,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-fish-speech' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "fish-speech" dockerfile: "./backend/Dockerfile.python" @@ -2351,7 +2351,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-voxcpm' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "voxcpm" dockerfile: "./backend/Dockerfile.python" @@ -2364,7 +2364,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-pocket-tts' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "pocket-tts" dockerfile: "./backend/Dockerfile.python" @@ -2377,7 +2377,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-faster-whisper' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "faster-whisper" dockerfile: "./backend/Dockerfile.python" @@ -2390,7 +2390,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-coqui' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "coqui" dockerfile: "./backend/Dockerfile.python" @@ -3854,7 +3854,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-whisper' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "whisper" @@ -3867,7 +3867,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-crispasr' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "crispasr" @@ -3976,7 +3976,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-parakeet-cpp' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "parakeet-cpp" @@ -4085,7 +4085,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-moss-transcribe-cpp' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "moss-transcribe-cpp" @@ -4233,7 +4233,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-ced' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "ced" @@ -4381,7 +4381,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-voice-detect' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "voice-detect" @@ -4529,7 +4529,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-face-detect' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "face-detect" @@ -4638,7 +4638,7 @@ include: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-acestep-cpp' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' skip-drivers: 'false' backend: "acestep-cpp" @@ -5571,7 +5571,7 @@ include: # platforms: 'linux/amd64' # tag-latest: 'auto' # tag-suffix: '-gpu-hipblas-rfdetr' - # base-image: "rocm/dev-ubuntu-24.04:7.2.1" + # base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" # runs-on: 'ubuntu-latest' # skip-drivers: 'false' # backend: "rfdetr" @@ -5643,7 +5643,7 @@ include: tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-neutts' runs-on: 'ubuntu-latest' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" skip-drivers: 'false' backend: "neutts" dockerfile: "./backend/Dockerfile.python" diff --git a/.github/workflows/base-images.yml b/.github/workflows/base-images.yml index 637b603a56fb..87ce9c3fded7 100644 --- a/.github/workflows/base-images.yml +++ b/.github/workflows/base-images.yml @@ -83,7 +83,7 @@ jobs: ubuntu-version: '2404' - tag: 'base-grpc-rocm-amd64' runs-on: 'ubuntu-latest' - base-image: 'rocm/dev-ubuntu-24.04:7.2.1' + base-image: 'rocm/dev-ubuntu-24.04:7.14.0-full' build-type: 'hipblas' cuda-major-version: '' cuda-minor-version: '' diff --git a/.github/workflows/image-pr.yml b/.github/workflows/image-pr.yml index 146ad22cffbe..b24a452b239d 100644 --- a/.github/workflows/image-pr.yml +++ b/.github/workflows/image-pr.yml @@ -59,7 +59,7 @@ platforms: 'linux/amd64' tag-latest: 'false' tag-suffix: '-hipblas' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' makeflags: "--jobs=3 --output-sync=target" ubuntu-version: '2404' diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index 655ef1b437dc..752634348b12 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -40,7 +40,7 @@ platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-hipblas' - base-image: "rocm/dev-ubuntu-24.04:7.2.1" + base-image: "rocm/dev-ubuntu-24.04:7.14.0-full" runs-on: 'ubuntu-latest' makeflags: "--jobs=3 --output-sync=target" ubuntu-version: '2404' diff --git a/Dockerfile b/Dockerfile index 4ca9b32791ab..1b5b319666ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -158,16 +158,21 @@ RUN if [ "${BUILD_TYPE}" = "clblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ ; fi RUN if [ "${BUILD_TYPE}" = "hipblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - hipblas-dev \ - hipblaslt-dev \ - rocblas-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ + # ROCm 7.x ("TheRock" packaging): the legacy hipblas-dev / hipblaslt-dev / + # rocblas-dev metapackages were removed — BLAS is consolidated and split + # per GPU arch (amdrocm-blas-gfx*, amdrocm-blas-dev, amdrocm-blas-host). + # The rocm/dev-ubuntu-*:*-full base image already ships those dev libraries + # and headers, so no apt install is needed here anymore. We still set the + # AMD capability marker. echo "amd" > /run/localai/capability && \ - # I have no idea why, but the ROCM lib packages don't trigger ldconfig after they install, which results in local-ai and others not being able - # to locate the libraries. We run ldconfig ourselves to work around this packaging deficiency + # TheRock ships the ROCm libs under /opt/rocm/core*/lib (reached via the + # /opt/rocm/lib alternatives symlink) but registers no ld.so.conf.d entry, + # so the dynamic linker can't find them. Register the path so ldconfig + # (and the ldd-based backend GPU-lib packaging) resolves librocm_kpack, + # rocBLAS, hipBLASLt, the bundled rocm_sysdeps (zlib/zstd/elf) and the + # LLVM runtime — otherwise the built backend fails at runtime with e.g. + # "librocm_kpack.so.0 / librocm_sysdeps_z.so.1: cannot open shared object". + printf '/opt/rocm/lib\n/opt/rocm/lib/rocm_sysdeps/lib\n/opt/rocm/llvm/lib\n' > /etc/ld.so.conf.d/rocm.conf && \ ldconfig \ ; fi diff --git a/backend/Dockerfile.base-grpc-builder b/backend/Dockerfile.base-grpc-builder index 8843a0a72c29..c7c681dde82b 100644 --- a/backend/Dockerfile.base-grpc-builder +++ b/backend/Dockerfile.base-grpc-builder @@ -26,7 +26,7 @@ # base-grpc-cuda-13-amd64 ubuntu:22.04 + CUDA 13.0 # base-grpc-cuda-13-arm64 ubuntu:24.04 + CUDA 13.0 (sbsa) # base-grpc-l4t-cuda-12-arm64 ubuntu:22.04 + CUDA 12.x (legacy JetPack) -# base-grpc-rocm-amd64 rocm/dev-ubuntu-24.04:7.2.1 + hipblas +# base-grpc-rocm-amd64 rocm/dev-ubuntu-24.04:7.14.0-full + hipblas # base-grpc-vulkan-amd64 ubuntu:24.04 + Vulkan SDK 1.4.335 # base-grpc-vulkan-arm64 ubuntu:24.04 + Vulkan SDK ARM 1.4.335 # base-grpc-intel-amd64 intel/oneapi-basekit:2025.3.2 (sycl) diff --git a/backend/Dockerfile.golang b/backend/Dockerfile.golang index c7dcac40068b..6b326c920df5 100644 --- a/backend/Dockerfile.golang +++ b/backend/Dockerfile.golang @@ -169,15 +169,12 @@ RUN if [ "${BUILD_TYPE}" = "clblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ ; fi RUN if [ "${BUILD_TYPE}" = "hipblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - hipblas-dev \ - hipblaslt-dev \ - rocblas-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - # I have no idea why, but the ROCM lib packages don't trigger ldconfig after they install, which results in local-ai and others not being able - # to locate the libraries. We run ldconfig ourselves to work around this packaging deficiency + # ROCm 7.x ("TheRock" packaging): the legacy hipblas-dev / hipblaslt-dev / + # rocblas-dev metapackages were removed (consolidated + arch-split). The + # rocm/dev-ubuntu-*:*-full base image already ships the BLAS dev libs and + # headers, so no apt install is needed. Register TheRock's library paths + # so Go backends using cgo can resolve the ROCm runtime libraries. + printf '/opt/rocm/lib\n/opt/rocm/lib/rocm_sysdeps/lib\n/opt/rocm/llvm/lib\n' > /etc/ld.so.conf.d/rocm.conf && \ ldconfig \ ; fi diff --git a/backend/Dockerfile.python b/backend/Dockerfile.python index 2522a6f56be0..ad363e320343 100644 --- a/backend/Dockerfile.python +++ b/backend/Dockerfile.python @@ -6,6 +6,11 @@ FROM ${BASE_IMAGE} AS builder ARG BACKEND=rerankers ARG BUILD_TYPE ENV BUILD_TYPE=${BUILD_TYPE} +# AMD GPU target arch(es) for ROCm builds (e.g. gfx1151, or a ';'-list). Exposed +# to install.sh so ROCm backends can select AMD's per-GPU torch and target the +# right arch when building from source, instead of hardcoding one. +ARG AMDGPU_TARGETS +ENV AMDGPU_TARGETS=${AMDGPU_TARGETS} ARG CUDA_MAJOR_VERSION ARG CUDA_MINOR_VERSION ARG SKIP_DRIVERS=false @@ -171,15 +176,14 @@ RUN if [ "${BUILD_TYPE}" = "clblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ ; fi RUN if [ "${BUILD_TYPE}" = "hipblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - hipblas-dev \ - hipblaslt-dev \ - rocblas-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - # I have no idea why, but the ROCM lib packages don't trigger ldconfig after they install, which results in local-ai and others not being able - # to locate the libraries. We run ldconfig ourselves to work around this packaging deficiency + # ROCm 7.x ("TheRock" packaging): the legacy hipblas-dev / hipblaslt-dev / + # rocblas-dev metapackages were removed (consolidated + arch-split). The + # rocm/dev-ubuntu-*:*-full base image already ships the BLAS dev libs and + # headers, so no apt install is needed. TheRock also scatters the ROCm libs + # across /opt/rocm/lib, /opt/rocm/lib/rocm_sysdeps/lib and /opt/rocm/llvm/lib + # with no ld.so.conf.d entry — register them so ldconfig (and Python + # backends that dlopen ROCm libs at runtime) can resolve them. + printf '/opt/rocm/lib\n/opt/rocm/lib/rocm_sysdeps/lib\n/opt/rocm/llvm/lib\n' > /etc/ld.so.conf.d/rocm.conf && \ ldconfig \ ; fi diff --git a/backend/python/diffusers/install.sh b/backend/python/diffusers/install.sh index 83703b1b2853..fc27a53c1a22 100755 --- a/backend/python/diffusers/install.sh +++ b/backend/python/diffusers/install.sh @@ -27,4 +27,21 @@ if [ "x${BUILD_PROFILE}" == "xl4t13" ]; then PY_STANDALONE_TAG="20251120" fi +# ROCm/gfx1151: the community whl/rocm7.0 torch wheels do not enumerate Strix +# Halo (device_count == 0). AMD ships stable ROCm 7.14 builds via its multi-arch +# index, selected per GPU with the torch[device-gfx] extra. Install torch + +# torchvision (the same pinned versions, from that index) in an isolated step: +# uv aborts on that index's 403-for-missing-package responses, so we must not let +# it resolve PyPI-only packages (diffusers, transformers, ...) there. Everything +# else then resolves from PyPI in installRequirements. +if [ "x${BUILD_PROFILE}" == "xhipblas" ]; then + # Arch from the build (AMDGPU_TARGETS); default gfx1151 -- the only arch we + # could validate on real hardware. The torch[device-...] extra takes one arch. + _gpu_arch="${AMDGPU_TARGETS:-gfx1151}"; _gpu_arch="${_gpu_arch%%;*}"; _gpu_arch="${_gpu_arch%% *}" + ensureVenv + uv pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ \ + "torch[device-${_gpu_arch}]==2.10.0+rocm7.14.0" \ + "torchvision==0.25.0+rocm7.14.0" +fi + installRequirements diff --git a/backend/python/diffusers/requirements-hipblas.txt b/backend/python/diffusers/requirements-hipblas.txt index f3666d5f504c..47846fc5c91c 100644 --- a/backend/python/diffusers/requirements-hipblas.txt +++ b/backend/python/diffusers/requirements-hipblas.txt @@ -1,6 +1,8 @@ ---extra-index-url https://download.pytorch.org/whl/rocm7.0 -torch==2.10.0+rocm7.0 -torchvision==0.25.0+rocm7.0 +# torch + torchvision are installed separately from AMD's multi-arch ROCm 7.14 +# index in install.sh (gfx1151 needs torch built against ROCm 7.13+; the +# community whl/rocm7.0 wheel does not enumerate Strix Halo). uv aborts on that +# index's 403-for-missing-pkg, so torch is pulled there; everything below resolves +# from PyPI. diffusers==0.38.0 opencv-python transformers==4.57.6 diff --git a/backend/python/kokoro/install.sh b/backend/python/kokoro/install.sh index 3e1ab8606480..c652ba6a8980 100755 --- a/backend/python/kokoro/install.sh +++ b/backend/python/kokoro/install.sh @@ -20,6 +20,23 @@ if [ "x${BUILD_PROFILE}" == "xl4t12" ]; then USE_PIP=true fi +# ROCm/gfx1151: the community whl/rocm7.0 torch wheels do not enumerate Strix +# Halo (device_count == 0). AMD ships stable ROCm 7.14 builds via its multi-arch +# index, selected per GPU with the torch[device-gfx] extra. Install torch + +# torchaudio (the same pinned versions, from that index) in an isolated step: uv +# aborts on that index's 403-for-missing-package responses, so we must not let it +# resolve PyPI-only packages (kokoro, transformers, ...) there. Everything else +# then resolves from PyPI in installRequirements. +if [ "x${BUILD_PROFILE}" == "xhipblas" ]; then + # Arch from the build (AMDGPU_TARGETS); default gfx1151 -- the only arch we + # could validate on real hardware. The torch[device-...] extra takes one arch. + _gpu_arch="${AMDGPU_TARGETS:-gfx1151}"; _gpu_arch="${_gpu_arch%%;*}"; _gpu_arch="${_gpu_arch%% *}" + ensureVenv + uv pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ \ + "torch[device-${_gpu_arch}]==2.10.0+rocm7.14.0" \ + "torchaudio==2.10.0+rocm7.14.0" +fi + installRequirements # spaCy is a dependency of misaki (used by kokoro for English phonemization). diff --git a/backend/python/kokoro/requirements-hipblas.txt b/backend/python/kokoro/requirements-hipblas.txt index 0ca1352282c9..9edeff371eeb 100644 --- a/backend/python/kokoro/requirements-hipblas.txt +++ b/backend/python/kokoro/requirements-hipblas.txt @@ -1,6 +1,8 @@ ---extra-index-url https://download.pytorch.org/whl/rocm7.0 -torch==2.10.0+rocm7.0 -torchaudio==2.10.0+rocm7.0 +# torch + torchaudio are installed separately from AMD's multi-arch ROCm 7.14 +# index in install.sh (gfx1151 needs torch built against ROCm 7.13+; the +# community whl/rocm7.0 wheel does not enumerate Strix Halo). uv aborts on that +# index's 403-for-missing-pkg, so torch is pulled there; everything below resolves +# from PyPI. transformers accelerate kokoro diff --git a/backend/python/transformers/install.sh b/backend/python/transformers/install.sh index 32befa8e6c03..e2084bb30744 100755 --- a/backend/python/transformers/install.sh +++ b/backend/python/transformers/install.sh @@ -16,4 +16,20 @@ if [ "x${BUILD_PROFILE}" == "xintel" ]; then EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match" fi +# ROCm/gfx1151: the community whl/rocm7.0 torch wheel does not enumerate Strix +# Halo (device_count == 0). AMD ships a stable ROCm 7.14 build via its multi-arch +# index, selected per GPU with the torch[device-gfx] extra. Install it (plus +# its rocm/triton deps, which all live on that same index) in an isolated step: +# uv aborts on that index's 403-for-missing-package responses, so we must not let +# it look up PyPI-only packages (accelerate, transformers, ...) there. Everything +# else then resolves from PyPI in installRequirements. +if [ "x${BUILD_PROFILE}" == "xhipblas" ]; then + # Arch from the build (AMDGPU_TARGETS); default gfx1151 -- the only arch we + # could validate on real hardware. The torch[device-...] extra takes one arch. + _gpu_arch="${AMDGPU_TARGETS:-gfx1151}"; _gpu_arch="${_gpu_arch%%;*}"; _gpu_arch="${_gpu_arch%% *}" + ensureVenv + uv pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ \ + "torch[device-${_gpu_arch}]==2.12.0+rocm7.14.0" +fi + installRequirements diff --git a/backend/python/transformers/requirements-hipblas.txt b/backend/python/transformers/requirements-hipblas.txt index 16d3c61933b3..54a000fbbeee 100644 --- a/backend/python/transformers/requirements-hipblas.txt +++ b/backend/python/transformers/requirements-hipblas.txt @@ -1,5 +1,7 @@ ---extra-index-url https://download.pytorch.org/whl/rocm7.0 -torch==2.10.0+rocm7.0 +# torch is installed separately from AMD's multi-arch ROCm index in install.sh +# (gfx1151 needs torch built against ROCm 7.13+; the community whl/rocm7.0 wheel +# does not enumerate Strix Halo). uv aborts on that index's 403-for-missing-pkg, +# so torch is pulled with pip there; everything below resolves from PyPI. accelerate transformers>=5.14.1 llvmlite==0.43.0 diff --git a/backend/python/vllm/install.sh b/backend/python/vllm/install.sh index ca9b98f6aa48..a72159920314 100755 --- a/backend/python/vllm/install.sh +++ b/backend/python/vllm/install.sh @@ -209,22 +209,87 @@ elif [ "x${BUILD_TYPE}" == "xintel" ]; then export CMAKE_PREFIX_PATH="$(python -c 'import site; print(site.getsitepackages()[0])'):${CMAKE_PREFIX_PATH:-}" VLLM_TARGET_DEVICE=xpu uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --no-deps . popd -# AMD ROCm: install vllm from its dedicated ROCm wheel index instead of the -# CUDA-only PyPI wheel. installRequirements brings the base ROCm -# torch/transformers (requirements-hipblas.txt), then we pull vllm (plus the -# matching ROCm torch, via --upgrade) from wheels.vllm.ai/rocm. This is the -# method upstream prescribes for AMD; the Python-3.12 pin is set above. -# There is intentionally no requirements-hipblas-after.txt: a bare `vllm` -# there would resolve to the CUDA wheel, and installRequirements never loads -# a ${BUILD_TYPE}-after file for hipblas anyway (BUILD_TYPE == BUILD_PROFILE). -# https://docs.vllm.ai/en/latest/getting_started/installation/gpu.html?device=rocm +# AMD ROCm / gfx1151 (RDNA 3.5, Strix Halo): there is NO prebuilt vllm wheel for +# this GPU. wheels.vllm.ai/rocm ships only a gfx942/gfx950 (CDNA) build pinned to +# a rocm7.2.3 torch that cannot even enumerate Strix Halo (device_count == 0), and +# AMD's gfx1151 wheel index carries torch/triton but no vllm. So build vllm from +# source against AMD's ROCm 7.14 torch, targeting gfx1151 — mirrors the intel/cpu +# source-build branches above. This reproduces the exact stack AMD's own +# rocm/vllm:*_rdna_* image ships and runs on gfx1151 (torch 2.11.0+rocm7.14.0, +# vllm 0.23.1.dev1 @ commit 9ddef7117). The Python-3.12 pin is set above. elif [ "x${BUILD_TYPE}" == "xhipblas" ]; then + # vllm's HIP/C++ compile is large; MAX_JOBS=1 (set above for CUDA hosts) would + # take hours. 8 jobs balances throughput vs the per-job RAM of the ROCm kernels. + export MAX_JOBS=8 + + # rocm_smi's CMake package (pulled by vllm's ROCm build) probes system libdrm + # via pkg-config; these are OS-level build tools, not ROCm. No-op outside the + # Docker builder stage. The ROCm toolchain itself comes entirely from pip below. + if command -v apt-get >/dev/null 2>&1 && [ "$(id -u)" = "0" ]; then + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + pkg-config libdrm-dev + fi + + # GPU arch to build for. AMDGPU_TARGETS comes from the build (Dockerfile.python + # ARG/ENV); default to gfx1151 -- the only arch we could validate on real + # hardware. AMD ships a per-GPU torch build for essentially every AMD arch on + # the multi-arch index, selected via the torch[device-gfx] extra, which + # takes a single arch, so use the first target for that selection while the + # source build below targets the full AMDGPU_TARGETS list. + _gpu_targets="${AMDGPU_TARGETS:-gfx1151}" + _gpu_arch="${_gpu_targets%%;*}"; _gpu_arch="${_gpu_arch%% *}" + + ensureVenv + # Self-contained ROCm 7.14 build -- reproduces AMD's own rocm/vllm rdna recipe. + # From AMD's multi-arch index: the per-GPU torch (torch[device-gfx]) AND + # the full devel SDK (rocm-sdk-devel: hipcc + the HIP CMake packages that the + # runtime _rocm_sdk_core lacks). Everything lives in the venv, so the build + # needs NO system ROCm at /opt/rocm and reproduces on any base image with pip. + uv pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ \ + "torch[device-${_gpu_arch}]==2.11.0+rocm7.14.0" \ + "torchvision==0.26.0+rocm7.14.0" \ + "rocm-sdk-devel==7.14.0" installRequirements - # --upgrade reconciles the base ROCm torch to whatever the vllm ROCm wheel - # pins; --extra-index-url adds the ROCm wheel repository on top of PyPI. - uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} \ - --extra-index-url https://wheels.vllm.ai/rocm/ --upgrade vllm + # Expand the devel tree and link the installed rocm-sdk-device-* wheels into + # it (required before the SDK's hipcc/CMake are usable), then resolve every + # path from the SDK's own CLI — no hardcoded location, no host dependency. + rocm-sdk init + ROCM_PATH="$(rocm-sdk path --root)" + export ROCM_PATH ROCM_HOME="$ROCM_PATH" + export CMAKE_PREFIX_PATH="$(rocm-sdk path --cmake)${CMAKE_PREFIX_PATH:+:$CMAKE_PREFIX_PATH}" + export PATH="$(rocm-sdk path --bin):$PATH" + + # vLLM detects ROCm at runtime via `import amdsmi` (platforms/__init__.py: + # amdsmi_init() + get_processor_handles()); without it vLLM falls back to + # UnspecifiedPlatform and dies with "Device string must not be empty". + # rocm-sdk-core ships amd_smi in-place under _rocm_sdk_core/share/amd_smi, + # where it resolves its native libamd_smi.so RELATIVE to that directory. + # A pip copy into site-packages breaks that relative lookup, and forcing the + # lib via LD_LIBRARY_PATH shadows torch's own bundled ROCm runtime and drops + # device_count to 0. So register the in-place package with a .pth entry -- + # `import amdsmi` then resolves where its libs do, no LD override needed. + # (AMD's rocm/vllm image likewise imports amdsmi in-place from this path.) + _sp="$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')" + printf '_rocm_sdk_core/share/amd_smi\n' > "${_sp}/amdsmi.pth" + + # Build vllm from source against this ROCm 7.14 torch. Pin to the release tag + # closest to what AMD's rocm/vllm rdna image ships (0.23.1.dev1+g9ddef7117 + # sits between v0.23.0 and v0.23.1rc0). + VLLM_REF="${VLLM_REF:-v0.23.0}" + _vllm_src=$(mktemp -d) + trap 'rm -rf "${_vllm_src}"' EXIT + git clone --depth 1 --branch "${VLLM_REF}" \ + https://github.com/vllm-project/vllm "${_vllm_src}/vllm" + pushd "${_vllm_src}/vllm" + # Strip vllm's own torch/triton pins so it builds against our ROCm wheels. + python use_existing_torch.py || true + uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} -r requirements/rocm.txt + # No GPU in the build container -> pin the target arch(es) explicitly. + export PYTORCH_ROCM_ARCH="${_gpu_targets}" GPU_TARGETS="${_gpu_targets}" GPU_ARCHS="${_gpu_targets}" + VLLM_TARGET_DEVICE=rocm uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --no-deps . + popd # FROM_SOURCE=true on a CPU build skips the prebuilt vllm wheel in # requirements-cpu-after.txt and compiles vllm locally against the host's # actual CPU. Not used by default because it takes ~30-40 minutes, but diff --git a/backend/python/vllm/requirements-hipblas.txt b/backend/python/vllm/requirements-hipblas.txt index 1163f1b647c6..c26500ea1ed3 100644 --- a/backend/python/vllm/requirements-hipblas.txt +++ b/backend/python/vllm/requirements-hipblas.txt @@ -1,5 +1,8 @@ ---extra-index-url https://download.pytorch.org/whl/nightly/rocm7.0 +# torch is installed separately from AMD's multi-arch ROCm 7.14 index in +# install.sh (gfx1151 needs torch built against ROCm 7.13+; the community +# whl/rocm7.0 wheel does not enumerate Strix Halo). vllm itself is built from +# source there against that torch. uv aborts on the AMD index's 403-for-missing +# -pkg, so torch is pulled there; everything below resolves from PyPI. accelerate -torch transformers bitsandbytes \ No newline at end of file diff --git a/backend/python/vllm/run.sh b/backend/python/vllm/run.sh index 83607fb8d659..7c3668a5d096 100755 --- a/backend/python/vllm/run.sh +++ b/backend/python/vllm/run.sh @@ -47,6 +47,20 @@ if [ -d "${EDIR}/toolchain/usr/bin" ]; then export LD_LIBRARY_PATH="${_libpath}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" fi +# ROCm (hipblas) profile: Triton JIT-compiles its launcher/util C modules at first +# inference (e.g. the top-k/top-p sampling kernel), and torch._inductor's ISA probe +# shells out to a C compiler too. The LocalAI runtime image ships hipcc but not +# cc/gcc, and the FROM-scratch backend omits them as well -- so Triton dies with +# "Failed to find C compiler". rocm-sdk-devel (installed for the gfx1151 build) +# already bundles amdclang, so just put it on PATH and expose it as CC/CXX. Mirrors +# the CPU-toolchain block above; no-op for other profiles (the dir doesn't exist). +_rocm_sdk_bin="$(ls -d ${EDIR}/venv/lib/python*/site-packages/_rocm_sdk_devel/bin 2>/dev/null | head -1)" +if [ -n "${_rocm_sdk_bin}" ] && [ -x "${_rocm_sdk_bin}/amdclang" ]; then + export PATH="${_rocm_sdk_bin}:${PATH}" + export CC="${CC:-${_rocm_sdk_bin}/amdclang}" + export CXX="${CXX:-${_rocm_sdk_bin}/amdclang++}" +fi + # Multi-node DP follower mode: when the first arg is `serve`, exec into # vllm's own CLI instead of LocalAI's backend.py gRPC server. The # follower speaks ZMQ directly to the head node's vllm ranks — there