diff --git a/.github/workflows/unsloth-macos-prebuilt.yml b/.github/workflows/unsloth-macos-prebuilt.yml deleted file mode 100644 index 544ca5c2ac6d..000000000000 --- a/.github/workflows/unsloth-macos-prebuilt.yml +++ /dev/null @@ -1,198 +0,0 @@ -# Build and publish Unsloth Studio's own macOS llama.cpp prebuilts once per day. -# Source is ggml-org/llama.cpp at the newest release that has been public for at -# least UNSLOTH_LLAMA_MIN_RELEASE_AGE_HOURS (supply-chain delay); the only change -# over their macos-cpu build is an explicit -DCMAKE_OSX_DEPLOYMENT_TARGET per -# slice, so the binaries declare their load floor instead of inheriting the -# runner OS: arm64 pins 14.0 (the floor of upstream's pre-macos-26 arm64 -# release), x64 pins 13.3 (matching upstream's own Intel leg, which still -# covers 2017 Intel Macs capped at Ventura). Publishes a macOS-only release on -# this fork that Studio installs from. - -name: Unsloth macOS llama.cpp prebuilt - -on: - schedule: - - cron: "13 22 * * *" # ~3PM San Francisco (UTC; not DST adjusted) - workflow_dispatch: - inputs: - upstream_tag: - description: "ggml-org/llama.cpp release tag to build (blank = newest aged release)" - required: false - default: "" - min_age_hours: - description: "Only build a release public for at least this many hours (blank = default)" - required: false - default: "" - -concurrency: - group: unsloth-macos-prebuilt - cancel-in-progress: false - -permissions: - contents: write - -env: - # Supply-chain aging: build only upstream releases that have been public for - # at least this many hours, so a malicious or broken release has time to be - # caught and yanked before we compile and ship it. Per-run override above. - UNSLOTH_LLAMA_MIN_RELEASE_AGE_HOURS: "6" - -jobs: - resolve: - runs-on: ubuntu-latest - outputs: - upstream_tag: ${{ steps.r.outputs.upstream_tag }} - source_commit: ${{ steps.r.outputs.source_commit }} - release_tag: ${{ steps.r.outputs.release_tag }} - skip: ${{ steps.r.outputs.skip }} - steps: - - name: Resolve upstream tag and check idempotency - id: r - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - REQ="${{ github.event.inputs.upstream_tag }}" - AGE_H="${{ github.event.inputs.min_age_hours }}" - [ -n "$AGE_H" ] || AGE_H="${UNSLOTH_LLAMA_MIN_RELEASE_AGE_HOURS:-6}" - if [ -n "$REQ" ]; then - TAG="$REQ" # explicit override skips the aging filter - else - # Newest published release that has been public for >= AGE_H hours. - CUTOFF="$(date -u -d "-${AGE_H} hours" +%s)" - TAG="$(gh api 'repos/ggml-org/llama.cpp/releases?per_page=100' \ - --jq "[.[] | select(.draft==false and .prerelease==false) | select((.published_at|fromdateiso8601) <= ${CUTOFF})] | first | .tag_name")" - if [ -z "$TAG" ] || [ "$TAG" = "null" ]; then - echo "::error::no ggml-org release older than ${AGE_H}h found in the last 100 releases" - exit 1 - fi - echo "selected $TAG (aged >= ${AGE_H}h)" - fi - SHA="$(gh api "repos/ggml-org/llama.cpp/commits/$TAG" --jq .sha)" - RTAG="llama-prebuilt-macos-$TAG" - SKIP=false - if gh release view "$RTAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then - SKIP=true - fi - { - echo "upstream_tag=$TAG" - echo "source_commit=$SHA" - echo "release_tag=$RTAG" - echo "skip=$SKIP" - } >> "$GITHUB_OUTPUT" - echo "upstream=$TAG commit=$SHA release=$RTAG skip=$SKIP" - - build: - needs: resolve - if: needs.resolve.outputs.skip != 'true' - strategy: - fail-fast: false - matrix: - include: - - build: arm64 - runner: macos-14 - defines: "-DGGML_METAL_USE_BF16=ON -DGGML_METAL_EMBED_LIBRARY=ON" - expect_arch: arm64 - deploy_target: "14.0" # no Apple Silicon Mac is capped below 14 - - build: x64 - runner: macos-15-intel - defines: "-DGGML_METAL=OFF" - expect_arch: x86_64 - deploy_target: "13.3" # matches upstream; covers 2017 Intel (Ventura) - runs-on: ${{ matrix.runner }} - steps: - - uses: actions/checkout@v6 - with: - persist-credentials: false - - - name: Clone ggml-org source at ${{ needs.resolve.outputs.upstream_tag }} - run: | - git clone --depth 1 --branch "${{ needs.resolve.outputs.upstream_tag }}" \ - https://github.com/ggml-org/llama.cpp src - - - name: Build (deployment target ${{ matrix.deploy_target }}) - working-directory: src - run: | - set -euo pipefail - cmake -B build \ - ${{ matrix.defines }} \ - -DCMAKE_OSX_DEPLOYMENT_TARGET=${{ matrix.deploy_target }} \ - -DCMAKE_INSTALL_RPATH='@loader_path' \ - -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ - -DLLAMA_FATAL_WARNINGS=ON \ - -DLLAMA_BUILD_BORINGSSL=ON \ - -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_TOOLS=ON \ - -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON - cmake --build build --config Release -j "$(sysctl -n hw.logicalcpu)" - - - name: Load gate (minos <= ${{ matrix.deploy_target }}, arch, launch) - run: bash scripts/unsloth/assert_macho_minos.sh src/build/bin "${{ matrix.expect_arch }}" "${{ matrix.deploy_target }}" - - - name: Package - working-directory: src - run: | - set -euo pipefail - TAG="${{ needs.resolve.outputs.upstream_tag }}" - cp LICENSE build/bin/ - tar -czf "../llama-${TAG}-bin-macos-${{ matrix.build }}.tar.gz" \ - -s ",^\.,llama-${TAG}," -C build/bin . - - - uses: actions/upload-artifact@v6 - with: - name: macos-${{ matrix.build }} - path: llama-*-bin-macos-${{ matrix.build }}.tar.gz - if-no-files-found: error - - publish: - needs: [resolve, build] - if: needs.resolve.outputs.skip != 'true' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - with: - persist-credentials: false - - - uses: actions/download-artifact@v7 - with: - path: dist - - - name: Stage slice tarballs and source archives - run: | - set -euo pipefail - TAG="${{ needs.resolve.outputs.upstream_tag }}" - SHA="${{ needs.resolve.outputs.source_commit }}" - mkdir -p assets - cp dist/macos-arm64/llama-*-bin-macos-arm64.tar.gz assets/ - cp dist/macos-x64/llama-*-bin-macos-x64.tar.gz assets/ - git clone --depth 1 --branch "$TAG" https://github.com/ggml-org/llama.cpp src - tar --exclude=.git -czf "assets/llama.cpp-source-${TAG}.tar.gz" -C src . - cp "assets/llama.cpp-source-${TAG}.tar.gz" "assets/llama.cpp-source-commit-${SHA}.tar.gz" - - - name: Generate manifest and checksums - run: | - set -euo pipefail - TAG="${{ needs.resolve.outputs.upstream_tag }}" - SHA="${{ needs.resolve.outputs.source_commit }}" - python3 scripts/unsloth/gen_manifest.py \ - --upstream-tag "$TAG" \ - --source-commit "$SHA" \ - --release-tag "${{ needs.resolve.outputs.release_tag }}" \ - --out-dir assets \ - --artifact "macos-arm64:assets/llama-${TAG}-bin-macos-arm64.tar.gz" \ - --artifact "macos-x64:assets/llama-${TAG}-bin-macos-x64.tar.gz" \ - --source-archive "assets/llama.cpp-source-${TAG}.tar.gz" \ - --exact-source-archive "assets/llama.cpp-source-commit-${SHA}.tar.gz" - - - name: Publish release - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - TAG="${{ needs.resolve.outputs.upstream_tag }}" - RTAG="${{ needs.resolve.outputs.release_tag }}" - gh release create "$RTAG" \ - --repo "$GITHUB_REPOSITORY" \ - --target master \ - --title "llama.cpp prebuilt macos $TAG" \ - --notes "Install-ready Unsloth Studio llama.cpp macOS bundles for $TAG." \ - assets/* diff --git a/.github/workflows/unsloth-prebuilt-cuda-windows.yml b/.github/workflows/unsloth-prebuilt-cuda-windows.yml new file mode 100644 index 000000000000..883ef286d1c5 --- /dev/null +++ b/.github/workflows/unsloth-prebuilt-cuda-windows.yml @@ -0,0 +1,181 @@ +name: "Unsloth prebuilt: CUDA Windows" + +# Reusable child of unsloth-prebuilt.yml. Builds the per-profile CUDA Windows +# bundles the parent's `resolve` job filtered into `matrix`. Each entry uploads +# a single app-*.zip artifact for the parent's assemble step to pick up. +# +# Mirrors unsloth-prebuilt-cuda.yml (Linux), adapted for Windows: MSVC + Ninja +# toolchain, BoringSSL instead of system OpenSSL, no RPATH (Windows resolves +# DLLs from the executable's directory), and a .zip archive. CUDA compiles +# entirely in software here -- no GPU is present on the runner. + +on: + workflow_call: + inputs: + tag: + description: 'Upstream llama.cpp release tag (b####), resolved by parent' + required: true + type: string + commit: + description: 'Upstream commit SHA for that tag, resolved by parent' + required: true + type: string + matrix: + description: 'Matrix JSON {include:[...]} produced by the parent resolve job' + required: true + type: string + +permissions: + contents: read + +jobs: + build: + name: x64/${{ matrix.profile }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(inputs.matrix) }} + defaults: + run: + shell: pwsh + steps: + - name: Checkout build tooling (this repo) + uses: actions/checkout@v6 + with: + path: tooling + + - name: Checkout upstream source @ ${{ inputs.tag }} + uses: actions/checkout@v6 + with: + repository: ggml-org/llama.cpp + ref: ${{ inputs.tag }} + path: src + # Full history: cmake/build-info.cmake runs `git rev-list --count HEAD` + # to bake LLAMA_BUILD_NUMBER into llama-server --version as b. + # A shallow clone would bake in "b1" instead. + fetch-depth: 0 + + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2.20 + with: + key: cuda-${{ matrix.cuda }}-windows-${{ matrix.profile }} + variant: ccache + evict-old-files: 14d + + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + + - name: Install Ninja + run: choco install ninja --no-progress + + - name: Setup MSVC + uses: ilammy/msvc-dev-cmd@v1 + + # Jimver's action has no mapping for CUDA 13.3 yet, so for that version we + # fall back to llama.cpp's own install method: curl the individual NVIDIA + # redist component archives and assemble the toolkit by hand. Block copied + # verbatim from ggml-org/llama.cpp .github/actions/windows-setup-cuda + # (the cuda_version == '13.3' case). Any other version uses Jimver. + - name: Install CUDA toolkit 13.3 (NVIDIA redist) + if: matrix.cuda == '13.3' + run: | + mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" + choco install unzip -y + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.3.33-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-13.3.29-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.3.33-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-13.3.33-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-13.5.1.27-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.3.33-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-13.3.29-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.3.27-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.3.27-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.3.3.1-archive.zip" + unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_crt-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_cudart-windows-x86_64-13.3.29-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvcc-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvrtc-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\libcublas-windows-x86_64-13.5.1.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\libnvvm-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvtx-windows-x86_64-13.3.29-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_profiler_api-windows-x86_64-13.3.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\visual_studio_integration-windows-x86_64-13.3.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cccl-windows-x86_64-13.3.3.3.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + echo "CUDA_PATH_V13_3=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + + - name: Install CUDA toolkit ${{ matrix.cuda }} (Jimver) + if: matrix.cuda != '13.3' + uses: Jimver/cuda-toolkit@v0.2.35 + id: cuda-toolkit + with: + cuda: ${{ matrix.cuda }} + method: 'network' + + - name: Set up CUDA environment (Jimver) + if: matrix.cuda != '13.3' + run: | + echo "CUDA_PATH=$env:CUDA_PATH" >> $env:GITHUB_ENV + echo "CUDA_HOME=$env:CUDA_PATH" >> $env:GITHUB_ENV + + - name: Verify CUDA + run: nvcc --version + + - name: Configure + working-directory: src + run: | + # CMAKE_CUDA_ARCHITECTURES is the explicit per-profile arch list (the + # whole point of the matrix). ccache launchers cache nvcc + cl.exe; + # no RPATH knobs -- Windows loads sibling DLLs from the binary's dir. + $archs = "${{ matrix.archs }}".Replace(' ', ';') + cmake -S . -B build -G Ninja ` + -DCMAKE_BUILD_TYPE=Release ` + -DGGML_NATIVE=OFF ` + -DGGML_BACKEND_DL=ON ` + -DGGML_CPU_ALL_VARIANTS=ON ` + -DGGML_RPC=ON ` + -DGGML_CUDA=ON ` + -DGGML_CUDA_CUB_3DOT2=ON ` + -DLLAMA_BUILD_TESTS=OFF ` + -DLLAMA_BUILD_EXAMPLES=OFF ` + -DLLAMA_BUILD_BORINGSSL=ON ` + -DCMAKE_CUDA_ARCHITECTURES="$archs" ` + -DCMAKE_C_COMPILER_LAUNCHER=ccache ` + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache ` + -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache + + - name: Build + working-directory: src + run: | + # -j 2: nvcc TUs with the full arch fanout can spike past 12 GB each, + # and the GitHub-hosted runners only have ~16 GB. + cmake --build build --config Release -j 2 ` + --target llama-server llama-quantize + + - name: Package bundle + env: + PLATFORM: windows + ARCH: x64 + BIN_DIR: ${{ github.workspace }}/src/build/bin + SRC_DIR: ${{ github.workspace }}/src + OUT_DIR: ${{ github.workspace }}/dist + TAG: ${{ inputs.tag }} + SOURCE_COMMIT: ${{ inputs.commit }} + PROFILE: ${{ matrix.profile }} + LINE: ${{ matrix.line }} + KLASS: ${{ matrix.klass }} + RANK: ${{ matrix.rank }} + TOOLKIT_LINE: ${{ matrix.toolkit_line }} + DOCKER_IMAGE: github-hosted ${{ matrix.runner }}, CUDA ${{ matrix.cuda }} ${{ matrix.cuda == '13.3' && '(NVIDIA redist)' || '(Jimver)' }} + ARCHS: ${{ matrix.archs }} + run: python tooling/scripts/unsloth/package_bundle.py + + - name: Upload bundle artifact + uses: actions/upload-artifact@v6 + with: + name: app-${{ inputs.tag }}-windows-x64-${{ matrix.profile }} + path: dist/app-${{ inputs.tag }}-windows-x64-${{ matrix.profile }}.zip + if-no-files-found: error diff --git a/.github/workflows/unsloth-prebuilt-cuda.yml b/.github/workflows/unsloth-prebuilt-cuda.yml new file mode 100644 index 000000000000..f495cb3dafef --- /dev/null +++ b/.github/workflows/unsloth-prebuilt-cuda.yml @@ -0,0 +1,205 @@ +name: "Unsloth prebuilt: CUDA" + +# Reusable child of unsloth-prebuilt.yml. Builds the per-profile CUDA bundles +# the parent's `resolve` job filtered into `matrix`. Each entry uploads a +# single app-*.tar.gz artifact for the parent's assemble step to pick up. + +on: + workflow_call: + inputs: + tag: + description: 'Upstream llama.cpp release tag (b####), resolved by parent' + required: true + type: string + commit: + description: 'Upstream commit SHA for that tag, resolved by parent' + required: true + type: string + matrix: + description: 'Matrix JSON {include:[...]} produced by the parent resolve job' + required: true + type: string + +permissions: + contents: read + +jobs: + build: + name: ${{ matrix.arch }}/${{ matrix.profile }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(inputs.matrix) }} + steps: + - name: Free disk space + uses: jlumbroso/free-disk-space@v1.3.1 + with: + tool-cache: true + android: true + dotnet: true + haskell: true + large-packages: false + swap-storage: true + + - name: Checkout build tooling (this repo) + uses: actions/checkout@v6 + with: + path: tooling + + - name: Checkout upstream source @ ${{ inputs.tag }} + uses: actions/checkout@v6 + with: + repository: ggml-org/llama.cpp + ref: ${{ inputs.tag }} + path: src + # Full history is required: cmake/build-info.cmake runs + # `git rev-list --count HEAD` to compute LLAMA_BUILD_NUMBER, + # which gets baked into llama-server --version as b. + # Shallow clone (default depth=1) would bake in "b1" instead. + fetch-depth: 0 + + - name: Install build dependencies + run: | + set -eux + sudo apt-get update + sudo apt-get install -y build-essential libssl-dev + + - name: Install ARM64 host compiler (gcc-14) + if: matrix.arch == 'arm64' + run: | + set -eux + sudo apt-get install -y gcc-14 g++-14 + { + echo "CC=gcc-14" + echo "CXX=g++-14" + echo "CUDAHOSTCXX=g++-14" + } >> "$GITHUB_ENV" + + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2.20 + with: + key: cuda-${{ matrix.cuda }}-${{ matrix.arch }}-${{ matrix.profile }} + variant: ccache + evict-old-files: 14d + + # Jimver has no mapping for CUDA 13.3 yet, so for that version we pull the + # toolkit components straight from NVIDIA's redist CDN (same source Jimver + # and ggml-org use) and assemble a prefix by hand. Component versions are + # from NVIDIA's redistrib_13.3.0.json. Runs on the same ubuntu runner, so + # the glibc floor is unchanged. Any other version uses Jimver. + - name: Install CUDA toolkit 13.3 (NVIDIA redist) + if: matrix.cuda == '13.3' + run: | + set -eux + case "${{ matrix.arch }}" in + x64) PLAT=linux-x86_64 ;; + arm64) PLAT=linux-sbsa ;; + *) echo "unsupported arch ${{ matrix.arch }}" >&2; exit 1 ;; + esac + PREFIX="$HOME/cuda-13.3" + mkdir -p "$PREFIX" + BASE="https://developer.download.nvidia.com/compute/cuda/redist" + for cv in \ + cuda_crt:13.3.33 cuda_cudart:13.3.29 cuda_nvcc:13.3.33 \ + cuda_nvrtc:13.3.33 libcublas:13.5.1.27 libnvvm:13.3.33 \ + cuda_nvtx:13.3.29 cuda_profiler_api:13.3.27 cccl:13.3.3.3.1; do + comp="${cv%:*}"; ver="${cv#*:}" + name="${comp}-${PLAT}-${ver}-archive" + curl -fsSL -o comp.tar.xz "$BASE/${comp}/${PLAT}/${name}.tar.xz" + tar -xf comp.tar.xz + cp -a "${name}/." "$PREFIX"/ + rm -rf comp.tar.xz "${name}" + done + # Redist ships libs under lib/; some CMake CUDA discovery expects lib64. + ln -sfn lib "$PREFIX/lib64" + { + echo "CUDA_PATH=$PREFIX" + echo "CUDA_HOME=$PREFIX" + echo "LD_LIBRARY_PATH=$PREFIX/lib:${LD_LIBRARY_PATH:-}" + } >> "$GITHUB_ENV" + echo "$PREFIX/bin" >> "$GITHUB_PATH" + + - name: Install CUDA toolkit ${{ matrix.cuda }} (Jimver) + if: matrix.cuda != '13.3' + uses: Jimver/cuda-toolkit@v0.2.35 + id: cuda-toolkit + with: + cuda: ${{ matrix.cuda }} + method: 'network' + + - name: Set up CUDA environment (Jimver) + if: matrix.cuda != '13.3' + run: | + echo "CUDA_PATH=${{ steps.cuda-toolkit.outputs.CUDA_PATH }}" >> "$GITHUB_ENV" + echo "CUDA_HOME=${{ steps.cuda-toolkit.outputs.CUDA_PATH }}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${{ steps.cuda-toolkit.outputs.CUDA_PATH }}/lib64:${LD_LIBRARY_PATH:-}" >> "$GITHUB_ENV" + + - name: Verify CUDA + run: nvcc --version + + - name: Configure + working-directory: src + run: | + set -eux + # Three non-obvious flags: + # - CMAKE_INSTALL_RPATH=$ORIGIN + RPATH knobs: the tar.gz layout + # ships sibling .so files in the same directory as the binaries. + # - CMAKE_CUDA_ARCHITECTURES: explicit per-profile arch list, the + # whole point of the matrix. + # - CMAKE_*_COMPILER_LAUNCHER=ccache: Jimver installs nvcc outside + # /usr/local/bin, so PATH-symlinked ccache misses CUDA TUs -- + # setting the launcher explicitly caches nvcc too. + cmake -S . -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DGGML_NATIVE=OFF \ + -DGGML_BACKEND_DL=ON \ + -DGGML_CPU_ALL_VARIANTS=ON \ + -DGGML_RPC=ON \ + -DGGML_CUDA=ON \ + -DGGML_CUDA_CUB_3DOT2=ON \ + -DLLAMA_BUILD_TESTS=OFF \ + -DLLAMA_BUILD_EXAMPLES=OFF \ + -DCMAKE_CUDA_ARCHITECTURES="$(echo '${{ matrix.archs }}' | tr ' ' ';')" \ + -DCMAKE_INSTALL_RPATH='$ORIGIN' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=OFF \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache + + - name: Build + working-directory: src + run: | + set -eux + # Backend modules (CPU variants, CUDA, RPC) build as ggml dependencies, + # so the server/quantize targets pull them in. + # -j 2: nvcc TUs with the full arch fanout can spike past 12 GB each, + # and the GitHub-hosted runners (x64 and arm64 alike) only have ~16 GB. + cmake --build build --config Release -j 2 \ + --target llama-server llama-quantize + strip build/bin/llama-server build/bin/llama-quantize || true + + - name: Package bundle + env: + PLATFORM: linux + ARCH: ${{ matrix.arch }} + BIN_DIR: ${{ github.workspace }}/src/build/bin + SRC_DIR: ${{ github.workspace }}/src + OUT_DIR: ${{ github.workspace }}/dist + TAG: ${{ inputs.tag }} + SOURCE_COMMIT: ${{ inputs.commit }} + PROFILE: ${{ matrix.profile }} + LINE: ${{ matrix.line }} + KLASS: ${{ matrix.klass }} + RANK: ${{ matrix.rank }} + TOOLKIT_LINE: ${{ matrix.toolkit_line }} + DOCKER_IMAGE: github-hosted ${{ matrix.runner }}, CUDA ${{ matrix.cuda }} ${{ matrix.cuda == '13.3' && '(NVIDIA redist)' || '(Jimver)' }} + ARCHS: ${{ matrix.archs }} + run: python3 tooling/scripts/unsloth/package_bundle.py + + - name: Upload bundle artifact + uses: actions/upload-artifact@v6 + with: + name: app-${{ inputs.tag }}-linux-${{ matrix.arch }}-${{ matrix.profile }} + path: dist/app-${{ inputs.tag }}-linux-${{ matrix.arch }}-${{ matrix.profile }}.tar.gz + if-no-files-found: error diff --git a/.github/workflows/unsloth-prebuilt-macos.yml b/.github/workflows/unsloth-prebuilt-macos.yml new file mode 100644 index 000000000000..8f15de1b5fc1 --- /dev/null +++ b/.github/workflows/unsloth-prebuilt-macos.yml @@ -0,0 +1,88 @@ +name: "Unsloth prebuilt: macOS" + +# Reusable child of unsloth-prebuilt.yml. Builds the macOS slices (arm64 Metal + +# x64 CPU) the parent's `resolve` job placed in `matrix`. Each entry uploads a +# single app-*.tar.gz artifact for the parent's assemble step to pick up. +# +# The only change over ggml-org's own macos build is an explicit +# -DCMAKE_OSX_DEPLOYMENT_TARGET per slice, so the binaries declare their load +# floor instead of inheriting the runner OS. Upstream stopped pinning this on +# arm64 (their macos-26 runner stamps minos=26, which fails to dyld-load on +# macOS < 26); owning the build is how we keep a loadable floor. + +on: + workflow_call: + inputs: + tag: + description: 'Upstream llama.cpp release tag (b####), resolved by parent' + required: true + type: string + matrix: + description: 'Matrix JSON {include:[...]} produced by the parent resolve job' + required: true + type: string + +permissions: + contents: read + +jobs: + build: + name: ${{ matrix.build }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(inputs.matrix) }} + steps: + - name: Checkout build tooling (this repo) + uses: actions/checkout@v6 + with: + path: tooling + persist-credentials: false + + - name: Checkout upstream source @ ${{ inputs.tag }} + uses: actions/checkout@v6 + with: + repository: ggml-org/llama.cpp + ref: ${{ inputs.tag }} + path: src + # Full history: cmake/build-info.cmake runs `git rev-list --count HEAD` + # to bake LLAMA_BUILD_NUMBER into llama-server --version as b. + # A shallow clone would bake in "b1" instead. + fetch-depth: 0 + + - name: Build (deployment target ${{ matrix.deploy_target }}) + working-directory: src + run: | + set -euo pipefail + cmake -B build \ + ${{ matrix.defines }} \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=${{ matrix.deploy_target }} \ + -DCMAKE_INSTALL_RPATH='@loader_path' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ + -DLLAMA_FATAL_WARNINGS=ON \ + -DLLAMA_BUILD_BORINGSSL=ON \ + -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_TOOLS=ON \ + -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON + cmake --build build --config Release -j "$(sysctl -n hw.logicalcpu)" + + - name: Load gate (minos <= ${{ matrix.deploy_target }}, arch, launch) + run: bash tooling/scripts/unsloth/assert_macho_minos.sh src/build/bin "${{ matrix.expect_arch }}" "${{ matrix.deploy_target }}" + + - name: Package bundle + working-directory: src + run: | + set -euo pipefail + TAG="${{ inputs.tag }}" + cp LICENSE build/bin/ + mkdir -p "$GITHUB_WORKSPACE/dist" + # BSD tar (-s) rewrites the leading ./ to llama-/ so the archive + # unpacks into a named dir, matching upstream's own macos tarball layout. + tar -czf "$GITHUB_WORKSPACE/dist/llama-${TAG}-bin-macos-${{ matrix.build }}.tar.gz" \ + -s ",^\.,llama-${TAG}," -C build/bin . + + - name: Upload bundle artifact + uses: actions/upload-artifact@v6 + with: + name: app-${{ inputs.tag }}-macos-${{ matrix.build }} + path: dist/llama-${{ inputs.tag }}-bin-macos-${{ matrix.build }}.tar.gz + if-no-files-found: error diff --git a/.github/workflows/unsloth-prebuilt-rocm.yml b/.github/workflows/unsloth-prebuilt-rocm.yml new file mode 100644 index 000000000000..9b6f642408e0 --- /dev/null +++ b/.github/workflows/unsloth-prebuilt-rocm.yml @@ -0,0 +1,701 @@ +name: "Unsloth prebuilt: ROCm" + +# Reusable child of unsloth-prebuilt.yml. Per-gfx-target ROCm bundles on +# Windows + Ubuntu. +# +# Huge thanks to the lemonade-sdk team -- this is adapted from their workflow: +# https://github.com/lemonade-sdk/llamacpp-rocm/blob/main/.github/workflows/build-llamacpp-rocm.yml +# +# Build mechanics taken verbatim from that file: TheRock S3 nightly download +# + version auto-detect, gfx target name mapping, HIP/clang cmake invocation, +# the full hardcoded ROCm runtime lib copy list, patchelf RPATH. +# +# Local adaptations: +# - tag is passed in (the parent resolves it against ggml-org upstream) +# instead of lemonade's auto-incrementing b1000+ scheme. +# - artifacts pre-packaged into app---x64-rocm-.{tar.gz|zip} +# so the parent assemble step picks them up via the same merge-multiple +# download pattern it uses for CUDA bundles. +# - test jobs (stx-halo, stx) dropped -- those need self-hosted AMD runners. +# - build-summary + post-build cleanup steps dropped (clutter). + +on: + workflow_call: + inputs: + tag: + description: 'Upstream llama.cpp release tag (b####), resolved by parent' + required: true + type: string + matrix: + description: 'gfx_target matrix JSON ({"gfx_target":[...]}), built by parent' + required: true + type: string + operating_systems: + description: 'OSes to build for (comma-separated: windows,ubuntu)' + required: false + default: 'windows,ubuntu' + type: string + rocm_version: + description: 'TheRock ROCm version (e.g., 7.13.0a20260318) or "latest"' + required: false + default: 'latest' + type: string + +permissions: + contents: read + +jobs: + build-windows: + name: windows/${{ matrix.gfx_target }} + runs-on: windows-latest + if: contains(inputs.operating_systems, 'windows') + strategy: + matrix: ${{ fromJson(inputs.matrix) }} + fail-fast: false + + steps: + - name: Clean up existing directories (safety precaution) + run: | + # Remove existing llama.cpp directory if it exists + if (Test-Path "llama.cpp") { + Write-Host "Removing existing llama.cpp directory..." + Remove-Item -Recurse -Force "llama.cpp" + } + + # Remove existing C:\opt\rocm directory if it exists + if (Test-Path "C:\opt\rocm") { + Write-Host "Removing existing C:\opt\rocm directory..." + Remove-Item -Recurse -Force "C:\opt\rocm" + } + + # Remove any existing ROCm tarball + if (Test-Path "rocm.tar.gz") { + Write-Host "Removing existing rocm.tar.gz..." + Remove-Item -Force "rocm.tar.gz" + } + + Write-Host "Cleanup completed successfully" + + - name: Install Visual Studio Build Tools + run: | + # Download and install Visual Studio Build Tools + $vsInstallerUrl = "https://aka.ms/vs/17/release/vs_buildtools.exe" + $vsInstallerPath = "$env:TEMP\vs_buildtools.exe" + + Write-Host "Downloading Visual Studio Build Tools..." + Invoke-WebRequest -Uri $vsInstallerUrl -OutFile $vsInstallerPath + + Write-Host "Installing Visual Studio Build Tools..." + Start-Process -FilePath $vsInstallerPath -ArgumentList "--quiet", "--wait", "--norestart", "--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", "--add", "Microsoft.VisualStudio.Component.VC.CMake.Project", "--add", "Microsoft.VisualStudio.Component.VC.ATL", "--add", "Microsoft.VisualStudio.Component.Windows11SDK.22621" -Wait + + # Clean up installer + Remove-Item $vsInstallerPath -Force + + - name: Install build dependencies + run: | + Write-Host "Installing build dependencies using manual methods..." + + # Install Ninja + Write-Host "Installing Ninja..." + $ninjaUrl = "https://github.com/ninja-build/ninja/releases/download/v1.11.1/ninja-win.zip" + $ninjaPath = "$env:TEMP\ninja-win.zip" + $ninjaDir = "C:\ninja" + New-Item -ItemType Directory -Force -Path $ninjaDir + Invoke-WebRequest -Uri $ninjaUrl -OutFile $ninjaPath + Expand-Archive -Path $ninjaPath -DestinationPath $ninjaDir -Force + + # Install Strawberry Perl + Write-Host "Installing Strawberry Perl..." + $perlUrl = "http://strawberryperl.com/download/5.32.1.1/strawberry-perl-5.32.1.1-64bit.msi" + $perlPath = "$env:TEMP\strawberry-perl-5.32.1.1-64bit.msi" + Invoke-WebRequest -Uri $perlUrl -OutFile $perlPath + Start-Process msiexec.exe -ArgumentList "/i $perlPath /quiet /norestart" -Wait + + # Verify installations + $env:PATH = "C:\ninja;C:\Strawberry\perl\bin;$env:PATH" + Write-Host "Verifying installations..." + ninja --version + perl --version + + Write-Host "Manual installation of build dependencies completed" + + - name: Download ROCm nightly tarball + run: | + # Determine ROCm version to use + $rocmVersion = "${{ inputs.rocm_version }}" + $currentTarget = "${{ matrix.gfx_target }}" + + # Add appropriate suffixes for different GPU targets + $s3Target = $currentTarget + if ($currentTarget -eq "gfx103X") { + $s3Target = "$currentTarget-dgpu" + Write-Host "Using S3 target with -dgpu suffix: $s3Target" + } elseif ($currentTarget -eq "gfx110X" -or $currentTarget -eq "gfx120X") { + $s3Target = "$currentTarget-all" + Write-Host "Using S3 target with -all suffix: $s3Target" + } + + if ($rocmVersion -eq "latest") { + Write-Host "Auto-detecting latest ROCm version for target: $currentTarget" + $s3Response = (Invoke-WebRequest "https://therock-nightly-tarball.s3.amazonaws.com/?prefix=therock-dist-windows-$s3Target-7").Content + $files = $s3Response -split '' | Where-Object {$_ -match ''} | ForEach-Object { ($_ -split '')[0] } + + # Extract versions and sort them properly using semantic versioning + $versionFiles = @() + foreach ($file in $files) { + if ($file -match "therock-dist-windows-$s3Target-.*?(\d+\.\d+\.\d+(?:a|rc)\d+)\.tar\.gz") { + $version = $matches[1] + $versionFiles += [PSCustomObject]@{ + File = $file + Version = $version + Major = [int]($version -split '\.')[0] + Minor = [int]($version -split '\.')[1] + Patch = [int](($version -split '\.')[2] -replace '(?:a|rc).*', '') + RC = [int]($version -replace '.*(?:a|rc)', '') + IsAlpha = $version -match 'a' + } + } + } + + # Sort by semantic version (major.minor.patch.rc, with alpha versions being newer than rc) + $latestFile = ($versionFiles | Sort-Object Major, Minor, Patch, @{Expression={if($_.IsAlpha){1}else{0}}}, RC | Select-Object -Last 1).File + + Write-Host "Found latest file: $latestFile" + + # Extract version from the filename for environment variable + if ($latestFile -match "therock-dist-windows-$s3Target-.*?(\d+\.\d+\.\d+(?:a|rc)\d+)\.tar\.gz") { + $rocmVersion = $matches[1] + Write-Host "Detected latest ROCm version: $rocmVersion" + } else { + Write-Error "Failed to extract ROCm version from latest file: $latestFile" + Write-Error "Expected pattern: therock-dist-windows-$s3Target-*.tar.gz" + exit 1 + } + + # Use the exact filename from S3 instead of reconstructing + $rocmUrl = "https://therock-nightly-tarball.s3.amazonaws.com/$latestFile" + } else { + # For specific versions, construct the URL as before + $rocmUrl = "https://therock-nightly-tarball.s3.amazonaws.com/therock-dist-windows-$s3Target-$rocmVersion.tar.gz" + } + + # Store the version for use in other steps + echo "DETECTED_ROCM_VERSION=$rocmVersion" >> $env:GITHUB_ENV + + Write-Host "Downloading ROCm from: $rocmUrl" + Invoke-WebRequest -Uri $rocmUrl -OutFile "rocm.tar.gz" + + - name: Extract ROCm to C:\opt\rocm + run: | + # Create directory if it doesn't exist + New-Item -ItemType Directory -Force -Path "C:\opt\rocm" + + # Extract the tarball + tar -xzf rocm.tar.gz -C C:\opt\rocm --strip-components=1 + + - name: Clone llama.cpp @ ${{ inputs.tag }} + run: | + $tag = "${{ inputs.tag }}" + Write-Host "Cloning llama.cpp version: $tag (full history so LLAMA_BUILD_NUMBER is correct)" + git clone --single-branch --branch $tag https://github.com/ggerganov/llama.cpp.git + cd llama.cpp + + # Show current commit info + Write-Host "Current llama.cpp commit:" + git log --oneline -1 + + - name: Build Llama.cpp + ROCm + shell: cmd + run: | + + REM Map GPU targets + set "current_target=${{ matrix.gfx_target }}" + echo Input target: %current_target% + + if "%current_target%"=="gfx110X" ( + set "mapped_target=gfx1100;gfx1101;gfx1102;gfx1103" + ) else if "%current_target%"=="gfx103X" ( + set "mapped_target=gfx1030;gfx1031;gfx1032;gfx1034" + ) else if "%current_target%"=="gfx1151" ( + set "mapped_target=gfx1151" + ) else if "%current_target%"=="gfx1150" ( + set "mapped_target=gfx1150" + ) else if "%current_target%"=="gfx120X" ( + set "mapped_target=gfx1200;gfx1201" + ) else ( + set "mapped_target=%current_target%" + ) + echo Mapped target: %mapped_target% + + REM Set up environment variables and PATH + set HIP_PATH=C:\opt\rocm + set HIP_PLATFORM=amd + set PATH=%HIP_PATH%\lib\llvm\bin;%HIP_PATH%\bin;%PATH% + + REM Set up x64 Native Tools Command Prompt environment + call "C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath > vs_path.txt + set /p VS_PATH=)[^<]*' | grep "therock-dist-linux-${s3_target}-") + + # Extract versions and sort them properly using semantic versioning + latest_file="" + latest_major=0 + latest_minor=0 + latest_patch=0 + latest_rc=0 + latest_is_alpha=false + + while IFS= read -r file; do + if [[ "$file" =~ therock-dist-linux-${s3_target}-.*?([0-9]+\.[0-9]+\.[0-9]+(a|rc)[0-9]+)\.tar\.gz ]]; then + version="${BASH_REMATCH[1]}" + major=$(echo "$version" | cut -d. -f1) + minor=$(echo "$version" | cut -d. -f2) + patch=$(echo "$version" | cut -d. -f3 | sed 's/\(a\|rc\).*//') + rc=$(echo "$version" | sed 's/.*\(a\|rc\)//') + is_alpha=false + if [[ "$version" =~ a ]]; then + is_alpha=true + fi + + # Compare versions (alpha versions are newer than rc versions) + is_newer=false + if [ "$major" -gt "$latest_major" ]; then + is_newer=true + elif [ "$major" -eq "$latest_major" ] && [ "$minor" -gt "$latest_minor" ]; then + is_newer=true + elif [ "$major" -eq "$latest_major" ] && [ "$minor" -eq "$latest_minor" ] && [ "$patch" -gt "$latest_patch" ]; then + is_newer=true + elif [ "$major" -eq "$latest_major" ] && [ "$minor" -eq "$latest_minor" ] && [ "$patch" -eq "$latest_patch" ]; then + # Same major.minor.patch, compare alpha vs rc + if [ "$is_alpha" = true ] && [ "$latest_is_alpha" = false ]; then + is_newer=true + elif [ "$is_alpha" = "$latest_is_alpha" ] && [ "$rc" -gt "$latest_rc" ]; then + is_newer=true + fi + fi + + if [ "$is_newer" = true ]; then + latest_file="$file" + latest_major="$major" + latest_minor="$minor" + latest_patch="$patch" + latest_rc="$rc" + latest_is_alpha="$is_alpha" + fi + fi + done <<< "$files" + + echo "Found latest file: $latest_file" + + # Extract version from the filename for environment variable + if [[ "$latest_file" =~ therock-dist-linux-${s3_target}-.*?([0-9]+\.[0-9]+\.[0-9]+(a|rc)[0-9]+)\.tar\.gz ]]; then + rocm_version="${BASH_REMATCH[1]}" + echo "Detected latest ROCm version: $rocm_version" + else + echo "Failed to extract ROCm version from latest file: $latest_file" + echo "Expected pattern: therock-dist-linux-${s3_target}-*.tar.gz" + exit 1 + fi + + # Use the exact filename from S3 instead of reconstructing + rocm_url="https://therock-nightly-tarball.s3.amazonaws.com/$latest_file" + else + # For specific versions, construct the URL as before + rocm_url="https://therock-nightly-tarball.s3.amazonaws.com/therock-dist-linux-${s3_target}-${rocm_version}.tar.gz" + fi + + # Store the version for use in other steps + echo "DETECTED_ROCM_VERSION=$rocm_version" >> $GITHUB_ENV + + echo "Streaming ROCm from: $rocm_url directly to extraction" + + # Create directory if it doesn't exist + sudo mkdir -p /opt/rocm + + # Stream download directly into tar extraction (no intermediate file) + curl -sL "$rocm_url" | sudo tar --use-compress-program=gzip -xf - -C /opt/rocm --strip-components=1 + + - name: Set ROCm environment variables + run: | + echo "Setting ROCm environment variables..." + + # Set environment variables for this step and subsequent steps + echo "HIP_PATH=/opt/rocm" >> $GITHUB_ENV + echo "ROCM_PATH=/opt/rocm" >> $GITHUB_ENV + echo "HIP_PLATFORM=amd" >> $GITHUB_ENV + echo "HIP_CLANG_PATH=/opt/rocm/llvm/bin" >> $GITHUB_ENV + echo "HIP_INCLUDE_PATH=/opt/rocm/include" >> $GITHUB_ENV + echo "HIP_LIB_PATH=/opt/rocm/lib" >> $GITHUB_ENV + echo "HIP_DEVICE_LIB_PATH=/opt/rocm/lib/llvm/amdgcn/bitcode" >> $GITHUB_ENV + + # Update PATH + echo "/opt/rocm/bin:/opt/rocm/llvm/bin:$PATH" >> $GITHUB_PATH + + # Set library paths + echo "LD_LIBRARY_PATH=/opt/rocm/lib:/opt/rocm/lib64:/opt/rocm/llvm/lib:${LD_LIBRARY_PATH:-}" >> $GITHUB_ENV + echo "LIBRARY_PATH=/opt/rocm/lib:/opt/rocm/lib64:${LIBRARY_PATH:-}" >> $GITHUB_ENV + echo "CPATH=/opt/rocm/include:${CPATH:-}" >> $GITHUB_ENV + echo "PKG_CONFIG_PATH=/opt/rocm/lib/pkgconfig:${PKG_CONFIG_PATH:-}" >> $GITHUB_ENV + + echo "ROCm environment variables set successfully" + + - name: Clone llama.cpp @ ${{ inputs.tag }} + run: | + tag="${{ inputs.tag }}" + echo "Cloning llama.cpp version: $tag (full history so LLAMA_BUILD_NUMBER is correct)" + git clone --single-branch --branch "$tag" https://github.com/ggerganov/llama.cpp.git + cd llama.cpp + + # Show current commit info + echo "Current llama.cpp commit:" + git log --oneline -1 + + - name: Build Llama.cpp + ROCm + run: | + # Map GPU targets + current_target="${{ matrix.gfx_target }}" + echo "Input target: $current_target" + + if [ "$current_target" = "gfx110X" ]; then + mapped_target="gfx1100;gfx1101;gfx1102;gfx1103" + elif [ "$current_target" = "gfx103X" ]; then + mapped_target="gfx1030;gfx1031;gfx1032;gfx1034" + elif [ "$current_target" = "gfx1151" ]; then + mapped_target="gfx1151" + elif [ "$current_target" = "gfx1150" ]; then + mapped_target="gfx1150" + elif [ "$current_target" = "gfx120X" ]; then + mapped_target="gfx1200;gfx1201" + else + mapped_target="$current_target" + fi + echo "Mapped target: $mapped_target" + + # Create build directory + cd llama.cpp + mkdir build + cd build + + # Configure the project + cmake .. -G Ninja \ + -DCMAKE_C_COMPILER=/opt/rocm/llvm/bin/clang \ + -DCMAKE_CXX_COMPILER=/opt/rocm/llvm/bin/clang++ \ + -DCMAKE_CXX_FLAGS="-I/opt/rocm/include" \ + -DCMAKE_CROSSCOMPILING=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DGPU_TARGETS="$mapped_target" \ + -DBUILD_SHARED_LIBS=ON \ + -DLLAMA_BUILD_TESTS=OFF \ + -DGGML_HIP=ON \ + -DGGML_OPENMP=OFF \ + -DGGML_CUDA_FORCE_CUBLAS=OFF \ + -DGGML_RPC=ON \ + -DGGML_HIP_ROCWMMA_FATTN=OFF \ + -DLLAMA_BUILD_BORINGSSL=ON \ + -DGGML_NATIVE=OFF \ + -DGGML_STATIC=OFF \ + -DCMAKE_SYSTEM_NAME=Linux + + # Build the project + cmake --build . -j $(nproc) + + - name: Copy ROCm core libs to build directory + run: | + build_bin_path="llama.cpp/build/bin" + rocm_bin_path="/opt/rocm/bin" + + # Copy the rocblas/library folder and all its contents + rocblas_lib_path="/opt/rocm/lib/rocblas/library" + if [ -d "$rocblas_lib_path" ]; then + echo "Copying rocblas/library folder and all contents..." + dest_rocblas_path="$build_bin_path/rocblas/library" + mkdir -p "$(dirname "$dest_rocblas_path")" + cp -r "$rocblas_lib_path" "$(dirname "$dest_rocblas_path")/" + echo "Copied: rocblas/library folder with all contents" + + # List the contents of the copied rocblas/library folder + echo "Contents of rocblas/library:" + find "$dest_rocblas_path" -type f -exec ls -la {} \; | head -20 + else + echo "Warning: rocblas/library folder not found at: $rocblas_lib_path" + fi + + # Copy the hipblaslt/library folder and all its contents + hipblaslt_lib_path="/opt/rocm/lib/hipblaslt/library" + if [ -d "$hipblaslt_lib_path" ]; then + echo "Copying hipblaslt/library folder and all contents..." + dest_hipblaslt_path="$build_bin_path/hipblaslt/library" + mkdir -p "$(dirname "$dest_hipblaslt_path")" + cp -r "$hipblaslt_lib_path" "$(dirname "$dest_hipblaslt_path")/" + echo "Copied: hipblaslt/library folder with all contents" + + # List the contents of the copied hipblaslt/library folder + echo "Contents of hipblaslt/library:" + find "$dest_hipblaslt_path" -type f -exec ls -la {} \; | head -20 + else + echo "Warning: hipblaslt/library folder not found at: $hipblaslt_lib_path" + fi + + # Copy required ROCm libraries to build directory + # If artifacts from ROCm or Llama.cpp change, you may need to update this list + # To get a new list of all libraries, run: + # gather_required_libs.py --rocm-dir /opt/rocm --dest-dir llama.cpp/build/bin + echo "Copying required ROCm libraries to build directory..." + cp -v /opt/rocm/lib/libhipblas.so* "$build_bin_path/" 2>/dev/null || echo "libhipblas.so* not found" + cp -v /opt/rocm/lib/librocblas.so* "$build_bin_path/" 2>/dev/null || echo "librocblas.so* not found" + cp -v /opt/rocm/lib/libamdhip64.so* "$build_bin_path/" 2>/dev/null || echo "libamdhip64.so* not found" + cp -v /opt/rocm/lib/librocsolver.so* "$build_bin_path/" 2>/dev/null || echo "librocsolver.so* not found" + cp -v /opt/rocm/lib/libroctx64.so* "$build_bin_path/" 2>/dev/null || echo "libroctx64.so* not found" + cp -v /opt/rocm/lib/libhipblaslt.so* "$build_bin_path/" 2>/dev/null || echo "libhipblaslt.so* not found" + cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_liblzma.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_liblzma.so* not found" + cp -v /opt/rocm/lib/librocprofiler-register.so* "$build_bin_path/" 2>/dev/null || echo "librocprofiler-register.so* not found" + cp -v /opt/rocm/lib/libamd_comgr.so* "$build_bin_path/" 2>/dev/null || echo "libamd_comgr.so* not found" + cp -v /opt/rocm/lib/libamd_comgr_loader.so* "$build_bin_path/" 2>/dev/null || echo "libamd_comgr_loader.so* not found" + cp -v /opt/rocm/lib/libhsa-runtime64.so* "$build_bin_path/" 2>/dev/null || echo "libhsa-runtime64.so* not found" + cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_numa.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_numa.so* not found" + cp -v /opt/rocm/lib/librocroller.so* "$build_bin_path/" 2>/dev/null || echo "librocroller.so* not found" + cp -v /opt/rocm/lib/librocm_kpack.so* "$build_bin_path/" 2>/dev/null || echo "librocm_kpack.so* not found" + cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_z.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_z.so* not found" + cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_zstd.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_zstd.so* not found" + cp -v /opt/rocm/lib/llvm/lib/libLLVM.so* "$build_bin_path/" 2>/dev/null || echo "libLLVM.so* not found" + cp -v /opt/rocm/lib/llvm/lib/libclang-cpp.so* "$build_bin_path/" 2>/dev/null || echo "libclang-cpp.so* not found" + + cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_elf.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_elf.so* not found" + cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_drm.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_drm.so* not found" + cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_drm_amdgpu.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_drm_amdgpu.so* not found" + cp -v /opt/rocm/lib/rocm_sysdeps/lib/librocm_sysdeps_bz2.so* "$build_bin_path/" 2>/dev/null || echo "librocm_sysdeps_bz2.so* not found" + + # libatomic.so.1 is a transitive dependency of librocm_sysdeps_numa + # (libhsa-runtime64 -> numa -> libatomic) but it is a system GCC runtime + # library, not shipped under /opt/rocm. Bundle it so the runtime stays + # self-contained on hosts that lack it. lemonade-sdk/lemonade#1349 hit + # exactly this ("libatomic.so.1: cannot open shared object file"). + sudo apt-get install -y libatomic1 >/dev/null 2>&1 || true + libatomic_path="$(ldconfig -p | awk -F'=> ' '/libatomic\.so\.1/{print $2; exit}')" + cp -v "$libatomic_path" "$build_bin_path/" 2>/dev/null || echo "libatomic.so.1 not found" + + echo "Finished copying required ROCm libraries" + + - name: Set RPATH for portable distribution + run: | + sudo apt-get install -y patchelf + cd llama.cpp/build/bin + # Set RPATH to $ORIGIN so all libraries (including the comgr stub loader) find deps locally + for file in *.so* llama-*; do + [ -f "$file" ] && [ ! -L "$file" ] && patchelf --set-rpath '$ORIGIN' "$file" 2>/dev/null || true + done + + - name: List build artifacts (including ROCm files) + run: | + cd llama.cpp/build/bin + echo "Final build artifacts (including ROCm library files):" + ls -la + + - name: Package bundle (tar.gz) + run: | + set -eux + ASSET="app-${{ inputs.tag }}-linux-x64-rocm-${{ matrix.gfx_target }}.tar.gz" + mkdir -p dist + (cd llama.cpp/build/bin && tar -czf "${GITHUB_WORKSPACE}/dist/${ASSET}" .) + ls -la dist + + - name: Upload bundle artifact + uses: actions/upload-artifact@v6 + with: + name: app-${{ inputs.tag }}-linux-x64-rocm-${{ matrix.gfx_target }} + path: dist/app-${{ inputs.tag }}-linux-x64-rocm-${{ matrix.gfx_target }}.tar.gz + if-no-files-found: error diff --git a/.github/workflows/unsloth-prebuilt.yml b/.github/workflows/unsloth-prebuilt.yml new file mode 100644 index 000000000000..c1d38c2de36a --- /dev/null +++ b/.github/workflows/unsloth-prebuilt.yml @@ -0,0 +1,352 @@ +name: Unsloth prebuilt (full release) + +# Atomic daily build for unslothai/llama.cpp. One cron, one workflow run, one +# release per upstream b#### tag. Splits the heavy build work into four reusable +# children: +# unsloth-prebuilt-cuda.yml -- Linux CUDA bundles (x64 + arm64, matrix profiles) +# unsloth-prebuilt-cuda-windows.yml -- CUDA Windows bundles (x64, matrix profiles) +# unsloth-prebuilt-rocm.yml -- ROCm bundles (Windows + Ubuntu, per gfx target) +# unsloth-prebuilt-macos.yml -- macOS bundles (arm64 Metal + x64 CPU) +# +# Atomicity: the `assemble` job depends on all children. GitHub's default +# `needs` semantics require all needs to succeed -- if any matrix entry in +# any child fails, `assemble` skips and no release is published. The +# installer needs the full bundle set at the same tag or it'll dispatch to +# something that isn't there. + +on: + schedule: + - cron: '13 22 * * *' # ~3PM San Francisco (UTC; not DST adjusted) + workflow_dispatch: + inputs: + tag: + description: 'ggml-org tag (b#### or "latest")' + default: 'latest' + required: true + type: string + min_age_hours: + description: 'For "latest": only build a release public for at least this many hours (blank = default 6)' + default: '' + required: false + type: string + only_profile: + description: 'CUDA profile to build' + default: 'all' + required: false + type: choice + options: [all, cuda12-older, cuda12-newer, cuda12-portable, cuda13-older, cuda13-newer, cuda13-portable] + operating_systems: + description: 'OSes for ROCm builds' + default: 'windows,ubuntu' + required: false + type: string + gfx_target: + description: 'GPU targets for ROCm builds' + default: 'gfx1151,gfx1150,gfx120X,gfx110X,gfx103X' + required: false + type: string + rocm_version: + description: 'ROCm version (or "latest")' + default: 'latest' + required: false + type: string + publish: + description: 'Publish to GitHub Releases' + default: false + required: false + type: boolean + +permissions: + contents: write + +env: + # Supply-chain aging: when resolving "latest", only build an upstream release + # that has been public for at least this many hours, so a malicious or broken + # release has time to be caught and yanked before we compile and ship it. An + # explicit b#### tag (manual run) skips it. Per-run override via min_age_hours. + UNSLOTH_LLAMA_MIN_RELEASE_AGE_HOURS: "6" + +concurrency: + group: ${{ github.workflow }}-${{ github.event.inputs.tag || 'scheduled' }} + cancel-in-progress: false + +jobs: + resolve: + name: Resolve tag + runs-on: ubuntu-22.04 + outputs: + tag: ${{ steps.r.outputs.tag }} + commit: ${{ steps.r.outputs.commit }} + exists: ${{ steps.r.outputs.exists }} + cuda_matrix: ${{ steps.r.outputs.cuda_matrix }} + win_cuda_matrix: ${{ steps.r.outputs.win_cuda_matrix }} + rocm_matrix: ${{ steps.r.outputs.rocm_matrix }} + macos_matrix: ${{ steps.r.outputs.macos_matrix }} + env: + GH_TOKEN: ${{ github.token }} + steps: + - id: r + run: | + set -euo pipefail + REQ='${{ github.event.inputs.tag || 'latest' }}' + ONLY='${{ github.event.inputs.only_profile || 'all' }}' + GFX_DEFAULT='gfx1151,gfx1150,gfx120X,gfx110X,gfx103X' + GFX='${{ github.event.inputs.gfx_target }}'; GFX="${GFX:-$GFX_DEFAULT}" + OS_LIST='${{ github.event.inputs.operating_systems || 'windows,ubuntu' }}' + AGE_H='${{ github.event.inputs.min_age_hours }}' + + # publish does delete-then-create on the whole release, and the installer + # treats the published manifest as the authoritative bundle set: a partial + # build drops bundles and silently downgrades uncovered hosts to slow + # source builds. Refuse to publish unless the full default set is built; + # publish=false test runs may use subsets. + if [ "${{ github.event_name }}" = "schedule" ] || [ "${{ inputs.publish }}" = "true" ]; then + [ "$ONLY" = "all" ] || { echo "refusing to publish only_profile=$ONLY: a partial CUDA set clobbers manifest coverage. Use only_profile=all to publish, or publish=false for an artifact-only test." >&2; exit 1; } + [ "$GFX" = "$GFX_DEFAULT" ] || { echo "refusing to publish gfx_target='$GFX': publish requires the full default set ($GFX_DEFAULT); use publish=false for a subset test." >&2; exit 1; } + case "$OS_LIST" in + *windows*ubuntu*|*ubuntu*windows*) : ;; + *) echo "refusing to publish operating_systems='$OS_LIST': both windows and ubuntu ROCm bundles are required. Use publish=false for a subset test." >&2; exit 1 ;; + esac + fi + [ -n "$AGE_H" ] || AGE_H="${UNSLOTH_LLAMA_MIN_RELEASE_AGE_HOURS:-6}" + if [ "$REQ" = "latest" ]; then + # Newest published (non-draft, non-prerelease) release that has been + # public for >= AGE_H hours -- the supply-chain aging window. GitHub + # does not guarantee the list order, so pick the max by published_at + # explicitly rather than trusting `first`. + CUTOFF="$(date -u -d "-${AGE_H} hours" +%s)" + TAG="$(gh api 'repos/ggml-org/llama.cpp/releases?per_page=100' \ + --jq "[.[] | select(.draft==false and .prerelease==false) | select((.published_at|fromdateiso8601) <= ${CUTOFF})] | max_by(.published_at|fromdateiso8601) | .tag_name")" + if [ -z "$TAG" ] || [ "$TAG" = "null" ]; then + echo "refusing: no ggml-org release older than ${AGE_H}h in the last 100 releases" >&2 + exit 1 + fi + echo "selected $TAG (aged >= ${AGE_H}h)" + else + TAG="$REQ" # explicit b#### override skips the aging filter + fi + printf '%s' "$TAG" | grep -qE '^b[0-9]+$' || { echo "refusing non-release tag '$TAG'" >&2; exit 1; } + COMMIT="$(gh api repos/ggml-org/llama.cpp/commits/"$TAG" --jq .sha)" + + # Only PUBLISHED releases count as "exists"; a leftover draft (from + # a failed prior publish) should not block today's rebuild. + EXISTS=false + if [ "$(gh release view "$TAG" --repo "$GITHUB_REPOSITORY" --json isDraft --jq .isDraft 2>/dev/null || true)" = "false" ]; then + EXISTS=true + fi + + # ubuntu-22.04 is x64 (glibc 2.35). arm64 only has ubuntu-24.04-arm + # available on GitHub-hosted runners (glibc 2.39). arm64 is cuda13-only + # (no cuda12 SBSA) and ships the single "portable" coverage class. + # cuda12 installs via Jimver; cuda13 is pinned to 13.3 (matching + # upstream) which Jimver lacks, so those install via NVIDIA redist in + # the build children -- on the same runners, so the glibc floor holds. + ALL='[ + {"profile":"cuda12-older", "arch":"x64", "runner":"ubuntu-22.04", "line":"cuda12","klass":"older", "rank":10,"cuda":"12.8.0","toolkit_line":"12.8","archs":"70 75 80 86 89"}, + {"profile":"cuda12-newer", "arch":"x64", "runner":"ubuntu-22.04", "line":"cuda12","klass":"newer", "rank":20,"cuda":"12.8.0","toolkit_line":"12.8","archs":"86 89 90 100 120"}, + {"profile":"cuda12-portable","arch":"x64", "runner":"ubuntu-22.04", "line":"cuda12","klass":"portable","rank":30,"cuda":"12.8.0","toolkit_line":"12.8","archs":"70 75 80 86 89 90 100 120"}, + {"profile":"cuda13-older", "arch":"x64", "runner":"ubuntu-22.04", "line":"cuda13","klass":"older", "rank":40,"cuda":"13.3","toolkit_line":"13.3","archs":"75 80 86 89"}, + {"profile":"cuda13-newer", "arch":"x64", "runner":"ubuntu-22.04", "line":"cuda13","klass":"newer", "rank":50,"cuda":"13.3","toolkit_line":"13.3","archs":"86 89 90 100 120"}, + {"profile":"cuda13-portable","arch":"x64", "runner":"ubuntu-22.04", "line":"cuda13","klass":"portable","rank":60,"cuda":"13.3","toolkit_line":"13.3","archs":"75 80 86 89 90 100 120"}, + {"profile":"cuda13-portable","arch":"arm64","runner":"ubuntu-24.04-arm","line":"cuda13","klass":"portable","rank":60,"cuda":"13.3","toolkit_line":"13.3","archs":"90 100 120 121"} + ]' + if [ "$ONLY" = "all" ]; then + CUDA_INCLUDE="$(echo "$ALL" | jq -c .)" + else + CUDA_INCLUDE="$(echo "$ALL" | jq -c --arg p "$ONLY" '[.[] | select(.profile==$p)]')" + fi + # CUDA Windows reuses the x64 profiles (same arch lists / CUDA + # versions), just on a Windows runner. arm64 has no CUDA Windows target. + WIN_CUDA_INCLUDE="$(echo "$CUDA_INCLUDE" | jq -c '[.[] | select(.arch=="x64") | .runner="windows-2022"]')" + [ -n "$GFX" ] || { echo "refusing empty gfx_target (would publish a CUDA-only release labeled CUDA + ROCm)" >&2; exit 1; } + ROCM_MATRIX="$(jq -cn --arg g "$GFX" '{gfx_target: ($g | split(",") | map(gsub("^\\s+|\\s+$"; "")))}')" + + # macOS slices are static: two fixed runners with per-slice deployment + # targets. arm64 builds on macos-26 (newest Metal SDK; avoids the + # M5/A19 "error compiling source" the macos-14 SDK emits) yet pins + # minos 14.0 via deploy_target, so it still loads on macOS 14+. x64 + # pins 13.3 (matches upstream's Intel leg, covers 2017 Intel Macs + # capped at Ventura). + MACOS_INCLUDE='[ + {"build":"arm64","runner":"macos-26", "expect_arch":"arm64", "deploy_target":"14.0","defines":"-DGGML_METAL_USE_BF16=ON -DGGML_METAL_EMBED_LIBRARY=ON"}, + {"build":"x64", "runner":"macos-15-intel","expect_arch":"x86_64","deploy_target":"13.3","defines":"-DGGML_METAL=OFF"} + ]' + MACOS_INCLUDE="$(echo "$MACOS_INCLUDE" | jq -c .)" + + { + echo "tag=$TAG" + echo "commit=$COMMIT" + echo "exists=$EXISTS" + echo "cuda_matrix={\"include\":$CUDA_INCLUDE}" + echo "win_cuda_matrix={\"include\":$WIN_CUDA_INCLUDE}" + echo "rocm_matrix=$ROCM_MATRIX" + echo "macos_matrix={\"include\":$MACOS_INCLUDE}" + } >> "$GITHUB_OUTPUT" + echo "Resolved $REQ -> $TAG ($COMMIT); release exists=$EXISTS; only=$ONLY; gfx=$GFX" + + build-cuda: + name: CUDA + needs: resolve + if: ${{ needs.resolve.outputs.exists != 'true' || github.event_name == 'workflow_dispatch' }} + uses: ./.github/workflows/unsloth-prebuilt-cuda.yml + with: + tag: ${{ needs.resolve.outputs.tag }} + commit: ${{ needs.resolve.outputs.commit }} + matrix: ${{ needs.resolve.outputs.cuda_matrix }} + + build-windows-cuda: + name: CUDA Windows + needs: resolve + if: ${{ needs.resolve.outputs.exists != 'true' || github.event_name == 'workflow_dispatch' }} + uses: ./.github/workflows/unsloth-prebuilt-cuda-windows.yml + with: + tag: ${{ needs.resolve.outputs.tag }} + commit: ${{ needs.resolve.outputs.commit }} + matrix: ${{ needs.resolve.outputs.win_cuda_matrix }} + + build-rocm: + name: ROCm + needs: resolve + if: ${{ needs.resolve.outputs.exists != 'true' || github.event_name == 'workflow_dispatch' }} + uses: ./.github/workflows/unsloth-prebuilt-rocm.yml + with: + tag: ${{ needs.resolve.outputs.tag }} + matrix: ${{ needs.resolve.outputs.rocm_matrix }} + operating_systems: ${{ github.event.inputs.operating_systems || 'windows,ubuntu' }} + rocm_version: ${{ github.event.inputs.rocm_version || 'latest' }} + + build-macos: + name: macOS + needs: resolve + if: ${{ needs.resolve.outputs.exists != 'true' || github.event_name == 'workflow_dispatch' }} + uses: ./.github/workflows/unsloth-prebuilt-macos.yml + with: + tag: ${{ needs.resolve.outputs.tag }} + matrix: ${{ needs.resolve.outputs.macos_matrix }} + + assemble: + name: Assemble metadata + publish + needs: [resolve, build-cuda, build-windows-cuda, build-rocm, build-macos] + # No `if: always()`. Atomicity: if any matrix entry in any child + # workflow failed, the corresponding `build-*` job reports failure, the + # default `needs` success check skips this assemble job, and nothing is + # published. The installer needs the full bundle set or it'll fail. + if: ${{ needs.resolve.outputs.exists != 'true' || github.event_name == 'workflow_dispatch' }} + runs-on: ubuntu-22.04 + env: + GH_TOKEN: ${{ github.token }} + steps: + - name: Checkout build tooling (this repo) + uses: actions/checkout@v6 + with: + path: tooling + + # The first download attempt occasionally hits a transient ECONNRESET + # on GitHub's ListArtifacts API. Retrying once is cheaper than + # re-running the whole hour-long build matrix. + - name: Download built bundles + id: download + uses: actions/download-artifact@v6 + continue-on-error: true + with: + path: dist + pattern: app-* + merge-multiple: true + + - name: Download built bundles (retry) + if: steps.download.outcome == 'failure' + uses: actions/download-artifact@v6 + with: + path: dist + pattern: app-* + merge-multiple: true + + # Ship the upstream source tarballs as release assets so the installer's + # source-build fallback has a tree to fetch. Downloaded into dist/ (not + # self-tarred) so the published bytes match the sha256 index, which + # assemble_metadata records by hashing these exact local files. + - name: Fetch upstream source archives + run: | + set -eux + TAG='${{ needs.resolve.outputs.tag }}' + SHA='${{ needs.resolve.outputs.commit }}' + mkdir -p dist + curl -fL --retry 3 -o "dist/llama.cpp-source-${TAG}.tar.gz" \ + "https://codeload.github.com/ggml-org/llama.cpp/tar.gz/refs/tags/${TAG}" + curl -fL --retry 3 -o "dist/llama.cpp-source-commit-${SHA}.tar.gz" \ + "https://codeload.github.com/ggml-org/llama.cpp/tar.gz/${SHA}" + + - name: Generate manifest + sha256 index + run: | + set -eux + python3 tooling/scripts/unsloth/assemble_metadata.py \ + --tag '${{ needs.resolve.outputs.tag }}' \ + --commit '${{ needs.resolve.outputs.commit }}' \ + --dist dist --out dist \ + --publish-repo "$GITHUB_REPOSITORY" + ls -la dist + + - name: Upload full release set (artifacts) + uses: actions/upload-artifact@v6 + with: + name: unsloth-prebuilt-${{ needs.resolve.outputs.tag }} + path: dist/* + if-no-files-found: error + # Short retention: the published GitHub release is the canonical + # storage; this artifact is just debug fallback for failed-publish + # runs. Default 90 days × ~8 GB/run is real storage cost. + retention-days: 7 + + - name: Verify full bundle coverage before publish + if: ${{ (github.event_name == 'schedule' || inputs.publish) && needs.resolve.outputs.exists != 'true' }} + run: | + set -eu + TAG='${{ needs.resolve.outputs.tag }}' + fail=0 + # A partial CUDA set (cuda13 without cuda12) silently strands the + # cuda12-runtime majority; require matching x64 coverage on both lines. + for os in linux windows; do + ext=$([ "$os" = windows ] && echo zip || echo tar.gz) + c12=$(ls dist/app-*-"$os"-x64-cuda12-*."$ext" 2>/dev/null | sed -E 's/.*cuda12-(older|newer|portable)\..*/\1/' | sort -u | paste -sd, -) + c13=$(ls dist/app-*-"$os"-x64-cuda13-*."$ext" 2>/dev/null | sed -E 's/.*cuda13-(older|newer|portable)\..*/\1/' | sort -u | paste -sd, -) + echo "$os x64 coverage: cuda12=[$c12] cuda13=[$c13]" + [ -n "$c12" ] && [ "$c12" = "$c13" ] || { echo "ERROR: $os x64 cuda12/cuda13 coverage mismatch" >&2; fail=1; } + done + # Children passing is not the same as files landing in dist/ (a + # download-artifact anomaly leaves green jobs and missing bundles), + # so assert presence of every non-CUDA-x64 line the installer + # selects from. The resolve-time input guard already pins publish + # runs to the full default matrix, so these names are exact. + for f in \ + "llama-${TAG}-bin-macos-arm64.tar.gz" \ + "llama-${TAG}-bin-macos-x64.tar.gz" \ + "app-${TAG}-linux-arm64-cuda13-portable.tar.gz"; do + [ -s "dist/$f" ] || { echo "ERROR: missing $f in dist/" >&2; fail=1; } + done + # Default ROCm set (mirrors the gfx_target input default): both OSes + # per family, or AMD hosts of that family silently lose their bundle. + for gfx in gfx1151 gfx1150 gfx120X gfx110X gfx103X; do + [ -s "dist/app-${TAG}-linux-x64-rocm-${gfx}.tar.gz" ] || { echo "ERROR: missing linux rocm ${gfx} bundle in dist/" >&2; fail=1; } + [ -s "dist/app-${TAG}-windows-x64-rocm-${gfx}.zip" ] || { echo "ERROR: missing windows rocm ${gfx} bundle in dist/" >&2; fail=1; } + done + [ "$fail" = 0 ] || { echo "ERROR: refusing to publish a partial release" >&2; exit 1; } + + - name: Publish GitHub release + if: ${{ (github.event_name == 'schedule' || inputs.publish) && needs.resolve.outputs.exists != 'true' }} + run: | + set -eux + TAG='${{ needs.resolve.outputs.tag }}' + REPO="$GITHUB_REPOSITORY" + + # Atomic publish: upload as draft (hidden from the anon GitHub API + # the installer uses), then flip draft=false only once every asset + # landed. Leftover drafts from failed runs are detected as "not + # exists" by resolve and rebuilt on the next run. + if [ "$(gh release view "$TAG" --repo "$REPO" --json isDraft --jq .isDraft 2>/dev/null || true)" = "true" ]; then + gh release delete "$TAG" --repo "$REPO" --yes + fi + gh release create "$TAG" --repo "$REPO" --draft \ + --title "llama.cpp prebuilt $TAG" \ + --notes "Automated Unsloth llama.cpp CUDA (Linux + Windows) + ROCm + macOS prebuild for upstream $TAG (ggml-org/llama.cpp@${{ needs.resolve.outputs.commit }})." \ + dist/* + gh release edit "$TAG" --repo "$REPO" --draft=false diff --git a/scripts/unsloth/assemble_metadata.py b/scripts/unsloth/assemble_metadata.py new file mode 100644 index 000000000000..b2d367615c6e --- /dev/null +++ b/scripts/unsloth/assemble_metadata.py @@ -0,0 +1,405 @@ +#!/usr/bin/env python3 +"""Assemble the release-level sidecars for an Unsloth llama.cpp prebuilt release. + +Produces, matching the schema consumed by unslothai/unsloth's installer: + - llama-prebuilt-manifest.json : describes every locally-built bundle in this + release (CUDA x64/arm64 profiles + ROCm Linux/Windows per gfx target), + with the dispatch metadata the installer needs to pick the right one. + - llama-prebuilt-sha256.json : a cross-OS integrity index covering both the + locally-built bundles AND the upstream ggml-org assets the installer pulls + for Windows/macOS/Linux-CPU + the source tarballs. + +Run after the build matrix has dropped the app-*.{tar.gz,zip} bundles into --dist. +""" +from __future__ import annotations + +import argparse +import datetime as _dt +import hashlib +import json +import os +import re +import sys +import tarfile +import time +import urllib.error +import urllib.request +import zipfile +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +UPSTREAM_REPO = "ggml-org/llama.cpp" +UPSTREAM_URL = f"https://github.com/{UPSTREAM_REPO}" + +BUNDLE_RE = re.compile( + r"^app-(?P[^/]+)-(?Plinux|windows)-(?Px64|arm64)-(?Pcuda1[23]-(?:older|newer|portable))\.(?Ptar\.gz|zip)$" +) + +ROCM_BUNDLE_RE = re.compile( + r"^app-(?P[^/]+)-(?Plinux|windows)-x64-rocm-(?Pgfx[0-9a-zA-Z]+)\.(?Ptar\.gz|zip)$" +) + +# macOS slices are built by unsloth-prebuilt-macos.yml and land in dist/ under +# upstream's own naming (the installer expects that name). They carry no +# embedded UNSLOTH_PREBUILT_INFO.json, so -- like ROCm -- everything is derived +# from the filename. +MACOS_BUNDLE_RE = re.compile( + r"^llama-(?P[^/]+)-bin-macos-(?Parm64|x64)\.tar\.gz$" +) + +# Per-(platform, arch) dispatch keys for the published manifest + sha256 index. +# Linux x64 keeps the historical "linux-cuda" so older unsloth installers stay +# compatible; the others get distinct kinds so installers cleanly ignore a +# bundle they can't run instead of trying to launch the wrong binary. +KIND_BY_CUDA = { + ("linux", "x64"): {"manifest": "linux-cuda", "sha": "linux-cuda-app"}, + ("linux", "arm64"): {"manifest": "linux-arm64-cuda", "sha": "linux-arm64-cuda-app"}, + ("windows", "x64"): {"manifest": "windows-cuda", "sha": "windows-cuda-app"}, +} + +KIND_BY_ROCM_PLATFORM = { + "linux": {"manifest": "linux-rocm", "sha": "linux-rocm-app"}, + "windows": {"manifest": "windows-rocm", "sha": "windows-rocm-app"}, +} + +# macOS slices: install_kind / sha-index kind / manifest bundle_profile per arch. +# We build these ourselves now (upstream's arm64 release stamps minos=26 and +# won't dyld-load on macOS < 26), so they are recorded as locally-built bundles +# rather than upstream passthroughs. +MACOS_SLICE = { + "arm64": {"manifest": "macos-arm64", "sha": "macos-arm64-app", "profile": "macos-metal-arm64"}, + "x64": {"manifest": "macos-x64", "sha": "macos-x64-app", "profile": "macos-cpu-x64"}, +} + +# Mapping from the umbrella gfx target name (as it appears in the asset +# filename) to the concrete gfx architectures it compiles for. Mirrors the +# `mapped_target` switch in unsloth-prebuilt-rocm.yml; kept duplicated so the +# manifest can stay self-describing without parsing the workflow. +ROCM_TARGET_MAP = { + "gfx1151": ["gfx1151"], + "gfx1150": ["gfx1150"], + "gfx120X": ["gfx1200", "gfx1201"], + "gfx110X": ["gfx1100", "gfx1101", "gfx1102", "gfx1103"], + "gfx103X": ["gfx1030", "gfx1031", "gfx1032", "gfx1034"], +} + + +def sha256_file(path: Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def read_bundle_info(bundle: Path) -> dict: + """Read the UNSLOTH_PREBUILT_INFO.json embedded in a built bundle. + + Linux/macOS bundles are .tar.gz; Windows bundles are .zip -- dispatch on the + extension so the Windows CUDA bundles can be read too. + """ + target = "UNSLOTH_PREBUILT_INFO.json" + if bundle.name.endswith(".zip"): + with zipfile.ZipFile(bundle) as zf: + for n in zf.namelist(): + if n.endswith(target): + return json.loads(zf.read(n)) + else: + with tarfile.open(bundle, "r:gz") as tar: + for m in tar.getmembers(): + if m.isfile() and m.name.endswith(target): + return json.loads(tar.extractfile(m).read()) + sys.exit(f"ERROR: {bundle.name} has no {target}") + + +def _request(url: str, token: str | None) -> urllib.request.Request: + req = urllib.request.Request(url, headers={"User-Agent": "unsloth-prebuilt-assembler"}) + if token and "api.github.com" in url: + req.add_header("Authorization", f"Bearer {token}") + req.add_header("Accept", "application/vnd.github+json") + return req + + +def _with_retry(fn, *, attempts: int = 4, base: float = 2.0): + for i in range(attempts): + try: + return fn() + except (urllib.error.URLError, TimeoutError, ConnectionError) as e: + code = getattr(e, "code", None) + # give up on the last try or on a non-transient 4xx (429 is transient) + if i == attempts - 1 or (code is not None and 400 <= code < 500 and code != 429): + raise + time.sleep(base * (2 ** i)) + + +def http_json(url: str, token: str | None) -> object: + def go(): + with urllib.request.urlopen(_request(url, token), timeout=120) as resp: + return json.loads(resp.read()) + return _with_retry(go) + + +def sha256_url(url: str, token: str | None) -> str: + def go(): + h = hashlib.sha256() + with urllib.request.urlopen(_request(url, token), timeout=300) as resp: + for chunk in iter(lambda: resp.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + return _with_retry(go) + + +def upstream_assets(tag: str, token: str | None) -> dict[str, dict]: + """name -> {url, digest} for the upstream release at `tag`.""" + data = http_json(f"https://api.github.com/repos/{UPSTREAM_REPO}/releases/tags/{tag}", token) + out: dict[str, dict] = {} + for asset in data.get("assets", []): # type: ignore[union-attr] + out[asset["name"]] = { + "url": asset["browser_download_url"], + "digest": asset.get("digest"), # "sha256:" since 2024, else None + } + return out + + +def asset_digest_or_hash(asset: dict, token: str | None) -> str: + """Prefer GitHub's published asset digest; stream-hash as fallback.""" + raw = (asset.get("digest") or "").strip().lower() + if raw.startswith("sha256:"): + h = raw.split(":", 1)[1] + if len(h) == 64 and all(c in "0123456789abcdef" for c in h): + return h + return sha256_url(asset["url"], token) + + +def build_manifest( + tag: str, + commit: str, + cuda_bundles: list[tuple[str, str, str, dict]], + rocm_bundles: list[tuple[str, str, str]], + macos_bundles: list[tuple[str, str]], +) -> dict: + """cuda_bundles: list of (asset_name, platform, arch, embedded UNSLOTH_PREBUILT_INFO). + rocm_bundles: list of (asset_name, platform, gfx_target). + macos_bundles: list of (asset_name, arch). + + CUDA fields come from each bundle's own embedded metadata, so the manifest + can never disagree with what was actually compiled. ROCm and macOS bundles + are raw archives (no embedded info), so their manifest entries are derived + from the filename + the ROCM_TARGET_MAP / MACOS_SLICE tables. + """ + artifacts = [] + for asset_name, platform, arch, info in cuda_bundles: + artifacts.append({ + "asset_name": asset_name, + "install_kind": KIND_BY_CUDA[(platform, arch)]["manifest"], + "bundle_profile": info["bundle_profile"], + "runtime_line": info["runtime_line"], + "coverage_class": info["coverage_class"], + "supported_sms": info["supported_sms"], + "min_sm": info["min_sm"], + "max_sm": info["max_sm"], + "rank": info["bundle_rank"], + "toolkit_version": info["toolkit_line"], + }) + for asset_name, platform, gfx in rocm_bundles: + artifacts.append({ + "asset_name": asset_name, + "install_kind": KIND_BY_ROCM_PLATFORM[platform]["manifest"], + "gfx_target": gfx, + "mapped_targets": ROCM_TARGET_MAP.get(gfx, [gfx]), + }) + for asset_name, arch in macos_bundles: + # No runtime_line/coverage_class for macOS (no CUDA/ROCm runtime to + # match); emitted as explicit null so the key set stays stable, and a + # fixed rank since there is a single slice per arch. + artifacts.append({ + "asset_name": asset_name, + "install_kind": MACOS_SLICE[arch]["manifest"], + "bundle_profile": MACOS_SLICE[arch]["profile"], + "runtime_line": None, + "coverage_class": None, + "rank": 50, + }) + return { + "schema_version": 1, + "component": "llama.cpp", + "source_repo": UPSTREAM_REPO, + "source_repo_url": UPSTREAM_URL, + "source_ref_kind": "tag", + "requested_source_ref": tag, + "resolved_source_ref": tag, + "source_commit": commit, + "source_commit_short": commit[:7], + "upstream_repo": UPSTREAM_REPO, + "upstream_tag": tag, + "generated_at_utc": _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "artifacts": artifacts, + } + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--tag", required=True) + ap.add_argument("--commit", required=True) + ap.add_argument("--dist", required=True, type=Path, help="dir holding the built app-*.tar.gz bundles") + ap.add_argument("--out", required=True, type=Path, help="dir to write the two JSON sidecars into") + ap.add_argument("--publish-repo", required=True, help="repo the bundles+manifest are published to") + ap.add_argument("--token", default=None, help="GitHub token (else $GH_TOKEN/$GITHUB_TOKEN)") + args = ap.parse_args() + + token = args.token or os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") + tag, commit, short = args.tag, args.commit, args.commit[:7] + args.out.mkdir(parents=True, exist_ok=True) + + def base_entry(kind: str, repo: str, digest: str) -> dict: + return { + "kind": kind, + "repo": repo, + "sha256": digest, + "source_commit": commit, + "source_commit_short": short, + "upstream_tag": tag, + } + + sha_artifacts: dict[str, dict] = {} + + # 1a) locally-built CUDA bundles (Linux x64/arm64 .tar.gz + Windows x64 + # .zip): hash in parallel. All carry embedded UNSLOTH_PREBUILT_INFO.json. + found: list[tuple[str, str, str, dict]] = [] + cuda_paths = sorted(args.dist.glob("app-*-linux-*.tar.gz")) + sorted(args.dist.glob("app-*-windows-*.zip")) + for p in cuda_paths: + m = BUNDLE_RE.match(p.name) + if not m: + continue + found.append((p.name, m.group("platform"), m.group("arch"), read_bundle_info(p))) + if not found: + print(f"ERROR: no app-* CUDA bundles in {args.dist}", file=sys.stderr) + return 1 + with ThreadPoolExecutor(max_workers=4) as pool: + local_digests = list(pool.map(lambda b: sha256_file(args.dist / b[0]), found)) + for (name, platform, arch, _info), digest in zip(found, local_digests): + sha_artifacts[name] = base_entry(KIND_BY_CUDA[(platform, arch)]["sha"], args.publish_repo, digest) + + # 1b) locally-built ROCm bundles (linux .tar.gz + windows .zip): hash in + # parallel. No embedded metadata; we derive everything from the filename. + rocm_found: list[tuple[str, str, str]] = [] + for p in sorted(list(args.dist.glob("app-*-rocm-*.tar.gz")) + list(args.dist.glob("app-*-rocm-*.zip"))): + m = ROCM_BUNDLE_RE.match(p.name) + if not m: + continue + rocm_found.append((p.name, m.group("platform"), m.group("gfx"))) + if rocm_found: + with ThreadPoolExecutor(max_workers=4) as pool: + rocm_digests = list(pool.map(lambda b: sha256_file(args.dist / b[0]), rocm_found)) + for (name, platform, _gfx), digest in zip(rocm_found, rocm_digests): + sha_artifacts[name] = base_entry(KIND_BY_ROCM_PLATFORM[platform]["sha"], args.publish_repo, digest) + else: + # Warning, not error: ROCm can legitimately be empty when a dispatch run + # narrows operating_systems to skip both Windows and Ubuntu. The daily + # schedule always builds the full set, so this fires only on manual runs. + print("WARNING: no app-*-rocm-*.{tar.gz,zip} bundles found", file=sys.stderr) + + # 1c) locally-built macOS slices (arm64 Metal + x64 CPU): hash in parallel. + # No embedded metadata; we derive everything from the filename. We build + # these ourselves now, so they are NOT recorded as upstream passthroughs in + # section 2. + macos_found: list[tuple[str, str]] = [] + for p in sorted(args.dist.glob("llama-*-bin-macos-*.tar.gz")): + m = MACOS_BUNDLE_RE.match(p.name) + if not m: + continue + macos_found.append((p.name, m.group("arch"))) + if macos_found: + with ThreadPoolExecutor(max_workers=4) as pool: + macos_digests = list(pool.map(lambda b: sha256_file(args.dist / b[0]), macos_found)) + for (name, arch), digest in zip(macos_found, macos_digests): + sha_artifacts[name] = base_entry(MACOS_SLICE[arch]["sha"], args.publish_repo, digest) + else: + # Like ROCm: warn rather than error, so a partial dispatch run still + # assembles. The daily schedule always builds both slices. + print("WARNING: no llama-*-bin-macos-*.tar.gz bundles found", file=sys.stderr) + + # 2) upstream per-OS bundles: read GitHub's published asset.digest from the + # API response; fall back to a streaming hash if a digest is missing. + # macOS is absent here on purpose -- we build our own slices in 1c. + assets = upstream_assets(tag, token) + wanted: list[tuple[str, str]] = [] # (name, kind) + for name in sorted(assets): + if re.fullmatch(r"cudart-llama-bin-win-cuda-\d+\.\d+-x64\.zip", name): + wanted.append((name, "windows-cuda-upstream")) + # The win-cuda BINARY zips must be recorded under their own names too: + # the installer resolves an attempt's hash by exact asset name first + # and only then falls back to the cudart alias, so without these + # entries every Windows CUDA binary gets paired with the cudart digest + # and fails download verification. + elif re.fullmatch( + rf"llama-{re.escape(tag)}-bin-win-cuda-\d+\.\d+-x64\.zip", name + ): + wanted.append((name, "windows-cuda-upstream")) + for name, kind in ( + (f"llama-{tag}-bin-ubuntu-x64.tar.gz", "linux-cpu-upstream"), + (f"llama-{tag}-bin-win-cpu-x64.zip", "windows-cpu-upstream"), + (f"llama-{tag}-bin-ubuntu-vulkan-x64.tar.gz", "linux-vulkan-upstream"), + (f"llama-{tag}-bin-win-vulkan-x64.zip", "windows-vulkan-upstream"), + (f"llama-{tag}-bin-ubuntu-arm64.tar.gz", "linux-arm64-upstream"), + (f"llama-{tag}-bin-win-cpu-arm64.zip", "windows-arm64-upstream"), + ): + if name not in assets: + print(f"WARNING: upstream asset {name} not found at {tag}; skipping", file=sys.stderr) + continue + wanted.append((name, kind)) + for name, kind in wanted: + sha_artifacts[name] = base_entry(kind, UPSTREAM_REPO, asset_digest_or_hash(assets[name], token)) + + # 3) source tarballs: prefer a local copy in dist -- the workflow downloads + # them from codeload so the published asset and its recorded checksum are + # the exact same bytes. Fall back to stream-hashing codeload if absent + # (e.g. a standalone/local run that didn't pre-fetch them). codeload + # doesn't expose pre-computed digests, so we always hash the content. + source_jobs = [ + (f"llama.cpp-source-{tag}.tar.gz", "upstream-source", + f"https://codeload.github.com/{UPSTREAM_REPO}/tar.gz/refs/tags/{tag}"), + (f"llama.cpp-source-commit-{commit}.tar.gz", "exact-source", + f"https://codeload.github.com/{UPSTREAM_REPO}/tar.gz/{commit}"), + ] + + def source_digest(name: str, url: str) -> str: + local = args.dist / name + return sha256_file(local) if local.is_file() else sha256_url(url, token) + + with ThreadPoolExecutor(max_workers=2) as pool: + source_digests = list(pool.map(lambda j: source_digest(j[0], j[2]), source_jobs)) + for (name, kind, _url), digest in zip(source_jobs, source_digests): + sha_artifacts[name] = base_entry(kind, UPSTREAM_REPO, digest) + + # 4) manifest, then hash it into the index + manifest = build_manifest(tag, commit, found, rocm_found, macos_found) + manifest_path = args.out / "llama-prebuilt-manifest.json" + manifest_path.write_text(json.dumps(manifest, indent=2)) + sha_artifacts["llama-prebuilt-manifest.json"] = base_entry( + "published-manifest", args.publish_repo, sha256_file(manifest_path) + ) + + sha256_doc = { + "artifacts": sha_artifacts, + "component": "llama.cpp", + "release_tag": tag, + "requested_source_ref": tag, + "resolved_source_ref": tag, + "schema_version": 1, + "source_commit": commit, + "source_commit_short": short, + "source_ref_kind": "tag", + "source_repo": UPSTREAM_REPO, + "source_repo_url": UPSTREAM_URL, + "upstream_tag": tag, + } + (args.out / "llama-prebuilt-sha256.json").write_text(json.dumps(sha256_doc, indent=2)) + + print(f"wrote manifest ({len(manifest['artifacts'])} artifacts) and sha256 index " + f"({len(sha_artifacts)} entries) to {args.out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/unsloth/assert_macho_minos.sh b/scripts/unsloth/assert_macho_minos.sh index a27e2999abc8..58fc1b38ff34 100755 --- a/scripts/unsloth/assert_macho_minos.sh +++ b/scripts/unsloth/assert_macho_minos.sh @@ -20,7 +20,9 @@ MAX_KEY="$(ver_key "$MAX_MINOS")" command -v vtool >/dev/null 2>&1 || fail "vtool not found (Xcode command line tools required)" -mapfile -t MACHOS < <(find "$BIN_DIR" -type f \( -name '*.dylib' -o -name 'llama-server' -o -name 'llama-quantize' -o -name 'llama-cli' \) 2>/dev/null) +# macOS ships bash 3.2, which has no `mapfile`; read into the array portably. +MACHOS=() +while IFS= read -r _macho; do MACHOS+=("$_macho"); done < <(find "$BIN_DIR" -type f \( -name '*.dylib' -o -name 'llama-server' -o -name 'llama-quantize' -o -name 'llama-cli' \) 2>/dev/null) [ "${#MACHOS[@]}" -gt 0 ] || fail "no Mach-O binaries found under $BIN_DIR" for macho in "${MACHOS[@]}"; do @@ -43,5 +45,9 @@ done CLI="$(find "$BIN_DIR" -type f -name llama-cli | head -1)" QUANT="$(find "$BIN_DIR" -type f -name llama-quantize | head -1)" "$CLI" --version >/dev/null 2>&1 || fail "llama-cli failed to launch (dyld load / symbol error)" -"$QUANT" --help >/dev/null 2>&1 || fail "llama-quantize failed to launch (dyld load / symbol error)" +# llama-quantize's usage() ends in exit(1), so --help is non-zero by design. +# A dyld/symbol failure dies before main and prints nothing, so verify the +# binary actually reached main (printed usage) rather than trusting the code. +q_out="$("$QUANT" --help 2>&1 || true)" +printf '%s\n' "$q_out" | grep -q "usage:" || fail "llama-quantize failed to launch (dyld load / symbol error)" echo "runtime launch passed: llama-cli --version and llama-quantize --help both ran" diff --git a/scripts/unsloth/gen_manifest.py b/scripts/unsloth/gen_manifest.py deleted file mode 100755 index 3b07535371b5..000000000000 --- a/scripts/unsloth/gen_manifest.py +++ /dev/null @@ -1,164 +0,0 @@ -#!/usr/bin/env python3 -"""Generate the Unsloth Studio llama.cpp release manifest + checksum assets. - -Writes llama-prebuilt-manifest.json and llama-prebuilt-sha256.json in the exact -schema studio/install_llama_prebuilt.py consumes (parse_published_release_bundle / -parse_approved_release_checksums). Used by unsloth-macos-prebuilt.yml after the -macOS slices are built and validated, before the release is published. -""" -from __future__ import annotations - -import argparse -import datetime as _dt -import hashlib -import json -from pathlib import Path - -# install_kind -> (manifest bundle_profile, sha256 kind tag) -SLICE_PROFILES = { - "macos-arm64": ("macos-metal-arm64", "macos-arm64-app"), - "macos-x64": ("macos-cpu-x64", "macos-x64-app"), -} -MANIFEST_ASSET = "llama-prebuilt-manifest.json" -SHA256_ASSET = "llama-prebuilt-sha256.json" - - -def sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def parse_slice(value: str) -> tuple[str, Path]: - # "macos-arm64:/path/to/llama-bNNNN-bin-macos-arm64.tar.gz" - kind, _, path = value.partition(":") - if kind not in SLICE_PROFILES or not path: - raise argparse.ArgumentTypeError(f"invalid --artifact {value!r}") - return kind, Path(path) - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("--upstream-tag", required=True, help="e.g. b9439") - ap.add_argument("--source-commit", required=True, help="full 40-hex commit sha") - ap.add_argument("--release-tag", required=True, help="fork release tag") - ap.add_argument("--source-repo", default="ggml-org/llama.cpp") - ap.add_argument("--out-dir", required=True, type=Path) - ap.add_argument( - "--artifact", - required=True, - action="append", - type=parse_slice, - metavar="KIND:PATH", - help="repeatable, e.g. macos-arm64:llama-b9439-bin-macos-arm64.tar.gz", - ) - ap.add_argument("--source-archive", type=Path, help="llama.cpp-source-.tar.gz") - ap.add_argument( - "--exact-source-archive", - type=Path, - help="llama.cpp-source-commit-.tar.gz", - ) - args = ap.parse_args() - - commit = args.source_commit.strip().lower() - short = commit[:7] - repo_url = f"https://github.com/{args.source_repo}" - out = args.out_dir - out.mkdir(parents=True, exist_ok=True) - - source_fields = { - "source_repo": args.source_repo, - "source_repo_url": repo_url, - "source_ref_kind": "tag", - "requested_source_ref": args.upstream_tag, - "resolved_source_ref": args.upstream_tag, - "source_commit": commit, - "source_commit_short": short, - } - - # Manifest: one artifacts[] entry per built slice. - manifest_artifacts = [] - checksum_artifacts: dict[str, dict] = {} - for kind, path in args.artifact: - if not path.is_file(): - ap.error(f"artifact file not found: {path}") - profile, hash_kind = SLICE_PROFILES[kind] - name = path.name - manifest_artifacts.append( - { - "asset_name": name, - "install_kind": kind, - "bundle_profile": profile, - "runtime_line": None, - "coverage_class": None, - "rank": 50, - } - ) - checksum_artifacts[name] = { - "sha256": sha256_file(path), - "repo": "unslothai/llama.cpp", - "kind": hash_kind, - "upstream_tag": args.upstream_tag, - "source_commit": commit, - "source_commit_short": short, - } - - manifest = { - "schema_version": 1, - "component": "llama.cpp", - "upstream_repo": args.source_repo, - "upstream_tag": args.upstream_tag, - "generated_at_utc": _dt.datetime.now(_dt.timezone.utc).isoformat( - timespec="seconds" - ), - **source_fields, - "artifacts": manifest_artifacts, - } - manifest_path = out / MANIFEST_ASSET - manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") - - # Source archives let the source-build fallback resolve an approved tree. - # The exact-commit entry alone satisfies validated_checksums_for_bundle. - if args.source_archive and args.source_archive.is_file(): - checksum_artifacts[f"llama.cpp-source-{args.upstream_tag}.tar.gz"] = { - "sha256": sha256_file(args.source_archive), - "repo": args.source_repo, - "kind": "upstream-source", - } - if args.exact_source_archive and args.exact_source_archive.is_file(): - checksum_artifacts[f"llama.cpp-source-commit-{commit}.tar.gz"] = { - "sha256": sha256_file(args.exact_source_archive), - "repo": args.source_repo, - "kind": "exact-source", - } - - # The manifest's own sha is cross-checked by validated_checksums_for_bundle, - # so hash it after it is written and add it to the checksum asset. - checksum_artifacts[MANIFEST_ASSET] = { - "sha256": sha256_file(manifest_path), - "repo": "unslothai/llama.cpp", - "kind": "published-manifest", - } - - checksums = { - "schema_version": 1, - "component": "llama.cpp", - "release_tag": args.release_tag, - "upstream_tag": args.upstream_tag, - **source_fields, - "artifacts": checksum_artifacts, - } - (out / SHA256_ASSET).write_text( - json.dumps(checksums, indent=2) + "\n", encoding="utf-8" - ) - - print(f"wrote {manifest_path}") - print(f"wrote {out / SHA256_ASSET}") - print(f"artifacts: {', '.join(sorted(checksum_artifacts))}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/unsloth/package_bundle.py b/scripts/unsloth/package_bundle.py new file mode 100644 index 000000000000..7b366d581041 --- /dev/null +++ b/scripts/unsloth/package_bundle.py @@ -0,0 +1,344 @@ +#!/usr/bin/env python3 +"""Cross-platform packager for Unsloth llama.cpp prebuilt bundles. + +Curates the shipped executables, their local dynamic-library closure, and the +dynamically-loaded ggml backend modules; writes the in-bundle metadata +(BUILD_INFO.txt / UNSLOTH_PREBUILT_INFO.json); archives the result. + +The curation and archive engine is OS-generic: adding a new OS means +implementing one PlatformStrategy (its dependency-walk tool, lib-name +convention, backend glob, and archive format), not writing a new packaging +script. + +The CUDA runtime (libcudart/libcublas, cudart DLLs) is intentionally NOT +bundled: the installer pairs it with the user's PyTorch runtime, selected by +runtime_line. + +Linux is the CI-validated path. macOS/Windows strategies follow the correct +platform conventions (otool/@loader_path/tar.gz; dir-local DLLs/zip) but have +not yet been exercised on their runners. + +Configuration is read from the environment (see read_config). Runs both inside +the build workflow and standalone for local testing. +""" +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +import zipfile +from datetime import datetime, timezone +from pathlib import Path + +# Force C locale so tool output (readelf/otool) is not localized. +_C_ENV = {**os.environ, "LC_ALL": "C", "LANG": "C"} + + +def _run(cmd: list[str]) -> str: + # Fail loudly: a missing/erroring readelf|otool would otherwise yield an + # empty closure and silently ship a bundle with missing libraries. + try: + r = subprocess.run(cmd, capture_output=True, text=True, env=_C_ENV) + except FileNotFoundError: + sys.exit(f"ERROR: required tool '{cmd[0]}' not found") + if r.returncode != 0: + sys.exit(f"ERROR: {' '.join(cmd)} failed (rc={r.returncode}): {r.stderr.strip()}") + return r.stdout + + +class PlatformStrategy: + name = "generic" + exe_suffix = "" + archive_ext = ".tar.gz" + rpath = "" + binaries = ["llama-server", "llama-quantize"] + + def shipped_binaries(self) -> list[str]: + return [b + self.exe_suffix for b in self.binaries] + + def local_needed(self, path: Path, bin_dir: Path) -> list[str]: + """Names of dynamic libs `path` needs that are *local* (live in bin_dir).""" + raise NotImplementedError + + def backend_patterns(self) -> list[str]: + """Globs for the dlopen'd ggml backend modules (not found via the walk).""" + raise NotImplementedError + + def supports_symlinks(self) -> bool: + return True + + def archive(self, stage: Path, out_path: Path) -> None: + raise NotImplementedError + + +class LinuxStrategy(PlatformStrategy): + name = "linux" + rpath = "$ORIGIN" + + def local_needed(self, path: Path, bin_dir: Path) -> list[str]: + # Locale-independent: key only on the (NEEDED) tag and the [name]. + needed = re.findall(r"\(NEEDED\)[^\[]*\[([^\]]+)\]", _run(["readelf", "-d", str(path)])) + return [n for n in needed if (bin_dir / n).exists() or (bin_dir / n).is_symlink()] + + def backend_patterns(self) -> list[str]: + return ["libggml-cpu-*.so*", "libggml-cuda.so*", "libggml-rpc.so*"] + + def archive(self, stage: Path, out_path: Path) -> None: + with tarfile.open(out_path, "w:gz") as tar: + tar.add(stage, arcname=".") + + +class MacOSStrategy(PlatformStrategy): + name = "macos" + rpath = "@loader_path" + + def local_needed(self, path: Path, bin_dir: Path) -> list[str]: + out = _run(["otool", "-L", str(path)]) + deps: list[str] = [] + for line in out.splitlines()[1:]: # first line echoes the file path + m = re.match(r"\s+(\S+)\s+\(", line) + if not m: + continue + ref = m.group(1) + base = os.path.basename(ref) + # @rpath/@loader_path/relative refs that exist locally are "ours" + if (ref.startswith("@") or not ref.startswith("/")) and (bin_dir / base).exists(): + deps.append(base) + return deps + + def backend_patterns(self) -> list[str]: + return ["libggml-*.dylib"] + + def archive(self, stage: Path, out_path: Path) -> None: + with tarfile.open(out_path, "w:gz") as tar: + tar.add(stage, arcname=".") + + +class WindowsStrategy(PlatformStrategy): + name = "windows" + exe_suffix = ".exe" + archive_ext = ".zip" + rpath = "" # Windows resolves DLLs from the executable's directory + + # No portable readelf/otool equivalent; the project's own DLLs live beside + # the binaries in build/bin/Release, so bundle those by name convention. + LOCAL_DLL_PREFIXES = ("ggml", "llama", "mtmd") + + def local_needed(self, path: Path, bin_dir: Path) -> list[str]: + return [ + p.name for p in bin_dir.glob("*.dll") + if p.name.lower().startswith(self.LOCAL_DLL_PREFIXES) + ] + + def backend_patterns(self) -> list[str]: + return ["ggml-cpu-*.dll", "ggml-cuda.dll", "ggml-rpc.dll"] + + def supports_symlinks(self) -> bool: + return False + + def archive(self, stage: Path, out_path: Path) -> None: + with zipfile.ZipFile(out_path, "w", zipfile.ZIP_DEFLATED) as z: + for p in sorted(stage.rglob("*")): + if p.is_file(): + z.write(p, p.relative_to(stage).as_posix()) + + +STRATEGIES = {s.name: s for s in (LinuxStrategy(), MacOSStrategy(), WindowsStrategy())} + + +def _copy_one(strategy: PlatformStrategy, bin_dir: Path, stage: Path, name: str) -> None: + src, dst = bin_dir / name, stage / name + if dst.exists() or dst.is_symlink(): + return + if strategy.supports_symlinks() and src.is_symlink(): + target = os.readlink(src) + os.symlink(target, dst) + _copy_one(strategy, bin_dir, stage, os.path.basename(target)) + elif src.exists(): + shutil.copy2(src, dst, follow_symlinks=True) + + +def curate(strategy: PlatformStrategy, bin_dir: Path, stage: Path) -> None: + roots: list[Path] = [] + for b in strategy.shipped_binaries(): + if not (bin_dir / b).exists(): + sys.exit(f"ERROR: missing {bin_dir / b}") + shutil.copy2(bin_dir / b, stage / b) + roots.append(stage / b) + + # Backend modules are dlopen'd, so they never appear in the NEEDED graph; + # copy them explicitly and treat them as extra roots so their own local + # dependencies get pulled into the closure too. + for pat in strategy.backend_patterns(): + for match in sorted(bin_dir.glob(pat)): + _copy_one(strategy, bin_dir, stage, match.name) + roots.append(stage / match.name) + + # Walk the local NEEDED closure from every root, scanning each lib once. + queue = list(roots) + while queue: + for need in strategy.local_needed(queue.pop(), bin_dir): + if not (stage / need).exists() and not (stage / need).is_symlink(): + _copy_one(strategy, bin_dir, stage, need) + queue.append(stage / need) + + +def detect_nvcc_sms() -> tuple[str, list[str], str]: + if not shutil.which("nvcc"): + return "unavailable", [], "nvcc not found" + r = subprocess.run(["nvcc", "--list-gpu-arch"], capture_output=True, text=True, env=_C_ENV) + if r.returncode != 0: + return "unavailable", [], f"nvcc failed (rc={r.returncode})" + sms = sorted(set(re.findall(r"compute_(\d+)", r.stdout)), key=int) + return "available", sms, f"detected {len(sms)} SM targets" + + +def write_metadata(stage: Path, strategy: PlatformStrategy, cfg: dict, archs: list[str]) -> None: + short = cfg["commit"][:7] + min_sm, max_sm = min(map(int, archs)), max(map(int, archs)) + nvcc_status, nvcc_sms, nvcc_msg = detect_nvcc_sms() + # sm_103 (B300 / GB300 Blackwell Ultra) has no native build, but it JIT-runs + # on the bundled compute_100 PTX, so any bundle that ships sm_100 also covers + # it. Declare it in supported_sms (not the native nvcc build) so every + # platform's manifest agrees -- Windows and arm64 reuse these x64 profiles. + supported_sms = list(archs) + if "100" in supported_sms and "103" not in supported_sms: + supported_sms = sorted([*supported_sms, "103"], key=int) + note = f"CUDA {cfg['line'].removeprefix('cuda')} {cfg['klass']} bundle." + + licenses = [f"Third-party licenses bundled with this llama.cpp prebuilt ({cfg['tag']}).", + f"Source: https://github.com/{cfg['source_repo']} @ {cfg['commit']}", ""] + src = Path(cfg["src_dir"]) + if (src / "LICENSE").is_file(): + licenses += ["=== llama.cpp LICENSE ===", (src / "LICENSE").read_text(), ""] + lic_dir = src / "licenses" + if lic_dir.is_dir(): + for lic in sorted(lic_dir.glob("*")): + if lic.is_file(): + licenses += [f"=== {lic.name} ===", lic.read_text(), ""] + (stage / "THIRD_PARTY_LICENSES.txt").write_text("\n".join(licenses)) + + info = { + "upstream_tag": cfg["tag"], + "source_repo": cfg["source_repo"], + "source_repo_url": f"https://github.com/{cfg['source_repo']}", + "source_ref_kind": "tag", + "requested_source_ref": cfg["tag"], + "resolved_source_ref": cfg["tag"], + "source_commit": cfg["commit"], + "source_commit_short": short, + "platform": f"{strategy.name}-{cfg['arch']}-cuda", + "bundle_profile": cfg["profile"], + "runtime_line": cfg["line"], + "coverage_class": cfg["klass"], + "bundle_rank": int(cfg["rank"]), + "toolkit_line": cfg["toolkit_line"], + "docker_image": cfg["docker_image"], + "supported_sms": supported_sms, + "nvcc_validation_status": nvcc_status, + "nvcc_detected_sms": nvcc_sms, + "nvcc_validation_message": nvcc_msg, + "min_sm": min_sm, + "max_sm": max_sm, + "notes": note, + "build_shared_libs": True, + "ggml_backend_dl": True, + "ggml_cpu_all_variants": True, + "rpath": strategy.rpath, + } + (stage / "UNSLOTH_PREBUILT_INFO.json").write_text(json.dumps(info, indent=2)) + + build_info = [ + f"llama.cpp version: {cfg['tag']}", + f"requested source ref: {cfg['tag']}", + f"resolved source ref: {cfg['tag']}", + f"variant: {cfg['profile']}", + f"runtime line: {cfg['line']}", + f"coverage class: {cfg['klass']}", + f"bundle rank: {cfg['rank']}", + f"docker image: {cfg['docker_image']}", + "backend: CUDA", + f"toolkit version: {cfg['toolkit_line']}", + f"supported sms: {','.join(supported_sms)}", + f"nvcc validation: {nvcc_status}", + f"min sm: {min_sm}", + f"max sm: {max_sm}", + f"os: {strategy.name}", + f"arch: {cfg['arch']}", + "build_shared_libs: ON", + "ggml_backend_dl: ON", + "ggml_cpu_all_variants: ON", + "ggml_cuda_nccl: OFF", + f"rpath: {strategy.rpath}", + "llama_openssl: ON", + "openssl_linkage: dynamic", + "cxx_runtime: dynamic", + f"source commit: {cfg['commit']}", + f"source commit short: {short}", + f"built at (UTC): {datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')}", + f"notes: {note}", + ] + (stage / "BUILD_INFO.txt").write_text("\n".join(build_info) + "\n") + + +def read_config() -> dict: + def need(k: str) -> str: + v = os.environ.get(k) + if not v: + sys.exit(f"ERROR: missing required env {k}") + return v + + return { + "bin_dir": need("BIN_DIR"), + "src_dir": need("SRC_DIR"), + "out_dir": need("OUT_DIR"), + "tag": need("TAG"), + "commit": need("SOURCE_COMMIT"), + "profile": need("PROFILE"), + "line": need("LINE"), + "klass": need("KLASS"), + "rank": need("RANK"), + "toolkit_line": need("TOOLKIT_LINE"), + "archs": need("ARCHS"), + "platform": os.environ.get("PLATFORM", "linux"), + "arch": os.environ.get("ARCH", "x64"), + "docker_image": os.environ.get("DOCKER_IMAGE", ""), + "source_repo": os.environ.get("SOURCE_REPO", "ggml-org/llama.cpp"), + } + + +def main() -> int: + cfg = read_config() + strategy = STRATEGIES.get(cfg["platform"]) + if strategy is None: + sys.exit(f"ERROR: unknown PLATFORM '{cfg['platform']}' (have {sorted(STRATEGIES)})") + + archs = [a for a in re.split(r"[ ;,]+", cfg["archs"]) if a] + bin_dir = Path(cfg["bin_dir"]) + out_dir = Path(cfg["out_dir"]) + out_dir.mkdir(parents=True, exist_ok=True) + + stage = Path(tempfile.mkdtemp()) + try: + curate(strategy, bin_dir, stage) + write_metadata(stage, strategy, cfg, archs) + + asset = f"app-{cfg['tag']}-{strategy.name}-{cfg['arch']}-{cfg['profile']}{strategy.archive_ext}" + out_path = out_dir / asset + strategy.archive(stage, out_path) + + print(f"wrote {out_path}") + for p in sorted(stage.iterdir()): + print(f" {p.name}") + finally: + shutil.rmtree(stage, ignore_errors=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())