diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index b6140ab13c8d..d185e1ac5c59 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -13,37 +13,95 @@ on:
types: [opened, synchronize, edited, reopened, review_requested, ready_for_review]
paths: ['CMakeLists.txt', 'Makefile', '**.h', '*.c', '**.cpp']
+env:
+ BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+
jobs:
ubuntu-latest:
runs-on: ubuntu-latest
steps:
- name: Clone
+ id: checkout
uses: actions/checkout@v1
- name: Dependencies
+ id: depends
run: |
sudo apt-get update
sudo apt-get install build-essential
- name: Build
+ id: make_build
run: |
make
- macOS-latest:
+ - name: Zip executables
+ run: zip alpaca-linux.zip chat
+
+ - name: Archive production artifacts
+ uses: actions/upload-artifact@v3
+ with:
+ name: linux
+ path: |
+ alpaca-linux.zip
+
+
+ macos-latest:
runs-on: macOS-latest
steps:
- name: Clone
+ id: checkout
uses: actions/checkout@v1
- name: Dependencies
+ id: depends
run: |
brew update
- name: Build
+ id: make_build
run: |
- make
+ make chat_mac
+
+ - name: Codesign executable
+ if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
+ env:
+ MACOS_CERTIFICATE: ${{ secrets.APPLE_DEVELOPER_CERTIFICATE_P12_BASE64 }}
+ MACOS_CERTIFICATE_PWD: ${{ secrets.APPLE_DEVELOPER_CERTIFICATE_PASSWORD }}
+ KEYCHAIN_PWD: ${{ secrets.KEYCHAIN_PASSWORD }}
+ MACOS_CERT_ID: ${{ secrets.APPLE_DEVELOPER_CERTIFICATE_ID }}
+
+ run: |
+ echo $MACOS_CERTIFICATE | base64 --decode > certificate.p12
+ security create-keychain -p $KEYCHAIN_PWD build.keychain
+ security default-keychain -s build.keychain
+ security unlock-keychain -p $KEYCHAIN_PWD build.keychain
+ security import certificate.p12 -k build.keychain -P $MACOS_CERTIFICATE_PWD -T /usr/bin/codesign
+ security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k $KEYCHAIN_PWD build.keychain
+ /usr/bin/codesign --options runtime --force -s $MACOS_CERT_ID ./chat_mac -v
+
+ - name: Zip executables
+ run: zip alpaca-mac.zip chat_mac
+
+ - name: Notarize executables
+ if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
+ env:
+ PROD_MACOS_NOTARIZATION_APPLE_ID: ${{ secrets.AC_USERNAME }}
+ PROD_MACOS_NOTARIZATION_PWD: ${{ secrets.AC_PASSWORD }}
+ PROD_MACOS_NOTARIZATION_TEAM_ID: ${{ secrets.PROD_MACOS_NOTARIZATION_TEAM_ID }}
+ run: |
+ xcrun notarytool store-credentials "notarytool-profile" --apple-id "$PROD_MACOS_NOTARIZATION_APPLE_ID" --team-id "$PROD_MACOS_NOTARIZATION_TEAM_ID" --password "$PROD_MACOS_NOTARIZATION_PWD"
+ xcrun notarytool submit "alpaca-mac.zip" --keychain-profile "notarytool-profile" --wait
+
+ - name: Archive production artifacts
+ uses: actions/upload-artifact@v3
+ with:
+ name: macos
+ path: |
+ alpaca-mac.zip
+
windows-latest:
runs-on: windows-latest
@@ -61,227 +119,70 @@ jobs:
cmake ..
cmake --build . --config Release
- - name: Set commit hash variables
- id: commit
- if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
- uses: pr-mpt/actions-commit-hash@v2
-
- name: Pack artifacts
id: pack_artifacts
- if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
run: |
- 7z a alpaca-bin-win-x64-${{ steps.commit.outputs.short }}.zip .\build\Release\*
+ 7z a alpaca-win.zip .\build\Release\*
+
+ - name: Archive production artifacts
+ uses: actions/upload-artifact@v3
+ with:
+ name: windows
+ path: |
+ alpaca-win.zip
+
+
+ release:
+ runs-on: ubuntu-latest
+ if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
+ needs:
+ - windows-latest
+ - macos-latest
+ - ubuntu-latest
+
+ steps:
+ - name: Download all workflow run artifacts
+ uses: actions/download-artifact@v3
+
+ - name: Set commit hash variables
+ id: commit
+ uses: pr-mpt/actions-commit-hash@v2
- name: Create release
id: create_release
- if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
uses: zendesk/action-create-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: ${{ steps.commit.outputs.short }}
- - name: Upload release
- id: upload_release
- if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
+ - name: Upload windows release
+ uses: actions/upload-release-asset@v1
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ with:
+ upload_url: ${{ steps.create_release.outputs.upload_url }}
+ asset_path: windows/alpaca-win.zip
+ asset_name: alpaca-win.zip
+ asset_content_type: application/octet-stream
+
+ - name: Upload mac release
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
- asset_path: .\alpaca-bin-win-x64-${{ steps.commit.outputs.short }}.zip
- asset_name: alpaca-bin-win-x64-${{ steps.commit.outputs.short }}.zip
+ asset_path: macos/alpaca-mac.zip
+ asset_name: alpaca-mac.zip
asset_content_type: application/octet-stream
-# ubuntu-latest-gcc:
-# runs-on: ubuntu-latest
-#
-# strategy:
-# matrix:
-# build: [Debug, Release]
-#
-# steps:
-# - name: Clone
-# uses: actions/checkout@v1
-#
-# - name: Dependencies
-# run: |
-# sudo apt-get update
-# sudo apt-get install build-essential
-# sudo apt-get install cmake
-#
-# - name: Configure
-# run: cmake . -DCMAKE_BUILD_TYPE=${{ matrix.build }}
-#
-# - name: Build
-# run: |
-# make
-#
-# ubuntu-latest-clang:
-# runs-on: ubuntu-latest
-#
-# strategy:
-# matrix:
-# build: [Debug, Release]
-#
-# steps:
-# - name: Clone
-# uses: actions/checkout@v1
-#
-# - name: Dependencies
-# run: |
-# sudo apt-get update
-# sudo apt-get install build-essential
-# sudo apt-get install cmake
-#
-# - name: Configure
-# run: cmake . -DCMAKE_BUILD_TYPE=${{ matrix.build }} -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang
-#
-# - name: Build
-# run: |
-# make
-#
-# ubuntu-latest-gcc-sanitized:
-# runs-on: ubuntu-latest
-#
-# strategy:
-# matrix:
-# sanitizer: [ADDRESS, THREAD, UNDEFINED]
-#
-# steps:
-# - name: Clone
-# uses: actions/checkout@v1
-#
-# - name: Dependencies
-# run: |
-# sudo apt-get update
-# sudo apt-get install build-essential
-# sudo apt-get install cmake
-#
-# - name: Configure
-# run: cmake . -DCMAKE_BUILD_TYPE=Debug -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON
-#
-# - name: Build
-# run: |
-# make
-#
-# windows:
-# runs-on: windows-latest
-#
-# strategy:
-# matrix:
-# build: [Release]
-# arch: [Win32, x64]
-# include:
-# - arch: Win32
-# s2arc: x86
-# - arch: x64
-# s2arc: x64
-#
-# steps:
-# - name: Clone
-# uses: actions/checkout@v1
-#
-# - name: Add msbuild to PATH
-# uses: microsoft/setup-msbuild@v1
-#
-# - name: Configure
-# run: >
-# cmake -S . -B ./build -A ${{ matrix.arch }}
-# -DCMAKE_BUILD_TYPE=${{ matrix.build }}
-#
-# - name: Build
-# run: |
-# cd ./build
-# msbuild ALL_BUILD.vcxproj -t:build -p:configuration=${{ matrix.build }} -p:platform=${{ matrix.arch }}
-#
-# - name: Upload binaries
-# uses: actions/upload-artifact@v1
-# with:
-# name: llama-bin-${{ matrix.arch }}
-# path: build/bin/${{ matrix.build }}
-#
-# windows-blas:
-# runs-on: windows-latest
-#
-# strategy:
-# matrix:
-# build: [Release]
-# arch: [Win32, x64]
-# blas: [ON]
-# include:
-# - arch: Win32
-# obzip: https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x86.zip
-# s2arc: x86
-# - arch: x64
-# obzip: https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x64.zip
-# s2arc: x64
-#
-# steps:
-# - name: Clone
-# uses: actions/checkout@v1
-#
-# - name: Add msbuild to PATH
-# uses: microsoft/setup-msbuild@v1
-#
-# - name: Fetch OpenBLAS
-# if: matrix.blas == 'ON'
-# run: |
-# C:/msys64/usr/bin/wget.exe -qO blas.zip ${{ matrix.obzip }}
-# 7z x blas.zip -oblas -y
-# copy blas/include/cblas.h .
-# copy blas/include/openblas_config.h .
-# echo "blasdir=$env:GITHUB_WORKSPACE/blas" >> $env:GITHUB_ENV
-#
-# - name: Configure
-# run: >
-# cmake -S . -B ./build -A ${{ matrix.arch }}
-# -DCMAKE_BUILD_TYPE=${{ matrix.build }}
-# -DLLAMA_SUPPORT_OPENBLAS=${{ matrix.blas }}
-# -DCMAKE_LIBRARY_PATH="$env:blasdir/lib"
-#
-# - name: Build
-# run: |
-# cd ./build
-# msbuild ALL_BUILD.vcxproj -t:build -p:configuration=${{ matrix.build }} -p:platform=${{ matrix.arch }}
-#
-# - name: Copy libopenblas.dll
-# if: matrix.blas == 'ON'
-# run: copy "$env:blasdir/bin/libopenblas.dll" build/bin/${{ matrix.build }}
-#
-# - name: Upload binaries
-# if: matrix.blas == 'ON'
-# uses: actions/upload-artifact@v1
-# with:
-# name: llama-blas-bin-${{ matrix.arch }}
-# path: build/bin/${{ matrix.build }}
-#
-# emscripten:
-# runs-on: ubuntu-latest
-#
-# strategy:
-# matrix:
-# build: [Release]
-#
-# steps:
-# - name: Clone
-# uses: actions/checkout@v1
-#
-# - name: Dependencies
-# run: |
-# wget -q https://github.com/emscripten-core/emsdk/archive/master.tar.gz
-# tar -xvf master.tar.gz
-# emsdk-master/emsdk update
-# emsdk-master/emsdk install latest
-# emsdk-master/emsdk activate latest
-#
-# - name: Configure
-# run: echo "tmp"
-#
-# - name: Build
-# run: |
-# pushd emsdk-master
-# source ./emsdk_env.sh
-# popd
-# emcmake cmake . -DCMAKE_BUILD_TYPE=${{ matrix.build }}
-# make
+ - name: Upload linux release
+
+ uses: actions/upload-release-asset@v1
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ with:
+ upload_url: ${{ steps.create_release.outputs.upload_url }}
+ asset_path: linux/alpaca-linux.zip
+ asset_name: alpaca-linux.zip
+ asset_content_type: application/octet-stream
diff --git a/.gitignore b/.gitignore
index 5eb1ff1b873f..da9e6bed8008 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
+/chat
+
*.o
*.a
.cache/
@@ -15,9 +17,22 @@ build-sanitize-addr/
build-sanitize-thread/
models/*
+*.bin
/main
/quantize
arm_neon.h
compile_commands.json
+
+# Windows CMake files
+*.vcxproj
+*.filters
+*.cmake
+*.sln
+x64/
+Debug/
+Release/
+CMakeFiles/
+CMakeCache.txt
+*.dir/
diff --git a/Makefile b/Makefile
index cbbfb158913a..93da626f8836 100644
--- a/Makefile
+++ b/Makefile
@@ -176,7 +176,7 @@ $(info I CC: $(CCV))
$(info I CXX: $(CXXV))
$(info )
-default: main quantize
+default: chat quantize
#
# Build library
@@ -191,13 +191,16 @@ utils.o: utils.cpp utils.h
clean:
rm -f *.o main quantize
-main: main.cpp ggml.o utils.o
- $(CXX) $(CXXFLAGS) main.cpp ggml.o utils.o -o main $(LDFLAGS)
- ./main -h
-
chat: chat.cpp ggml.o utils.o
$(CXX) $(CXXFLAGS) chat.cpp ggml.o utils.o -o chat $(LDFLAGS)
+chat_mac: chat.cpp ggml.c utils.cpp
+ $(CC) $(CFLAGS) -c ggml.c -o ggml_x86.o -target x86_64-apple-macos
+ $(CC) $(CFLAGS) -c ggml.c -o ggml_arm.o -target arm64-apple-macos
+
+ $(CXX) $(CXXFLAGS) chat.cpp ggml_x86.o utils.cpp -o chat_x86 $(LDFLAGS) -target x86_64-apple-macos
+ $(CXX) $(CXXFLAGS) chat.cpp ggml_arm.o utils.cpp -o chat_arm $(LDFLAGS) -target arm64-apple-macos
+ lipo -create -output chat_mac chat_x86 chat_arm
quantize: quantize.cpp ggml.o utils.o
$(CXX) $(CXXFLAGS) quantize.cpp ggml.o utils.o -o quantize $(LDFLAGS)
diff --git a/README.md b/README.md
index 14b294f1a56b..aec100114a5a 100644
--- a/README.md
+++ b/README.md
@@ -8,9 +8,60 @@ Run a fast ChatGPT-like model locally on your device. The screencast below is no
This combines the [LLaMA foundation model](https://github.com/facebookresearch/llama) with an [open reproduction](https://github.com/tloen/alpaca-lora) of [Stanford Alpaca](https://github.com/tatsu-lab/stanford_alpaca) a fine-tuning of the base model to obey instructions (akin to the [RLHF](https://huggingface.co/blog/rlhf) used to train ChatGPT) and a set of modifications to [llama.cpp](https://github.com/ggerganov/llama.cpp) to add a chat interface.
-## Get started
+## Get Started (7B)
+Download the zip file corresponding to your operating system from the [latest release](https://github.com/antimatter15/alpaca.cpp/releases/latest). On Windows, download `alpaca-win.zip`, on Mac (both Intel or ARM) download `alpaca-mac.zip`, and on Linux (x64) download `alpaca-linux.zip`.
+
+Download `ggml-alpaca-7b-q4.bin` and place it in the same folder as the `chat` executable in the zip file. There are several options:
+
+```
+# Any of these commands will work.
+curl -o ggml-alpaca-7b-q4.bin -C - https://gateway.estuary.tech/gw/ipfs/QmQ1bf2BTnYxq73MFJWu1B7bQ2UD6qG7D7YDCxhTndVkPC
+curl -o ggml-alpaca-7b-q4.bin -C - https://ipfs.io/ipfs/QmQ1bf2BTnYxq73MFJWu1B7bQ2UD6qG7D7YDCxhTndVkPC
+curl -o ggml-alpaca-7b-q4.bin -C - https://cloudflare-ipfs.com/ipfs/QmQ1bf2BTnYxq73MFJWu1B7bQ2UD6qG7D7YDCxhTndVkPC
+
+# BitTorrent
+magnet:?xt=urn:btih:5aaceaec63b03e51a98f04fd5c42320b2a033010&dn=ggml-alpaca-7b-q4.bin&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce&tr=udp%3A%2F%2Fopentracker.i2p.rocks%3A6969%2Fannounce
+https://btcache.me/torrent/5AACEAEC63B03E51A98F04FD5C42320B2A033010
+https://torrage.info/torrent.php?h=5aaceaec63b03e51a98f04fd5c42320b2a033010
+```
+
+Once you've downloaded the model weights and placed them into the same directory as the `chat` or `chat.exe` executable, run:
+
+```
+./chat
```
+
+The weights are based on the published fine-tunes from `alpaca-lora`, converted back into a pytorch checkpoint with a [modified script](https://github.com/tloen/alpaca-lora/pull/19) and then quantized with llama.cpp the regular way.
+
+## Getting Started (13B)
+
+If you have more than 10GB of RAM, you can use the higher quality 13B `ggml-alpaca-13b-q4.bin` model. To download the weights, you can use
+
+```
+
+# Any of these commands will work.
+curl -o ggml-alpaca-13b-q4.bin -C - https://gateway.estuary.tech/gw/ipfs/Qme6wyw9MzqbrUMpFNVq42rC1kSdko7MGT9CL7o1u9Cv9G
+curl -o ggml-alpaca-13b-q4.bin -C - https://ipfs.io/ipfs/Qme6wyw9MzqbrUMpFNVq42rC1kSdko7MGT9CL7o1u9Cv9G
+curl -o ggml-alpaca-13b-q4.bin -C - https://cloudflare-ipfs.com/ipfs/Qme6wyw9MzqbrUMpFNVq42rC1kSdko7MGT9CL7o1u9Cv9G
+
+# BitTorrent
+magnet:?xt=urn:btih:053b3d54d2e77ff020ebddf51dad681f2a651071&dn=ggml-alpaca-13b-q4.bin&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce&tr=udp%3A%2F%2Fopentracker.i2p.rocks%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.openbittorrent.com%3A6969%2Fannounce&tr=udp%3A%2F%2F9.rarbg.com%3A2810%2Fannounce
+https://btcache.me/torrent/053B3D54D2E77FF020EBDDF51DAD681F2A651071
+https://torrage.info/torrent.php?h=053b3d54d2e77ff020ebddf51dad681f2a651071
+```
+
+Once you've downloaded the weights, you can run the following command to enter chat
+
+```
+./chat -m ggml-alpaca-13b-q4.bin
+```
+
+
+## Building from Source (MacOS/Linux)
+
+
+```sh
git clone https://github.com/antimatter15/alpaca.cpp
cd alpaca.cpp
@@ -18,25 +69,31 @@ make chat
./chat
```
-You can download the weights for `ggml-alpaca-7b-q4.bin` with BitTorrent `magnet:?xt=urn:btih:5aaceaec63b03e51a98f04fd5c42320b2a033010&dn=ggml-alpaca-7b-q4.bin&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce&tr=udp%3A%2F%2Fopentracker.i2p.rocks%3A6969%2Fannounce`
+## Building from Source (Windows)
-Alternatively you can download them with IPFS.
+- Download and install CMake:
+- Download and install `git`. If you've never used git before, consider a GUI client like
+- Clone this repo using your git client of choice (for GitHub Desktop, go to File -> Clone repository -> From URL and paste `https://github.com/antimatter15/alpaca.cpp` in as the URL)
+- Open a Windows Terminal inside the folder you cloned the repository to
+- Run the following commands one by one:
-```
-# any of these will work
-wget -O ggml-alpaca-7b-q4.bin -c https://gateway.estuary.tech/gw/ipfs/QmQ1bf2BTnYxq73MFJWu1B7bQ2UD6qG7D7YDCxhTndVkPC
-wget -O ggml-alpaca-7b-q4.bin -c https://ipfs.io/ipfs/QmQ1bf2BTnYxq73MFJWu1B7bQ2UD6qG7D7YDCxhTndVkPC
-wget -O ggml-alpaca-7b-q4.bin -c https://cloudflare-ipfs.com/ipfs/QmQ1bf2BTnYxq73MFJWu1B7bQ2UD6qG7D7YDCxhTndVkPC
+```ps1
+cmake .
+cmake --build . --config Release
```
-Save the `ggml-alpaca-7b-q4.bin` file in the same directory as your `./chat` executable.
-
-The weights are based on the published fine-tunes from `alpaca-lora`, converted back into a pytorch checkpoint with a [modified script](https://github.com/tloen/alpaca-lora/pull/19) and then quantized with llama.cpp the regular way.
+- Download the weights via any of the links in "Get started" above, and save the file as `ggml-alpaca-7b-q4.bin` in the main Alpaca directory.
+- In the terminal window, run this command:
+```ps1
+.\Release\chat.exe
+```
+- (You can add other launch options like `--n 8` as preferred onto the same line)
+- You can now type to the AI in the terminal and it will reply. Enjoy!
## Credit
-This combines [Facebook's LLaMA](https://github.com/facebookresearch/llama), [Stanford Alpaca](https://crfm.stanford.edu/2023/03/13/alpaca.html), [alpaca-lora](https://github.com/tloen/alpaca-lora) and [corresponding weights](https://huggingface.co/tloen/alpaca-lora-7b/tree/main) by Eric Wang (which uses [Jason Phang's implementation of LLaMA](https://github.com/huggingface/transformers/pull/21955) on top of Hugging Face Transformers), and [llama.cpp](https://github.com/ggerganov/llama.cpp) by Georgi Gerganov. The chat implementation is based on Matvey Soloviev's [Interactive Mode](https://github.com/ggerganov/llama.cpp/pull/61) for llama.cpp. Inspired by [Simon Willison's](https://til.simonwillison.net/llms/llama-7b-m2) getting started guide for LLaMA.
+This combines [Facebook's LLaMA](https://github.com/facebookresearch/llama), [Stanford Alpaca](https://crfm.stanford.edu/2023/03/13/alpaca.html), [alpaca-lora](https://github.com/tloen/alpaca-lora) and [corresponding weights](https://huggingface.co/tloen/alpaca-lora-7b/tree/main) by Eric Wang (which uses [Jason Phang's implementation of LLaMA](https://github.com/huggingface/transformers/pull/21955) on top of Hugging Face Transformers), and [llama.cpp](https://github.com/ggerganov/llama.cpp) by Georgi Gerganov. The chat implementation is based on Matvey Soloviev's [Interactive Mode](https://github.com/ggerganov/llama.cpp/pull/61) for llama.cpp. Inspired by [Simon Willison's](https://til.simonwillison.net/llms/llama-7b-m2) getting started guide for LLaMA. [Andy Matuschak](https://twitter.com/andy_matuschak/status/1636769182066053120)'s thread on adapting this to 13B, using fine tuning weights by [Sam Witteveen](https://huggingface.co/samwit/alpaca13B-lora).
## Disclaimer
diff --git a/chat.cpp b/chat.cpp
index 885d1f69a1e5..38b39771ad98 100644
--- a/chat.cpp
+++ b/chat.cpp
@@ -16,6 +16,7 @@
#include
#elif defined (_WIN32)
#include
+#include
#endif
#define ANSI_COLOR_RED "\x1b[31m"
@@ -30,7 +31,7 @@
// determine number of model parts based on the dimension
static const std::map LLAMA_N_PARTS = {
{ 4096, 1 },
- { 5120, 2 },
+ { 5120, 1 },
{ 6656, 4 },
{ 8192, 8 },
};
@@ -798,6 +799,7 @@ int main(int argc, char ** argv) {
params.temp = 0.1f;
params.top_p = 0.95f;
+ params.n_ctx = 2048;
params.interactive = true;
params.interactive_start = true;
params.use_color = true;
@@ -814,9 +816,9 @@ int main(int argc, char ** argv) {
fprintf(stderr, "%s: seed = %d\n", __func__, params.seed);
std::mt19937 rng(params.seed);
- if (params.prompt.empty()) {
- params.prompt = gpt_random_prompt(rng);
- }
+ // if (params.prompt.empty()) {
+ // params.prompt = gpt_random_prompt(rng);
+ // }
// params.prompt = R"(// this function checks if the number n is prime
//bool is_prime(int n) {)";
@@ -851,6 +853,8 @@ int main(int argc, char ** argv) {
std::vector logits;
+ // Add a space in front of the first character to match OG llama tokenizer behavior
+ // params.prompt.insert(0, 1, ' ');
// tokenize the prompt
std::vector embd_inp;// = ::llama_tokenize(vocab, params.prompt, true);
@@ -883,6 +887,12 @@ int main(int argc, char ** argv) {
sigaction(SIGINT, &sigint_action, NULL);
#elif defined (_WIN32)
signal(SIGINT, sigint_handler);
+
+ // Windows console ANSI color fix
+ DWORD mode = 0;
+ HANDLE hConsole = GetStdHandle(STD_OUTPUT_HANDLE);
+ if (hConsole && hConsole != INVALID_HANDLE_VALUE && GetConsoleMode(hConsole, &mode))
+ SetConsoleMode(hConsole, mode | ENABLE_VIRTUAL_TERMINAL_PROCESSING);
#endif
fprintf(stderr, "%s: interactive mode on.\n", __func__);
@@ -915,11 +925,12 @@ int main(int argc, char ** argv) {
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32)
" - Press Ctrl+C to interject at any time.\n"
#endif
- " - Press Return to return control to LLaMa.\n"
+ " - Press Return to return control to LLaMA.\n"
" - If you want to submit another line, end your input in '\\'.\n");
}
- int remaining_tokens = params.n_predict;
+ // we may want to slide the input window along with the context, but for now we restrict to the context length
+ int remaining_tokens = model.hparams.n_ctx - embd_inp.size();
int input_consumed = 0;
bool input_noecho = true;
@@ -935,7 +946,7 @@ int main(int argc, char ** argv) {
- while (true) {
+ while (remaining_tokens > 0) {
// predict
if (embd.size() > 0) {
const int64_t t_start_us = ggml_time_us();
@@ -980,7 +991,7 @@ int main(int argc, char ** argv) {
input_noecho = false;
// decrement remaining sampling budget
- // --remaining_tokens;
+ --remaining_tokens;
} else {
// some user input remains from prompt or interaction, forward it to processing
while (embd_inp.size() > input_consumed) {
@@ -1054,6 +1065,8 @@ int main(int argc, char ** argv) {
embd_inp.insert(embd_inp.end(), line_inp.begin(), line_inp.end());
embd_inp.insert(embd_inp.end(), response_inp.begin(), response_inp.end());
+ remaining_tokens -= prompt_inp.size() + line_inp.size() + response_inp.size();
+
input_noecho = true; // do not echo this again
}
diff --git a/utils.cpp b/utils.cpp
index aa3ad1053da0..d739b5d48923 100644
--- a/utils.cpp
+++ b/utils.cpp
@@ -275,41 +275,57 @@ std::vector gpt_tokenize(const gpt_vocab & vocab, const std::stri
return tokens;
}
+// TODO: Calculate this constant from the vocabulary
+#define MAX_TOKEN_LEN 18
+// SentencePiece implementation after https://guillaume-be.github.io/2020-05-30/sentence_piece
std::vector llama_tokenize(const gpt_vocab & vocab, const std::string & text, bool bos) {
- //auto res = gpt_tokenize(vocab, text);
-
- //if (bos) {
- // res.insert(res.begin(), 1); // TODO: replace with vocab.bos
- //}
-
std::vector res;
-
- if (bos) {
- res.push_back(1); // TODO: replace with vocab.bos
- }
-
- //find the longest token that matches the text
- int pos = 0;
- while (true) {
- int l = 0;
- int t = 0;
- for (const auto & kv : vocab.id_to_token) {
- if (kv.second.size() < l) continue;
- if (kv.second.size() > text.size() - pos) continue;
- if (text.substr(pos, kv.second.size()) == kv.second) {
- l = kv.second.size();
- t = kv.first;
+ std::vector score;
+ std::vector prev;
+ int len = text.length();
+
+ score.resize(len + 1);
+ prev.resize(len + 1);
+
+ // Forward pass
+ for (int i = 0; i < len; i++) {
+ int max_len = std::min(len - i, MAX_TOKEN_LEN);
+ for (int sub_len = 1; sub_len <= len - i; sub_len++) {
+ auto sub = text.substr(i, sub_len);
+ auto token = vocab.token_to_id.find(sub);
+ if (token != vocab.token_to_id.end()) {
+ int token_score = sub.length() * sub.length();
+ int local_score = score[i] + token_score;
+ int next = i + sub_len;
+ if (score[next] < local_score) {
+ score[next] = local_score;
+ prev[next] = (*token).second;
+ }
}
}
+ }
- if (l == 0) {
- break;
+ // Backward pass
+ int i = len;
+ while (i > 0) {
+ gpt_vocab::id token_id = prev[i];
+ if (token_id == 0) {
+ // TODO: Return error or something more meaningful
+ printf("failed to tokenize string!\n");
+ break;
}
+ res.push_back(token_id);
+ auto token = (*vocab.id_to_token.find(token_id)).second;
+ i -= token.length();
+ }
- res.push_back(t);
- pos += l;
+ if (bos) {
+ res.push_back(1); // TODO: replace with vocab.bos
}
+ // Pieces are in reverse order so correct that
+ std::reverse(res.begin(), res.end());
+
return res;
}