KeyArgo
/
llama-gpt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Set 70B's GQA to 8

This commit is contained in:
Mayank Chhabra 2023-08-18 00:18:19 +07:00
parent bc1300fba5
commit 2a24cb9e60
1 changed files with 6 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
docker-compose-70b.yml
View File

@ -9,6 +9,12 @@ services:
restart: on-failure restart: on-failure
environment: environment:
MODEL: '/models/llama-2-70b-chat.bin' MODEL: '/models/llama-2-70b-chat.bin'
# Llama 2 70B's grouping factor is 8 compared to 7B and 13B's 1. Currently,
# it's not possible to change this using --n_gqa with llama-cpp-python in
# run.sh, so we expose it as an environment variable.
# See: https://github.com/abetlen/llama-cpp-python/issues/528
# and: https://github.com/facebookresearch/llama/issues/407
N_GQA: '8'
llama-gpt-ui: llama-gpt-ui:
image: 'ghcr.io/getumbrel/llama-gpt-ui:latest' image: 'ghcr.io/getumbrel/llama-gpt-ui:latest'