llama-cpp-docker/docker-task at main · fboulnois/llama-cpp-docker · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
#!/bin/sh

set -eu

MODEL_LIST=$(cat << EOF
https://huggingface.co/TheBloke/Llama-2-13B-chat-GGUF
https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF
https://huggingface.co/bartowski/mistralai_Magistral-Small-2509-GGUF
https://huggingface.co/bartowski/mistralai_Ministral-3-8B-Instruct-2512-GGUF
https://huggingface.co/bartowski/mistralai_Ministral-3-8B-Reasoning-2512-GGUF
https://huggingface.co/bartowski/mistralai_Ministral-3-14B-Instruct-2512-GGUF
https://huggingface.co/bartowski/mistralai_Ministral-3-14B-Reasoning-2512-GGUF
https://huggingface.co/bartowski/google_gemma-4-26B-A4B-it-GGUF
https://huggingface.co/bartowski/google_gemma-4-31B-it-GGUF
https://huggingface.co/bartowski/Qwen_Qwen3.5-0.8B-GGUF
https://huggingface.co/bartowski/Qwen_Qwen3.5-2B-GGUF
https://huggingface.co/bartowski/Qwen_Qwen3.5-4B-GGUF
https://huggingface.co/bartowski/Qwen_Qwen3.5-9B-GGUF
https://huggingface.co/bartowski/Qwen_Qwen3.5-27B-GGUF
https://huggingface.co/bartowski/Qwen_Qwen3.5-35B-A3B-GGUF
https://huggingface.co/bartowski/deepseek-ai_DeepSeek-R1-0528-Qwen3-8B-GGUF
https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF
https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-14B-GGUF
https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-32B-GGUF
EOF
)

use_sudo() {
  if command -v docker > /dev/null 2>&1 && docker ps 2>&1 | grep -q "permission denied"; then
    DOCKER="sudo docker"
  else
    DOCKER="docker"
  fi
}

has_nvidia_gpu() {
  if command -v nvidia-smi > /dev/null 2>&1 && nvidia-smi --query --display=COMPUTE > /dev/null 2>&1; then
    return 0
  else
    return 1
  fi
}

usage() {
  MODEL_NAMES=$(echo "$MODEL_LIST" | sed 's|https://huggingface.co/||' | sort)
  cat << EOF
Set the LLAMA_ARG_HF_REPO environment variable in docker-compose.yml to
automatically download and use the model from HuggingFace.

Popular models to download and use:

$MODEL_NAMES

EOF
  exit 0
}

parse_args_check_docker() {
  if [ ! -f /.dockerenv ] && [ ! -f /run/.containerenv ]; then
    usage
  fi
}

set_default_env_vars() {
  if [ -z ${LLAMA_ARG_HOST+x} ]; then
    export LLAMA_ARG_HOST="0.0.0.0"
  fi
}

set_cache_permissions() {
  UID=$(id -u llama)
  chown -R "$UID:$UID" /home/llama/.cache
  chmod 1777 /home/llama/.cache
}

docker_entrypoint() {
  parse_args_check_docker
  set_default_env_vars
  set_cache_permissions

  set -x
  exec gosu "$UID:$UID" llama-server
}

docker_build() {
  DIRNAME=$(basename "$PWD")
  if has_nvidia_gpu; then
    $DOCKER build --target env-build  . --tag "${DIRNAME}-build"
	  $DOCKER build --target env-deploy . --tag "${DIRNAME}"
  else
    $DOCKER build --target env-build  . --file Dockerfile.cpu --tag "${DIRNAME}-build"
	  $DOCKER build --target env-deploy . --file Dockerfile.cpu --tag "${DIRNAME}"
  fi
}

docker_opt() {
  if has_nvidia_gpu; then
    $DOCKER compose --file docker-compose.yml --file docker-compose.gpu.yml "$1"
  else
    $DOCKER compose --file docker-compose.yml "$1"
  fi
}

docker_up() {
  docker_opt up
}

docker_down() {
  docker_opt down
}

use_sudo

if [ -z "${1+:}" ]; then
  usage
fi

case $1 in
  entrypoint) docker_entrypoint ;;
  build) docker_build ;;
  up) docker_up ;;
  down) docker_down ;;
  --help|-h) usage ;;
  *) echo "Unknown command: $1"; exit 1 ;;
esac