Repository navigation
Expand file tree
/
Copy pathserver.sh
More file actions
executable file
·81 lines (70 loc) · 2.13 KB
/
Copy pathserver.sh
File metadata and controls
executable file
·81 lines (70 loc) · 2.13 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
#!/bin/bash
set -euo pipefail
# Llama server launcher
# All configurations are set in /root/.env or in the .env file in the same directory as this script.
# Load environment variables if .env exists
if [ -f "/root/.env" ]; then
set -a
source "/root/.env"
set +a
elif [ -f "$(dirname "$0")/.env" ]; then
set -a
source "$(dirname "$0")/.env"
set +a
fi
# Kill any existing llama-server processes and free up ports 8080 and 7600
pkill -9 -t tty llama-server || true
lsof -ti :8080,7600 | xargs -r kill -9 2>/dev/null || true
LOGFILE=/root/logs/llama
mkdir -p "$LOGFILE"
export CUDA_VISIBLE_DEVICES="${LLAMA_GPU:-0}"
export LLAMA_ARG_PORT="${LLAMA_PORT:-7600}"
export LLAMA_ARG_HOST="${LLAMA_HOST:-0.0.0.0}"
#LLAMA_TEMP="${LLAMA_TEMP:-0.7}"
#LLAMA_TOP_P="${LLAMA_TOP_P:-0.80}"
#LLAMA_TOP_K="${LLAMA_TOP_K:-20}"
#LLAMA_MIN_P="${LLAMA_MIN_P:-0.0}"
#LLAMA_PRESENCE_PENALTY="${LLAMA_PRESENCE_PENALTY:-1.5}"
#LLAMA_REPEAT_PENALTY="${LLAMA_REPEAT_PENALTY:-1.0}"
#LLAMA_CTX_SIZE="${LLAMA_CTX_SIZE:-32768}"
#LLAMA_REASONING="${LLAMA_REASONING:-off}"
LLAMA_SERVER_BIN="/root/llama.cpp/build/bin/llama-server"
if [ ! -f "$LLAMA_SERVER_BIN" ]; then
LLAMA_SERVER_BIN="/root/llama.cpp/llama-server"
fi
API_KEY_ARG=""
if [ -n "${LLAMA_KEY:-}" ]; then
API_KEY_ARG="--api-key $LLAMA_KEY"
fi
#nohup sh -c '
"$LLAMA_SERVER_BIN" \
-m /root/unsloth/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-Q4_K_XL.gguf \
-md /root/unsloth/Qwen3.8-27B-GGUF/mtp-Qwen3.8-27B-Q4_0.gguf \
--mmproj /root/unsloth/Qwen3.8-27B-GGUF/mmproj-F16.gguf \
-ngl 999 \
-c 163840 \
-n 16384 \
-np 2 \
-b 2048 \
-ub 1024 \
-fa on \
--timeout 300 \
-ctk q8_0 \
-ctv q8_0 \
-t 8 \
--temp 0.7 \
--top-p 0.80 \
--top-k 20 \
--min-p 0.05 \
--presence-penalty 0.0 \
--repeat-penalty 1.05 \
--reasoning auto \
--reasoning-budget 1024 \
--jinja \
$API_KEY_ARG \
2>&1 | multilog t s5000000 n3 "$LOGFILE" &
#' >/dev/null 2>&1 &
#mkdir -p /root/logs/gost83 && gost -L tcp://:8384/:7600 2>&1 | multilog t s500000 n3 /root/logs/gost83 &
#echo "Server started. Checking log for GPU offload:"
#sleep 2
#grep -E "CUDA|offload|vram" server.log || tail -n 25 server.log