Your Name
0b5043a9f3
gates / consistency-and-conventions (push) Successful in 26s
User report after the llamacpp app install: 'installed llama.cpp unknown', valid flags rejected (randomly per run), 'Model not found' for the HF downloader's own layout, and a systemd user-bus failure over SSH. Detective (real b10822 binary, FACT) found four independent causes: - version: llama-server --version prints to STDERR; detect_llama_version's 2>/dev/null swallowed it -> always 'unknown'. Now captures 2>&1 + accepts semver/build tokens (incl. build 1.2.3 edge) - validation: printf|grep -q under pipefail -> SIGPIPE rc=141 race randomly rejected flags present in the 59 KB --help. Now pipe-less grep (no race); 20x determinism regression test - model resolution: resolve_model accepted files only, but the HF downloader creates <models>/<repo>/file.gguf dirs. Now expands a dir with exactly one *.gguf (never silently picks; multi-gguf lists + errs) - port: llama.cpp default 8080 vs tool/adapter 8088; validation reliability means --port is now always pinned in the unit - user bus: headless/SSH sessions lack XDG_RUNTIME_DIR -> ensure_user_bus in lib/common.sh pre-flights all three systemctl --user tools with remediation text; pos ai server --no-unit direct-run escape hatch (pidfile) for boxes with no bus - find_llamacpp narrowed to llama-server/llama-server-cuda (bare 'server' fallback hazard); installer post-install sanity (version+help execute, symlink targets resolve) Architect decisions DQ1-DQ6 recorded. Tester: 4 new regression files (version-from-stderr, 25x flag-validation determinism, model dir expansion, bus pre-flight + E2E) + 3 fixture updates; suite 16 files / 269 checks. Verified: make gen idempotent; make check OK; make lint 0 FAIL, 0 WARN; make test 269/269 (~49s); bash -n clean; git diff --check clean.
880 lines
35 KiB
Bash
Executable File
880 lines
35 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
# POS: ai server — llama.cpp local inference server (start, stop, status, models, logs)
|
|
# POS_SUBCMDS: start stop status models logs
|
|
# POS_FLAGS: --port --host --model --ctx --gpu --threads --gpu-layers --gpu-threads --tensor-split --n-gpu-layers --batch-size --ubatch-size --temperature --top-k --top-p --repetition-penalty --mmap --mlock --kv-cache --ctx-size --metrics --health --slots --no-unit
|
|
# POS_DEPS: curl jq
|
|
|
|
source "$(dirname "$0")/../lib/common.sh" 2>/dev/null || source "$(dirname "$0")/common.sh"
|
|
|
|
# Shared config loader (canonical KEY=VALUE parser, env-wins precedence)
|
|
source "$(dirname "$0")/../lib/config-ui.sh" 2>/dev/null || source "$(dirname "$0")/config-ui.sh"
|
|
|
|
# ── Dependencies (before --help) ───────────────────────────────
|
|
command -v curl &>/dev/null || err "curl not found (install curl)"
|
|
command -v jq &>/dev/null || err "jq not found (install jq)"
|
|
|
|
# ── Config / seams ─────────────────────────────────────────────
|
|
CONFIG_FILE="${CONFIG_FILE:-$CONFIG_DIR/ai.env}"
|
|
USER_SYSTEMD_DIR="${USER_SYSTEMD_DIR:-${XDG_CONFIG_HOME:-$HOME/.config}/systemd/user}"
|
|
SERVICE="pos-ai-server.service"
|
|
HF_DOWNLOAD_DIR="${HF_DOWNLOAD_DIR:-$HOME/.local/share/linux_post_install/ai/models}"
|
|
|
|
# Direct-run (--no-unit) escape hatch for headless/SSH boxes without a user
|
|
# systemd bus: pidfile + log live under $XDG_RUNTIME_DIR when set, else /tmp.
|
|
NO_UNIT=0
|
|
NO_UNIT_LOG="${NO_UNIT_LOG:-${XDG_RUNTIME_DIR:-/tmp}/pos-ai-server.log}"
|
|
NO_UNIT_PIDFILE="${NO_UNIT_PIDFILE:-${XDG_RUNTIME_DIR:-/tmp}/pos-ai-server.pid}"
|
|
|
|
# ── Config loader (canonical env-var precedence, same pattern as pos-ai-hf) ──
|
|
# Loaded keys are also recorded in LOADED_ENV_KEYS (see below); the D-F
|
|
# requested-from-config tracking uses the exported values, so it is
|
|
# unaffected by the loader implementation.
|
|
load_config() {
|
|
load_env_file "$CONFIG_FILE"
|
|
}
|
|
|
|
load_config
|
|
|
|
# ── Config/env-requested flags (D-F) ───────────────────────────
|
|
# Flags the user explicitly configured (via ai.env or exported env) are
|
|
# validated as "requested" (hard error if unsupported), distinct from the
|
|
# always-emitted tool defaults (warn + omit). This is captured here, BEFORE
|
|
# the CLI parse overwrites the LLAMACPP_* variables, so it reflects genuine
|
|
# user intent rather than the final defaulted values.
|
|
CONFIG_REQUESTED_FLAGS=()
|
|
requested_from_env_config() {
|
|
local var="$1" flag="$2"
|
|
[ -n "${!var:-}" ] || return 0
|
|
case " ${CONFIG_REQUESTED_FLAGS[*]:-} " in
|
|
*" $flag "*) ;; # dedupe
|
|
*) CONFIG_REQUESTED_FLAGS+=("$flag") ;;
|
|
esac
|
|
}
|
|
requested_from_env_config LLAMACPP_PORT --port
|
|
requested_from_env_config LLAMACPP_HOST --host
|
|
requested_from_env_config LLAMACPP_CTX_SIZE --ctx-size
|
|
requested_from_env_config LLAMACPP_GPU_LAYERS --n-gpu-layers
|
|
requested_from_env_config LLAMACPP_THREADS --threads
|
|
|
|
# ── Binary detection ───────────────────────────────────────────
|
|
find_llamacpp() {
|
|
# Deliberately narrow (F5): the bare `server` candidate previously picked
|
|
# an UNRELATED on-PATH binary named `server`, after which every default
|
|
# flag was rejected as "unsupported". Anonymous `llama.cpp/server` is not
|
|
# a real on-PATH name either. Only real llama.cpp server names remain.
|
|
local candidates=("llama-server" "llama-server-cuda")
|
|
local bin
|
|
for bin in "${candidates[@]}"; do
|
|
command -v "$bin" &>/dev/null && { echo "$bin"; return 0; }
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# ── Version detection ──────────────────────────────────────────
|
|
# detect_llama_version <binary> → X.Y.Z, a build token, or "unknown". Guarded:
|
|
# a missing binary or unreadable --version output yields "unknown", never an
|
|
# errexit. Real llama.cpp prints `version: 0.4.0-dev (build 10822, commit …)`
|
|
# to STDERR (common/build-info.h), so both streams are captured; the regex
|
|
# accepts semver OR a build token (`build 10822` / `b10822`) so very old
|
|
# builds that print no semver still resolve. The build number is displayed
|
|
# without the "build " phrase.
|
|
detect_llama_version() {
|
|
local bin="${1:-llama-server}"
|
|
command -v "$bin" &>/dev/null || { echo "unknown"; return 0; }
|
|
local version
|
|
version="$("$bin" --version 2>&1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+|build [0-9]+(\.[0-9]+)*|b[0-9]+' | head -1 || true)"
|
|
version="${version#build }"
|
|
[ -n "$version" ] || version="unknown"
|
|
echo "$version"
|
|
}
|
|
|
|
# ── Validate explicitly requested flags ────────────────────────
|
|
# validate_requested_flags <binary> <version> <flag...> — for every flag the
|
|
# user explicitly requested, check its token appears in the binary's --help
|
|
# output and err (version-aware) on the first unsupported one. If --help
|
|
# cannot be read, warn once and proceed instead of hard-failing.
|
|
validate_requested_flags() {
|
|
local bin="$1" version="$2"
|
|
shift 2
|
|
[ $# -gt 0 ] || return 0
|
|
|
|
local help_text
|
|
help_text="$("$bin" --help 2>/dev/null)" || {
|
|
warn "Cannot obtain llama-server --help output — skipping flag validation"
|
|
return 0
|
|
}
|
|
|
|
local seen=() flag
|
|
for flag in "$@"; do
|
|
case " ${seen[*]:-} " in
|
|
*" $flag "*) continue ;; # dedupe alias-mapped flags (e.g. --gpu → --n-gpu-layers)
|
|
esac
|
|
seen+=("$flag")
|
|
# Word-boundary match: the flag token must appear as a whole word in
|
|
# --help, not as a substring of a longer flag (D4 — e.g. --mmap must
|
|
# not match a --no-mmap entry). NON-quiet grep: with `grep -q` under
|
|
# set -o pipefail, grep exits at the first match and printf dies of
|
|
# SIGPIPE → rc=141 → valid flags randomly judged unsupported (F2).
|
|
# Here ``<<<`` needs no pipe and non-q grep consumes the whole input,
|
|
# so no early-exit race exists.
|
|
if ! grep -E -- "(^|[[:space:]])${flag}([[:space:]]|=|$)" <<<"$help_text" >/dev/null; then
|
|
err "installed llama.cpp ${version} does not expose ${flag} — remove it or upgrade llama.cpp"
|
|
fi
|
|
done
|
|
}
|
|
|
|
# ── Validate always-emitted default flags (D-F) ────────────────
|
|
# validate_default_flags <binary> <version> <requested-flag...> — for every
|
|
# flag the tool emits BY DEFAULT (with no explicit user intent), check it is
|
|
# supported in --help. Requested flags (CLI or config/env) are excluded — they
|
|
# were hard-validated upstream and must be kept. An unsupported DEFAULT is
|
|
# omitted from ExecStart with a single warn (never a hard error — the tool
|
|
# chose the default, not the user). Readable --help sets globals:
|
|
# DEFAULT_PORT_OK DEFAULT_HOST_OK DEFAULT_GPU_OK DEFAULT_CTX_OK DEFAULT_THREADS_OK
|
|
# (all 1 = keep; 0 = omit). Unreadable --help text sets all OK = 1 (warn).
|
|
validate_default_flags() {
|
|
local bin="$1" version="$2"
|
|
shift 2
|
|
local requested=("$@")
|
|
local requested_str=" ${requested[*]:-} "
|
|
|
|
DEFAULT_PORT_OK=1; DEFAULT_HOST_OK=1; DEFAULT_GPU_OK=1; DEFAULT_CTX_OK=1; DEFAULT_THREADS_OK=1
|
|
|
|
local help_text
|
|
help_text="$("$bin" --help 2>/dev/null)" || {
|
|
warn "Cannot obtain llama-server --help output — skipping default flag validation"
|
|
return 0
|
|
}
|
|
|
|
local spec flag ok_var
|
|
# flag | ok_var — the always-emitted defaults a user may not have requested.
|
|
local specs=(
|
|
"--port|DEFAULT_PORT_OK"
|
|
"--host|DEFAULT_HOST_OK"
|
|
"--n-gpu-layers|DEFAULT_GPU_OK"
|
|
"--ctx-size|DEFAULT_CTX_OK"
|
|
"--threads|DEFAULT_THREADS_OK"
|
|
)
|
|
for spec in "${specs[@]}"; do
|
|
flag="${spec%%|*}"
|
|
ok_var="${spec#*|}"
|
|
case "$requested_str" in
|
|
*" $flag "*) continue ;; # requested → already hard-validated, keep
|
|
esac
|
|
# NON-quiet grep (see validate_requested_flags — F2): no pipe, no
|
|
# early-grep-exit, no printf SIGPIPE/rc=141 race under pipefail.
|
|
if grep -E -- "(^|[[:space:]])${flag}([[:space:]]|=|$)" <<<"$help_text" >/dev/null; then
|
|
eval "$ok_var=1"
|
|
else
|
|
warn "installed llama.cpp ${version} does not support default flag ${flag} — omitting it from the unit"
|
|
eval "$ok_var=0"
|
|
fi
|
|
done
|
|
}
|
|
|
|
# ── GPU detection ──────────────────────────────────────────────
|
|
detect_gpu() {
|
|
if command -v nvidia-smi &>/dev/null && nvidia-smi &>/dev/null 2>&1; then
|
|
echo "cuda"
|
|
else
|
|
echo "cpu"
|
|
fi
|
|
}
|
|
|
|
resolve_gpu_layers() {
|
|
local configured="${LLAMACPP_GPU_LAYERS:-}"
|
|
if [ -n "$configured" ] && [ "$configured" != "-1" ]; then
|
|
echo "$configured"
|
|
return
|
|
fi
|
|
# Auto-detect
|
|
local gpu
|
|
gpu="$(detect_gpu)"
|
|
case "$gpu" in
|
|
cuda) echo "-1" ;;
|
|
*) echo "0" ;;
|
|
esac
|
|
}
|
|
|
|
# ── Human-readable size ────────────────────────────────────────
|
|
human_size() {
|
|
local bytes="$1"
|
|
if [ "$bytes" -ge 1073741824 ]; then
|
|
awk "BEGIN { printf \"%.1f GB\", $bytes / 1073741824 }"
|
|
elif [ "$bytes" -ge 1048576 ]; then
|
|
awk "BEGIN { printf \"%.1f MB\", $bytes / 1048576 }"
|
|
elif [ "$bytes" -ge 1024 ]; then
|
|
awk "BEGIN { printf \"%.1f KB\", $bytes / 1024 }"
|
|
else
|
|
printf '%d B' "$bytes"
|
|
fi
|
|
}
|
|
|
|
# ── Health check ───────────────────────────────────────────────
|
|
check_health() {
|
|
# Probe the SAME host/port the unit binds (HOST/PORT, defaults
|
|
# 127.0.0.1/8088) — previously probes hardcoded 127.0.0.1 and missed a
|
|
# non-localhost LLAMACPP_HOST bind.
|
|
local resp
|
|
resp="$(curl -sf "http://$HOST:$PORT/health" 2>/dev/null)" || { echo "not running"; return 1; }
|
|
local status
|
|
status="$(printf '%s' "$resp" | jq -r '.status // "unknown"' 2>/dev/null)"
|
|
echo "$status"
|
|
}
|
|
|
|
# ── Interactive model picker (reads /dev/tty, not stdin) ───────
|
|
pick_model() {
|
|
local models=() i
|
|
while IFS= read -r f; do
|
|
[ -f "$f" ] || continue
|
|
models+=("$f")
|
|
done < <(find "$HF_DOWNLOAD_DIR" -name '*.gguf' -type f 2>/dev/null | sort)
|
|
|
|
[ ${#models[@]} -gt 0 ] || err "No GGUF models found — run 'pos ai hf download <repo> --gguf'"
|
|
|
|
echo "Available models:"
|
|
for ((i = 0; i < ${#models[@]}; i++)); do
|
|
local name size
|
|
name="$(basename "${models[$i]}")"
|
|
size="$(stat -c%s "${models[$i]}" 2>/dev/null || echo 0)"
|
|
printf ' %2d) %-50s %s\n' "$((i + 1))" "$name" "$(human_size "$size")"
|
|
done
|
|
echo
|
|
local choice
|
|
printf 'Pick a model [1-%d]: ' "${#models[@]}"
|
|
IFS= read -r choice </dev/tty || choice=""
|
|
[[ "$choice" =~ ^[0-9]+$ ]] && [ "$choice" -ge 1 ] && [ "$choice" -le "${#models[@]}" ] || err "Invalid selection"
|
|
printf '%s' "${models[$((choice - 1))]}"
|
|
}
|
|
|
|
# ── Model resolution ───────────────────────────────────────────
|
|
# resolve_gguf_in_dir <dir> — resolve a model DIRECTORY to its single top-level
|
|
# *.gguf (case-insensitive, no recursion, matching the HF downloader's
|
|
# $HF_DOWNLOAD_DIR/<repo-slug>/<file>.gguf layout). Exactly one → print it and
|
|
# return 0. Multiple → list every candidate as <dirname>/<file> on stderr and
|
|
# err "pick one" — NEVER silently pick. Zero → return 1 (caller falls through
|
|
# to the standard "Model not found" error).
|
|
resolve_gguf_in_dir() {
|
|
local dir="$1" matches=() f
|
|
while IFS= read -r f; do
|
|
[ -f "$f" ] && matches+=("$f")
|
|
done < <(find "$dir" -maxdepth 1 -type f -iname '*.gguf' 2>/dev/null | sort)
|
|
if [ "${#matches[@]}" -eq 1 ]; then
|
|
printf '%s' "${matches[0]}"
|
|
return 0
|
|
fi
|
|
if [ "${#matches[@]}" -gt 1 ]; then
|
|
local dirname="${dir##*/}"
|
|
for f in "${matches[@]}"; do
|
|
echo " $dirname/$(basename "$f")" >&2
|
|
done
|
|
err "model dir $dirname contains multiple .gguf files — pick one (e.g. 'pos ai server start $dirname/<file>.gguf')"
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
resolve_model() {
|
|
local explicit="${1:-}"
|
|
# 1. Explicit argument
|
|
if [ -n "$explicit" ]; then
|
|
# Absolute path
|
|
if [[ "$explicit" == /* ]]; then
|
|
[ -e "$explicit" ] || err "Model not found: $explicit"
|
|
if [ -f "$explicit" ]; then
|
|
printf '%s' "$explicit"
|
|
return
|
|
fi
|
|
# Absolute DIRECTORY — same single-.gguf expansion as below.
|
|
local abs_resolved
|
|
if abs_resolved="$(resolve_gguf_in_dir "$explicit")"; then
|
|
printf '%s' "$abs_resolved"
|
|
return
|
|
fi
|
|
err "Model not found: $explicit"
|
|
fi
|
|
# Relative to HF_DOWNLOAD_DIR
|
|
local candidate="$HF_DOWNLOAD_DIR/$explicit"
|
|
if [ -f "$candidate" ]; then
|
|
printf '%s' "$candidate"
|
|
return
|
|
fi
|
|
# Directory under HF_DOWNLOAD_DIR (repo slug form) — the HF downloader
|
|
# writes $HF_DOWNLOAD_DIR/<repo-slug>/<file>.gguf; a single top-level
|
|
# .gguf resolves to it, multiple err with the pick-one list, zero
|
|
# falls through to the not-found error below (F3).
|
|
if [ -d "$candidate" ]; then
|
|
local slug_resolved
|
|
if slug_resolved="$(resolve_gguf_in_dir "$candidate")"; then
|
|
printf '%s' "$slug_resolved"
|
|
return
|
|
fi
|
|
fi
|
|
# Also try with the name as-is (could be a relative path)
|
|
[ -f "$explicit" ] && { printf '%s' "$explicit"; return; }
|
|
err "Model not found: $explicit (also searched $HF_DOWNLOAD_DIR)"
|
|
fi
|
|
# 2. Config
|
|
if [ -n "${LLAMACPP_MODEL:-}" ]; then
|
|
[ -f "$LLAMACPP_MODEL" ] || err "Configured model not found: $LLAMACPP_MODEL"
|
|
printf '%s' "$LLAMACPP_MODEL"
|
|
return
|
|
fi
|
|
# 3. Interactive pick (only on TTY)
|
|
if [ -t 0 ] || [ -w /dev/tty ]; then
|
|
local picked
|
|
picked="$(pick_model)"
|
|
printf '%s' "$picked"
|
|
return
|
|
fi
|
|
err "No model specified and no LLAMACPP_MODEL configured — run 'pos ai server start <model>' or set LLAMACPP_MODEL in ai.env"
|
|
}
|
|
|
|
# ── Usage ──────────────────────────────────────────────────────
|
|
usage() {
|
|
cat <<'EOF'
|
|
Usage: pos ai server <command> [args]
|
|
|
|
Manage a local llama.cpp inference server via systemd user service.
|
|
|
|
Commands:
|
|
start [model] Start the server (model: argument, config, or interactive pick)
|
|
stop Stop and disable the server
|
|
status Show service state, config, and health
|
|
models List available GGUF files
|
|
logs [lines] Show recent server logs
|
|
|
|
Options:
|
|
--port <port> Server port (default: 8088)
|
|
--host <addr> Bind address (default: 127.0.0.1)
|
|
--model <path> Model path (overrides argument and config)
|
|
--ctx <size> Context window size (default: 4096)
|
|
--gpu <layers> GPU layers: -1=auto, 0=CPU, N=explicit (default: -1)
|
|
--threads <n> CPU threads (default: nproc)
|
|
--gpu-layers <n> GPU layers (overrides --gpu)
|
|
--gpu-threads <n> GPU threads (default: auto)
|
|
--tensor-split <n> Tensor split configuration
|
|
--n-gpu-layers <n> GPU layers (alternative to --gpu)
|
|
--batch-size <n> Batch size for processing
|
|
--ubatch-size <n> UBatch size for processing
|
|
--temperature <n> Sampling temperature (default: 0.8)
|
|
--top-k <n> Top-K sampling parameter
|
|
--top-p <n> Top-P sampling parameter
|
|
--repetition-penalty <n> Repetition penalty for sampling
|
|
--mmap Use memory mapping
|
|
--mlock Lock memory
|
|
--kv-cache <size> KV cache size
|
|
--ctx-size <n> Context window size (alternative to --ctx)
|
|
--metrics Enable metrics endpoint
|
|
--health Enable health endpoint
|
|
--slots <n> Concurrent request slots
|
|
--no-unit Run the server directly (nohup + pidfile) instead of
|
|
installing a systemd unit — for headless/SSH boxes whose
|
|
user systemd bus is unreachable; stop/status still work
|
|
|
|
Examples:
|
|
pos ai server start mistral-7b-v0.1.Q4_K_M.gguf
|
|
pos ai server start /path/to/model.gguf --port 9090 --gpu 0
|
|
pos ai server status
|
|
pos ai server logs 50
|
|
pos ai server models
|
|
pos ai server stop
|
|
pos ai server start --model model.gguf --gpu-layers 35 --ctx-size 4096 --temperature 0.7
|
|
pos ai server start --model model.gguf --mmap --mlock --batch-size 512
|
|
|
|
Config (~/.config/linux_post_install/ai.env):
|
|
LLAMACPP_PORT Server port (default 8088)
|
|
LLAMACPP_HOST Bind address (default 127.0.0.1)
|
|
LLAMACPP_MODEL Default model path (GGUF file)
|
|
LLAMACPP_CTX_SIZE Context window size (default 4096)
|
|
LLAMACPP_GPU_LAYERS GPU layers: -1=auto, 0=CPU only (default -1)
|
|
LLAMACPP_THREADS CPU threads (default: nproc)
|
|
|
|
Requires: llama-server binary — install llama.cpp with the app installer: 'apps/ai/llamacpp.sh' (run 'bash apps/install.sh llamacpp', or pass '--apps'/'--full' to install.sh), see 'pos help ai server' (https://github.com/ggerganov/llama.cpp)
|
|
EOF
|
|
exit 0
|
|
}
|
|
|
|
# ── Parse flags ────────────────────────────────────────────────
|
|
PORT="${LLAMACPP_PORT:-8088}"
|
|
HOST="${LLAMACPP_HOST:-127.0.0.1}"
|
|
CTX_SIZE="${LLAMACPP_CTX_SIZE:-4096}"
|
|
GPU_LAYERS="${LLAMACPP_GPU_LAYERS:--1}"
|
|
THREADS="${LLAMACPP_THREADS:-}"
|
|
MODEL_ARG=""
|
|
SUBCMD=""
|
|
SUBCMD_ARGS=()
|
|
|
|
# New GPU and performance options
|
|
GPU_LAYERS_FLAG=""
|
|
GPU_THREADS=""
|
|
TENSOR_SPLIT=""
|
|
BATCH_SIZE=""
|
|
UBATCH_SIZE=""
|
|
TEMPERATURE=""
|
|
TOP_K=""
|
|
TOP_P=""
|
|
REPETITION_PENALTY=""
|
|
MAPPING=""
|
|
LOCKING=""
|
|
KV_CACHE_SIZE=""
|
|
METRICS=""
|
|
HEALTH=""
|
|
SLOTS=""
|
|
|
|
# Canonical flag tokens the user explicitly requested (defaults excluded) —
|
|
# validated against the installed binary's --help in cmd_start.
|
|
REQUESTED_FLAGS=()
|
|
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
-h|--help) usage ;;
|
|
--port)
|
|
[ $# -ge 2 ] || err "--port requires a value"
|
|
PORT="$2"; REQUESTED_FLAGS+=("--port"); shift 2 ;;
|
|
--host)
|
|
[ $# -ge 2 ] || err "--host requires a value"
|
|
HOST="$2"; REQUESTED_FLAGS+=("--host"); shift 2 ;;
|
|
--model)
|
|
[ $# -ge 2 ] || err "--model requires a value"
|
|
MODEL_ARG="$2"; REQUESTED_FLAGS+=("--model"); shift 2 ;;
|
|
--ctx)
|
|
[ $# -ge 2 ] || err "--ctx requires a value"
|
|
CTX_SIZE="$2"; REQUESTED_FLAGS+=("--ctx-size"); shift 2 ;;
|
|
--gpu)
|
|
[ $# -ge 2 ] || err "--gpu requires a value"
|
|
GPU_LAYERS="$2"; REQUESTED_FLAGS+=("--n-gpu-layers"); shift 2 ;;
|
|
--threads)
|
|
[ $# -ge 2 ] || err "--threads requires a value"
|
|
THREADS="$2"; REQUESTED_FLAGS+=("--threads"); shift 2 ;;
|
|
--gpu-layers)
|
|
[ $# -ge 2 ] || err "--gpu-layers requires a value"
|
|
GPU_LAYERS_FLAG="$2"; REQUESTED_FLAGS+=("--n-gpu-layers"); shift 2 ;;
|
|
--gpu-threads)
|
|
[ $# -ge 2 ] || err "--gpu-threads requires a value"
|
|
GPU_THREADS="$2"; REQUESTED_FLAGS+=("--gpu-threads"); shift 2 ;;
|
|
--tensor-split)
|
|
[ $# -ge 2 ] || err "--tensor-split requires a value"
|
|
TENSOR_SPLIT="$2"; REQUESTED_FLAGS+=("--tensor-split"); shift 2 ;;
|
|
--n-gpu-layers)
|
|
[ $# -ge 2 ] || err "--n-gpu-layers requires a value"
|
|
GPU_LAYERS_FLAG="$2"; REQUESTED_FLAGS+=("--n-gpu-layers"); shift 2 ;;
|
|
--batch-size)
|
|
[ $# -ge 2 ] || err "--batch-size requires a value"
|
|
BATCH_SIZE="$2"; REQUESTED_FLAGS+=("--batch-size"); shift 2 ;;
|
|
--ubatch-size)
|
|
[ $# -ge 2 ] || err "--ubatch-size requires a value"
|
|
UBATCH_SIZE="$2"; REQUESTED_FLAGS+=("--ubatch-size"); shift 2 ;;
|
|
--temperature)
|
|
[ $# -ge 2 ] || err "--temperature requires a value"
|
|
TEMPERATURE="$2"; REQUESTED_FLAGS+=("--temperature"); shift 2 ;;
|
|
--top-k)
|
|
[ $# -ge 2 ] || err "--top-k requires a value"
|
|
TOP_K="$2"; REQUESTED_FLAGS+=("--top-k"); shift 2 ;;
|
|
--top-p)
|
|
[ $# -ge 2 ] || err "--top-p requires a value"
|
|
TOP_P="$2"; REQUESTED_FLAGS+=("--top-p"); shift 2 ;;
|
|
--repetition-penalty)
|
|
[ $# -ge 2 ] || err "--repetition-penalty requires a value"
|
|
REPETITION_PENALTY="$2"; REQUESTED_FLAGS+=("--repetition-penalty"); shift 2 ;;
|
|
--mmap)
|
|
MAPPING="true"; REQUESTED_FLAGS+=("--mmap"); shift ;;
|
|
--mlock)
|
|
LOCKING="true"; REQUESTED_FLAGS+=("--mlock"); shift ;;
|
|
--kv-cache)
|
|
[ $# -ge 2 ] || err "--kv-cache requires a value"
|
|
KV_CACHE_SIZE="$2"; REQUESTED_FLAGS+=("--kv-cache"); shift 2 ;;
|
|
--ctx-size)
|
|
[ $# -ge 2 ] || err "--ctx-size requires a value"
|
|
CTX_SIZE="$2"; REQUESTED_FLAGS+=("--ctx-size"); shift 2 ;;
|
|
--metrics)
|
|
METRICS="true"; REQUESTED_FLAGS+=("--metrics"); shift ;;
|
|
--health)
|
|
HEALTH="true"; REQUESTED_FLAGS+=("--health"); shift ;;
|
|
--slots)
|
|
[ $# -ge 2 ] || err "--slots requires a value"
|
|
SLOTS="$2"; REQUESTED_FLAGS+=("--slots"); shift 2 ;;
|
|
--no-unit)
|
|
NO_UNIT=1; shift ;;
|
|
-*)
|
|
err "Unknown option '$1' (see --help)" ;;
|
|
*)
|
|
if [ -z "$SUBCMD" ]; then
|
|
SUBCMD="$1"
|
|
else
|
|
SUBCMD_ARGS+=("$1")
|
|
fi
|
|
shift ;;
|
|
esac
|
|
done
|
|
|
|
# Apply flag overrides back to config defaults (flags > env > file default)
|
|
LLAMACPP_PORT="$PORT"
|
|
LLAMACPP_HOST="$HOST"
|
|
LLAMACPP_CTX_SIZE="$CTX_SIZE"
|
|
LLAMACPP_GPU_LAYERS="$GPU_LAYERS"
|
|
if [ -z "$THREADS" ]; then
|
|
THREADS="$(nproc 2>/dev/null || echo 4)"
|
|
fi
|
|
LLAMACPP_THREADS="$THREADS"
|
|
|
|
# ── Subcommands ────────────────────────────────────────────────
|
|
|
|
# systemd_quote <value> — wrap a path in double quotes for systemd's
|
|
# ExecStart word-splitting (systemd.service(5)), escaping embedded `"` as
|
|
# `\"`. Only tokens that may legally contain spaces need this (binary and
|
|
# model path); plain numeric/flag tokens like `--port 8088` stay unquoted.
|
|
systemd_quote() {
|
|
local value="$1"
|
|
value="${value//\"/\\\"}"
|
|
printf '"%s"' "$value"
|
|
}
|
|
|
|
cmd_start() {
|
|
# Resolve the llama-server binary
|
|
local llamacpp_bin
|
|
llamacpp_bin="$(find_llamacpp)" || err "llama-server not found — install llama.cpp with the app installer: 'apps/ai/llamacpp.sh' (run 'bash apps/install.sh llamacpp', or pass '--apps'/'--full' to install.sh), see 'pos help ai server' (https://github.com/ggerganov/llama.cpp)"
|
|
local llamacpp_full
|
|
llamacpp_full="$(command -v "$llamacpp_bin")"
|
|
|
|
# Detect version (guarded — never crashes; returns "unknown" when
|
|
# unreadable, then basic defaults are used)
|
|
local version
|
|
version="$(detect_llama_version "$llamacpp_bin")"
|
|
|
|
# Validate EVERY flag that will appear in ExecStart (D-F):
|
|
# - Requested flags (CLI OR config/env) → hard error if unsupported.
|
|
# - Always-emitted defaults → warn + omit if unsupported.
|
|
local all_requested=("${REQUESTED_FLAGS[@]}" "${CONFIG_REQUESTED_FLAGS[@]}")
|
|
local deduped=() flag
|
|
for flag in "${all_requested[@]}"; do
|
|
case " ${deduped[*]:-} " in
|
|
*" $flag "*) continue ;;
|
|
esac
|
|
deduped+=("$flag")
|
|
done
|
|
if [ "${#deduped[@]}" -gt 0 ]; then
|
|
validate_requested_flags "$llamacpp_bin" "$version" "${deduped[@]}"
|
|
fi
|
|
validate_default_flags "$llamacpp_bin" "$version" "${deduped[@]}"
|
|
|
|
# Resolve model
|
|
local explicit_model="${SUBCMD_ARGS[0]:-}"
|
|
# Flag --model takes precedence over positional arg
|
|
[ -n "$MODEL_ARG" ] && explicit_model="$MODEL_ARG"
|
|
local model
|
|
model="$(resolve_model "$explicit_model")"
|
|
|
|
# Resolve GPU layers
|
|
local gpu_layers
|
|
gpu_layers="$(resolve_gpu_layers)"
|
|
# Use the flag value if provided, otherwise use resolved value
|
|
[ -n "$GPU_LAYERS_FLAG" ] && gpu_layers="$GPU_LAYERS_FLAG"
|
|
|
|
# Warn if no GPU detected and auto-detect resolved to CPU
|
|
if [ "$gpu_layers" = "0" ] && [ "${LLAMACPP_GPU_LAYERS:--1}" = "-1" ]; then
|
|
warn "No NVIDIA GPU detected — running in CPU mode"
|
|
fi
|
|
|
|
# Check port availability (best-effort)
|
|
if command -v ss &>/dev/null; then
|
|
if ss -tlnp 2>/dev/null | grep -q ":${PORT} "; then
|
|
# Port might be our own old instance — only warn
|
|
warn "Port $PORT may already be in use — check with 'ss -tlnp'"
|
|
fi
|
|
fi
|
|
|
|
# Build ONE command line: binary + model + ALL resolved flags. A single
|
|
# string keeps the systemd unit's ExecStart on one line (systemd requires
|
|
# trailing `\` for multi-line continuations) and makes dry-run show
|
|
# exactly what the unit will contain. systemd splits ExecStart on
|
|
# unquoted whitespace, so the binary and the model path — the only tokens
|
|
# that may contain spaces — are systemd_quote()d; plain flag/number
|
|
# tokens stay unquoted.
|
|
# Build the command with ONLY the flags that passed validation. Requested
|
|
# flags (hard-validated) and supported defaults are always emitted; an
|
|
# unsupported DEFAULT is omitted here (validate_default_flags set the
|
|
# DEFAULT_*_OK globals) so the unit never carries an unsupported flag.
|
|
local exec_cmd
|
|
exec_cmd="$(systemd_quote "$llamacpp_full") -m $(systemd_quote "$model")"
|
|
if [ "$DEFAULT_PORT_OK" -eq 1 ]; then exec_cmd+=" --port $PORT"; fi
|
|
if [ "$DEFAULT_HOST_OK" -eq 1 ]; then exec_cmd+=" --host $HOST"; fi
|
|
if [ "$DEFAULT_GPU_OK" -eq 1 ]; then exec_cmd+=" --n-gpu-layers $gpu_layers"; fi
|
|
if [ "$DEFAULT_CTX_OK" -eq 1 ]; then exec_cmd+=" --ctx-size $CTX_SIZE"; fi
|
|
if [ "$DEFAULT_THREADS_OK" -eq 1 ]; then exec_cmd+=" --threads $THREADS"; fi
|
|
if [ -n "$GPU_THREADS" ]; then
|
|
exec_cmd+=" --gpu-threads $GPU_THREADS"
|
|
fi
|
|
if [ -n "$TENSOR_SPLIT" ]; then
|
|
exec_cmd+=" --tensor-split $TENSOR_SPLIT"
|
|
fi
|
|
if [ -n "$BATCH_SIZE" ]; then
|
|
exec_cmd+=" --batch-size $BATCH_SIZE"
|
|
fi
|
|
if [ -n "$UBATCH_SIZE" ]; then
|
|
exec_cmd+=" --ubatch-size $UBATCH_SIZE"
|
|
fi
|
|
if [ -n "$TEMPERATURE" ]; then
|
|
exec_cmd+=" --temperature $TEMPERATURE"
|
|
fi
|
|
if [ -n "$TOP_K" ]; then
|
|
exec_cmd+=" --top-k $TOP_K"
|
|
fi
|
|
if [ -n "$TOP_P" ]; then
|
|
exec_cmd+=" --top-p $TOP_P"
|
|
fi
|
|
if [ -n "$REPETITION_PENALTY" ]; then
|
|
exec_cmd+=" --repetition-penalty $REPETITION_PENALTY"
|
|
fi
|
|
if [ -n "$MAPPING" ]; then
|
|
exec_cmd+=" --mmap"
|
|
fi
|
|
if [ -n "$LOCKING" ]; then
|
|
exec_cmd+=" --mlock"
|
|
fi
|
|
if [ -n "$KV_CACHE_SIZE" ]; then
|
|
exec_cmd+=" --kv-cache $KV_CACHE_SIZE"
|
|
fi
|
|
if [ -n "$METRICS" ]; then
|
|
exec_cmd+=" --metrics"
|
|
fi
|
|
if [ -n "$HEALTH" ]; then
|
|
exec_cmd+=" --health"
|
|
fi
|
|
if [ -n "$SLOTS" ]; then
|
|
exec_cmd+=" --slots $SLOTS"
|
|
fi
|
|
|
|
# ── User-bus pre-flight + stale-unit warning (unit path only) ──
|
|
# ensure_user_bus aborts with remediation BEFORE the unit is written, so a
|
|
# broken user bus leaves NO orphaned unit. --no-unit skips the bus check —
|
|
# that is its whole purpose (headless/SSH boxes without a user bus). An
|
|
# existing unit is warned about but never deleted here (removal stays in
|
|
# cmd_stop); it may be stale from a previous failed start.
|
|
if [ "$NO_UNIT" -ne 1 ]; then
|
|
ensure_user_bus
|
|
if [ -f "$USER_SYSTEMD_DIR/$SERVICE" ]; then
|
|
warn "existing unit $USER_SYSTEMD_DIR/$SERVICE will be overwritten — it may be stale from a previous failed start"
|
|
fi
|
|
fi
|
|
|
|
if [ "${DRY_RUN:-0}" -eq 1 ]; then
|
|
if [ "$NO_UNIT" -eq 1 ]; then
|
|
log "(dry-run) nohup $exec_cmd >$NO_UNIT_LOG 2>&1 &"
|
|
log "(dry-run) pidfile: $NO_UNIT_PIDFILE"
|
|
else
|
|
log "(dry-run) generate systemd unit $USER_SYSTEMD_DIR/$SERVICE"
|
|
log "(dry-run) ExecStart: $exec_cmd"
|
|
log "(dry-run) systemctl --user daemon-reload && enable --now $SERVICE"
|
|
fi
|
|
return 0
|
|
fi
|
|
|
|
# ── Direct run (--no-unit) ──
|
|
# Run the exact ExecStart command directly under nohup with a pidfile
|
|
# (under $XDG_RUNTIME_DIR, else /tmp) so stop/status keep working without
|
|
# a systemd unit. The string is the same one the unit would carry; only
|
|
# the binary and model tokens are quoted, and both are validated paths
|
|
# (command -v / existing file) resolved above, so eval is the faithful
|
|
# way to honor its systemd-style quoting.
|
|
if [ "$NO_UNIT" -eq 1 ]; then
|
|
mkdir -p "$(dirname "$NO_UNIT_PIDFILE")"
|
|
: >"$NO_UNIT_LOG"
|
|
eval "nohup $exec_cmd >'$NO_UNIT_LOG' 2>&1 &"
|
|
local direct_pid=$!
|
|
printf '%s\n' "$direct_pid" >"$NO_UNIT_PIDFILE"
|
|
log "Server starting (direct run, no systemd) — pid $direct_pid, model: $(basename "$model"), port: $PORT"
|
|
log "log: $NO_UNIT_LOG"
|
|
log "stop: kill \$(cat $NO_UNIT_PIDFILE) (or 'pos ai server stop')"
|
|
return 0
|
|
fi
|
|
|
|
# Generate systemd unit — ExecStart is a single line with the full command
|
|
mkdir -p "$USER_SYSTEMD_DIR"
|
|
cat > "$USER_SYSTEMD_DIR/$SERVICE" <<EOF
|
|
[Unit]
|
|
Description=pos llama.cpp inference server (linux-post-install)
|
|
After=network-online.target
|
|
|
|
[Service]
|
|
Type=simple
|
|
ExecStart=$exec_cmd
|
|
Restart=on-failure
|
|
RestartSec=5
|
|
TimeoutStopSec=10
|
|
KillMode=control-group
|
|
EnvironmentFile=-%h/.config/linux_post_install/ai.env
|
|
|
|
[Install]
|
|
WantedBy=default.target
|
|
EOF
|
|
chmod 644 "$USER_SYSTEMD_DIR/$SERVICE"
|
|
|
|
# Enable and start
|
|
systemctl --user daemon-reload
|
|
systemctl --user enable --now "$SERVICE"
|
|
|
|
log "Server starting — model: $(basename "$model"), port: $PORT"
|
|
|
|
# Linger warning
|
|
if command -v loginctl >/dev/null 2>&1; then
|
|
if ! loginctl show-user "$(id -un)" 2>/dev/null | grep -q '^Linger=yes'; then
|
|
warn "enable linger so the server survives logout: sudo loginctl enable-linger $(id -un)"
|
|
fi
|
|
fi
|
|
|
|
# Health check (wait briefly)
|
|
sleep 2
|
|
local health
|
|
health="$(check_health)" || true
|
|
if [ "$health" != "not running" ]; then
|
|
ok "Server healthy (status: $health)"
|
|
else
|
|
warn "Server may not be ready yet — check with 'pos ai server status'"
|
|
fi
|
|
}
|
|
|
|
cmd_stop() {
|
|
# Direct-run (--no-unit) pidfile first: kill the recorded pid, remove the
|
|
# pidfile. Then fall through to the normal unit path (both can exist if a
|
|
# unit was installed after a direct run).
|
|
local stopped_direct=0
|
|
if [ -f "$NO_UNIT_PIDFILE" ]; then
|
|
stopped_direct=1
|
|
local direct_pid
|
|
direct_pid="$(cat "$NO_UNIT_PIDFILE" 2>/dev/null || true)"
|
|
if [ "${DRY_RUN:-0}" -eq 1 ]; then
|
|
log "(dry-run) kill $direct_pid; rm -f $NO_UNIT_PIDFILE"
|
|
else
|
|
if [[ "$direct_pid" =~ ^[0-9]+$ ]]; then
|
|
kill "$direct_pid" 2>/dev/null || true
|
|
fi
|
|
rm -f "$NO_UNIT_PIDFILE"
|
|
log "llama.cpp server stopped (direct run)"
|
|
fi
|
|
fi
|
|
if [ ! -f "$USER_SYSTEMD_DIR/$SERVICE" ]; then
|
|
if [ "$stopped_direct" -eq 0 ]; then
|
|
warn "No llama.cpp server service installed ($SERVICE)"
|
|
fi
|
|
return 0
|
|
fi
|
|
if [ "${DRY_RUN:-0}" -eq 1 ]; then
|
|
log "(dry-run) systemctl --user disable --now $SERVICE; remove unit"
|
|
else
|
|
systemctl --user disable --now "$SERVICE" 2>/dev/null || true
|
|
rm -f "$USER_SYSTEMD_DIR/$SERVICE"
|
|
systemctl --user daemon-reload
|
|
fi
|
|
log "llama.cpp server stopped and removed"
|
|
}
|
|
|
|
cmd_status() {
|
|
# llama-server must be present for the version probe below — same
|
|
# actionable deps message as `start`
|
|
if ! find_llamacpp >/dev/null 2>&1; then
|
|
err "llama-server not found — install llama.cpp with the app installer: 'apps/ai/llamacpp.sh' (run 'bash apps/install.sh llamacpp', or pass '--apps'/'--full' to install.sh), see 'pos help ai server' (https://github.com/ggerganov/llama.cpp)"
|
|
fi
|
|
|
|
# Service state — systemd unit, or the direct-run (--no-unit) pidfile
|
|
local svc_state="stopped"
|
|
if [ -f "$NO_UNIT_PIDFILE" ]; then
|
|
local direct_pid
|
|
direct_pid="$(cat "$NO_UNIT_PIDFILE" 2>/dev/null || true)"
|
|
if [[ "$direct_pid" =~ ^[0-9]+$ ]] && kill -0 "$direct_pid" 2>/dev/null; then
|
|
svc_state="running"
|
|
fi
|
|
fi
|
|
if systemctl --user is-active "$SERVICE" &>/dev/null; then
|
|
svc_state="running"
|
|
fi
|
|
printf 'service: %s\n' "$svc_state"
|
|
|
|
# Model (from health endpoint if running)
|
|
if [ "$svc_state" = "running" ]; then
|
|
local models_resp
|
|
models_resp="$(curl -sf "http://$HOST:$PORT/v1/models" 2>/dev/null)" || true
|
|
local model_id
|
|
model_id="$(printf '%s' "$models_resp" | jq -r '.data[0].id // "unknown"' 2>/dev/null)" || model_id="unknown"
|
|
printf 'model: %s\n' "$model_id"
|
|
else
|
|
printf 'model: (not loaded)\n'
|
|
fi
|
|
|
|
# Config
|
|
printf 'port: %s\n' "$PORT"
|
|
printf 'host: %s\n' "$HOST"
|
|
|
|
# GPU
|
|
local gpu_type
|
|
gpu_type="$(detect_gpu)"
|
|
printf 'gpu: %s (%s layers)\n' "${gpu_type^^}" "$GPU_LAYERS"
|
|
|
|
printf 'context: %s\n' "$CTX_SIZE"
|
|
printf 'threads: %s\n' "$THREADS"
|
|
|
|
# Autostart
|
|
if systemctl --user is-enabled "$SERVICE" &>/dev/null; then
|
|
printf 'autostart: enabled\n'
|
|
else
|
|
printf 'autostart: disabled\n'
|
|
fi
|
|
|
|
# Endpoint
|
|
printf 'endpoint: http://%s:%s\n' "$HOST" "$PORT"
|
|
|
|
# Health
|
|
if [ "$svc_state" = "running" ]; then
|
|
local health
|
|
health="$(check_health)" || health="not responding"
|
|
printf 'health: %s\n' "$health"
|
|
else
|
|
printf 'health: not running\n'
|
|
fi
|
|
|
|
# Version info (probe the resolved binary; "unknown" if unreadable)
|
|
local llamacpp_bin version
|
|
llamacpp_bin="$(find_llamacpp)"
|
|
version="$(detect_llama_version "$llamacpp_bin")"
|
|
printf 'version: %s\n' "$version"
|
|
}
|
|
|
|
cmd_models() {
|
|
local dir="${HF_DOWNLOAD_DIR}"
|
|
[ -d "$dir" ] || { warn "No models directory — run 'pos ai hf download' first"; return 0; }
|
|
|
|
local found=0
|
|
echo "Available GGUF models:"
|
|
while IFS= read -r gguf; do
|
|
[ -f "$gguf" ] || continue
|
|
found=1
|
|
local name size
|
|
name="$(basename "$gguf")"
|
|
local dir_name
|
|
dir_name="$(basename "$(dirname "$gguf")")"
|
|
size="$(stat -c%s "$gguf" 2>/dev/null || echo 0)"
|
|
local hsize
|
|
hsize="$(human_size "$size")"
|
|
printf ' %-50s %s\n' "$dir_name/$name" "$hsize"
|
|
done < <(find "$dir" -name '*.gguf' -type f 2>/dev/null | sort)
|
|
|
|
[ "$found" -eq 0 ] && warn "No .gguf files found — download with 'pos ai hf download <repo> --gguf'"
|
|
}
|
|
|
|
cmd_logs() {
|
|
local lines="${SUBCMD_ARGS[0]:-50}"
|
|
[[ "$lines" =~ ^[0-9]+$ ]] || err "lines must be a number"
|
|
journalctl --user -u "$SERVICE" -n "$lines" --no-pager 2>/dev/null || warn "No logs found — server may not have been started"
|
|
}
|
|
|
|
# ── Dispatch ───────────────────────────────────────────────────
|
|
case "${SUBCMD:-}" in
|
|
"") usage ;;
|
|
start) cmd_start ;;
|
|
stop) cmd_stop ;;
|
|
status) cmd_status ;;
|
|
models) cmd_models ;;
|
|
logs) cmd_logs ;;
|
|
*) err "Unknown subcommand '$SUBCMD' (see --help)" ;;
|
|
esac
|