#!/usr/bin/env bash
set -euo pipefail
# POS: ai server — llama.cpp local inference server (start, stop, status, models, logs)
# POS_SUBCMDS: start stop status models logs
# POS_FLAGS: --port --host --model --ctx --gpu --threads --gpu-layers --gpu-threads --tensor-split --n-gpu-layers --batch-size --ubatch-size --temperature --top-k --top-p --repetition-penalty --mmap --mlock --kv-cache --ctx-size --metrics --health --slots --no-unit
# POS_DEPS: curl jq

source "$(dirname "$0")/../lib/common.sh" 2>/dev/null || source "$(dirname "$0")/common.sh"

# Shared config loader (canonical KEY=VALUE parser, env-wins precedence)
source "$(dirname "$0")/../lib/config-ui.sh" 2>/dev/null || source "$(dirname "$0")/config-ui.sh"

# ── Dependencies (before --help) ───────────────────────────────
command -v curl &>/dev/null || err "curl not found (install curl)"
command -v jq &>/dev/null || err "jq not found (install jq)"

# ── Config / seams ─────────────────────────────────────────────
CONFIG_FILE="${CONFIG_FILE:-$CONFIG_DIR/ai.env}"
USER_SYSTEMD_DIR="${USER_SYSTEMD_DIR:-${XDG_CONFIG_HOME:-$HOME/.config}/systemd/user}"
SERVICE="pos-ai-server.service"
HF_DOWNLOAD_DIR="${HF_DOWNLOAD_DIR:-$HOME/.local/share/linux_post_install/ai/models}"

# Direct-run (--no-unit) escape hatch for headless/SSH boxes without a user
# systemd bus: pidfile + log live under $XDG_RUNTIME_DIR when set, else /tmp.
NO_UNIT=0
NO_UNIT_LOG="${NO_UNIT_LOG:-${XDG_RUNTIME_DIR:-/tmp}/pos-ai-server.log}"
NO_UNIT_PIDFILE="${NO_UNIT_PIDFILE:-${XDG_RUNTIME_DIR:-/tmp}/pos-ai-server.pid}"

# ── Config loader (canonical env-var precedence, same pattern as pos-ai-hf) ──
# Loaded keys are also recorded in LOADED_ENV_KEYS (see below); the D-F
# requested-from-config tracking uses the exported values, so it is
# unaffected by the loader implementation.
load_config() {
    load_env_file "$CONFIG_FILE"
}

load_config

# ── Config/env-requested flags (D-F) ───────────────────────────
# Flags the user explicitly configured (via ai.env or exported env) are
# validated as "requested" (hard error if unsupported), distinct from the
# always-emitted tool defaults (warn + omit). This is captured here, BEFORE
# the CLI parse overwrites the LLAMACPP_* variables, so it reflects genuine
# user intent rather than the final defaulted values.
CONFIG_REQUESTED_FLAGS=()
requested_from_env_config() {
    local var="$1" flag="$2"
    [ -n "${!var:-}" ] || return 0
    case " ${CONFIG_REQUESTED_FLAGS[*]:-} " in
        *" $flag "*) ;;                # dedupe
        *) CONFIG_REQUESTED_FLAGS+=("$flag") ;;
    esac
}
requested_from_env_config LLAMACPP_PORT       --port
requested_from_env_config LLAMACPP_HOST       --host
requested_from_env_config LLAMACPP_CTX_SIZE   --ctx-size
requested_from_env_config LLAMACPP_GPU_LAYERS --n-gpu-layers
requested_from_env_config LLAMACPP_THREADS    --threads

# ── Binary detection ───────────────────────────────────────────
find_llamacpp() {
    # Deliberately narrow (F5): the bare `server` candidate previously picked
    # an UNRELATED on-PATH binary named `server`, after which every default
    # flag was rejected as "unsupported". Anonymous `llama.cpp/server` is not
    # a real on-PATH name either. Only real llama.cpp server names remain.
    local candidates=("llama-server" "llama-server-cuda")
    local bin
    for bin in "${candidates[@]}"; do
        command -v "$bin" &>/dev/null && { echo "$bin"; return 0; }
    done
    return 1
}

# ── Version detection ──────────────────────────────────────────
# detect_llama_version <binary> → X.Y.Z, a build token, or "unknown". Guarded:
# a missing binary or unreadable --version output yields "unknown", never an
# errexit. Real llama.cpp prints `version: 0.4.0-dev (build 10822, commit …)`
# to STDERR (common/build-info.h), so both streams are captured; the regex
# accepts semver OR a build token (`build 10822` / `b10822`) so very old
# builds that print no semver still resolve. The build number is displayed
# without the "build " phrase.
detect_llama_version() {
    local bin="${1:-llama-server}"
    command -v "$bin" &>/dev/null || { echo "unknown"; return 0; }
    local version
    version="$("$bin" --version 2>&1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+|build [0-9]+(\.[0-9]+)*|b[0-9]+' | head -1 || true)"
    version="${version#build }"
    [ -n "$version" ] || version="unknown"
    echo "$version"
}

# ── Validate explicitly requested flags ────────────────────────
# validate_requested_flags <binary> <version> <flag...> — for every flag the
# user explicitly requested, check its token appears in the binary's --help
# output and err (version-aware) on the first unsupported one. If --help
# cannot be read, warn once and proceed instead of hard-failing.
validate_requested_flags() {
    local bin="$1" version="$2"
    shift 2
    [ $# -gt 0 ] || return 0

    local help_text
    help_text="$("$bin" --help 2>/dev/null)" || {
        warn "Cannot obtain llama-server --help output — skipping flag validation"
        return 0
    }

    local seen=() flag
    for flag in "$@"; do
        case " ${seen[*]:-} " in
            *" $flag "*) continue ;;   # dedupe alias-mapped flags (e.g. --gpu → --n-gpu-layers)
        esac
        seen+=("$flag")
        # Word-boundary match: the flag token must appear as a whole word in
        # --help, not as a substring of a longer flag (D4 — e.g. --mmap must
        # not match a --no-mmap entry). NON-quiet grep: with `grep -q` under
        # set -o pipefail, grep exits at the first match and printf dies of
        # SIGPIPE → rc=141 → valid flags randomly judged unsupported (F2).
        # Here ``<<<`` needs no pipe and non-q grep consumes the whole input,
        # so no early-exit race exists.
        if ! grep -E -- "(^|[[:space:]])${flag}([[:space:]]|=|$)" <<<"$help_text" >/dev/null; then
            err "installed llama.cpp ${version} does not expose ${flag} — remove it or upgrade llama.cpp"
        fi
    done
}

# ── Validate always-emitted default flags (D-F) ────────────────
# validate_default_flags <binary> <version> <requested-flag...> — for every
# flag the tool emits BY DEFAULT (with no explicit user intent), check it is
# supported in --help. Requested flags (CLI or config/env) are excluded — they
# were hard-validated upstream and must be kept. An unsupported DEFAULT is
# omitted from ExecStart with a single warn (never a hard error — the tool
# chose the default, not the user). Readable --help sets globals:
#   DEFAULT_PORT_OK DEFAULT_HOST_OK DEFAULT_GPU_OK DEFAULT_CTX_OK DEFAULT_THREADS_OK
# (all 1 = keep; 0 = omit). Unreadable --help text sets all OK = 1 (warn).
validate_default_flags() {
    local bin="$1" version="$2"
    shift 2
    local requested=("$@")
    local requested_str=" ${requested[*]:-} "

    DEFAULT_PORT_OK=1; DEFAULT_HOST_OK=1; DEFAULT_GPU_OK=1; DEFAULT_CTX_OK=1; DEFAULT_THREADS_OK=1

    local help_text
    help_text="$("$bin" --help 2>/dev/null)" || {
        warn "Cannot obtain llama-server --help output — skipping default flag validation"
        return 0
    }

    local spec flag ok_var
    # flag | ok_var — the always-emitted defaults a user may not have requested.
    local specs=(
        "--port|DEFAULT_PORT_OK"
        "--host|DEFAULT_HOST_OK"
        "--n-gpu-layers|DEFAULT_GPU_OK"
        "--ctx-size|DEFAULT_CTX_OK"
        "--threads|DEFAULT_THREADS_OK"
    )
    for spec in "${specs[@]}"; do
        flag="${spec%%|*}"
        ok_var="${spec#*|}"
        case "$requested_str" in
            *" $flag "*) continue ;;   # requested → already hard-validated, keep
        esac
        # NON-quiet grep (see validate_requested_flags — F2): no pipe, no
        # early-grep-exit, no printf SIGPIPE/rc=141 race under pipefail.
        if grep -E -- "(^|[[:space:]])${flag}([[:space:]]|=|$)" <<<"$help_text" >/dev/null; then
            eval "$ok_var=1"
        else
            warn "installed llama.cpp ${version} does not support default flag ${flag} — omitting it from the unit"
            eval "$ok_var=0"
        fi
    done
}

# ── GPU detection ──────────────────────────────────────────────
detect_gpu() {
    if command -v nvidia-smi &>/dev/null && nvidia-smi &>/dev/null 2>&1; then
        echo "cuda"
    else
        echo "cpu"
    fi
}

resolve_gpu_layers() {
    local configured="${LLAMACPP_GPU_LAYERS:-}"
    if [ -n "$configured" ] && [ "$configured" != "-1" ]; then
        echo "$configured"
        return
    fi
    # Auto-detect
    local gpu
    gpu="$(detect_gpu)"
    case "$gpu" in
        cuda) echo "-1" ;;
        *)    echo "0" ;;
    esac
}

# ── Human-readable size ────────────────────────────────────────
human_size() {
    local bytes="$1"
    if [ "$bytes" -ge 1073741824 ]; then
        awk "BEGIN { printf \"%.1f GB\", $bytes / 1073741824 }"
    elif [ "$bytes" -ge 1048576 ]; then
        awk "BEGIN { printf \"%.1f MB\", $bytes / 1048576 }"
    elif [ "$bytes" -ge 1024 ]; then
        awk "BEGIN { printf \"%.1f KB\", $bytes / 1024 }"
    else
        printf '%d B' "$bytes"
    fi
}

# ── Health check ───────────────────────────────────────────────
check_health() {
    # Probe the SAME host/port the unit binds (HOST/PORT, defaults
    # 127.0.0.1/8088) — previously probes hardcoded 127.0.0.1 and missed a
    # non-localhost LLAMACPP_HOST bind.
    local resp
    resp="$(curl -sf "http://$HOST:$PORT/health" 2>/dev/null)" || { echo "not running"; return 1; }
    local status
    status="$(printf '%s' "$resp" | jq -r '.status // "unknown"' 2>/dev/null)"
    echo "$status"
}

# ── Interactive model picker (reads /dev/tty, not stdin) ───────
pick_model() {
    local models=() i
    while IFS= read -r f; do
        [ -f "$f" ] || continue
        models+=("$f")
    done < <(find "$HF_DOWNLOAD_DIR" -name '*.gguf' -type f 2>/dev/null | sort)

    [ ${#models[@]} -gt 0 ] || err "No GGUF models found — run 'pos ai hf download <repo> --gguf'"

    echo "Available models:"
    for ((i = 0; i < ${#models[@]}; i++)); do
        local name size
        name="$(basename "${models[$i]}")"
        size="$(stat -c%s "${models[$i]}" 2>/dev/null || echo 0)"
        printf '  %2d) %-50s %s\n' "$((i + 1))" "$name" "$(human_size "$size")"
    done
    echo
    local choice
    printf 'Pick a model [1-%d]: ' "${#models[@]}"
    IFS= read -r choice </dev/tty || choice=""
    [[ "$choice" =~ ^[0-9]+$ ]] && [ "$choice" -ge 1 ] && [ "$choice" -le "${#models[@]}" ] || err "Invalid selection"
    printf '%s' "${models[$((choice - 1))]}"
}

# ── Model resolution ───────────────────────────────────────────
# resolve_gguf_in_dir <dir> — resolve a model DIRECTORY to its single top-level
# *.gguf (case-insensitive, no recursion, matching the HF downloader's
# $HF_DOWNLOAD_DIR/<repo-slug>/<file>.gguf layout). Exactly one → print it and
# return 0. Multiple → list every candidate as <dirname>/<file> on stderr and
# err "pick one" — NEVER silently pick. Zero → return 1 (caller falls through
# to the standard "Model not found" error).
resolve_gguf_in_dir() {
    local dir="$1" matches=() f
    while IFS= read -r f; do
        [ -f "$f" ] && matches+=("$f")
    done < <(find "$dir" -maxdepth 1 -type f -iname '*.gguf' 2>/dev/null | sort)
    if [ "${#matches[@]}" -eq 1 ]; then
        printf '%s' "${matches[0]}"
        return 0
    fi
    if [ "${#matches[@]}" -gt 1 ]; then
        local dirname="${dir##*/}"
        for f in "${matches[@]}"; do
            echo "  $dirname/$(basename "$f")" >&2
        done
        err "model dir $dirname contains multiple .gguf files — pick one (e.g. 'pos ai server start $dirname/<file>.gguf')"
    fi
    return 1
}

resolve_model() {
    local explicit="${1:-}"
    # 1. Explicit argument
    if [ -n "$explicit" ]; then
        # Absolute path
        if [[ "$explicit" == /* ]]; then
            [ -e "$explicit" ] || err "Model not found: $explicit"
            if [ -f "$explicit" ]; then
                printf '%s' "$explicit"
                return
            fi
            # Absolute DIRECTORY — same single-.gguf expansion as below.
            local abs_resolved
            if abs_resolved="$(resolve_gguf_in_dir "$explicit")"; then
                printf '%s' "$abs_resolved"
                return
            fi
            err "Model not found: $explicit"
        fi
        # Relative to HF_DOWNLOAD_DIR
        local candidate="$HF_DOWNLOAD_DIR/$explicit"
        if [ -f "$candidate" ]; then
            printf '%s' "$candidate"
            return
        fi
        # Directory under HF_DOWNLOAD_DIR (repo slug form) — the HF downloader
        # writes $HF_DOWNLOAD_DIR/<repo-slug>/<file>.gguf; a single top-level
        # .gguf resolves to it, multiple err with the pick-one list, zero
        # falls through to the not-found error below (F3).
        if [ -d "$candidate" ]; then
            local slug_resolved
            if slug_resolved="$(resolve_gguf_in_dir "$candidate")"; then
                printf '%s' "$slug_resolved"
                return
            fi
        fi
        # Also try with the name as-is (could be a relative path)
        [ -f "$explicit" ] && { printf '%s' "$explicit"; return; }
        err "Model not found: $explicit (also searched $HF_DOWNLOAD_DIR)"
    fi
    # 2. Config
    if [ -n "${LLAMACPP_MODEL:-}" ]; then
        [ -f "$LLAMACPP_MODEL" ] || err "Configured model not found: $LLAMACPP_MODEL"
        printf '%s' "$LLAMACPP_MODEL"
        return
    fi
    # 3. Interactive pick (only on TTY)
    if [ -t 0 ] || [ -w /dev/tty ]; then
        local picked
        picked="$(pick_model)"
        printf '%s' "$picked"
        return
    fi
    err "No model specified and no LLAMACPP_MODEL configured — run 'pos ai server start <model>' or set LLAMACPP_MODEL in ai.env"
}

# ── Usage ──────────────────────────────────────────────────────
usage() {
    cat <<'EOF'
Usage: pos ai server <command> [args]

Manage a local llama.cpp inference server via systemd user service.

Commands:
  start [model]       Start the server (model: argument, config, or interactive pick)
  stop                Stop and disable the server
  status              Show service state, config, and health
  models              List available GGUF files
  logs [lines]        Show recent server logs

Options:
  --port <port>       Server port (default: 8088)
  --host <addr>       Bind address (default: 127.0.0.1)
  --model <path>      Model path (overrides argument and config)
  --ctx <size>        Context window size (default: 4096)
  --gpu <layers>      GPU layers: -1=auto, 0=CPU, N=explicit (default: -1)
  --threads <n>       CPU threads (default: nproc)
  --gpu-layers <n>    GPU layers (overrides --gpu)
  --gpu-threads <n>   GPU threads (default: auto)
  --tensor-split <n>  Tensor split configuration
  --n-gpu-layers <n>  GPU layers (alternative to --gpu)
  --batch-size <n>    Batch size for processing
  --ubatch-size <n>   UBatch size for processing
  --temperature <n>   Sampling temperature (default: 0.8)
  --top-k <n>         Top-K sampling parameter
  --top-p <n>         Top-P sampling parameter
  --repetition-penalty <n> Repetition penalty for sampling
  --mmap              Use memory mapping
  --mlock             Lock memory
  --kv-cache <size>   KV cache size
  --ctx-size <n>      Context window size (alternative to --ctx)
  --metrics           Enable metrics endpoint
  --health            Enable health endpoint
  --slots <n>         Concurrent request slots
  --no-unit           Run the server directly (nohup + pidfile) instead of
                      installing a systemd unit — for headless/SSH boxes whose
                      user systemd bus is unreachable; stop/status still work

Examples:
  pos ai server start mistral-7b-v0.1.Q4_K_M.gguf
  pos ai server start /path/to/model.gguf --port 9090 --gpu 0
  pos ai server status
  pos ai server logs 50
  pos ai server models
  pos ai server stop
  pos ai server start --model model.gguf --gpu-layers 35 --ctx-size 4096 --temperature 0.7
  pos ai server start --model model.gguf --mmap --mlock --batch-size 512

Config (~/.config/linux_post_install/ai.env):
  LLAMACPP_PORT       Server port (default 8088)
  LLAMACPP_HOST       Bind address (default 127.0.0.1)
  LLAMACPP_MODEL      Default model path (GGUF file)
  LLAMACPP_CTX_SIZE   Context window size (default 4096)
  LLAMACPP_GPU_LAYERS GPU layers: -1=auto, 0=CPU only (default -1)
  LLAMACPP_THREADS    CPU threads (default: nproc)

Requires: llama-server binary — install llama.cpp with the app installer: 'apps/ai/llamacpp.sh' (run 'bash apps/install.sh llamacpp', or pass '--apps'/'--full' to install.sh), see 'pos help ai server' (https://github.com/ggerganov/llama.cpp)
EOF
    exit 0
}

# ── Parse flags ────────────────────────────────────────────────
PORT="${LLAMACPP_PORT:-8088}"
HOST="${LLAMACPP_HOST:-127.0.0.1}"
CTX_SIZE="${LLAMACPP_CTX_SIZE:-4096}"
GPU_LAYERS="${LLAMACPP_GPU_LAYERS:--1}"
THREADS="${LLAMACPP_THREADS:-}"
MODEL_ARG=""
SUBCMD=""
SUBCMD_ARGS=()

# New GPU and performance options
GPU_LAYERS_FLAG=""
GPU_THREADS=""
TENSOR_SPLIT=""
BATCH_SIZE=""
UBATCH_SIZE=""
TEMPERATURE=""
TOP_K=""
TOP_P=""
REPETITION_PENALTY=""
MAPPING=""
LOCKING=""
KV_CACHE_SIZE=""
METRICS=""
HEALTH=""
SLOTS=""

# Canonical flag tokens the user explicitly requested (defaults excluded) —
# validated against the installed binary's --help in cmd_start.
REQUESTED_FLAGS=()

while [ $# -gt 0 ]; do
    case "$1" in
        -h|--help) usage ;;
        --port)
            [ $# -ge 2 ] || err "--port requires a value"
            PORT="$2"; REQUESTED_FLAGS+=("--port"); shift 2 ;;
        --host)
            [ $# -ge 2 ] || err "--host requires a value"
            HOST="$2"; REQUESTED_FLAGS+=("--host"); shift 2 ;;
        --model)
            [ $# -ge 2 ] || err "--model requires a value"
            MODEL_ARG="$2"; REQUESTED_FLAGS+=("--model"); shift 2 ;;
        --ctx)
            [ $# -ge 2 ] || err "--ctx requires a value"
            CTX_SIZE="$2"; REQUESTED_FLAGS+=("--ctx-size"); shift 2 ;;
        --gpu)
            [ $# -ge 2 ] || err "--gpu requires a value"
            GPU_LAYERS="$2"; REQUESTED_FLAGS+=("--n-gpu-layers"); shift 2 ;;
        --threads)
            [ $# -ge 2 ] || err "--threads requires a value"
            THREADS="$2"; REQUESTED_FLAGS+=("--threads"); shift 2 ;;
        --gpu-layers)
            [ $# -ge 2 ] || err "--gpu-layers requires a value"
            GPU_LAYERS_FLAG="$2"; REQUESTED_FLAGS+=("--n-gpu-layers"); shift 2 ;;
        --gpu-threads)
            [ $# -ge 2 ] || err "--gpu-threads requires a value"
            GPU_THREADS="$2"; REQUESTED_FLAGS+=("--gpu-threads"); shift 2 ;;
        --tensor-split)
            [ $# -ge 2 ] || err "--tensor-split requires a value"
            TENSOR_SPLIT="$2"; REQUESTED_FLAGS+=("--tensor-split"); shift 2 ;;
        --n-gpu-layers)
            [ $# -ge 2 ] || err "--n-gpu-layers requires a value"
            GPU_LAYERS_FLAG="$2"; REQUESTED_FLAGS+=("--n-gpu-layers"); shift 2 ;;
        --batch-size)
            [ $# -ge 2 ] || err "--batch-size requires a value"
            BATCH_SIZE="$2"; REQUESTED_FLAGS+=("--batch-size"); shift 2 ;;
        --ubatch-size)
            [ $# -ge 2 ] || err "--ubatch-size requires a value"
            UBATCH_SIZE="$2"; REQUESTED_FLAGS+=("--ubatch-size"); shift 2 ;;
        --temperature)
            [ $# -ge 2 ] || err "--temperature requires a value"
            TEMPERATURE="$2"; REQUESTED_FLAGS+=("--temperature"); shift 2 ;;
        --top-k)
            [ $# -ge 2 ] || err "--top-k requires a value"
            TOP_K="$2"; REQUESTED_FLAGS+=("--top-k"); shift 2 ;;
        --top-p)
            [ $# -ge 2 ] || err "--top-p requires a value"
            TOP_P="$2"; REQUESTED_FLAGS+=("--top-p"); shift 2 ;;
        --repetition-penalty)
            [ $# -ge 2 ] || err "--repetition-penalty requires a value"
            REPETITION_PENALTY="$2"; REQUESTED_FLAGS+=("--repetition-penalty"); shift 2 ;;
        --mmap)
            MAPPING="true"; REQUESTED_FLAGS+=("--mmap"); shift ;;
        --mlock)
            LOCKING="true"; REQUESTED_FLAGS+=("--mlock"); shift ;;
        --kv-cache)
            [ $# -ge 2 ] || err "--kv-cache requires a value"
            KV_CACHE_SIZE="$2"; REQUESTED_FLAGS+=("--kv-cache"); shift 2 ;;
        --ctx-size)
            [ $# -ge 2 ] || err "--ctx-size requires a value"
            CTX_SIZE="$2"; REQUESTED_FLAGS+=("--ctx-size"); shift 2 ;;
        --metrics)
            METRICS="true"; REQUESTED_FLAGS+=("--metrics"); shift ;;
        --health)
            HEALTH="true"; REQUESTED_FLAGS+=("--health"); shift ;;
        --slots)
            [ $# -ge 2 ] || err "--slots requires a value"
            SLOTS="$2"; REQUESTED_FLAGS+=("--slots"); shift 2 ;;
        --no-unit)
            NO_UNIT=1; shift ;;
        -*)
            err "Unknown option '$1' (see --help)" ;;
        *)
            if [ -z "$SUBCMD" ]; then
                SUBCMD="$1"
            else
                SUBCMD_ARGS+=("$1")
            fi
            shift ;;
    esac
done

# Apply flag overrides back to config defaults (flags > env > file default)
LLAMACPP_PORT="$PORT"
LLAMACPP_HOST="$HOST"
LLAMACPP_CTX_SIZE="$CTX_SIZE"
LLAMACPP_GPU_LAYERS="$GPU_LAYERS"
if [ -z "$THREADS" ]; then
    THREADS="$(nproc 2>/dev/null || echo 4)"
fi
LLAMACPP_THREADS="$THREADS"

# ── Subcommands ────────────────────────────────────────────────

# systemd_quote <value> — wrap a path in double quotes for systemd's
# ExecStart word-splitting (systemd.service(5)), escaping embedded `"` as
# `\"`. Only tokens that may legally contain spaces need this (binary and
# model path); plain numeric/flag tokens like `--port 8088` stay unquoted.
systemd_quote() {
    local value="$1"
    value="${value//\"/\\\"}"
    printf '"%s"' "$value"
}

cmd_start() {
    # Resolve the llama-server binary
    local llamacpp_bin
    llamacpp_bin="$(find_llamacpp)" || err "llama-server not found — install llama.cpp with the app installer: 'apps/ai/llamacpp.sh' (run 'bash apps/install.sh llamacpp', or pass '--apps'/'--full' to install.sh), see 'pos help ai server' (https://github.com/ggerganov/llama.cpp)"
    local llamacpp_full
    llamacpp_full="$(command -v "$llamacpp_bin")"

    # Detect version (guarded — never crashes; returns "unknown" when
    # unreadable, then basic defaults are used)
    local version
    version="$(detect_llama_version "$llamacpp_bin")"

    # Validate EVERY flag that will appear in ExecStart (D-F):
    #  - Requested flags (CLI OR config/env) → hard error if unsupported.
    #  - Always-emitted defaults → warn + omit if unsupported.
    local all_requested=("${REQUESTED_FLAGS[@]}" "${CONFIG_REQUESTED_FLAGS[@]}")
    local deduped=() flag
    for flag in "${all_requested[@]}"; do
        case " ${deduped[*]:-} " in
            *" $flag "*) continue ;;
        esac
        deduped+=("$flag")
    done
    if [ "${#deduped[@]}" -gt 0 ]; then
        validate_requested_flags "$llamacpp_bin" "$version" "${deduped[@]}"
    fi
    validate_default_flags "$llamacpp_bin" "$version" "${deduped[@]}"

    # Resolve model
    local explicit_model="${SUBCMD_ARGS[0]:-}"
    # Flag --model takes precedence over positional arg
    [ -n "$MODEL_ARG" ] && explicit_model="$MODEL_ARG"
    local model
    model="$(resolve_model "$explicit_model")"

    # Resolve GPU layers
    local gpu_layers
    gpu_layers="$(resolve_gpu_layers)"
    # Use the flag value if provided, otherwise use resolved value
    [ -n "$GPU_LAYERS_FLAG" ] && gpu_layers="$GPU_LAYERS_FLAG"

    # Warn if no GPU detected and auto-detect resolved to CPU
    if [ "$gpu_layers" = "0" ] && [ "${LLAMACPP_GPU_LAYERS:--1}" = "-1" ]; then
        warn "No NVIDIA GPU detected — running in CPU mode"
    fi

    # Check port availability (best-effort)
    if command -v ss &>/dev/null; then
        if ss -tlnp 2>/dev/null | grep -q ":${PORT} "; then
            # Port might be our own old instance — only warn
            warn "Port $PORT may already be in use — check with 'ss -tlnp'"
        fi
    fi

    # Build ONE command line: binary + model + ALL resolved flags. A single
    # string keeps the systemd unit's ExecStart on one line (systemd requires
    # trailing `\` for multi-line continuations) and makes dry-run show
    # exactly what the unit will contain. systemd splits ExecStart on
    # unquoted whitespace, so the binary and the model path — the only tokens
    # that may contain spaces — are systemd_quote()d; plain flag/number
    # tokens stay unquoted.
    # Build the command with ONLY the flags that passed validation. Requested
    # flags (hard-validated) and supported defaults are always emitted; an
    # unsupported DEFAULT is omitted here (validate_default_flags set the
    # DEFAULT_*_OK globals) so the unit never carries an unsupported flag.
    local exec_cmd
    exec_cmd="$(systemd_quote "$llamacpp_full") -m $(systemd_quote "$model")"
    if [ "$DEFAULT_PORT_OK" -eq 1 ]; then exec_cmd+=" --port $PORT"; fi
    if [ "$DEFAULT_HOST_OK" -eq 1 ]; then exec_cmd+=" --host $HOST"; fi
    if [ "$DEFAULT_GPU_OK" -eq 1 ]; then exec_cmd+=" --n-gpu-layers $gpu_layers"; fi
    if [ "$DEFAULT_CTX_OK" -eq 1 ]; then exec_cmd+=" --ctx-size $CTX_SIZE"; fi
    if [ "$DEFAULT_THREADS_OK" -eq 1 ]; then exec_cmd+=" --threads $THREADS"; fi
    if [ -n "$GPU_THREADS" ]; then
        exec_cmd+=" --gpu-threads $GPU_THREADS"
    fi
    if [ -n "$TENSOR_SPLIT" ]; then
        exec_cmd+=" --tensor-split $TENSOR_SPLIT"
    fi
    if [ -n "$BATCH_SIZE" ]; then
        exec_cmd+=" --batch-size $BATCH_SIZE"
    fi
    if [ -n "$UBATCH_SIZE" ]; then
        exec_cmd+=" --ubatch-size $UBATCH_SIZE"
    fi
    if [ -n "$TEMPERATURE" ]; then
        exec_cmd+=" --temperature $TEMPERATURE"
    fi
    if [ -n "$TOP_K" ]; then
        exec_cmd+=" --top-k $TOP_K"
    fi
    if [ -n "$TOP_P" ]; then
        exec_cmd+=" --top-p $TOP_P"
    fi
    if [ -n "$REPETITION_PENALTY" ]; then
        exec_cmd+=" --repetition-penalty $REPETITION_PENALTY"
    fi
    if [ -n "$MAPPING" ]; then
        exec_cmd+=" --mmap"
    fi
    if [ -n "$LOCKING" ]; then
        exec_cmd+=" --mlock"
    fi
    if [ -n "$KV_CACHE_SIZE" ]; then
        exec_cmd+=" --kv-cache $KV_CACHE_SIZE"
    fi
    if [ -n "$METRICS" ]; then
        exec_cmd+=" --metrics"
    fi
    if [ -n "$HEALTH" ]; then
        exec_cmd+=" --health"
    fi
    if [ -n "$SLOTS" ]; then
        exec_cmd+=" --slots $SLOTS"
    fi

    # ── User-bus pre-flight + stale-unit warning (unit path only) ──
    # ensure_user_bus aborts with remediation BEFORE the unit is written, so a
    # broken user bus leaves NO orphaned unit. --no-unit skips the bus check —
    # that is its whole purpose (headless/SSH boxes without a user bus). An
    # existing unit is warned about but never deleted here (removal stays in
    # cmd_stop); it may be stale from a previous failed start.
    if [ "$NO_UNIT" -ne 1 ]; then
        ensure_user_bus
        if [ -f "$USER_SYSTEMD_DIR/$SERVICE" ]; then
            warn "existing unit $USER_SYSTEMD_DIR/$SERVICE will be overwritten — it may be stale from a previous failed start"
        fi
    fi

    if [ "${DRY_RUN:-0}" -eq 1 ]; then
        if [ "$NO_UNIT" -eq 1 ]; then
            log "(dry-run) nohup $exec_cmd >$NO_UNIT_LOG 2>&1 &"
            log "(dry-run) pidfile: $NO_UNIT_PIDFILE"
        else
            log "(dry-run) generate systemd unit $USER_SYSTEMD_DIR/$SERVICE"
            log "(dry-run) ExecStart: $exec_cmd"
            log "(dry-run) systemctl --user daemon-reload && enable --now $SERVICE"
        fi
        return 0
    fi

    # ── Direct run (--no-unit) ──
    # Run the exact ExecStart command directly under nohup with a pidfile
    # (under $XDG_RUNTIME_DIR, else /tmp) so stop/status keep working without
    # a systemd unit. The string is the same one the unit would carry; only
    # the binary and model tokens are quoted, and both are validated paths
    # (command -v / existing file) resolved above, so eval is the faithful
    # way to honor its systemd-style quoting.
    if [ "$NO_UNIT" -eq 1 ]; then
        mkdir -p "$(dirname "$NO_UNIT_PIDFILE")"
        : >"$NO_UNIT_LOG"
        eval "nohup $exec_cmd >'$NO_UNIT_LOG' 2>&1 &"
        local direct_pid=$!
        printf '%s\n' "$direct_pid" >"$NO_UNIT_PIDFILE"
        log "Server starting (direct run, no systemd) — pid $direct_pid, model: $(basename "$model"), port: $PORT"
        log "log: $NO_UNIT_LOG"
        log "stop: kill \$(cat $NO_UNIT_PIDFILE)  (or 'pos ai server stop')"
        return 0
    fi

    # Generate systemd unit — ExecStart is a single line with the full command
    mkdir -p "$USER_SYSTEMD_DIR"
    cat > "$USER_SYSTEMD_DIR/$SERVICE" <<EOF
[Unit]
Description=pos llama.cpp inference server (linux-post-install)
After=network-online.target

[Service]
Type=simple
ExecStart=$exec_cmd
Restart=on-failure
RestartSec=5
TimeoutStopSec=10
KillMode=control-group
EnvironmentFile=-%h/.config/linux_post_install/ai.env

[Install]
WantedBy=default.target
EOF
    chmod 644 "$USER_SYSTEMD_DIR/$SERVICE"

    # Enable and start
    systemctl --user daemon-reload
    systemctl --user enable --now "$SERVICE"

    log "Server starting — model: $(basename "$model"), port: $PORT"

    # Linger warning
    if command -v loginctl >/dev/null 2>&1; then
        if ! loginctl show-user "$(id -un)" 2>/dev/null | grep -q '^Linger=yes'; then
            warn "enable linger so the server survives logout: sudo loginctl enable-linger $(id -un)"
        fi
    fi

    # Health check (wait briefly)
    sleep 2
    local health
    health="$(check_health)" || true
    if [ "$health" != "not running" ]; then
        ok "Server healthy (status: $health)"
    else
        warn "Server may not be ready yet — check with 'pos ai server status'"
    fi
}

cmd_stop() {
    # Direct-run (--no-unit) pidfile first: kill the recorded pid, remove the
    # pidfile. Then fall through to the normal unit path (both can exist if a
    # unit was installed after a direct run).
    local stopped_direct=0
    if [ -f "$NO_UNIT_PIDFILE" ]; then
        stopped_direct=1
        local direct_pid
        direct_pid="$(cat "$NO_UNIT_PIDFILE" 2>/dev/null || true)"
        if [ "${DRY_RUN:-0}" -eq 1 ]; then
            log "(dry-run) kill $direct_pid; rm -f $NO_UNIT_PIDFILE"
        else
            if [[ "$direct_pid" =~ ^[0-9]+$ ]]; then
                kill "$direct_pid" 2>/dev/null || true
            fi
            rm -f "$NO_UNIT_PIDFILE"
            log "llama.cpp server stopped (direct run)"
        fi
    fi
    if [ ! -f "$USER_SYSTEMD_DIR/$SERVICE" ]; then
        if [ "$stopped_direct" -eq 0 ]; then
            warn "No llama.cpp server service installed ($SERVICE)"
        fi
        return 0
    fi
    if [ "${DRY_RUN:-0}" -eq 1 ]; then
        log "(dry-run) systemctl --user disable --now $SERVICE; remove unit"
    else
        systemctl --user disable --now "$SERVICE" 2>/dev/null || true
        rm -f "$USER_SYSTEMD_DIR/$SERVICE"
        systemctl --user daemon-reload
    fi
    log "llama.cpp server stopped and removed"
}

cmd_status() {
    # llama-server must be present for the version probe below — same
    # actionable deps message as `start`
    if ! find_llamacpp >/dev/null 2>&1; then
        err "llama-server not found — install llama.cpp with the app installer: 'apps/ai/llamacpp.sh' (run 'bash apps/install.sh llamacpp', or pass '--apps'/'--full' to install.sh), see 'pos help ai server' (https://github.com/ggerganov/llama.cpp)"
    fi

    # Service state — systemd unit, or the direct-run (--no-unit) pidfile
    local svc_state="stopped"
    if [ -f "$NO_UNIT_PIDFILE" ]; then
        local direct_pid
        direct_pid="$(cat "$NO_UNIT_PIDFILE" 2>/dev/null || true)"
        if [[ "$direct_pid" =~ ^[0-9]+$ ]] && kill -0 "$direct_pid" 2>/dev/null; then
            svc_state="running"
        fi
    fi
    if systemctl --user is-active "$SERVICE" &>/dev/null; then
        svc_state="running"
    fi
    printf 'service:   %s\n' "$svc_state"

    # Model (from health endpoint if running)
    if [ "$svc_state" = "running" ]; then
        local models_resp
        models_resp="$(curl -sf "http://$HOST:$PORT/v1/models" 2>/dev/null)" || true
        local model_id
        model_id="$(printf '%s' "$models_resp" | jq -r '.data[0].id // "unknown"' 2>/dev/null)" || model_id="unknown"
        printf 'model:     %s\n' "$model_id"
    else
        printf 'model:     (not loaded)\n'
    fi

    # Config
    printf 'port:      %s\n' "$PORT"
    printf 'host:      %s\n' "$HOST"

    # GPU
    local gpu_type
    gpu_type="$(detect_gpu)"
    printf 'gpu:       %s (%s layers)\n' "${gpu_type^^}" "$GPU_LAYERS"

    printf 'context:   %s\n' "$CTX_SIZE"
    printf 'threads:   %s\n' "$THREADS"

    # Autostart
    if systemctl --user is-enabled "$SERVICE" &>/dev/null; then
        printf 'autostart: enabled\n'
    else
        printf 'autostart: disabled\n'
    fi

    # Endpoint
    printf 'endpoint:  http://%s:%s\n' "$HOST" "$PORT"

    # Health
    if [ "$svc_state" = "running" ]; then
        local health
        health="$(check_health)" || health="not responding"
        printf 'health:    %s\n' "$health"
    else
        printf 'health:    not running\n'
    fi

    # Version info (probe the resolved binary; "unknown" if unreadable)
    local llamacpp_bin version
    llamacpp_bin="$(find_llamacpp)"
    version="$(detect_llama_version "$llamacpp_bin")"
    printf 'version:   %s\n' "$version"
}

cmd_models() {
    local dir="${HF_DOWNLOAD_DIR}"
    [ -d "$dir" ] || { warn "No models directory — run 'pos ai hf download' first"; return 0; }

    local found=0
    echo "Available GGUF models:"
    while IFS= read -r gguf; do
        [ -f "$gguf" ] || continue
        found=1
        local name size
        name="$(basename "$gguf")"
        local dir_name
        dir_name="$(basename "$(dirname "$gguf")")"
        size="$(stat -c%s "$gguf" 2>/dev/null || echo 0)"
        local hsize
        hsize="$(human_size "$size")"
        printf '  %-50s %s\n' "$dir_name/$name" "$hsize"
    done < <(find "$dir" -name '*.gguf' -type f 2>/dev/null | sort)

    [ "$found" -eq 0 ] && warn "No .gguf files found — download with 'pos ai hf download <repo> --gguf'"
}

cmd_logs() {
    local lines="${SUBCMD_ARGS[0]:-50}"
    [[ "$lines" =~ ^[0-9]+$ ]] || err "lines must be a number"
    journalctl --user -u "$SERVICE" -n "$lines" --no-pager 2>/dev/null || warn "No logs found — server may not have been started"
}

# ── Dispatch ───────────────────────────────────────────────────
case "${SUBCMD:-}" in
    "")       usage ;;
    start)    cmd_start ;;
    stop)     cmd_stop ;;
    status)   cmd_status ;;
    models)   cmd_models ;;
    logs)     cmd_logs ;;
    *)        err "Unknown subcommand '$SUBCMD' (see --help)" ;;
esac
