#!/usr/bin/env bash set -euo pipefail # t-ai-server-bus.sh — F4 + E2E regression for the user systemd-bus pre-flight. # # F4 contract: # - `pos ai server start` (unit path) calls ensure_user_bus BEFORE the unit # is written; with no user bus (SSH/headless: XDG_RUNTIME_DIR and # DBUS_SESSION_BUS_ADDRESS unset, systemctl --user fails) it errs with BOTH # remediation lines and leaves NO orphaned unit file. # - with a reachable bus the command proceeds past the pre-flight. # - `--no-unit` is the bus-less escape hatch: it skips the pre-flight, runs # llama-server directly under nohup with a pidfile, and stop/status can # find it via the pidfile. # # E2E regression (the reported user scenario): # - install-shaped stubs + working bus: `start Qwen-Qwen3-1.7B-GGUF` resolves # the model dir (F3), writes a correct unit (F6: all 5 flags + --port # 8088), reports a resolvable version (F1: 0.4.0, not "unknown"), and the # unit passes `systemd-analyze verify`. # - SSH-shaped no-bus env: clean pre-flight err, remediation lines, NO unit. run_test() { local sandbox stubs models unitdir runtime sandbox="$(mksandbox ai-server-bus)" stubs="$sandbox/stubs" models="$sandbox/models" unitdir="$sandbox/userunits" runtime="$sandbox/runtime" mkdir -p "$stubs" "$models" "$unitdir" "$runtime" # ── User's real HF layout (F3 fixture) ── mkdir -p "$models/Qwen-Qwen3-1.7B-GGUF" touch "$models/Qwen-Qwen3-1.7B-GGUF/Qwen3-1.7B-Q8_0.gguf" # ── Stubs ── # llama-server: b10822-shaped (version → stderr, full-support help). cat > "$stubs/llama-server" <<'STUB' #!/usr/bin/env bash case "$1" in --version) printf 'version: 0.4.0-dev (build 10822, commit c457e3bf7)\n' >&2 ;; --help) cat <<'HELP' usage: llama-server [options] options: --host bind address --port server port --n-gpu-layers layers to offload --ctx-size context size --threads cpu threads HELP ;; *) exit 0 ;; esac STUB # systemctl: FAILS exactly like the real bus-missing case (SSH). cat > "$stubs/systemctl" <<'STUB' #!/usr/bin/env bash echo "Failed to connect to user scope bus via local transport: \$DBUS_SESSION_BUS_ADDRESS and \$XDG_RUNTIME_DIR not defined" >&2 exit 1 STUB # nvidia-smi: no GPU → deterministic CPU path. printf '#!/usr/bin/env bash\nexit 1\n' > "$stubs/nvidia-smi" # curl: healthy JSON (health + models probes). printf '#!/usr/bin/env bash\nprintf "%%s" '"'"'{"status":"ok"}'"'"'\n' > "$stubs/curl" # loginctl: Linger=no (so the linger warn path is deterministic). printf '#!/usr/bin/env bash\nprintf "Linger=no\\n"\n' > "$stubs/loginctl" chmod +x "$stubs/llama-server" "$stubs/systemctl" "$stubs/nvidia-smi" "$stubs/curl" "$stubs/loginctl" local server="$ROOT/bin/pos-ai-server" local nobus_env=(-u XDG_RUNTIME_DIR -u DBUS_SESSION_BUS_ADDRESS PATH="$stubs:/usr/bin:/bin" HF_DOWNLOAD_DIR="$models" USER_SYSTEMD_DIR="$unitdir" NO_UNIT_PIDFILE="$sandbox/pos-ai-server.pid" NO_UNIT_LOG="$sandbox/pos-ai-server.log") local bus_env=(-u XDG_RUNTIME_DIR -u DBUS_SESSION_BUS_ADDRESS PATH="$stubs:/usr/bin:/bin" HF_DOWNLOAD_DIR="$models" USER_SYSTEMD_DIR="$unitdir" XDG_RUNTIME_DIR="$runtime" DBUS_SESSION_BUS_ADDRESS="unix:path=$runtime/bus" NO_UNIT_PIDFILE="$sandbox/pos-ai-server.pid" NO_UNIT_LOG="$sandbox/pos-ai-server.log") # ═══ F4 pre-flight: bus missing → err + remediation + NO unit ═══ test_run_env "${nobus_env[@]}" -- timeout 30 "$server" start Qwen-Qwen3-1.7B-GGUF check_contains "F4 bus-missing start names the bus problem" "user systemd bus" "$TR_OUT" check_contains "F4 remediation: export XDG_RUNTIME_DIR" "export XDG_RUNTIME_DIR=/run/user/" "$TR_OUT" check_contains "F4 remediation: sudo loginctl enable-linger" "sudo loginctl enable-linger" "$TR_OUT" check_contains "F4 states no unit was written" "no unit was written" "$TR_OUT" [ "$TR_RC" -ne 0 ] && printf ' PASS F4 bus-missing start exits nonzero\n' || printf ' FAIL F4 bus-missing start exited 0\n' check_file_absent "F4 no orphaned unit written" "$unitdir/pos-ai-server.service" # ═══ F4 pre-flight: bus present → proceeds past pre-flight (DRY_RUN) ═══ # Replace the failing systemctl with a succeeding one. printf '#!/usr/bin/env bash\nexit 0\n' > "$stubs/systemctl" chmod +x "$stubs/systemctl" test_run_env "${bus_env[@]}" DRY_RUN=1 -- timeout 30 "$server" start Qwen-Qwen3-1.7B-GGUF check_rc "F4 bus-present DRY_RUN start exits 0" 0 "$TR_RC" check_contains "F4 bus-present proceeds past pre-flight (ExecStart)" "ExecStart:" "$TR_OUT" # ═══ F4 --no-unit escape hatch (bus-less direct run) ═══ # A real --no-unit start must skip the bus check and launch the stub under # nohup with a pidfile; status finds it; stop kills it. The stub sleeps via # `exec sleep` so the recorded pid IS the sleeper (no orphan children). cat > "$stubs/llama-server" <<'STUB' #!/usr/bin/env bash case "$1" in --version) printf 'version: 0.4.0-dev (build 10822, commit c457e3bf7)\n' >&2 ;; --help) cat <<'HELP' usage: llama-server [options] options: --host bind address --port server port --n-gpu-layers layers to offload --ctx-size context size --threads cpu threads HELP ;; *) exec sleep 300 ;; esac STUB chmod +x "$stubs/llama-server" # failing systemctl stub again: --no-unit must NOT care. cat > "$stubs/systemctl" <<'STUB' #!/usr/bin/env bash echo "no bus" >&2 exit 1 STUB chmod +x "$stubs/systemctl" test_run_env "${nobus_env[@]}" -- timeout 30 "$server" start Qwen-Qwen3-1.7B-GGUF --no-unit check_rc "F4 --no-unit bus-less start exits 0" 0 "$TR_RC" check_file_exists "F4 --no-unit writes pidfile" "$sandbox/pos-ai-server.pid" check_contains "F4 --no-unit prints log path hint" "log:" "$TR_OUT" local direct_pid direct_pid="$(cat "$sandbox/pos-ai-server.pid" 2>/dev/null || true)" if [[ "$direct_pid" =~ ^[0-9]+$ ]] && kill -0 "$direct_pid" 2>/dev/null; then printf ' PASS F4 --no-unit launched a live server process (pid %s)\n' "$direct_pid" else printf ' FAIL F4 --no-unit pid %s not alive\n' "${direct_pid:-EMPTY}" fi # status via the pidfile reports running (deps + curl stubs present). test_run_env "${nobus_env[@]}" -- timeout 30 "$server" status check_rc "F4 status exits 0 after --no-unit start" 0 "$TR_RC" check_contains "F4 status finds direct-run server running" "service: running" "$TR_OUT" # stop kills the pid and removes the pidfile. test_run_env "${nobus_env[@]}" -- timeout 30 "$server" stop check_rc "F4 --no-unit stop exits 0" 0 "$TR_RC" check_file_absent "F4 stop removes pidfile" "$sandbox/pos-ai-server.pid" if [[ "$direct_pid" =~ ^[0-9]+$ ]] && ! kill -0 "$direct_pid" 2>/dev/null; then printf ' PASS F4 stop killed the direct-run process\n' else printf ' FAIL F4 direct-run process still alive after stop\n' fi # ═══ E2E positive: user scenario with a working bus ═══ # Restore full-support stub + succeeding systemctl. cat > "$stubs/llama-server" <<'STUB' #!/usr/bin/env bash case "$1" in --version) printf 'version: 0.4.0-dev (build 10822, commit c457e3bf7)\n' >&2 ;; --help) cat <<'HELP' usage: llama-server [options] options: --host bind address --port server port --n-gpu-layers layers to offload --ctx-size context size --threads cpu threads HELP ;; *) exit 0 ;; esac STUB printf '#!/usr/bin/env bash\nexit 0\n' > "$stubs/systemctl" chmod +x "$stubs/llama-server" "$stubs/systemctl" test_run_env "${bus_env[@]}" -- timeout 60 "$server" start Qwen-Qwen3-1.7B-GGUF check_rc "E2E user scenario start exits 0" 0 "$TR_RC" check_file_exists "E2E unit written" "$unitdir/pos-ai-server.service" local unit="$unitdir/pos-ai-server.service" local unit_exec unit_exec="$(grep '^ExecStart=' "$unit")" check_contains "E2E unit: model expanded from dir" "Qwen3-1.7B-Q8_0.gguf" "$unit_exec" check_contains "E2E unit: --port 8088 pinned" "--port 8088" "$unit_exec" check_contains "E2E unit: --host present" "--host 127.0.0.1" "$unit_exec" check_contains "E2E unit: --n-gpu-layers present" "--n-gpu-layers 0" "$unit_exec" check_contains "E2E unit: --ctx-size present" "--ctx-size 4096" "$unit_exec" check_contains "E2E unit: --threads present" "--threads" "$unit_exec" # Version resolvable through the real CLI on the same run's stub. test_run_env "${bus_env[@]}" -- timeout 30 "$server" status check_contains "E2E version is resolvable (0.4.0)" "version: 0.4.0" "$TR_OUT" check_not_contains "E2E version is NOT unknown" "version: unknown" "$TR_OUT" # Unit must pass systemd-analyze verify. if command -v systemd-analyze >/dev/null 2>&1; then test_run systemd-analyze verify "$unit" check_rc "E2E systemd-analyze verify accepts unit" 0 "$TR_RC" else skip_case "E2E systemd-analyze verify" "systemd-analyze not available" fi # stop cleans up (unit removed). test_run_env "${bus_env[@]}" -- timeout 30 "$server" stop check_rc "E2E stop exits 0" 0 "$TR_RC" check_file_absent "E2E stop removes unit" "$unitdir/pos-ai-server.service" # ═══ E2E SSH-shaped: no bus → clean pre-flight err, remediation, NO unit ═══ cat > "$stubs/systemctl" <<'STUB' #!/usr/bin/env bash echo "Failed to connect to user scope bus via local transport: \$DBUS_SESSION_BUS_ADDRESS and \$XDG_RUNTIME_DIR not defined" >&2 exit 1 STUB chmod +x "$stubs/systemctl" test_run_env "${nobus_env[@]}" -- timeout 30 "$server" start Qwen-Qwen3-1.7B-GGUF check_rc "E2E SSH-shaped start exits nonzero" 1 "$TR_RC" check_contains "E2E SSH-shaped: bus problem named" "user systemd bus" "$TR_OUT" check_contains "E2E SSH-shaped: export XDG_RUNTIME_DIR remediation" "export XDG_RUNTIME_DIR=/run/user/" "$TR_OUT" check_contains "E2E SSH-shaped: loginctl enable-linger remediation" "sudo loginctl enable-linger" "$TR_OUT" check_contains "E2E SSH-shaped: no unit was written" "no unit was written" "$TR_OUT" check_file_absent "E2E SSH-shaped: no orphaned unit" "$unitdir/pos-ai-server.service" }