#!/usr/bin/env bash set -euo pipefail # t-ai-server-validate.sh — F1/F2/F5/F6 regression for `pos ai server`: # F1 detect_llama_version reads llama.cpp's version string from STDERR # (real builds print `version: 0.4.0-dev (build 10822, …)` to stderr), # and falls back to "unknown" (no errexit) when unreadable/missing. # F2 flag validation is pipe-less (no printf|grep -q SIGPIPE/rc=141 race): # validate_default_flags / validate_requested_flags are deterministic — # all 5 defaults kept every run on a real-shaped ~59 KB help, and a # requested flag is always accepted/rejected the same way. # F5 find_llamacpp no longer matches an unrelated bare `server` binary. # F6 --port $PORT (and the 4 other defaults) always appear in ExecStart. # # Strategy: F1/F5/F2-internals call the REAL shipped functions directly # (brace-matching extraction, so production logic is never re-typed); F6 # exercises the full CLI with the real-shaped help fixture. # extract_fn — print one brace-delimited function body. # Used to unit-test a self-contained function that lives in a thick script # whose dispatch we must not trigger. extract_fn() { local file="$1" fn="$2" awk -v fn="$fn" ' BEGIN { found=0; depth=0 } { if (!found && $0 ~ ("^" fn "\\(\\)")) { found=1; depth=0 } if (found) { n_open = gsub(/\{/, "{") n_close = gsub(/\}/, "}") depth = depth + n_open - n_close print if (depth <= 0) exit } } ' "$file" } # ── F1/F2 help fixture: 732-line ~59 KB body carrying the 5 defaults at the # REAL llama.cpp line offsets (7/25/140/503/506) — the shape that used to # trip the printf|grep -q SIGPIPE race under pipefail. build_real_help() { local total=732 i for ((i = 1; i <= total; i++)); do case "$i" in 7) printf '%s\n' ' -t, --threads N' ;; 25) printf '%s\n' ' -c, --ctx-size N' ;; 140) printf '%s\n' ' -ngl, --gpu-layers, --n-gpu-layers N' ;; 503) printf '%s\n' '--host HOST' ;; 506) printf '%s\n' '--port PORT' ;; *) if [ "$i" -le 2 ]; then printf 'usage: llama-server [options]\n' else printf ' --general-option-%04d example option number %d with a longish description\n' "$i" "$i" fi ;; esac done } new_stub() { # new_stub — full-support llama-server stub (stderr version) local dir="$1" help_fixture help_fixture="$(build_real_help)" cat > "$dir/llama-server" <&2 ;; --help) cat <<'EOF_HELP' $help_fixture EOF_HELP ;; *) exit 0 ;; esac STUB chmod +x "$dir/llama-server" } run_test() { source "$ROOT/lib/common.sh" # err/warn/log for the extracted functions local sandbox stubs sandbox="$(mksandbox ai-server-validate)" stubs="$sandbox/stubs" mkdir -p "$stubs" local aid="$ROOT/bin/pos-ai-server" local fn_file="$sandbox/fns.sh" # ═══ F1: detect_llama_version (STDERR capture + "unknown" guard) ═══ extract_fn "$aid" detect_llama_version > "$fn_file" source "$fn_file" cat > "$stubs/f1-stderr" <<'STUB' #!/usr/bin/env bash printf 'version: 0.4.0-dev (build 10822, commit c457e3bf7)\n' >&2 STUB chmod +x "$stubs/f1-stderr" test_run detect_llama_version "$stubs/f1-stderr" check_rc "F1 stderr semver -> rc 0" 0 "$TR_RC" check_eq "F1 stderr semver resolved (not unknown)" "0.4.0" "$TR_OUT" cat > "$stubs/f1-build" <<'STUB' #!/usr/bin/env bash printf 'version: build 10822\n' >&2 STUB chmod +x "$stubs/f1-build" test_run detect_llama_version "$stubs/f1-build" check_rc "F1 build-only -> rc 0" 0 "$TR_RC" check_eq "F1 build 10822 -> 10822" "10822" "$TR_OUT" cat > "$stubs/f1-b" <<'STUB' #!/usr/bin/env bash printf 'version: b10822\n' >&2 STUB chmod +x "$stubs/f1-b" test_run detect_llama_version "$stubs/f1-b" check_eq "F1 b10822 -> b10822" "b10822" "$TR_OUT" # missing binary → "unknown", rc 0, NO errexit. test_run detect_llama_version "$sandbox/does-not-exist-llama-server" check_rc "F1 missing binary -> rc 0 (no errexit)" 0 "$TR_RC" check_eq "F1 missing binary -> unknown" "unknown" "$TR_OUT" # binary exists but --version prints nothing parseable → unknown. printf '#!/usr/bin/env bash\nexit 0\n' > "$stubs/f1-noop" chmod +x "$stubs/f1-noop" test_run detect_llama_version "$stubs/f1-noop" check_eq "F1 unreadable version -> unknown" "unknown" "$TR_OUT" # ═══ F5: find_llamacpp candidate narrowing ═══ extract_fn "$aid" find_llamacpp > "$fn_file" source "$fn_file" local onlyserver="$sandbox/onlyserver" mkdir -p "$onlyserver" ln -s /usr/bin/false "$onlyserver/server" ( PATH="$onlyserver:/usr/bin:/bin" test_run find_llamacpp printf '%s' "$TR_RC" > "$sandbox/f5.rc" printf '%s' "$TR_OUT" > "$sandbox/f5.out" ) check_rc "F5 bare 'server' alone NOT found (rc != 0)" 1 "$(cat "$sandbox/f5.rc")" check_eq "F5 'server' candidate yields nothing" "" "$(cat "$sandbox/f5.out")" # llama-server IS a legitimate candidate; found first (rc 0 name). local havebin="$sandbox/havebin" mkdir -p "$havebin" new_stub "$havebin" ( PATH="$havebin:/usr/bin:/bin" test_run find_llamacpp printf '%s' "$TR_RC" > "$sandbox/f5b.rc" printf '%s' "$TR_OUT" > "$sandbox/f5b.out" ) check_rc "F5 llama-server present -> found (rc 0)" 0 "$(cat "$sandbox/f5b.rc")" check_eq "F5 returns llama-server (not server)" "llama-server" "$(cat "$sandbox/f5b.out")" # ═══ F2: deterministic flag validation (pipe-less, no rc=141) ═══ # Extract both validation functions; they use err/warn from common.sh. { extract_fn "$aid" validate_requested_flags extract_fn "$aid" validate_default_flags } > "$fn_file" local com="$ROOT/lib/common.sh" # Default-flag determinism: 25 runs over the real-shaped help; every run # must keep all 5 defaults (the pre-fix SIGPIPE race dropped a random # subset per run). Run in a subshell sourcing the exact function bodies + # common.sh, capturing the globals it sets. local run stable=1 gvals run_rc for run in $(seq 1 25); do gvals="$(bash -c " source '$com' source '$fn_file' validate_default_flags '$havebin/llama-server' 0.4.0 '' printf '%s|%s|%s|%s|%s' \"\${DEFAULT_PORT_OK}\" \"\${DEFAULT_HOST_OK}\" \\ \"\${DEFAULT_GPU_OK}\" \"\${DEFAULT_CTX_OK}\" \"\${DEFAULT_THREADS_OK}\" " 2>/dev/null)" run_rc=$? if [ "$run_rc" -ne 0 ]; then stable=0 printf ' FAIL F2 default-flag validation run %s: whole invocation rc=%s (possible rc=141 SIGPIPE component)\n' "$run" "$run_rc" break fi if [ "$gvals" != "1|1|1|1|1" ]; then stable=0 printf ' FAIL F2 run %s dropped a default (globals=%s)\n' "$run" "$gvals" break fi done [ "$stable" -eq 1 ] && printf ' PASS F2 validate_default_flags keeps all 5 defaults across 25 pipefail runs (whole-invocation rc=0 every run)\n' # Requested-flag determinism (20+ runs each): supported always rc 0, # unsupported always hard-errors (never a flaky pass). local ok=0 bad=0 for run in $(seq 1 20); do if bash -c "source '$com'; source '$fn_file'; validate_requested_flags '$havebin/llama-server' 0.4.0 --port" 2>/dev/null; then ok=$((ok + 1)) fi if bash -c "source '$com'; source '$fn_file'; validate_requested_flags '$havebin/llama-server' 0.4.0 --tensor-split 1" 2>/dev/null; then bad=$((bad + 1)) fi done check_eq "F2 requested supported --port accepted every run" 20 "$ok" check_eq "F2 no flaky pass on unsupported requested flag" 0 "$bad" test_run bash -c "source '$com'; source '$fn_file'; validate_requested_flags '$havebin/llama-server' 0.4.0 --tensor-split 1" check_rc "F2 unsupported requested flag exits 1" 1 "$TR_RC" check_contains "F2 unsupported requested flag names it" "--tensor-split" "$TR_OUT" check_contains "F2 unsupported requested flag is version-aware" "0.4.0" "$TR_OUT" # ═══ F6: full CLI — port pinning + version not "unknown" ═══ local models="$sandbox/models" mkdir -p "$models" touch "$models/my-model.gguf" printf '#!/usr/bin/env bash\nexit 1\n' > "$stubs/nvidia-smi" printf '#!/usr/bin/env bash\nexit 0\n' > "$stubs/systemctl" chmod +x "$stubs/nvidia-smi" "$stubs/systemctl" local server="$ROOT/bin/pos-ai-server" local base_env=(PATH="$havebin:$stubs:/usr/bin:/bin" DRY_RUN=1 USER_SYSTEMD_DIR="$sandbox/userunits") test_run_env "${base_env[@]}" -- "$server" start "$models/my-model.gguf" check_rc "F6 DRY_RUN start exits 0" 0 "$TR_RC" local exec_line exec_line="$(printf '%s\n' "$TR_OUT" | grep 'dry-run) ExecStart:' | sed 's/.*ExecStart: //')" check_contains "F6 --port 8088 pinned in ExecStart" "--port 8088" "$exec_line" check_contains "F6 --host present" "--host 127.0.0.1" "$exec_line" check_contains "F6 --n-gpu-layers present" "--n-gpu-layers 0" "$exec_line" check_contains "F6 --ctx-size present" "--ctx-size 4096" "$exec_line" check_contains "F6 --threads present" "--threads" "$exec_line" # Version via the same b10822-shaped stub: status shows 0.4.0, never # "unknown" (F1 proven through the real CLI on a real-shaped build). test_run_env "${base_env[@]}" -- "$server" status check_rc "F1 CLI status exits 0" 0 "$TR_RC" check_contains "F1 CLI status resolves version 0.4.0" "version: 0.4.0" "$TR_OUT" check_not_contains "F1 CLI status version is NOT unknown" "version: unknown" "$TR_OUT" }