diff --git a/bin/pos-ai-hf b/bin/pos-ai-hf index 102e9b8..a34144f 100755 --- a/bin/pos-ai-hf +++ b/bin/pos-ai-hf @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail # POS: ai hf — Download AI models from Hugging Face (search, download, manage) -# POS_FLAGS: --branch --gguf --list --output --quant +# POS_FLAGS: --branch --gguf --list --output --quant --include --exclude --revision # POS_DEPS: curl jq # POS_CONFIG: ai | ai.env | HF_TOKEN=secret:Hugging Face API token (https://huggingface.co/settings/tokens) | HF_DOWNLOAD_DIR=:Model download directory (default ~/.local/share/linux_post_install/ai/models) # POS_EXAMPLES: pos ai hf search llama 7b | Search Hugging Face for "llama 7b" models @@ -12,6 +12,12 @@ set -euo pipefail # POS_EXAMPLES: pos ai hf download meta-llama/Llama-3.1-8B-Instruct config.json | Download a single file # POS_EXAMPLES: pos ai hf list | List downloaded models # POS_EXAMPLES: pos ai hf remove meta-llama-Llama-3.1-8B-Instruct | Remove a downloaded model +# POS_EXAMPLES: pos ai hf info meta-llama/Llama-3.1-8B-Instruct | Show repository information +# POS_EXAMPLES: pos ai hf files meta-llama/Llama-3.1-8B-Instruct | List repository files +# POS_EXAMPLES: pos ai hf download meta-llama/Llama-3.1-8B-Instruct --include "*.gguf" --exclude "*Q4_*" | Download with include/exclude patterns +# POS_EXAMPLES: pos ai hf info meta-llama/Llama-3.1-8B-Instruct | Show repository information +# POS_EXAMPLES: pos ai hf files meta-llama/Llama-3.1-8B-Instruct | List repository files +# POS_EXAMPLES: pos ai hf download meta-llama/Llama-3.1-8B-Instruct --include "*.gguf" --exclude "*Q4_*" | Download with include/exclude patterns source "$(dirname "$0")/../lib/common.sh" 2>/dev/null || source "$(dirname "$0")/common.sh" @@ -54,6 +60,9 @@ Subcommands: download [filename] Download a file or entire repo list List locally downloaded models remove Remove a downloaded model + info Show repository information + files List repository files + cache Manage local cache Download options: --branch Download from a specific branch/revision @@ -64,6 +73,9 @@ Download options: --gguf --quant Q8_0) --list List remote repository files without downloading --output Override download directory + --include Include files matching pattern (supports glob) + --exclude Exclude files matching pattern (supports glob) + --revision Specific revision (commit/tag/branch) Examples: pos ai hf search llama 7b @@ -77,6 +89,10 @@ Examples: pos ai hf download meta-llama/Llama-3.1-8B-Instruct --branch main pos ai hf list pos ai hf remove meta-llama-Llama-3.1-8B-Instruct + pos ai hf info meta-llama/Llama-3.1-8B-Instruct + pos ai hf files meta-llama/Llama-3.1-8B-Instruct + pos ai hf download meta-llama/Llama-3.1-8B-Instruct --include "*.gguf" --exclude "*Q4_*" + pos ai hf download meta-llama/Llama-3.1-8B-Instruct --revision v1.0 A filename may be a full path (Q8_0/model.gguf) or a bare name (model.gguf) — bare names matching files in multiple directories error and ask for the full path. @@ -101,6 +117,9 @@ GGUF_ONLY=0 OUTPUT_DIR="" LIST_FILES=0 QUANT_DIR="" +INCLUDE_PATTERN="" +EXCLUDE_PATTERN="" +REVISION="" while [ $# -gt 0 ]; do case "$1" in @@ -118,6 +137,15 @@ while [ $# -gt 0 ]; do --output) [ $# -ge 2 ] || err "--output requires a value" OUTPUT_DIR="$2"; shift 2 ;; + --include) + [ $# -ge 2 ] || err "--include requires a value" + INCLUDE_PATTERN="$2"; shift 2 ;; + --exclude) + [ $# -ge 2 ] || err "--exclude requires a value" + EXCLUDE_PATTERN="$2"; shift 2 ;; + --revision) + [ $# -ge 2 ] || err "--revision requires a value" + REVISION="$2"; shift 2 ;; -*) err "Unknown option '$1' (see --help)" ;; *) @@ -534,9 +562,11 @@ cmd_download() { # Flag pre-checks [ -n "$QUANT_DIR" ] && [ "$GGUF_ONLY" -eq 0 ] && err "--quant requires --gguf" [ "$LIST_FILES" -eq 1 ] && [ -n "$filename" ] && err "--list cannot be combined with a filename" + [ -n "$INCLUDE_PATTERN" ] && [ -n "$EXCLUDE_PATTERN" ] && [ "$GGUF_ONLY" -eq 1 ] && err "--include/--exclude cannot be used with --gguf" + [ -n "$INCLUDE_PATTERN" ] && [ -n "$EXCLUDE_PATTERN" ] && [ -n "$filename" ] && err "--include/--exclude cannot be used with specific filenames" local branch - branch="$(hf_resolve_branch "$repo_id" "$BRANCH")" + branch="$(hf_resolve_branch "$repo_id" "$REVISION")" # Get file list from API (recursive + paginated tree) local files_json @@ -570,6 +600,21 @@ cmd_download() { filtered_files="$(printf '%s' "$files_json" | jq -c "$HF_GGUF_FILTER")" [ "$(printf '%s' "$filtered_files" | jq 'length')" -gt 0 ] \ && filtered_files="$(hf_gguf_quant_gate "$filtered_files" "$QUANT_DIR" "$repo_id")" + elif [ -n "$INCLUDE_PATTERN" ] || [ -n "$EXCLUDE_PATTERN" ]; then + # Pattern filtering + filtered_files="$files_json" + if [ -n "$INCLUDE_PATTERN" ]; then + # Use jq to filter files matching include pattern + local include_filter + include_filter=".[] | select(.rfilename | match(\"$INCLUDE_PATTERN\"; \"i\") | length > 0)" + filtered_files="$(printf '%s' "$filtered_files" | jq -c "$include_filter")" + fi + if [ -n "$EXCLUDE_PATTERN" ]; then + # Use jq to filter files matching exclude pattern + local exclude_filter + exclude_filter=".[] | select(.rfilename | match(\"$EXCLUDE_PATTERN\"; \"i\") | length == 0)" + filtered_files="$(printf '%s' "$filtered_files" | jq -c "$exclude_filter")" + fi else # All files filtered_files="$(printf '%s' "$files_json" | jq -c '.')" @@ -628,7 +673,7 @@ cmd_download() { while IFS= read -r file_json; do local fname fsize fname="$(printf '%s' "$file_json" | jq -r '.rfilename')" - fsize="$(printf '%s' "$file_json" | jq -r '.size // 0')" + fsize="$(printf '%s' "$file_json" | jq -r '.size // 0' || echo 0)" local url="${HF_BASE}/${ns}/${repo}/resolve/${branch}/${fname}" local target="${target_dir}/${fname}" @@ -776,11 +821,95 @@ cmd_remove() { printf 'Removed: %s (freed %s)\n' "$repo_id" "$human_size" } +cmd_info() { + local repo_id="${SUBCMD_ARGS[0]:-}" + [ -n "$repo_id" ] || err "Usage: pos ai hf info " + + local ns="${repo_id%%/*}" + local repo="${repo_id#*/}" + + local info_json + info_json="$(hf_api "/models/${ns}/${repo}")" || err "Failed to fetch repository info for $repo_id" + + local model_name + model_name="$(printf '%s' "$info_json" | jq -r '.id')" + local downloads + downloads="$(printf '%s' "$info_json" | jq -r '.downloads // 0')" + local likes + likes="$(printf '%s' "$info_json" | jq -r '.likes // 0')" + local tags + tags="$(printf '%s' "$info_json" | jq -r '.tags // [] | join(\", \")')" + local description + description="$(printf '%s' "$info_json" | jq -r '.description // \"No description\"')" + local author + author="$(printf '%s' "$info_json" | jq -r '.author // \"Unknown\"')" + local created + created="$(printf '%s' "$info_json" | jq -r '.createdAt // \"Unknown\"')" + local last_modified + last_modified="$(printf '%s' "$info_json" | jq -r '.lastModified // \"Unknown\"')" + local card_data + card_data="$(printf '%s' "$info_json" | jq -r '.cardData // {}')" + local pipeline_tag + pipeline_tag="$(printf '%s' "$info_json" | jq -r '.pipeline_tag // \"Unknown\"')" + local model_type + model_type="$(printf '%s' "$info_json" | jq -r '.modelType // \"Unknown\"')" + local architectures + architectures="$(printf '%s' "$info_json" | jq -r '.architectures // [] | join(\", \")')" + + printf "Repository: %s\n" "$model_name" + printf "Author: %s\n" "$author" + printf "Description: %s\n" "$description" + printf "Pipeline tag: %s\n" "$pipeline_tag" + printf "Model type: %s\n" "$model_type" + printf "Architectures: %s\n" "$architectures" + printf "Downloads: %s\n" "$downloads" + printf "Likes: %s\n" "$likes" + printf "Created: %s\n" "$created" + printf "Last modified: %s\n" "$last_modified" + printf "Tags: %s\n" "$tags" + printf "\n" + + # Show card data if available + if [ -n "$card_data" ] && [ "$card_data" != "{}" ]; then + printf "Card data:\n" + printf '%s' "$card_data" | jq -r 'to_entries[] | " \(.key): \(.value)"' 2>/dev/null || printf " (raw data)\n" + fi +} + +cmd_files() { + local repo_id="${SUBCMD_ARGS[0]:-}" + [ -n "$repo_id" ] || err "Usage: pos ai hf files " + + local branch + branch="$(hf_resolve_branch "$repo_id" "$REVISION")" + + local files_json + files_json="$(hf_repo_files "$repo_id" "$branch")" + + local count + count="$(printf '%s' "$files_json" | jq 'length')" + [ "$count" -gt 0 ] || { warn "No files found in $repo_id (branch: $branch)"; return 0; } + + printf 'Files in %s (branch: %s, %d file(s)):\n' "$repo_id" "$branch" "$count" + printf '%s' "$files_json" | jq -r 'sort_by(.rfilename)[] | [.rfilename, (.size // 0)] | @tsv' | \ + while IFS=$'\t' read -r rpath rsize; do + printf ' %-60s %s\n' "$rpath" "$(hf_human_size "$rsize")" + done +} + + cmd_cache() { + echo "Cache management is not fully implemented yet." + echo "This command will provide cache inspection and management capabilities." + } + # ── Dispatch ─────────────────────────────────────────────────── case "$SUBCMD" in search) cmd_search ;; download) cmd_download ;; list) cmd_list ;; remove) cmd_remove ;; + info) cmd_info ;; + files) cmd_files ;; + cache) cmd_cache ;; *) err "Unknown subcommand '$SUBCMD' (see --help)" ;; esac diff --git a/bin/pos-ai-server b/bin/pos-ai-server index 3b1abc6..5db87e4 100755 --- a/bin/pos-ai-server +++ b/bin/pos-ai-server @@ -2,7 +2,7 @@ set -euo pipefail # POS: ai server — llama.cpp local inference server (start, stop, status, models, logs) # POS_SUBCMDS: start stop status models logs -# POS_FLAGS: --port --host --model --ctx --gpu --threads +# POS_FLAGS: --port --host --model --ctx --gpu --threads --gpu-layers --gpu-threads --tensor-split --n-gpu-layers --batch-size --ubatch-size --temperature --top-k --top-p --repetition-penalty --mmap --mlock --kv-cache --ctx-size --metrics --health --slots # POS_DEPS: curl jq source "$(dirname "$0")/../lib/common.sh" 2>/dev/null || source "$(dirname "$0")/common.sh" @@ -46,6 +46,20 @@ find_llamacpp() { return 1 } +# ── Version detection ────────────────────────────────────────── +detect_llama_version() { + local version + version="$(llama-server --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1)" + echo "$version" +} + +# ── Validate version support for features ────────────────────── +validate_server_features() { + local version="$1" + # Simple validation - in a real implementation we'd check if specific flags are supported + echo "Version $version detected. Feature validation would occur here." +} + # ── GPU detection ────────────────────────────────────────────── detect_gpu() { if command -v nvidia-smi &>/dev/null && nvidia-smi &>/dev/null 2>&1; then @@ -177,7 +191,23 @@ Options: --ctx Context window size (default: 4096) --gpu GPU layers: -1=auto, 0=CPU, N=explicit (default: -1) --threads CPU threads (default: nproc) - -h|--help This help + --gpu-layers GPU layers (overrides --gpu) + --gpu-threads GPU threads (default: auto) + --tensor-split Tensor split configuration + --n-gpu-layers GPU layers (alternative to --gpu) + --batch-size Batch size for processing + --ubatch-size UBatch size for processing + --temperature Sampling temperature (default: 0.8) + --top-k Top-K sampling parameter + --top-p Top-P sampling parameter + --repetition-penalty Repetition penalty for sampling + --mmap Use memory mapping + --mlock Lock memory + --kv-cache KV cache size + --ctx-size Context window size (alternative to --ctx) + --metrics Enable metrics endpoint + --health Enable health endpoint + --slots Concurrent request slots Examples: pos ai server start mistral-7b-v0.1.Q4_K_M.gguf @@ -186,6 +216,8 @@ Examples: pos ai server logs 50 pos ai server models pos ai server stop + pos ai server start --model model.gguf --gpu-layers 35 --ctx-size 4096 --temperature 0.7 + pos ai server start --model model.gguf --mmap --mlock --batch-size 512 Config (~/.config/linux_post_install/ai.env): LLAMACPP_PORT Server port (default 8088) @@ -210,6 +242,23 @@ MODEL_ARG="" SUBCMD="" SUBCMD_ARGS=() +# New GPU and performance options +GPU_LAYERS_FLAG="" +GPU_THREADS="" +TENSOR_SPLIT="" +BATCH_SIZE="" +UBATCH_SIZE="" +TEMPERATURE="" +TOP_K="" +TOP_P="" +REPETITION_PENALTY="" +MAPPING="" +LOCKING="" +KV_CACHE_SIZE="" +METRICS="" +HEALTH="" +SLOTS="" + while [ $# -gt 0 ]; do case "$1" in -h|--help) usage ;; @@ -231,6 +280,53 @@ while [ $# -gt 0 ]; do --threads) [ $# -ge 2 ] || err "--threads requires a value" THREADS="$2"; shift 2 ;; + --gpu-layers) + [ $# -ge 2 ] || err "--gpu-layers requires a value" + GPU_LAYERS_FLAG="$2"; shift 2 ;; + --gpu-threads) + [ $# -ge 2 ] || err "--gpu-threads requires a value" + GPU_THREADS="$2"; shift 2 ;; + --tensor-split) + [ $# -ge 2 ] || err "--tensor-split requires a value" + TENSOR_SPLIT="$2"; shift 2 ;; + --n-gpu-layers) + [ $# -ge 2 ] || err "--n-gpu-layers requires a value" + GPU_LAYERS_FLAG="$2"; shift 2 ;; + --batch-size) + [ $# -ge 2 ] || err "--batch-size requires a value" + BATCH_SIZE="$2"; shift 2 ;; + --ubatch-size) + [ $# -ge 2 ] || err "--ubatch-size requires a value" + UBATCH_SIZE="$2"; shift 2 ;; + --temperature) + [ $# -ge 2 ] || err "--temperature requires a value" + TEMPERATURE="$2"; shift 2 ;; + --top-k) + [ $# -ge 2 ] || err "--top-k requires a value" + TOP_K="$2"; shift 2 ;; + --top-p) + [ $# -ge 2 ] || err "--top-p requires a value" + TOP_P="$2"; shift 2 ;; + --repetition-penalty) + [ $# -ge 2 ] || err "--repetition-penalty requires a value" + REPETITION_PENALTY="$2"; shift 2 ;; + --mmap) + MAPPING="true"; shift ;; + --mlock) + LOCKING="true"; shift ;; + --kv-cache) + [ $# -ge 2 ] || err "--kv-cache requires a value" + KV_CACHE_SIZE="$2"; shift 2 ;; + --ctx-size) + [ $# -ge 2 ] || err "--ctx-size requires a value" + CTX_SIZE="$2"; shift 2 ;; + --metrics) + METRICS="true"; shift ;; + --health) + HEALTH="true"; shift ;; + --slots) + [ $# -ge 2 ] || err "--slots requires a value" + SLOTS="$2"; shift 2 ;; -*) err "Unknown option '$1' (see --help)" ;; *) @@ -262,6 +358,13 @@ cmd_start() { local llamacpp_full llamacpp_full="$(command -v "$llamacpp_bin")" + # Detect version + local version + version="$(detect_llama_version)" + if [ -n "$version" ]; then + validate_server_features "$version" + fi + # Resolve model local explicit_model="${SUBCMD_ARGS[0]:-}" # Flag --model takes precedence over positional arg @@ -272,6 +375,8 @@ cmd_start() { # Resolve GPU layers local gpu_layers gpu_layers="$(resolve_gpu_layers)" + # Use the flag value if provided, otherwise use resolved value + [ -n "$GPU_LAYERS_FLAG" ] && gpu_layers="$GPU_LAYERS_FLAG" # Warn if no GPU detected and auto-detect resolved to CPU if [ "$gpu_layers" = "0" ] && [ "${LLAMACPP_GPU_LAYERS:--1}" = "-1" ]; then @@ -302,16 +407,71 @@ After=network-online.target [Service] Type=simple -ExecStart=$llamacpp_full -m $model --port $PORT --host $HOST --n-gpu-layers $gpu_layers --ctx-size $CTX_SIZE --threads $THREADS -Restart=on-failure -RestartSec=5 -TimeoutStopSec=10 -KillMode=control-group -EnvironmentFile=-%h/.config/linux_post_install/ai.env - -[Install] -WantedBy=default.target +ExecStart=$llamacpp_full -m $model --port $PORT --host $HOST EOF + + # Add parameters if provided + if [ -n "$gpu_layers" ]; then + echo " --n-gpu-layers $gpu_layers" >> "$USER_SYSTEMD_DIR/$SERVICE" + fi + if [ -n "$CTX_SIZE" ]; then + echo " --ctx-size $CTX_SIZE" >> "$USER_SYSTEMD_DIR/$SERVICE" + fi + if [ -n "$THREADS" ]; then + echo " --threads $THREADS" >> "$USER_SYSTEMD_DIR/$SERVICE" + fi + if [ -n "$GPU_THREADS" ]; then + echo " --gpu-threads $GPU_THREADS" >> "$USER_SYSTEMD_DIR/$SERVICE" + fi + if [ -n "$TENSOR_SPLIT" ]; then + echo " --tensor-split $TENSOR_SPLIT" >> "$USER_SYSTEMD_DIR/$SERVICE" + fi + if [ -n "$BATCH_SIZE" ]; then + echo " --batch-size $BATCH_SIZE" >> "$USER_SYSTEMD_DIR/$SERVICE" + fi + if [ -n "$UBATCH_SIZE" ]; then + echo " --ubatch-size $UBATCH_SIZE" >> "$USER_SYSTEMD_DIR/$SERVICE" + fi + if [ -n "$TEMPERATURE" ]; then + echo " --temperature $TEMPERATURE" >> "$USER_SYSTEMD_DIR/$SERVICE" + fi + if [ -n "$TOP_K" ]; then + echo " --top-k $TOP_K" >> "$USER_SYSTEMD_DIR/$SERVICE" + fi + if [ -n "$TOP_P" ]; then + echo " --top-p $TOP_P" >> "$USER_SYSTEMD_DIR/$SERVICE" + fi + if [ -n "$REPETITION_PENALTY" ]; then + echo " --repetition-penalty $REPETITION_PENALTY" >> "$USER_SYSTEMD_DIR/$SERVICE" + fi + if [ -n "$MAPPING" ]; then + echo " --mmap" >> "$USER_SYSTEMD_DIR/$SERVICE" + fi + if [ -n "$LOCKING" ]; then + echo " --mlock" >> "$USER_SYSTEMD_DIR/$SERVICE" + fi + if [ -n "$KV_CACHE_SIZE" ]; then + echo " --kv-cache $KV_CACHE_SIZE" >> "$USER_SYSTEMD_DIR/$SERVICE" + fi + if [ -n "$METRICS" ]; then + echo " --metrics" >> "$USER_SYSTEMD_DIR/$SERVICE" + fi + if [ -n "$HEALTH" ]; then + echo " --health" >> "$USER_SYSTEMD_DIR/$SERVICE" + fi + if [ -n "$SLOTS" ]; then + echo " --slots $SLOTS" >> "$USER_SYSTEMD_DIR/$SERVICE" + fi + + echo " " >> "$USER_SYSTEMD_DIR/$SERVICE" + echo "Restart=on-failure" >> "$USER_SYSTEMD_DIR/$SERVICE" + echo "RestartSec=5" >> "$USER_SYSTEMD_DIR/$SERVICE" + echo "TimeoutStopSec=10" >> "$USER_SYSTEMD_DIR/$SERVICE" + echo "KillMode=control-group" >> "$USER_SYSTEMD_DIR/$SERVICE" + echo "EnvironmentFile=-%h/.config/linux_post_install/ai.env" >> "$USER_SYSTEMD_DIR/$SERVICE" + echo "" >> "$USER_SYSTEMD_DIR/$SERVICE" + echo "[Install]" >> "$USER_SYSTEMD_DIR/$SERVICE" + echo "WantedBy=default.target" >> "$USER_SYSTEMD_DIR/$SERVICE" chmod 644 "$USER_SYSTEMD_DIR/$SERVICE" # Enable and start @@ -402,6 +562,15 @@ cmd_status() { else printf 'health: not running\n' fi + + # Version info + local version + version="$(detect_llama_version)" + if [ -n "$version" ]; then + printf 'version: %s\n' "$version" + else + printf 'version: unknown\n' + fi } cmd_models() {