Your Name
0b7fdb4954
Add bin/pos-config + lib/config-ui.sh: reads the '# POS_CONFIG:' headers across tools into a unified runtime config (~/.config/linux_post_install/ <scope>.env, chmod 600). Secret masking with show/hide, digits:/num:/url: validation, '-' clears, blank keeps; '*plugins' expands entertainment plugin vars from entertainment-lib.sh. pos config with no scope opens a picker (scope list when not a TTY); completion offers scopes from the registry at runtime. gen-docs.sh now handles category-less tools (pos-config) instead of emitting pos-config-config; fix set -e/pipefail abort that truncated the header registry mid-read. install.sh ships config-ui.sh.
276 lines
9.8 KiB
Bash
Executable File
276 lines
9.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
# POS: system health — Host health dashboard (disk, RAM, services, backup age, fail2ban, docker); exit 1 if any FAIL
|
|
# POS_FLAGS: --send --markdown
|
|
# POS_CONFIG: system | system.env | BACKUP_SERVICE_ROOTS=:Roots scanned by backup --service and the health backup-age check (default: /srv $HOME/srv) | HEALTH_BACKUP_MAX_AGE_DAYS=num:Max backup age in days before health warns (default 2)
|
|
# POS_CONFIG: notify | notify.env | NOTIFY_PLATFORM=:Comma-separated notify platforms (default telegram)
|
|
source "$(dirname "$0")/../lib/common.sh" 2>/dev/null || source "$(dirname "$0")/common.sh"
|
|
source "$(dirname "$0")/../lib/notify.sh" 2>/dev/null || source "$(dirname "$0")/notify.sh"
|
|
|
|
load_system_env
|
|
|
|
# Effective dynamic values (environment > system.env > defaults) shown in --help
|
|
EFF_PLATFORMS="$(notify_platforms)"
|
|
EFF_MAX_AGE="${HEALTH_BACKUP_MAX_AGE_DAYS:-2}"
|
|
EFF_ROOTS="${BACKUP_SERVICE_ROOTS:-/srv $HOME/srv}"
|
|
|
|
usage() {
|
|
cat <<EOF
|
|
Usage: pos system health [--send] [--markdown]
|
|
|
|
Host health dashboard: disk, RAM/swap, failed systemd units, backup age,
|
|
fail2ban, docker containers. Exits 1 if any check FAILs.
|
|
|
|
Flags:
|
|
--send Send the summary via the configured notify platform(s) and do
|
|
NOT print the dashboard (notification-only — for cron/timers or
|
|
wrappers like the Telegram listener that reply with output)
|
|
(uses lib/notify.sh; see NOTIFY_PLATFORM below)
|
|
--markdown Same as --send, with markdown parse mode
|
|
-h, --help Show this help
|
|
|
|
Environment (effective values):
|
|
NOTIFY_PLATFORM ${EFF_PLATFORMS}
|
|
HEALTH_BACKUP_MAX_AGE_DAYS ${EFF_MAX_AGE}
|
|
BACKUP_SERVICE_ROOTS ${EFF_ROOTS}
|
|
|
|
Loaded from ~/.config/linux_post_install/system.env (health/backup settings)
|
|
and ~/.config/linux_post_install/notify.env (NOTIFY_PLATFORM) unless already
|
|
exported. Defaults apply when neither is set.
|
|
|
|
Examples:
|
|
pos system health
|
|
pos system health --send
|
|
EOF
|
|
exit 0
|
|
}
|
|
|
|
SEND=0
|
|
MARKDOWN=0
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
-h|--help) usage ;;
|
|
--send) SEND=1 ;;
|
|
--markdown) MARKDOWN=1 ;;
|
|
*) err "Unknown option '$arg' (see --help)" ;;
|
|
esac
|
|
done
|
|
|
|
if [ "$MARKDOWN" -eq 1 ]; then
|
|
SEND=1
|
|
fi
|
|
|
|
# --send is notification-only: suppress the dashboard output (so a wrapper
|
|
# such as the Telegram listener doesn't echo it back as a duplicate reply).
|
|
if [ "$SEND" -eq 1 ]; then
|
|
exec 3>&1
|
|
exec 1>/dev/null
|
|
fi
|
|
|
|
FAILURES=0
|
|
WARNINGS=0
|
|
REPORT=()
|
|
|
|
report() {
|
|
local st="$1" label="$2" detail="$3" line
|
|
case "$st" in
|
|
ok)
|
|
echo " ${GREEN}${BOLD}[ OK ]${RESET} ${label}: ${detail}"
|
|
printf -v line "[ OK ] %s: %s" "$label" "$detail"
|
|
;;
|
|
warn)
|
|
echo " ${YELLOW}${BOLD}[WARN]${RESET} ${label}: ${detail}"
|
|
printf -v line "[WARN] %s: %s" "$label" "$detail"
|
|
WARNINGS=$((WARNINGS + 1))
|
|
;;
|
|
fail)
|
|
echo " ${RED}${BOLD}[FAIL]${RESET} ${label}: ${detail}"
|
|
printf -v line "[FAIL] %s: %s" "$label" "$detail"
|
|
FAILURES=$((FAILURES + 1))
|
|
;;
|
|
esac
|
|
REPORT+=("$line")
|
|
}
|
|
|
|
host="$(hostname -s 2>/dev/null || echo "localhost")"
|
|
ip_public="$(curl -fsS -m 5 https://api.ipify.org 2>/dev/null || echo "unreachable")"
|
|
loadavg="$(cut -d' ' -f1-3 /proc/loadavg 2>/dev/null || echo "?")"
|
|
uptime_s="$(uptime -p 2>/dev/null || echo "?")"
|
|
|
|
section "System Health — ${host}"
|
|
echo " ${BLUE}${BOLD}date${RESET} $(date '+%Y-%m-%d %H:%M:%S')"
|
|
echo " ${BLUE}${BOLD}uptime${RESET} ${uptime_s}"
|
|
echo " ${BLUE}${BOLD}load${RESET} ${loadavg}"
|
|
echo " ${BLUE}${BOLD}ip${RESET} ${ip_public}"
|
|
echo
|
|
|
|
# ── Disk ──────────────────────────────────────────────────────
|
|
DISK_PCT_MAX=0
|
|
DISK_ISSUES=()
|
|
while read -r fs use mount; do
|
|
pct=${use%\%}
|
|
if [ "$pct" -gt "$DISK_PCT_MAX" ]; then
|
|
DISK_PCT_MAX="$pct"
|
|
fi
|
|
if [ "$pct" -ge 90 ]; then
|
|
DISK_ISSUES+=("${mount} ${pct}%")
|
|
elif [ "$pct" -ge 85 ]; then
|
|
DISK_ISSUES+=("${mount} ${pct}%")
|
|
fi
|
|
done < <(df -P -x tmpfs -x devtmpfs -x squashfs -x overlay -x efivarfs -x proc -x sysfs -x cgroup2 -x zfs -x aufs 2>/dev/null | awk 'NR>1 && $1 !~ /loop|sr[0-9]/ {print $1, $5, $6}')
|
|
|
|
if [ ${#DISK_ISSUES[@]} -gt 0 ]; then
|
|
report fail "disk" "${DISK_ISSUES[*]}"
|
|
else
|
|
report ok "disk" "max usage ${DISK_PCT_MAX}%"
|
|
fi
|
|
|
|
# ── RAM / swap ────────────────────────────────────────────────
|
|
mem=($(free -m | awk '/^Mem:/{print $2, $7}'))
|
|
swap=($(free -m | awk '/^Swap:/{print $2, $3}'))
|
|
MEM_TOTAL=${mem[0]:-0}
|
|
MEM_AVAIL=${mem[1]:-0}
|
|
MEM_PCT=100
|
|
if [ "$MEM_TOTAL" -gt 0 ]; then
|
|
MEM_PCT=$(( (MEM_TOTAL - MEM_AVAIL) * 100 / MEM_TOTAL ))
|
|
fi
|
|
SWAP_PCT=0
|
|
if [ "${swap[0]:-0}" -gt 0 ]; then
|
|
SWAP_PCT=$(( swap[1] * 100 / swap[0] ))
|
|
fi
|
|
if [ "$MEM_PCT" -ge 95 ] || [ "$SWAP_PCT" -ge 90 ]; then
|
|
report fail "ram" "mem ${MEM_PCT}% used, swap ${SWAP_PCT}%"
|
|
elif [ "$MEM_PCT" -ge 85 ]; then
|
|
report warn "ram" "mem ${MEM_PCT}% used, swap ${SWAP_PCT}%"
|
|
else
|
|
report ok "ram" "mem ${MEM_PCT}% used, swap ${SWAP_PCT}%"
|
|
fi
|
|
|
|
# ── Failed systemd units ──────────────────────────────────────
|
|
if command -v systemctl >/dev/null 2>&1; then
|
|
failed_units=()
|
|
while IFS= read -r u; do
|
|
[ -n "$u" ] && failed_units+=("${u%% *}")
|
|
done < <(systemctl --failed --plain --no-legend --no-pager 2>/dev/null || true)
|
|
if [ "${#failed_units[@]}" -gt 0 ]; then
|
|
report fail "services" "${#failed_units[@]} failed unit(s): ${failed_units[*]}"
|
|
else
|
|
report ok "services" "no failed units"
|
|
fi
|
|
else
|
|
report warn "services" "systemd not available"
|
|
fi
|
|
|
|
# ── Backup age ────────────────────────────────────────────────
|
|
MAX_AGE="${HEALTH_BACKUP_MAX_AGE_DAYS:-2}"
|
|
roots=()
|
|
if [ -n "${BACKUP_SERVICE_ROOTS:-}" ]; then
|
|
read -r -a roots <<< "$BACKUP_SERVICE_ROOTS"
|
|
else
|
|
roots=(/srv "$HOME/srv")
|
|
fi
|
|
newest=""
|
|
newest_ts=0
|
|
for root in "${roots[@]}"; do
|
|
[ -d "$root" ] || continue
|
|
while IFS= read -r line; do
|
|
ts="${line%% *}"
|
|
if [ "${ts%%.*}" -gt "$newest_ts" ]; then
|
|
newest_ts="${ts%%.*}"
|
|
newest="${line#* }"
|
|
fi
|
|
done < <(find "$root" -maxdepth 2 -type f -name '*.tar.gz.gpg' -printf '%T@ %p\n' 2>/dev/null || true)
|
|
done
|
|
if [ -z "$newest" ]; then
|
|
report warn "backup" "no backups found under ${roots[*]}"
|
|
else
|
|
now="$(date +%s)"
|
|
days=$(( (now - newest_ts) / 86400 ))
|
|
if [ "$days" -gt "$MAX_AGE" ]; then
|
|
report warn "backup" "${days}d old: ${newest}"
|
|
else
|
|
report ok "backup" "${days}d old: ${newest}"
|
|
fi
|
|
fi
|
|
|
|
# ── fail2ban ──────────────────────────────────────────────────
|
|
if command -v fail2ban-client >/dev/null 2>&1; then
|
|
if fail2ban-client ping >/dev/null 2>&1; then
|
|
banned=0
|
|
jails=$(fail2ban-client status 2>/dev/null | awk -F': *' '/Jail list/{print $2}')
|
|
for j in $jails; do
|
|
n=$(fail2ban-client status "$j" 2>/dev/null | awk -F': *' '/Total banned/{print $2}')
|
|
n="${n:-0}"
|
|
banned=$(( banned + (n > 0 ? n : 0) ))
|
|
done
|
|
report ok "fail2ban" "running, ${banned} total banned across ${jails:-0} jail(s)"
|
|
else
|
|
report warn "fail2ban" "installed but not running"
|
|
fi
|
|
fi
|
|
|
|
# ── Docker ────────────────────────────────────────────────────
|
|
if command -v docker >/dev/null 2>&1; then
|
|
bad_containers=()
|
|
running=0
|
|
while IFS= read -r name state; do
|
|
case "$state" in
|
|
restarting|unhealthy) bad_containers+=("${name}(${state})") ;;
|
|
esac
|
|
done < <(docker ps -a --format '{{.Names}} {{.State}}' 2>/dev/null || true)
|
|
if [ "${#bad_containers[@]}" -gt 0 ]; then
|
|
report fail "docker" "${#bad_containers[@]} problem container(s): ${bad_containers[*]}"
|
|
else
|
|
report ok "docker" "no unhealthy/restarting containers"
|
|
fi
|
|
fi
|
|
|
|
echo
|
|
if [ "$FAILURES" -gt 0 ]; then
|
|
verdict="${FAILURES} FAIL, ${WARNINGS} WARN"
|
|
echo " ${RED}${BOLD}VERDICT${RESET} ${verdict}"
|
|
elif [ "$WARNINGS" -gt 0 ]; then
|
|
verdict="${WARNINGS} WARN — mostly healthy"
|
|
echo " ${YELLOW}${BOLD}VERDICT${RESET} ${verdict}"
|
|
else
|
|
verdict="all good"
|
|
echo " ${GREEN}${BOLD}VERDICT${RESET} ${verdict}"
|
|
fi
|
|
|
|
if [ "$SEND" -eq 1 ]; then
|
|
exec 1>&3 3>&-
|
|
msg="POS Health — ${host}
|
|
Verdict: ${verdict}"
|
|
for line in "${REPORT[@]}"; do
|
|
msg+="
|
|
${line}"
|
|
done
|
|
if [ "$MARKDOWN" -eq 1 ]; then
|
|
notify_send "$msg" --markdown
|
|
else
|
|
notify_send "$msg"
|
|
fi
|
|
# notify_send is silent-fail by design — confirm delivery only when a
|
|
# platform sender actually exists, otherwise warn on stderr (still visible
|
|
# here since only stdout was redirected while --send ran).
|
|
delivered=0
|
|
IFS=',' read -r -a platforms <<< "$(notify_platforms)"
|
|
for p in "${platforms[@]}"; do
|
|
p="${p// /}"
|
|
[ -n "$p" ] || continue
|
|
sname="$(notify_sender_name "$p")"
|
|
if command -v "pos-communication-$sname" &>/dev/null \
|
|
|| [ -x "$(dirname "$0")/pos-communication-$sname" ] \
|
|
|| [ -x "$(dirname "$0")/../bin/pos-communication-$sname" ]; then
|
|
delivered=1
|
|
fi
|
|
done
|
|
if [ "$delivered" -eq 1 ]; then
|
|
echo "sent: POS Health — ${host} (${verdict})"
|
|
else
|
|
warn "notify: no platform sender found (NOTIFY_PLATFORM=$(notify_platforms)) — digest not delivered"
|
|
fi
|
|
fi
|
|
|
|
[ "$FAILURES" -eq 0 ]
|