469 lines
14 KiB
Bash
Executable file
469 lines
14 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# koopa-login-health — read-only GOA/koopa service health.
|
|
# Inventory: koopa-admin-log host/overview/LIVE.md (+ ports.md).
|
|
#
|
|
# Architecture:
|
|
# collect — run probes, write snapshot (systemd user timer; NOT on login)
|
|
# show — print last snapshot only (default; used by bashrc on SSH login)
|
|
#
|
|
# Usage:
|
|
# koopa-login-health # show compact snapshot (login path)
|
|
# koopa-login-health show -v # show verbose snapshot
|
|
# koopa-login-health show -q # overall line only
|
|
# koopa-login-health collect # refresh snapshot now
|
|
# koopa-login-health collect -v # same (verbose always stored)
|
|
# KOOPA_SKIP_HEALTH=1 … # bashrc skip
|
|
# KOOPA_HEALTH_FROM_LOGIN=1 # always exit 0 (login hook)
|
|
#
|
|
# Never mutates host services. Login never runs probes.
|
|
|
|
set +e
|
|
set +u
|
|
set +o pipefail 2>/dev/null || true
|
|
|
|
FROM_LOGIN=0
|
|
[ -n "${KOOPA_HEALTH_FROM_LOGIN:-}" ] && FROM_LOGIN=1
|
|
|
|
_exit_health() {
|
|
local c="${1:-0}"
|
|
[ "${FROM_LOGIN}" = "1" ] && exit 0
|
|
exit "${c}"
|
|
}
|
|
|
|
_bail_login_safe() {
|
|
echo "koopa health: ${1:-error} (ignored; login continues)" >&2
|
|
_exit_health 0
|
|
}
|
|
|
|
# --- paths (XDG) ---
|
|
STATE_DIR="${KOOPA_HEALTH_STATE_DIR:-${XDG_STATE_HOME:-${HOME:-/tmp}/.local/state}/koopa-login-health}"
|
|
COMPACT_FILE="${STATE_DIR}/status.compact"
|
|
VERBOSE_FILE="${STATE_DIR}/status.verbose"
|
|
META_FILE="${STATE_DIR}/status.meta"
|
|
# Max age for "current" snapshot (seconds). Older → WARN stale on show.
|
|
STALE_SEC="${KOOPA_HEALTH_STALE_SEC:-900}" # 15 min (timer default 5 min)
|
|
CURL_M="${KOOPA_HEALTH_CURL_M:-2}"
|
|
|
|
CMD="show"
|
|
VERBOSE=0
|
|
QUIET=0
|
|
|
|
while [ "${1:-}" != "" ]; do
|
|
case "$1" in
|
|
show|collect|status)
|
|
CMD="$1"
|
|
[ "$CMD" = "status" ] && CMD="show"
|
|
;;
|
|
-v|--verbose) VERBOSE=1 ;;
|
|
-q|--quiet) QUIET=1 ;;
|
|
-h|--help)
|
|
sed -n '2,22p' "$0" 2>/dev/null | sed 's/^# \?//' || true
|
|
_exit_health 0
|
|
;;
|
|
*)
|
|
echo "unknown arg: $1 (try: show | collect | -v | -q | -h)" >&2
|
|
_exit_health 2
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
# ---------- SHOW: never probe ----------
|
|
do_show() {
|
|
if [ ! -f "${COMPACT_FILE}" ] && [ ! -f "${META_FILE}" ]; then
|
|
if [ "${QUIET}" -eq 1 ]; then
|
|
echo "unknown (no snapshot — run: systemctl --user start koopa-login-health.service)"
|
|
else
|
|
echo "koopa health: no snapshot yet"
|
|
echo " Collect runs on a user timer (not at login). Refresh once:"
|
|
echo " systemctl --user start koopa-login-health.service"
|
|
echo " or: koopa-login-health collect"
|
|
fi
|
|
_exit_health 1
|
|
fi
|
|
|
|
local overall="unknown" collected_at="" age="?" age_sec=-1 now
|
|
if [ -f "${META_FILE}" ]; then
|
|
# shellcheck disable=SC1090
|
|
overall=$(grep '^overall=' "${META_FILE}" 2>/dev/null | tail -1 | cut -d= -f2-)
|
|
collected_at=$(grep '^collected_at_rfc=' "${META_FILE}" 2>/dev/null | tail -1 | cut -d= -f2-)
|
|
local ts
|
|
ts=$(grep '^collected_at=' "${META_FILE}" 2>/dev/null | tail -1 | cut -d= -f2-)
|
|
now=$(date +%s 2>/dev/null || echo 0)
|
|
if [ -n "${ts}" ] && [ "${ts}" -gt 0 ] 2>/dev/null; then
|
|
age_sec=$((now - ts))
|
|
if [ "${age_sec}" -lt 0 ] 2>/dev/null; then age_sec=0; fi
|
|
if [ "${age_sec}" -lt 120 ] 2>/dev/null; then
|
|
age="${age_sec}s ago"
|
|
elif [ "${age_sec}" -lt 7200 ] 2>/dev/null; then
|
|
age="$((age_sec / 60))m ago"
|
|
else
|
|
age="$((age_sec / 3600))h ago"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
local stale=0
|
|
if [ "${age_sec}" -ge 0 ] 2>/dev/null && [ "${age_sec}" -gt "${STALE_SEC}" ] 2>/dev/null; then
|
|
stale=1
|
|
fi
|
|
|
|
if [ "${QUIET}" -eq 1 ]; then
|
|
if [ "${stale}" -eq 1 ]; then
|
|
echo "${overall} (stale ${age})"
|
|
else
|
|
echo "${overall}"
|
|
fi
|
|
[ "${stale}" -eq 1 ] && _exit_health 1
|
|
case "${overall}" in ok|warn) _exit_health 0 ;; *) _exit_health 1 ;; esac
|
|
fi
|
|
|
|
if [ "${VERBOSE}" -eq 1 ] && [ -f "${VERBOSE_FILE}" ]; then
|
|
cat "${VERBOSE_FILE}" 2>/dev/null || true
|
|
elif [ -f "${COMPACT_FILE}" ]; then
|
|
cat "${COMPACT_FILE}" 2>/dev/null || true
|
|
else
|
|
echo "koopa health: snapshot incomplete (meta only) overall=${overall}"
|
|
fi
|
|
|
|
echo " checked: ${collected_at:-?} (${age}) [timer snapshot; not live at login]"
|
|
if [ "${stale}" -eq 1 ]; then
|
|
echo " WARN: snapshot older than ${STALE_SEC}s — is koopa-login-health.timer active?"
|
|
echo " systemctl --user status koopa-login-health.timer"
|
|
fi
|
|
|
|
[ "${stale}" -eq 1 ] && _exit_health 1
|
|
case "${overall}" in ok|warn) _exit_health 0 ;; *) _exit_health 1 ;; esac
|
|
}
|
|
|
|
# ---------- COLLECT: probes → write snapshot ----------
|
|
CRITICAL_CONTAINERS=(
|
|
taler-hacktivism
|
|
taler-hacktivism-exchange-ansible
|
|
taler-hacktivism-bank
|
|
koopa-forgejo
|
|
)
|
|
|
|
EXPECT_CONTAINERS=(
|
|
taler-hacktivism
|
|
taler-hacktivism-exchange-ansible
|
|
taler-hacktivism-bank
|
|
koopa-castopod
|
|
koopa-bonfire
|
|
koopa-prime-jellyfin
|
|
koopa-prime-qbittorrent
|
|
koopa-forgejo
|
|
koopa-paivana
|
|
koopa-tor-relay
|
|
koopa-nym
|
|
koopa-tops-ng1
|
|
koopa-tops-ng2
|
|
koopa-tops-ng3
|
|
)
|
|
|
|
OPTIONAL_CONTAINERS=(
|
|
koopa-prime-jellyfin
|
|
koopa-prime-qbittorrent
|
|
koopa-nym
|
|
koopa-tor-relay
|
|
koopa-tops-ng1
|
|
koopa-tops-ng2
|
|
koopa-tops-ng3
|
|
koopa-castopod
|
|
koopa-bonfire
|
|
koopa-paivana
|
|
)
|
|
|
|
# merchant nginx on :9010 is HTTPS-only (plain HTTP → 400)
|
|
PROBES=(
|
|
"merchant|https://127.0.0.1:9010/config|200"
|
|
"exchange|http://127.0.0.1:9011/keys|200"
|
|
"bank|http://127.0.0.1:9012/config|200"
|
|
"forgejo|http://127.0.0.1:9024/|200,302,303"
|
|
"castopod|http://127.0.0.1:9020/|200,302,303,307,308"
|
|
"bonfire|http://127.0.0.1:9021/|200,302,303,307,308"
|
|
"prime|http://127.0.0.1:9022/|200,302,303,307,308"
|
|
"bt|http://127.0.0.1:9023/|200,302,303,307,308,401"
|
|
"paivana|http://127.0.0.1:9025/|200,302,303,307,308"
|
|
"tops-ng1|http://127.0.0.1:9090/|200,302,303"
|
|
"tops-ng2|http://127.0.0.1:9091/|200,302,303"
|
|
"tops-ng3|http://127.0.0.1:9092/|200,302,303"
|
|
)
|
|
|
|
OPTIONAL_PROBES=(castopod bonfire prime bt paivana tops-ng1 tops-ng2 tops-ng3)
|
|
|
|
C_OK=; C_WARN=; C_FAIL=; C_DIM=; C_RST=; C_BOLD=
|
|
# colors only when writing for a tty later; snapshots are plain + optional ANSI if COLLECT_COLOR=1
|
|
if [ "${KOOPA_HEALTH_COLLECT_COLOR:-0}" = "1" ] && [ -t 1 ]; then
|
|
C_OK=$'\033[32m'; C_WARN=$'\033[33m'; C_FAIL=$'\033[31m'
|
|
C_DIM=$'\033[2m'; C_RST=$'\033[0m'; C_BOLD=$'\033[1m'
|
|
fi
|
|
|
|
OK_N=0
|
|
WARN_N=0
|
|
FAIL_N=0
|
|
LINES=()
|
|
|
|
is_optional_container() {
|
|
local n="$1" x
|
|
for x in "${OPTIONAL_CONTAINERS[@]}"; do [ "$x" = "$n" ] && return 0; done
|
|
return 1
|
|
}
|
|
|
|
is_optional_probe() {
|
|
local n="$1" x
|
|
for x in "${OPTIONAL_PROBES[@]}"; do [ "$x" = "$n" ] && return 0; done
|
|
return 1
|
|
}
|
|
|
|
record() {
|
|
local level="$1" msg="$2"
|
|
case "$level" in
|
|
OK) OK_N=$((OK_N + 1)); LINES+=(" ${C_OK}OK ${C_RST}${msg}") ;;
|
|
WARN) WARN_N=$((WARN_N + 1)); LINES+=(" ${C_WARN}WARN${C_RST} ${msg}") ;;
|
|
FAIL) FAIL_N=$((FAIL_N + 1)); LINES+=(" ${C_FAIL}FAIL${C_RST} ${msg}") ;;
|
|
INFO) LINES+=(" ${C_DIM}··· ${C_RST}${msg}") ;;
|
|
esac
|
|
}
|
|
|
|
code_in_list() {
|
|
local code="$1" list="$2" part
|
|
local IFS=','
|
|
# shellcheck disable=SC2086
|
|
for part in $list; do
|
|
[ "$code" = "$part" ] && return 0
|
|
done
|
|
return 1
|
|
}
|
|
|
|
do_collect() {
|
|
mkdir -p "${STATE_DIR}" 2>/dev/null || {
|
|
echo "koopa health collect: cannot mkdir ${STATE_DIR}" >&2
|
|
return 1
|
|
}
|
|
|
|
local HOST NOW caddy_st st sock listen_blob port
|
|
local PODMAN running_names name dep tmpdir pids pid
|
|
local pname purl pok outf code overall line
|
|
|
|
HOST=$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo koopa)
|
|
NOW=$(date -R 2>/dev/null || date 2>/dev/null || echo "")
|
|
local NOW_UNIX
|
|
NOW_UNIX=$(date +%s 2>/dev/null || echo 0)
|
|
|
|
if [ "$(id -u 2>/dev/null)" = "0" ]; then
|
|
record INFO "running as root (podman rootless via hernani if available)"
|
|
fi
|
|
|
|
if command -v systemctl >/dev/null 2>&1; then
|
|
caddy_st=$(systemctl is-active caddy 2>/dev/null || echo unknown)
|
|
if [ "${caddy_st}" = "active" ]; then
|
|
record OK "caddy.service active"
|
|
else
|
|
record FAIL "caddy.service is ${caddy_st} (edge HTTPS)"
|
|
fi
|
|
for sock in http-proxy.socket https-proxy.socket; do
|
|
st=$(systemctl is-active "${sock}" 2>/dev/null || echo unknown)
|
|
if [ "${st}" = "active" ] || [ "${st}" = "listening" ]; then
|
|
record OK "${sock} ${st}"
|
|
else
|
|
record WARN "${sock} is ${st}"
|
|
fi
|
|
done
|
|
else
|
|
record WARN "systemctl not available"
|
|
fi
|
|
|
|
if command -v ss >/dev/null 2>&1; then
|
|
listen_blob=$(ss -lnt 2>/dev/null | awk '{print $4}' || true)
|
|
for port in 9000 9001 9010 9011 9012 9024 9200; do
|
|
if printf '%s\n' "${listen_blob}" | grep -qE "[:.]${port}\$"; then
|
|
record OK "listen :${port}"
|
|
else
|
|
record FAIL "nothing listening on :${port}"
|
|
fi
|
|
done
|
|
for port in 9020 9021 9022 9023 9025 9090 9091 9092 8080 9080; do
|
|
if printf '%s\n' "${listen_blob}" | grep -qE "[:.]${port}\$"; then
|
|
record OK "listen :${port} (optional)"
|
|
else
|
|
record WARN "nothing listening on :${port}"
|
|
fi
|
|
done
|
|
fi
|
|
|
|
PODMAN=(podman)
|
|
if [ "$(id -u 2>/dev/null)" = "0" ] && command -v runuser >/dev/null 2>&1; then
|
|
if id hernani >/dev/null 2>&1; then
|
|
PODMAN=(runuser -u hernani -- podman)
|
|
fi
|
|
fi
|
|
|
|
if ! command -v "${PODMAN[0]}" >/dev/null 2>&1 && [ "${PODMAN[0]}" = "podman" ]; then
|
|
record FAIL "podman not found"
|
|
running_names=""
|
|
else
|
|
running_names=$("${PODMAN[@]}" ps --format '{{.Names}}' 2>/dev/null || true)
|
|
if [ -z "${running_names}" ]; then
|
|
record FAIL "podman ps empty or failed"
|
|
fi
|
|
for name in "${EXPECT_CONTAINERS[@]}"; do
|
|
if printf '%s\n' "${running_names}" | grep -qxF "${name}"; then
|
|
st=$("${PODMAN[@]}" ps --format '{{.Names}}\t{{.Status}}' 2>/dev/null \
|
|
| awk -F'\t' -v n="${name}" '$1==n {print $2; exit}')
|
|
record OK "container ${name} (${st:-up})"
|
|
else
|
|
if is_optional_container "${name}"; then
|
|
record WARN "container ${name} not running"
|
|
else
|
|
record FAIL "container ${name} not running"
|
|
fi
|
|
fi
|
|
done
|
|
for dep in koopa-forgejo-db koopa-castopod-mariadb koopa-castopod-redis koopa-bonfire-db; do
|
|
if printf '%s\n' "${running_names}" | grep -qxF "${dep}"; then
|
|
record INFO "dep ${dep} running"
|
|
else
|
|
record INFO "dep ${dep} not in running set (name may differ)"
|
|
fi
|
|
done
|
|
fi
|
|
|
|
tmpdir=$(mktemp -d 2>/dev/null || true)
|
|
if [ -n "${tmpdir}" ] && [ -d "${tmpdir}" ]; then
|
|
probe_one() {
|
|
local _url="$1" _out="$2" _code
|
|
# curl still prints http_code on connect fail; do not append another "000" via ||
|
|
_code=$(curl -sk -m "${CURL_M}" -o /dev/null -w '%{http_code}' "${_url}" 2>/dev/null)
|
|
[ -n "${_code}" ] || _code="000"
|
|
printf '%s\n' "${_code}" >"${_out}"
|
|
}
|
|
pids=()
|
|
for spec in "${PROBES[@]}"; do
|
|
IFS='|' read -r pname purl pok <<EOF
|
|
${spec}
|
|
EOF
|
|
probe_one "${purl}" "${tmpdir}/${pname}" &
|
|
pids+=($!)
|
|
done
|
|
for pid in "${pids[@]:-}"; do
|
|
wait "${pid}" 2>/dev/null || true
|
|
done
|
|
for spec in "${PROBES[@]}"; do
|
|
IFS='|' read -r pname purl pok <<EOF
|
|
${spec}
|
|
EOF
|
|
code="000"
|
|
[ -f "${tmpdir}/${pname}" ] && code=$(awk '{print $1; exit}' "${tmpdir}/${pname}" 2>/dev/null || echo "000")
|
|
if code_in_list "${code}" "${pok}"; then
|
|
record OK "http ${pname} → ${code} (${purl})"
|
|
else
|
|
if is_optional_probe "${pname}"; then
|
|
record WARN "http ${pname} → ${code} (${purl})"
|
|
else
|
|
record FAIL "http ${pname} → ${code} (${purl})"
|
|
fi
|
|
fi
|
|
done
|
|
rm -rf "${tmpdir}" 2>/dev/null || true
|
|
else
|
|
record WARN "mktemp failed; skipped HTTP probes"
|
|
fi
|
|
|
|
if command -v timeout >/dev/null 2>&1; then
|
|
if timeout 1 bash -c 'echo >/dev/tcp/127.0.0.1/9200' 2>/dev/null; then
|
|
record OK "forgejo git-SSH TCP :9200 (git.hacktivism.ch)"
|
|
else
|
|
record FAIL "forgejo git-SSH TCP :9200 closed"
|
|
fi
|
|
fi
|
|
|
|
overall="ok"
|
|
[ "${WARN_N}" -gt 0 ] 2>/dev/null && overall="warn"
|
|
[ "${FAIL_N}" -gt 0 ] 2>/dev/null && overall="fail"
|
|
|
|
local header verbose_body compact_body
|
|
header="koopa health ${HOST} ${NOW}
|
|
git.hacktivism.ch = Forgejo :9024/:9200 · edge Caddy :9000/:9001"
|
|
|
|
verbose_body="${header}
|
|
"
|
|
if [ "${#LINES[@]}" -gt 0 ]; then
|
|
printf -v _lines '%s\n' "${LINES[@]}"
|
|
verbose_body+="${_lines}"
|
|
fi
|
|
case "${overall}" in
|
|
ok) verbose_body+="overall: OK ok=${OK_N} warn=${WARN_N} fail=${FAIL_N}"$'\n' ;;
|
|
warn) verbose_body+="overall: WARN ok=${OK_N} warn=${WARN_N} fail=${FAIL_N}"$'\n' ;;
|
|
*) verbose_body+="overall: FAIL ok=${OK_N} warn=${WARN_N} fail=${FAIL_N}"$'\n' ;;
|
|
esac
|
|
|
|
compact_body="${header}
|
|
"
|
|
local has_issue=0
|
|
for line in "${LINES[@]:-}"; do
|
|
case "${line}" in
|
|
*FAIL*|*WARN*) compact_body+="${line}"$'\n'; has_issue=1 ;;
|
|
esac
|
|
done
|
|
if [ "${has_issue}" -eq 0 ]; then
|
|
compact_body+=" OK all critical checks passed (${OK_N} ok)"$'\n'
|
|
else
|
|
compact_body+=" (${OK_N} ok — details: koopa-login-health show -v)"$'\n'
|
|
fi
|
|
case "${overall}" in
|
|
ok) compact_body+="overall: OK ok=${OK_N} warn=${WARN_N} fail=${FAIL_N}"$'\n' ;;
|
|
warn) compact_body+="overall: WARN ok=${OK_N} warn=${WARN_N} fail=${FAIL_N}"$'\n' ;;
|
|
*) compact_body+="overall: FAIL ok=${OK_N} warn=${WARN_N} fail=${FAIL_N}"$'\n' ;;
|
|
esac
|
|
|
|
# Atomic writes
|
|
local tmpc tmpv tmpm
|
|
tmpc="${STATE_DIR}/.status.compact.$$"
|
|
tmpv="${STATE_DIR}/.status.verbose.$$"
|
|
tmpm="${STATE_DIR}/.status.meta.$$"
|
|
printf '%s' "${compact_body}" >"${tmpc}" 2>/dev/null || return 1
|
|
printf '%s' "${verbose_body}" >"${tmpv}" 2>/dev/null || return 1
|
|
{
|
|
echo "overall=${overall}"
|
|
echo "collected_at=${NOW_UNIX}"
|
|
echo "collected_at_rfc=${NOW}"
|
|
echo "ok=${OK_N}"
|
|
echo "warn=${WARN_N}"
|
|
echo "fail=${FAIL_N}"
|
|
echo "host=${HOST}"
|
|
} >"${tmpm}" 2>/dev/null || return 1
|
|
mv -f "${tmpc}" "${COMPACT_FILE}" 2>/dev/null || return 1
|
|
mv -f "${tmpv}" "${VERBOSE_FILE}" 2>/dev/null || return 1
|
|
mv -f "${tmpm}" "${META_FILE}" 2>/dev/null || return 1
|
|
|
|
# Optional: print when run interactively
|
|
if [ -t 1 ] || [ "${VERBOSE}" -eq 1 ]; then
|
|
printf '%s' "${compact_body}"
|
|
echo " wrote snapshot → ${STATE_DIR}/"
|
|
fi
|
|
case "${overall}" in ok|warn) return 0 ;; *) return 1 ;; esac
|
|
}
|
|
|
|
case "${CMD}" in
|
|
show)
|
|
if [ "${FROM_LOGIN}" = "1" ]; then
|
|
( do_show ) 2>&1 || true
|
|
_exit_health 0
|
|
else
|
|
do_show
|
|
fi
|
|
;;
|
|
collect)
|
|
# Never invoke collect from login hook
|
|
if [ "${FROM_LOGIN}" = "1" ]; then
|
|
_bail_login_safe "collect refused under FROM_LOGIN"
|
|
fi
|
|
do_collect
|
|
_exit_health $?
|
|
;;
|
|
*)
|
|
echo "unknown command ${CMD}" >&2
|
|
_exit_health 2
|
|
;;
|
|
esac
|