#!/usr/bin/env bash # koopa-login-health — read-only GOA/koopa service health. # Inventory: koopa-admin-log host/overview/LIVE.md (+ ports.md). # # Architecture: # collect — run probes, write snapshot (systemd user timer; NOT on login) # show — print last snapshot only (default; used by bashrc on SSH login) # # Usage: # koopa-login-health # show compact snapshot (login path) # koopa-login-health show -v # show verbose snapshot # koopa-login-health show -q # overall line only # koopa-login-health collect # refresh snapshot now # koopa-login-health collect -v # same (verbose always stored) # KOOPA_SKIP_HEALTH=1 … # bashrc skip # KOOPA_HEALTH_FROM_LOGIN=1 # always exit 0 (login hook) # # Never mutates host services. Login never runs probes. set +e set +u set +o pipefail 2>/dev/null || true FROM_LOGIN=0 [ -n "${KOOPA_HEALTH_FROM_LOGIN:-}" ] && FROM_LOGIN=1 _exit_health() { local c="${1:-0}" [ "${FROM_LOGIN}" = "1" ] && exit 0 exit "${c}" } _bail_login_safe() { echo "koopa health: ${1:-error} (ignored; login continues)" >&2 _exit_health 0 } # --- paths (XDG) --- STATE_DIR="${KOOPA_HEALTH_STATE_DIR:-${XDG_STATE_HOME:-${HOME:-/tmp}/.local/state}/koopa-login-health}" COMPACT_FILE="${STATE_DIR}/status.compact" VERBOSE_FILE="${STATE_DIR}/status.verbose" META_FILE="${STATE_DIR}/status.meta" # Max age for "current" snapshot (seconds). Older → WARN stale on show. STALE_SEC="${KOOPA_HEALTH_STALE_SEC:-900}" # 15 min (timer default 5 min) CURL_M="${KOOPA_HEALTH_CURL_M:-2}" CMD="show" VERBOSE=0 QUIET=0 while [ "${1:-}" != "" ]; do case "$1" in show|collect|status) CMD="$1" [ "$CMD" = "status" ] && CMD="show" ;; -v|--verbose) VERBOSE=1 ;; -q|--quiet) QUIET=1 ;; -h|--help) sed -n '2,22p' "$0" 2>/dev/null | sed 's/^# \?//' || true _exit_health 0 ;; *) echo "unknown arg: $1 (try: show | collect | -v | -q | -h)" >&2 _exit_health 2 ;; esac shift done # ---------- SHOW: never probe ---------- do_show() { if [ ! -f "${COMPACT_FILE}" ] && [ ! -f "${META_FILE}" ]; then if [ "${QUIET}" -eq 1 ]; then echo "unknown (no snapshot — run: systemctl --user start koopa-login-health.service)" else echo "koopa health: no snapshot yet" echo " Collect runs on a user timer (not at login). Refresh once:" echo " systemctl --user start koopa-login-health.service" echo " or: koopa-login-health collect" fi _exit_health 1 fi local overall="unknown" collected_at="" age="?" age_sec=-1 now if [ -f "${META_FILE}" ]; then # shellcheck disable=SC1090 overall=$(grep '^overall=' "${META_FILE}" 2>/dev/null | tail -1 | cut -d= -f2-) collected_at=$(grep '^collected_at_rfc=' "${META_FILE}" 2>/dev/null | tail -1 | cut -d= -f2-) local ts ts=$(grep '^collected_at=' "${META_FILE}" 2>/dev/null | tail -1 | cut -d= -f2-) now=$(date +%s 2>/dev/null || echo 0) if [ -n "${ts}" ] && [ "${ts}" -gt 0 ] 2>/dev/null; then age_sec=$((now - ts)) if [ "${age_sec}" -lt 0 ] 2>/dev/null; then age_sec=0; fi if [ "${age_sec}" -lt 120 ] 2>/dev/null; then age="${age_sec}s ago" elif [ "${age_sec}" -lt 7200 ] 2>/dev/null; then age="$((age_sec / 60))m ago" else age="$((age_sec / 3600))h ago" fi fi fi local stale=0 if [ "${age_sec}" -ge 0 ] 2>/dev/null && [ "${age_sec}" -gt "${STALE_SEC}" ] 2>/dev/null; then stale=1 fi if [ "${QUIET}" -eq 1 ]; then if [ "${stale}" -eq 1 ]; then echo "${overall} (stale ${age})" else echo "${overall}" fi [ "${stale}" -eq 1 ] && _exit_health 1 case "${overall}" in ok|warn) _exit_health 0 ;; *) _exit_health 1 ;; esac fi if [ "${VERBOSE}" -eq 1 ] && [ -f "${VERBOSE_FILE}" ]; then cat "${VERBOSE_FILE}" 2>/dev/null || true elif [ -f "${COMPACT_FILE}" ]; then cat "${COMPACT_FILE}" 2>/dev/null || true else echo "koopa health: snapshot incomplete (meta only) overall=${overall}" fi echo " checked: ${collected_at:-?} (${age}) [timer snapshot; not live at login]" if [ "${stale}" -eq 1 ]; then echo " WARN: snapshot older than ${STALE_SEC}s — is koopa-login-health.timer active?" echo " systemctl --user status koopa-login-health.timer" fi [ "${stale}" -eq 1 ] && _exit_health 1 case "${overall}" in ok|warn) _exit_health 0 ;; *) _exit_health 1 ;; esac } # ---------- COLLECT: probes → write snapshot ---------- CRITICAL_CONTAINERS=( taler-hacktivism taler-hacktivism-exchange-ansible taler-hacktivism-bank koopa-forgejo ) EXPECT_CONTAINERS=( taler-hacktivism taler-hacktivism-exchange-ansible taler-hacktivism-bank koopa-castopod koopa-bonfire koopa-lemmy-proxy koopa-prime-jellyfin koopa-prime-qbittorrent koopa-forgejo koopa-paivana koopa-tor-relay koopa-nym koopa-tops-ng1 koopa-tops-ng2 koopa-tops-ng3 ) OPTIONAL_CONTAINERS=( koopa-prime-jellyfin koopa-prime-qbittorrent koopa-nym koopa-tor-relay koopa-tops-ng1 koopa-tops-ng2 koopa-tops-ng3 koopa-castopod koopa-bonfire koopa-lemmy-proxy koopa-paivana ) # merchant nginx on :9010 is HTTPS-only (plain HTTP → 400) PROBES=( "merchant|https://127.0.0.1:9010/config|200" "exchange|http://127.0.0.1:9011/keys|200" "bank|http://127.0.0.1:9012/config|200" "forgejo|http://127.0.0.1:9024/|200,302,303" "castopod|http://127.0.0.1:9020/|200,302,303,307,308" "bonfire|http://127.0.0.1:9021/|200,302,303,307,308" "lemmy|http://127.0.0.1:9026/|200,302,303,307,308" "prime|http://127.0.0.1:9022/|200,302,303,307,308" "bt|http://127.0.0.1:9023/|200,302,303,307,308,401" "paivana|http://127.0.0.1:9025/|200,302,303,307,308" "tops-ng1|http://127.0.0.1:9090/|200,302,303" "tops-ng2|http://127.0.0.1:9091/|200,302,303" "tops-ng3|http://127.0.0.1:9092/|200,302,303" ) OPTIONAL_PROBES=(castopod bonfire lemmy prime bt paivana tops-ng1 tops-ng2 tops-ng3) C_OK=; C_WARN=; C_FAIL=; C_DIM=; C_RST=; C_BOLD= # colors only when writing for a tty later; snapshots are plain + optional ANSI if COLLECT_COLOR=1 if [ "${KOOPA_HEALTH_COLLECT_COLOR:-0}" = "1" ] && [ -t 1 ]; then C_OK=$'\033[32m'; C_WARN=$'\033[33m'; C_FAIL=$'\033[31m' C_DIM=$'\033[2m'; C_RST=$'\033[0m'; C_BOLD=$'\033[1m' fi OK_N=0 WARN_N=0 FAIL_N=0 LINES=() is_optional_container() { local n="$1" x for x in "${OPTIONAL_CONTAINERS[@]}"; do [ "$x" = "$n" ] && return 0; done return 1 } is_optional_probe() { local n="$1" x for x in "${OPTIONAL_PROBES[@]}"; do [ "$x" = "$n" ] && return 0; done return 1 } record() { local level="$1" msg="$2" case "$level" in OK) OK_N=$((OK_N + 1)); LINES+=(" ${C_OK}OK ${C_RST}${msg}") ;; WARN) WARN_N=$((WARN_N + 1)); LINES+=(" ${C_WARN}WARN${C_RST} ${msg}") ;; FAIL) FAIL_N=$((FAIL_N + 1)); LINES+=(" ${C_FAIL}FAIL${C_RST} ${msg}") ;; INFO) LINES+=(" ${C_DIM}··· ${C_RST}${msg}") ;; esac } code_in_list() { local code="$1" list="$2" part local IFS=',' # shellcheck disable=SC2086 for part in $list; do [ "$code" = "$part" ] && return 0 done return 1 } do_collect() { mkdir -p "${STATE_DIR}" 2>/dev/null || { echo "koopa health collect: cannot mkdir ${STATE_DIR}" >&2 return 1 } local HOST NOW caddy_st st sock listen_blob port local PODMAN running_names name dep tmpdir pids pid local pname purl pok outf code overall line HOST=$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo koopa) NOW=$(date -R 2>/dev/null || date 2>/dev/null || echo "") local NOW_UNIX NOW_UNIX=$(date +%s 2>/dev/null || echo 0) if [ "$(id -u 2>/dev/null)" = "0" ]; then record INFO "running as root (podman rootless via hernani if available)" fi if command -v systemctl >/dev/null 2>&1; then caddy_st=$(systemctl is-active caddy 2>/dev/null || echo unknown) if [ "${caddy_st}" = "active" ]; then record OK "caddy.service active" else record FAIL "caddy.service is ${caddy_st} (edge HTTPS)" fi for sock in http-proxy.socket https-proxy.socket; do st=$(systemctl is-active "${sock}" 2>/dev/null || echo unknown) if [ "${st}" = "active" ] || [ "${st}" = "listening" ]; then record OK "${sock} ${st}" else record WARN "${sock} is ${st}" fi done else record WARN "systemctl not available" fi if command -v ss >/dev/null 2>&1; then listen_blob=$(ss -lnt 2>/dev/null | awk '{print $4}' || true) for port in 9000 9001 9010 9011 9012 9024 9200; do if printf '%s\n' "${listen_blob}" | grep -qE "[:.]${port}\$"; then record OK "listen :${port}" else record FAIL "nothing listening on :${port}" fi done for port in 9020 9021 9022 9023 9025 9026 9090 9091 9092 8080 9080; do if printf '%s\n' "${listen_blob}" | grep -qE "[:.]${port}\$"; then record OK "listen :${port} (optional)" else record WARN "nothing listening on :${port}" fi done fi PODMAN=(podman) if [ "$(id -u 2>/dev/null)" = "0" ] && command -v runuser >/dev/null 2>&1; then if id hernani >/dev/null 2>&1; then PODMAN=(runuser -u hernani -- podman) fi fi if ! command -v "${PODMAN[0]}" >/dev/null 2>&1 && [ "${PODMAN[0]}" = "podman" ]; then record FAIL "podman not found" running_names="" else running_names=$("${PODMAN[@]}" ps --format '{{.Names}}' 2>/dev/null || true) if [ -z "${running_names}" ]; then record FAIL "podman ps empty or failed" fi for name in "${EXPECT_CONTAINERS[@]}"; do if printf '%s\n' "${running_names}" | grep -qxF "${name}"; then st=$("${PODMAN[@]}" ps --format '{{.Names}}\t{{.Status}}' 2>/dev/null \ | awk -F'\t' -v n="${name}" '$1==n {print $2; exit}') record OK "container ${name} (${st:-up})" else if is_optional_container "${name}"; then record WARN "container ${name} not running" else record FAIL "container ${name} not running" fi fi done for dep in koopa-forgejo-db koopa-castopod-mariadb koopa-castopod-redis koopa-bonfire-db koopa-lemmy-db; do if printf '%s\n' "${running_names}" | grep -qxF "${dep}"; then record INFO "dep ${dep} running" else record INFO "dep ${dep} not in running set (name may differ)" fi done fi tmpdir=$(mktemp -d 2>/dev/null || true) if [ -n "${tmpdir}" ] && [ -d "${tmpdir}" ]; then probe_one() { local _url="$1" _out="$2" _code # curl still prints http_code on connect fail; do not append another "000" via || _code=$(curl -sk -m "${CURL_M}" -o /dev/null -w '%{http_code}' "${_url}" 2>/dev/null) [ -n "${_code}" ] || _code="000" printf '%s\n' "${_code}" >"${_out}" } pids=() for spec in "${PROBES[@]}"; do IFS='|' read -r pname purl pok </dev/null || true done for spec in "${PROBES[@]}"; do IFS='|' read -r pname purl pok </dev/null || echo "000") if code_in_list "${code}" "${pok}"; then record OK "http ${pname} → ${code} (${purl})" else if is_optional_probe "${pname}"; then record WARN "http ${pname} → ${code} (${purl})" else record FAIL "http ${pname} → ${code} (${purl})" fi fi done rm -rf "${tmpdir}" 2>/dev/null || true else record WARN "mktemp failed; skipped HTTP probes" fi if command -v timeout >/dev/null 2>&1; then if timeout 1 bash -c 'echo >/dev/tcp/127.0.0.1/9200' 2>/dev/null; then record OK "forgejo git-SSH TCP :9200 (git.hacktivism.ch)" else record FAIL "forgejo git-SSH TCP :9200 closed" fi fi overall="ok" [ "${WARN_N}" -gt 0 ] 2>/dev/null && overall="warn" [ "${FAIL_N}" -gt 0 ] 2>/dev/null && overall="fail" local header verbose_body compact_body header="koopa health ${HOST} ${NOW} git.hacktivism.ch = Forgejo :9024/:9200 · edge Caddy :9000/:9001" verbose_body="${header} " if [ "${#LINES[@]}" -gt 0 ]; then printf -v _lines '%s\n' "${LINES[@]}" verbose_body+="${_lines}" fi case "${overall}" in ok) verbose_body+="overall: OK ok=${OK_N} warn=${WARN_N} fail=${FAIL_N}"$'\n' ;; warn) verbose_body+="overall: WARN ok=${OK_N} warn=${WARN_N} fail=${FAIL_N}"$'\n' ;; *) verbose_body+="overall: FAIL ok=${OK_N} warn=${WARN_N} fail=${FAIL_N}"$'\n' ;; esac compact_body="${header} " local has_issue=0 for line in "${LINES[@]:-}"; do case "${line}" in *FAIL*|*WARN*) compact_body+="${line}"$'\n'; has_issue=1 ;; esac done if [ "${has_issue}" -eq 0 ]; then compact_body+=" OK all critical checks passed (${OK_N} ok)"$'\n' else compact_body+=" (${OK_N} ok — details: koopa-login-health show -v)"$'\n' fi case "${overall}" in ok) compact_body+="overall: OK ok=${OK_N} warn=${WARN_N} fail=${FAIL_N}"$'\n' ;; warn) compact_body+="overall: WARN ok=${OK_N} warn=${WARN_N} fail=${FAIL_N}"$'\n' ;; *) compact_body+="overall: FAIL ok=${OK_N} warn=${WARN_N} fail=${FAIL_N}"$'\n' ;; esac # Atomic writes local tmpc tmpv tmpm tmpc="${STATE_DIR}/.status.compact.$$" tmpv="${STATE_DIR}/.status.verbose.$$" tmpm="${STATE_DIR}/.status.meta.$$" printf '%s' "${compact_body}" >"${tmpc}" 2>/dev/null || return 1 printf '%s' "${verbose_body}" >"${tmpv}" 2>/dev/null || return 1 { echo "overall=${overall}" echo "collected_at=${NOW_UNIX}" echo "collected_at_rfc=${NOW}" echo "ok=${OK_N}" echo "warn=${WARN_N}" echo "fail=${FAIL_N}" echo "host=${HOST}" } >"${tmpm}" 2>/dev/null || return 1 mv -f "${tmpc}" "${COMPACT_FILE}" 2>/dev/null || return 1 mv -f "${tmpv}" "${VERBOSE_FILE}" 2>/dev/null || return 1 mv -f "${tmpm}" "${META_FILE}" 2>/dev/null || return 1 # Optional: print when run interactively if [ -t 1 ] || [ "${VERBOSE}" -eq 1 ]; then printf '%s' "${compact_body}" echo " wrote snapshot → ${STATE_DIR}/" fi case "${overall}" in ok|warn) return 0 ;; *) return 1 ;; esac } case "${CMD}" in show) if [ "${FROM_LOGIN}" = "1" ]; then ( do_show ) 2>&1 || true _exit_health 0 else do_show fi ;; collect) # Never invoke collect from login hook if [ "${FROM_LOGIN}" = "1" ]; then _bail_login_safe "collect refused under FROM_LOGIN" fi do_collect _exit_health $? ;; *) echo "unknown command ${CMD}" >&2 _exit_health 2 ;; esac