koopa-admin-log/scripts/login-health/koopa-login-health.sh

#!/usr/bin/env bash
# koopa-login-health — read-only GOA/koopa service health.
# Inventory: koopa-admin-log host/overview/LIVE.md (+ ports.md).
#
# Architecture:
#   collect  — run probes, write snapshot (systemd user timer; NOT on login)
#   show     — print last snapshot only (default; used by bashrc on SSH login)
#
# Usage:
#   koopa-login-health              # show compact snapshot (login path)
#   koopa-login-health show -v      # show verbose snapshot
#   koopa-login-health show -q      # overall line only
#   koopa-login-health collect      # refresh snapshot now
#   koopa-login-health collect -v   # same (verbose always stored)
#   KOOPA_SKIP_HEALTH=1 …           # bashrc skip
#   KOOPA_HEALTH_FROM_LOGIN=1       # always exit 0 (login hook)
#
# Never mutates host services. Login never runs probes.

set +e
set +u
set +o pipefail 2>/dev/null || true

FROM_LOGIN=0
[ -n "${KOOPA_HEALTH_FROM_LOGIN:-}" ] && FROM_LOGIN=1

_exit_health() {
  local c="${1:-0}"
  [ "${FROM_LOGIN}" = "1" ] && exit 0
  exit "${c}"
}

_bail_login_safe() {
  echo "koopa health: ${1:-error} (ignored; login continues)" >&2
  _exit_health 0
}

# --- paths (XDG) ---
STATE_DIR="${KOOPA_HEALTH_STATE_DIR:-${XDG_STATE_HOME:-${HOME:-/tmp}/.local/state}/koopa-login-health}"
COMPACT_FILE="${STATE_DIR}/status.compact"
VERBOSE_FILE="${STATE_DIR}/status.verbose"
META_FILE="${STATE_DIR}/status.meta"
# Max age for "current" snapshot (seconds). Older → WARN stale on show.
STALE_SEC="${KOOPA_HEALTH_STALE_SEC:-900}"   # 15 min (timer default 5 min)
CURL_M="${KOOPA_HEALTH_CURL_M:-2}"

CMD="show"
VERBOSE=0
QUIET=0

while [ "${1:-}" != "" ]; do
  case "$1" in
    show|collect|status)
      CMD="$1"
      [ "$CMD" = "status" ] && CMD="show"
      ;;
    -v|--verbose) VERBOSE=1 ;;
    -q|--quiet) QUIET=1 ;;
    -h|--help)
      sed -n '2,22p' "$0" 2>/dev/null | sed 's/^# \?//' || true
      _exit_health 0
      ;;
    *)
      echo "unknown arg: $1 (try: show | collect | -v | -q | -h)" >&2
      _exit_health 2
      ;;
  esac
  shift
done

# ---------- SHOW: never probe ----------
do_show() {
  if [ ! -f "${COMPACT_FILE}" ] && [ ! -f "${META_FILE}" ]; then
    if [ "${QUIET}" -eq 1 ]; then
      echo "unknown (no snapshot — run: systemctl --user start koopa-login-health.service)"
    else
      echo "koopa health: no snapshot yet"
      echo "  Collect runs on a user timer (not at login). Refresh once:"
      echo "    systemctl --user start koopa-login-health.service"
      echo "  or:  koopa-login-health collect"
    fi
    _exit_health 1
  fi

  local overall="unknown" collected_at="" age="?" age_sec=-1 now
  if [ -f "${META_FILE}" ]; then
    # shellcheck disable=SC1090
    overall=$(grep '^overall=' "${META_FILE}" 2>/dev/null | tail -1 | cut -d= -f2-)
    collected_at=$(grep '^collected_at_rfc=' "${META_FILE}" 2>/dev/null | tail -1 | cut -d= -f2-)
    local ts
    ts=$(grep '^collected_at=' "${META_FILE}" 2>/dev/null | tail -1 | cut -d= -f2-)
    now=$(date +%s 2>/dev/null || echo 0)
    if [ -n "${ts}" ] && [ "${ts}" -gt 0 ] 2>/dev/null; then
      age_sec=$((now - ts))
      if [ "${age_sec}" -lt 0 ] 2>/dev/null; then age_sec=0; fi
      if [ "${age_sec}" -lt 120 ] 2>/dev/null; then
        age="${age_sec}s ago"
      elif [ "${age_sec}" -lt 7200 ] 2>/dev/null; then
        age="$((age_sec / 60))m ago"
      else
        age="$((age_sec / 3600))h ago"
      fi
    fi
  fi

  local stale=0
  if [ "${age_sec}" -ge 0 ] 2>/dev/null && [ "${age_sec}" -gt "${STALE_SEC}" ] 2>/dev/null; then
    stale=1
  fi

  if [ "${QUIET}" -eq 1 ]; then
    if [ "${stale}" -eq 1 ]; then
      echo "${overall} (stale ${age})"
    else
      echo "${overall}"
    fi
    [ "${stale}" -eq 1 ] && _exit_health 1
    case "${overall}" in ok|warn) _exit_health 0 ;; *) _exit_health 1 ;; esac
  fi

  if [ "${VERBOSE}" -eq 1 ] && [ -f "${VERBOSE_FILE}" ]; then
    cat "${VERBOSE_FILE}" 2>/dev/null || true
  elif [ -f "${COMPACT_FILE}" ]; then
    cat "${COMPACT_FILE}" 2>/dev/null || true
  else
    echo "koopa health: snapshot incomplete (meta only) overall=${overall}"
  fi

  echo "  checked: ${collected_at:-?} (${age})  [timer snapshot; not live at login]"
  if [ "${stale}" -eq 1 ]; then
    echo "  WARN: snapshot older than ${STALE_SEC}s — is koopa-login-health.timer active?"
    echo "        systemctl --user status koopa-login-health.timer"
  fi

  [ "${stale}" -eq 1 ] && _exit_health 1
  case "${overall}" in ok|warn) _exit_health 0 ;; *) _exit_health 1 ;; esac
}

# ---------- COLLECT: probes → write snapshot ----------
CRITICAL_CONTAINERS=(
  taler-hacktivism
  taler-hacktivism-exchange-ansible
  taler-hacktivism-bank
  koopa-forgejo
)

EXPECT_CONTAINERS=(
  taler-hacktivism
  taler-hacktivism-exchange-ansible
  taler-hacktivism-bank
  koopa-castopod
  koopa-bonfire
  koopa-lemmy-proxy
  koopa-prime-jellyfin
  koopa-prime-qbittorrent
  koopa-forgejo
  koopa-paivana
  koopa-tor-relay
  koopa-nym
  koopa-tops-ng1
  koopa-tops-ng2
  koopa-tops-ng3
)

OPTIONAL_CONTAINERS=(
  koopa-prime-jellyfin
  koopa-prime-qbittorrent
  koopa-nym
  koopa-tor-relay
  koopa-tops-ng1
  koopa-tops-ng2
  koopa-tops-ng3
  koopa-castopod
  koopa-bonfire
  koopa-lemmy-proxy
  koopa-paivana
)

# merchant nginx on :9010 is HTTPS-only (plain HTTP → 400)
PROBES=(
  "merchant|https://127.0.0.1:9010/config|200"
  "exchange|http://127.0.0.1:9011/keys|200"
  "bank|http://127.0.0.1:9012/config|200"
  "forgejo|http://127.0.0.1:9024/|200,302,303"
  "castopod|http://127.0.0.1:9020/|200,302,303,307,308"
  "bonfire|http://127.0.0.1:9021/|200,302,303,307,308"
  "lemmy|http://127.0.0.1:9026/|200,302,303,307,308"
  "prime|http://127.0.0.1:9022/|200,302,303,307,308"
  "bt|http://127.0.0.1:9023/|200,302,303,307,308,401"
  "paivana|http://127.0.0.1:9025/|200,302,303,307,308"
  "tops-ng1|http://127.0.0.1:9090/|200,302,303"
  "tops-ng2|http://127.0.0.1:9091/|200,302,303"
  "tops-ng3|http://127.0.0.1:9092/|200,302,303"
)

OPTIONAL_PROBES=(castopod bonfire lemmy prime bt paivana tops-ng1 tops-ng2 tops-ng3)

C_OK=; C_WARN=; C_FAIL=; C_DIM=; C_RST=; C_BOLD=
# colors only when writing for a tty later; snapshots are plain + optional ANSI if COLLECT_COLOR=1
if [ "${KOOPA_HEALTH_COLLECT_COLOR:-0}" = "1" ] && [ -t 1 ]; then
  C_OK=$'\033[32m'; C_WARN=$'\033[33m'; C_FAIL=$'\033[31m'
  C_DIM=$'\033[2m'; C_RST=$'\033[0m'; C_BOLD=$'\033[1m'
fi

OK_N=0
WARN_N=0
FAIL_N=0
LINES=()

is_optional_container() {
  local n="$1" x
  for x in "${OPTIONAL_CONTAINERS[@]}"; do [ "$x" = "$n" ] && return 0; done
  return 1
}

is_optional_probe() {
  local n="$1" x
  for x in "${OPTIONAL_PROBES[@]}"; do [ "$x" = "$n" ] && return 0; done
  return 1
}

record() {
  local level="$1" msg="$2"
  case "$level" in
    OK)   OK_N=$((OK_N + 1));   LINES+=("  ${C_OK}OK  ${C_RST}${msg}") ;;
    WARN) WARN_N=$((WARN_N + 1)); LINES+=("  ${C_WARN}WARN${C_RST} ${msg}") ;;
    FAIL) FAIL_N=$((FAIL_N + 1)); LINES+=("  ${C_FAIL}FAIL${C_RST} ${msg}") ;;
    INFO) LINES+=("  ${C_DIM}··· ${C_RST}${msg}") ;;
  esac
}

code_in_list() {
  local code="$1" list="$2" part
  local IFS=','
  # shellcheck disable=SC2086
  for part in $list; do
    [ "$code" = "$part" ] && return 0
  done
  return 1
}

do_collect() {
  mkdir -p "${STATE_DIR}" 2>/dev/null || {
    echo "koopa health collect: cannot mkdir ${STATE_DIR}" >&2
    return 1
  }

  local HOST NOW caddy_st st sock listen_blob port
  local PODMAN running_names name dep tmpdir pids pid
  local pname purl pok outf code overall line

  HOST=$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo koopa)
  NOW=$(date -R 2>/dev/null || date 2>/dev/null || echo "")
  local NOW_UNIX
  NOW_UNIX=$(date +%s 2>/dev/null || echo 0)

  if [ "$(id -u 2>/dev/null)" = "0" ]; then
    record INFO "running as root (podman rootless via hernani if available)"
  fi

  if command -v systemctl >/dev/null 2>&1; then
    caddy_st=$(systemctl is-active caddy 2>/dev/null || echo unknown)
    if [ "${caddy_st}" = "active" ]; then
      record OK "caddy.service active"
    else
      record FAIL "caddy.service is ${caddy_st} (edge HTTPS)"
    fi
    for sock in http-proxy.socket https-proxy.socket; do
      st=$(systemctl is-active "${sock}" 2>/dev/null || echo unknown)
      if [ "${st}" = "active" ] || [ "${st}" = "listening" ]; then
        record OK "${sock} ${st}"
      else
        record WARN "${sock} is ${st}"
      fi
    done
  else
    record WARN "systemctl not available"
  fi

  if command -v ss >/dev/null 2>&1; then
    listen_blob=$(ss -lnt 2>/dev/null | awk '{print $4}' || true)
    for port in 9000 9001 9010 9011 9012 9024 9200; do
      if printf '%s\n' "${listen_blob}" | grep -qE "[:.]${port}\$"; then
        record OK "listen :${port}"
      else
        record FAIL "nothing listening on :${port}"
      fi
    done
    for port in 9020 9021 9022 9023 9025 9026 9090 9091 9092 8080 9080; do
      if printf '%s\n' "${listen_blob}" | grep -qE "[:.]${port}\$"; then
        record OK "listen :${port} (optional)"
      else
        record WARN "nothing listening on :${port}"
      fi
    done
  fi

  PODMAN=(podman)
  if [ "$(id -u 2>/dev/null)" = "0" ] && command -v runuser >/dev/null 2>&1; then
    if id hernani >/dev/null 2>&1; then
      PODMAN=(runuser -u hernani -- podman)
    fi
  fi

  if ! command -v "${PODMAN[0]}" >/dev/null 2>&1 && [ "${PODMAN[0]}" = "podman" ]; then
    record FAIL "podman not found"
    running_names=""
  else
    running_names=$("${PODMAN[@]}" ps --format '{{.Names}}' 2>/dev/null || true)
    if [ -z "${running_names}" ]; then
      record FAIL "podman ps empty or failed"
    fi
    for name in "${EXPECT_CONTAINERS[@]}"; do
      if printf '%s\n' "${running_names}" | grep -qxF "${name}"; then
        st=$("${PODMAN[@]}" ps --format '{{.Names}}\t{{.Status}}' 2>/dev/null \
          | awk -F'\t' -v n="${name}" '$1==n {print $2; exit}')
        record OK "container ${name} (${st:-up})"
      else
        if is_optional_container "${name}"; then
          record WARN "container ${name} not running"
        else
          record FAIL "container ${name} not running"
        fi
      fi
    done
    for dep in koopa-forgejo-db koopa-castopod-mariadb koopa-castopod-redis koopa-bonfire-db koopa-lemmy-db; do
      if printf '%s\n' "${running_names}" | grep -qxF "${dep}"; then
        record INFO "dep ${dep} running"
      else
        record INFO "dep ${dep} not in running set (name may differ)"
      fi
    done
  fi

  tmpdir=$(mktemp -d 2>/dev/null || true)
  if [ -n "${tmpdir}" ] && [ -d "${tmpdir}" ]; then
    probe_one() {
      local _url="$1" _out="$2" _code
      # curl still prints http_code on connect fail; do not append another "000" via ||
      _code=$(curl -sk -m "${CURL_M}" -o /dev/null -w '%{http_code}' "${_url}" 2>/dev/null)
      [ -n "${_code}" ] || _code="000"
      printf '%s\n' "${_code}" >"${_out}"
    }
    pids=()
    for spec in "${PROBES[@]}"; do
      IFS='|' read -r pname purl pok <<EOF
${spec}
EOF
      probe_one "${purl}" "${tmpdir}/${pname}" &
      pids+=($!)
    done
    for pid in "${pids[@]:-}"; do
      wait "${pid}" 2>/dev/null || true
    done
    for spec in "${PROBES[@]}"; do
      IFS='|' read -r pname purl pok <<EOF
${spec}
EOF
      code="000"
      [ -f "${tmpdir}/${pname}" ] && code=$(awk '{print $1; exit}' "${tmpdir}/${pname}" 2>/dev/null || echo "000")
      if code_in_list "${code}" "${pok}"; then
        record OK "http ${pname} → ${code} (${purl})"
      else
        if is_optional_probe "${pname}"; then
          record WARN "http ${pname} → ${code} (${purl})"
        else
          record FAIL "http ${pname} → ${code} (${purl})"
        fi
      fi
    done
    rm -rf "${tmpdir}" 2>/dev/null || true
  else
    record WARN "mktemp failed; skipped HTTP probes"
  fi

  if command -v timeout >/dev/null 2>&1; then
    if timeout 1 bash -c 'echo >/dev/tcp/127.0.0.1/9200' 2>/dev/null; then
      record OK "forgejo git-SSH TCP :9200 (git.hacktivism.ch)"
    else
      record FAIL "forgejo git-SSH TCP :9200 closed"
    fi
  fi

  overall="ok"
  [ "${WARN_N}" -gt 0 ] 2>/dev/null && overall="warn"
  [ "${FAIL_N}" -gt 0 ] 2>/dev/null && overall="fail"

  local header verbose_body compact_body
  header="koopa health  ${HOST}  ${NOW}
git.hacktivism.ch = Forgejo :9024/:9200 · edge Caddy :9000/:9001"

  verbose_body="${header}
"
  if [ "${#LINES[@]}" -gt 0 ]; then
    printf -v _lines '%s\n' "${LINES[@]}"
    verbose_body+="${_lines}"
  fi
  case "${overall}" in
    ok)   verbose_body+="overall: OK   ok=${OK_N} warn=${WARN_N} fail=${FAIL_N}"$'\n' ;;
    warn) verbose_body+="overall: WARN ok=${OK_N} warn=${WARN_N} fail=${FAIL_N}"$'\n' ;;
    *)    verbose_body+="overall: FAIL ok=${OK_N} warn=${WARN_N} fail=${FAIL_N}"$'\n' ;;
  esac

  compact_body="${header}
"
  local has_issue=0
  for line in "${LINES[@]:-}"; do
    case "${line}" in
      *FAIL*|*WARN*) compact_body+="${line}"$'\n'; has_issue=1 ;;
    esac
  done
  if [ "${has_issue}" -eq 0 ]; then
    compact_body+="  OK  all critical checks passed (${OK_N} ok)"$'\n'
  else
    compact_body+="  (${OK_N} ok — details: koopa-login-health show -v)"$'\n'
  fi
  case "${overall}" in
    ok)   compact_body+="overall: OK   ok=${OK_N} warn=${WARN_N} fail=${FAIL_N}"$'\n' ;;
    warn) compact_body+="overall: WARN ok=${OK_N} warn=${WARN_N} fail=${FAIL_N}"$'\n' ;;
    *)    compact_body+="overall: FAIL ok=${OK_N} warn=${WARN_N} fail=${FAIL_N}"$'\n' ;;
  esac

  # Atomic writes
  local tmpc tmpv tmpm
  tmpc="${STATE_DIR}/.status.compact.$$"
  tmpv="${STATE_DIR}/.status.verbose.$$"
  tmpm="${STATE_DIR}/.status.meta.$$"
  printf '%s' "${compact_body}" >"${tmpc}" 2>/dev/null || return 1
  printf '%s' "${verbose_body}" >"${tmpv}" 2>/dev/null || return 1
  {
    echo "overall=${overall}"
    echo "collected_at=${NOW_UNIX}"
    echo "collected_at_rfc=${NOW}"
    echo "ok=${OK_N}"
    echo "warn=${WARN_N}"
    echo "fail=${FAIL_N}"
    echo "host=${HOST}"
  } >"${tmpm}" 2>/dev/null || return 1
  mv -f "${tmpc}" "${COMPACT_FILE}" 2>/dev/null || return 1
  mv -f "${tmpv}" "${VERBOSE_FILE}" 2>/dev/null || return 1
  mv -f "${tmpm}" "${META_FILE}" 2>/dev/null || return 1

  # Optional: print when run interactively
  if [ -t 1 ] || [ "${VERBOSE}" -eq 1 ]; then
    printf '%s' "${compact_body}"
    echo "  wrote snapshot → ${STATE_DIR}/"
  fi
  case "${overall}" in ok|warn) return 0 ;; *) return 1 ;; esac
}

case "${CMD}" in
  show)
    if [ "${FROM_LOGIN}" = "1" ]; then
      ( do_show ) 2>&1 || true
      _exit_health 0
    else
      do_show
    fi
    ;;
  collect)
    # Never invoke collect from login hook
    if [ "${FROM_LOGIN}" = "1" ]; then
      _bail_login_safe "collect refused under FROM_LOGIN"
    fi
    do_collect
    _exit_health $?
    ;;
  *)
    echo "unknown command ${CMD}" >&2
    _exit_health 2
    ;;
esac