feat: amanita external probe (timer + vx-only key)

scripts/amanita-probe/koopa-amanita-probe.sh

@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+# koopa-amanita-probe — external ICMP+SSH last-alive for amanita freezes.
+# Runs as hernani on koopa (~15s user timer). Log is hernani-readable on koopa.
+# SSH uses dedicated ed25519 → vx only (forced command: cat heartbeat.last).
+#
+# Usage:
+#   koopa-amanita-probe           # one probe cycle (default; timer path)
+#   koopa-amanita-probe once      # same
+#   koopa-amanita-probe show      # tail last lines
+#   koopa-amanita-probe status    # last line + timer hint
+#
+# Env:
+#   AMANITA_PROBE_HOST   default 192.168.100.6
+#   AMANITA_PROBE_USER   default vx
+#   AMANITA_PROBE_KEY    default ~/.ssh/id_ed25519_amanita_probe
+#   AMANITA_PROBE_STATE  default ~/.local/state/amanita-probe
+set +e
+set +u
+
+CMD="${1:-once}"
+HOST="${AMANITA_PROBE_HOST:-192.168.100.6}"
+USER_R="${AMANITA_PROBE_USER:-vx}"
+KEY="${AMANITA_PROBE_KEY:-${HOME}/.ssh/id_ed25519_amanita_probe}"
+STATE="${AMANITA_PROBE_STATE:-${XDG_STATE_HOME:-${HOME}/.local/state}/amanita-probe}"
+LOG="${STATE}/amanita-probe.log"
+LAST="${STATE}/amanita-probe.last"
+META="${STATE}/amanita-probe.meta"
+SSH_TO="${AMANITA_PROBE_SSH_TIMEOUT:-5}"
+PING_TO="${AMANITA_PROBE_PING_TIMEOUT:-2}"
+
+mkdir -p "${STATE}" 2>/dev/null || true
+
+do_show() {
+  if [ ! -f "${LOG}" ]; then
+    echo "amanita-probe: no log yet (${LOG})"
+    echo "  start: systemctl --user start koopa-amanita-probe.service"
+    return 0
+  fi
+  tail -n "${1:-20}" "${LOG}"
+}
+
+do_status() {
+  if [ -f "${LAST}" ]; then
+    echo "last: $(cat "${LAST}")"
+  else
+    echo "last: (none)"
+  fi
+  if [ -f "${META}" ]; then
+    cat "${META}"
+  fi
+  systemctl --user is-active koopa-amanita-probe.timer 2>/dev/null \
+    | awk '{print "timer=" $0}'
+}
+
+do_once() {
+  local ts icmp ssh hb gap prev_ts prev_epoch now_epoch
+  ts="$(date -Iseconds)"
+  now_epoch="$(date +%s)"
+  icmp=fail
+  ssh=fail
+  hb="-"
+
+  if ping -c1 -W "${PING_TO}" "${HOST}" >/dev/null 2>&1; then
+    icmp=ok
+  fi
+
+  if [ ! -f "${KEY}" ]; then
+    ssh=nokey
+  else
+    # Forced command on amanita returns heartbeat.last; ignore remote argv.
+    hb="$(
+      ssh -i "${KEY}" -o IdentitiesOnly=yes -o BatchMode=yes \
+        -o StrictHostKeyChecking=accept-new \
+        -o ConnectTimeout="${SSH_TO}" -o ConnectionAttempts=1 \
+        -o PreferredAuthentications=publickey \
+        "${USER_R}@${HOST}" true 2>/dev/null \
+        | tr '\n' ' ' | tr -s ' ' | head -c 400
+    )"
+    if [ -n "${hb}" ]; then
+      ssh=ok
+    else
+      # Distinguish timeout/refuse when ICMP ok
+      if [ "${icmp}" = "ok" ]; then
+        ssh=fail
+      else
+        ssh=down
+      fi
+      hb="-"
+    fi
+  fi
+
+  gap="?"
+  if [ -f "${LAST}" ]; then
+    prev_ts="$(awk '{print $1}' "${LAST}" 2>/dev/null)"
+    if prev_epoch="$(date -d "${prev_ts}" +%s 2>/dev/null)"; then
+      gap=$((now_epoch - prev_epoch))
+    fi
+  fi
+
+  local line
+  line="$(printf '%s icmp=%s ssh=%s gap_s=%s host=%s user=%s hb=%s' \
+    "${ts}" "${icmp}" "${ssh}" "${gap}" "${HOST}" "${USER_R}" "${hb}")"
+
+  printf '%s\n' "${line}" >>"${LOG}"
+  printf '%s\n' "${line}" >"${LAST}"
+  {
+    echo "updated=${ts}"
+    echo "log=${LOG}"
+    echo "icmp=${icmp}"
+    echo "ssh=${ssh}"
+    echo "gap_s=${gap}"
+  } >"${META}"
+
+  # Keep log bounded (~14d at 15s ≈ 80k lines; rotate soft at 200k lines)
+  if [ -f "${LOG}" ]; then
+    local lines
+    lines="$(wc -l <"${LOG}" 2>/dev/null | tr -d ' ')"
+    if [ "${lines:-0}" -gt 200000 ] 2>/dev/null; then
+      tail -n 100000 "${LOG}" >"${LOG}.tmp" && mv "${LOG}.tmp" "${LOG}"
+    fi
+  fi
+
+  # Exit 0 always for timer; FAIL is data, not unit failure
+  return 0
+}
+
+case "${CMD}" in
+  once|probe|"") do_once ;;
+  show) do_show "${2:-20}" ;;
+  status) do_status ;;
+  -h|--help)
+    sed -n '2,20p' "$0" | sed 's/^# \?//'
+    ;;
+  *)
+    echo "unknown: ${CMD} (once|show|status)" >&2
+    exit 2
+    ;;
+esac