docs: update goa-regio-ng login-health monitoring

2026-09-20 23:41:26 +02:00 · 2026-09-20 23:41:26 +02:00 · 986e9a3090
commit 986e9a3090
parent 0348f4025a
5 changed files with 376 additions and 0 deletions
--- a/scripts/firecuda-probe/README.md
+++ b/scripts/firecuda-probe/README.md
@ -0,0 +1,80 @@
+# firecuda-probe — external last-alive from koopa
+
+**Why:** firecuda panics (e.g. 1GbE PCIe completion timeout 2026-09-20) and hard hangs leave the internal HB dead with the box. External ICMP+SSH from **koopa** distinguishes:
+
+| Pattern | Likely meaning |
+|---------|----------------|
+| ICMP fail + SSH fail, long gap | hang, power cut, or NIC/kernel down |
+| ICMP ok + SSH fail | userspace/sshd stuck, or max lock |
+| both ok, HB stamp stale vs wall | rare clock skew / forced-cmd path oddity / HB LaunchDaemon stopped |
+| both ok, HB fresh | alive |
+
+Log is **hernani-readable** on koopa (`~/.local/state/firecuda-probe/`).
+SSH key is **max-only** on firecuda, restricted to koopa LAN IP + forced `cat` of HB mirror.
+
+Requires firecuda crash-debug HB live (`/usr/local/var/lib/firecuda/heartbeat.last`) — see `firecuda-admin-log/configs/crash-debug/`.
+
+## Pieces
+
+| Piece | Path |
+|-------|------|
+| Script | `scripts/firecuda-probe/koopa-firecuda-probe.sh` → `~/.local/bin/koopa-firecuda-probe` |
+| Timer | `configs/systemd/user/koopa-firecuda-probe.{service,timer}` (~15 s) |
+| Key | `~/.ssh/id_ed25519_firecuda_probe` (hernani on koopa) |
+| Log | `~/.local/state/firecuda-probe/firecuda-probe.log` (+ `.last` / `.meta`) |
+| SSH Host | `firecuda-probe` → `max@192.168.100.84` with that IdentityFile |
+
+## Install (koopa as hernani) — Freigabe
+
+Laptop SoT: `$HOME/git/admin-logs/local/koopa-admin-log`. Live mirror: `~/src/koopa/koopa-admin-log` (no git commit on koopa).
+
+```bash
+# on koopa after syncing the tree:
+cd ~/src/koopa/koopa-admin-log
+./scripts/firecuda-probe/install-firecuda-probe.sh --keygen
+# copy the printed authorized_keys line → firecuda max (see below)
+./scripts/firecuda-probe/install-firecuda-probe.sh   # units + first probe
+```
+
+Linger already yes for hernani → timer survives logout.
+
+## firecuda max authorized_keys (one line)
+
+After `--keygen`, install prints the exact line. Template:
+
+```text
+from="192.168.100.95",no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-pty,command="/bin/cat /usr/local/var/lib/firecuda/heartbeat.last" ssh-ed25519 <PUBKEY> koopa-firecuda-probe@hernani-koopa
+```
+
+- **User:** `max` only (not root).
+- **from=:** only koopa LAN `192.168.100.95`.
+- **forced command:** read-only HB mirror (world-readable after crash-debug install).
+- Daylog: `firecuda-admin-log/2026/2026-09-20--pcie-lan-panic+crash-debug.md`.
+
+## Verify
+
+```bash
+# koopa
+systemctl --user list-timers koopa-firecuda-probe.timer
+koopa-firecuda-probe status
+koopa-firecuda-probe show
+ssh firecuda-probe   # should print one HB line, no shell
+
+# gaps after a crash: look for icmp=fail/ssh=fail with large gap_s=
+```
+
+## Uninstall
+
+```bash
+systemctl --user disable --now koopa-firecuda-probe.timer
+rm -f ~/.local/bin/koopa-firecuda-probe \
+  ~/.config/systemd/user/koopa-firecuda-probe.{service,timer}
+systemctl --user daemon-reload
+# optional: remove key + Host block + max authorized_keys line
+```
+
+## Do not
+
+- Put this key on **root** authorized_keys.
+- Enable timer / write authorized_keys without Freigabe.
+- Expect this to fix PCIe/Ethernet panics — it only timestamps last-alive.
--- a/scripts/firecuda-probe/install-firecuda-probe.sh
+++ b/scripts/firecuda-probe/install-firecuda-probe.sh
@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+# Install koopa-firecuda-probe (hernani on koopa):
+#   - ~/.local/bin/koopa-firecuda-probe
+#   - user systemd timer ~15s → ICMP+SSH last-alive log
+#   - optional: generate ~/.ssh/id_ed25519_firecuda_probe + Host firecuda-probe
+#   - optional: print restricted pubkey line for firecuda max authorized_keys
+#
+# On koopa as hernani (after laptop push + pull into ~/src/koopa/koopa-admin-log):
+#   ./scripts/firecuda-probe/install-firecuda-probe.sh
+#   ./scripts/firecuda-probe/install-firecuda-probe.sh --keygen   # if key missing
+#   ./scripts/firecuda-probe/install-firecuda-probe.sh --print-pubkey
+#
+# Key install on firecuda is separate (max authorized_keys) — see README.
+# Live install = Freigabe.
+
+set -euo pipefail
+
+ROOT=$(cd "$(dirname "$0")" && pwd)
+REPO_ROOT=$(cd "$ROOT/../.." && pwd)
+SRC="$ROOT/koopa-firecuda-probe.sh"
+UNIT_SRC_DIR="$REPO_ROOT/configs/systemd/user"
+BIN_DIR="${HOME}/.local/bin"
+DEST="$BIN_DIR/koopa-firecuda-probe"
+UNIT_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/systemd/user"
+KEY="${FIRECUDA_PROBE_KEY:-${HOME}/.ssh/id_ed25519_firecuda_probe}"
+SSH_CONFIG="${HOME}/.ssh/config"
+HOST_MARKER_BEGIN="# >>> firecuda-probe (koopa-admin-log) >>>"
+HOST_MARKER_END="# <<< firecuda-probe (koopa-admin-log) <<<"
+DO_KEYGEN=0
+DO_PRINT=0
+DO_UNITS=1
+
+for arg in "$@"; do
+  case "$arg" in
+    --keygen) DO_KEYGEN=1 ;;
+    --print-pubkey) DO_PRINT=1 ;;
+    --no-units) DO_UNITS=0 ;;
+    -h|--help)
+      sed -n '2,16p' "$0" | sed 's/^# \?//'
+      exit 0
+      ;;
+    *)
+      echo "unknown arg: $arg" >&2
+      exit 2
+      ;;
+  esac
+done
+
+if [ ! -f "$SRC" ]; then
+  echo "missing $SRC" >&2
+  exit 1
+fi
+
+mkdir -p "$BIN_DIR" "$UNIT_DIR" "${HOME}/.ssh"
+chmod 700 "${HOME}/.ssh" 2>/dev/null || true
+
+install -m 0755 "$SRC" "$DEST"
+echo "installed $DEST"
+
+if [ "$DO_KEYGEN" -eq 1 ]; then
+  if [ -f "$KEY" ]; then
+    echo "key exists: $KEY (skip --keygen)"
+  else
+    ssh-keygen -t ed25519 -a 64 -f "$KEY" -N "" \
+      -C "koopa-firecuda-probe@hernani-$(hostname -s 2>/dev/null || echo koopa)"
+    chmod 600 "$KEY" "${KEY}.pub"
+    echo "generated $KEY"
+  fi
+fi
+
+if [ -f "$KEY" ]; then
+  touch "$SSH_CONFIG"
+  chmod 600 "$SSH_CONFIG" 2>/dev/null || true
+  if grep -qF "$HOST_MARKER_BEGIN" "$SSH_CONFIG" 2>/dev/null; then
+    tmp=$(mktemp)
+    awk -v b="$HOST_MARKER_BEGIN" -v e="$HOST_MARKER_END" '
+      $0 == b {skip=1; next}
+      $0 == e {skip=0; next}
+      !skip {print}
+    ' "$SSH_CONFIG" >"$tmp"
+    mv "$tmp" "$SSH_CONFIG"
+  fi
+  {
+    echo ""
+    echo "$HOST_MARKER_BEGIN"
+    echo "Host firecuda-probe"
+    echo "  Hostname 192.168.100.84"
+    echo "  User max"
+    echo "  IdentityFile $KEY"
+    echo "  IdentitiesOnly yes"
+    echo "  BatchMode yes"
+    echo "  PreferredAuthentications publickey"
+    echo "  ConnectTimeout 5"
+    echo "$HOST_MARKER_END"
+  } >>"$SSH_CONFIG"
+  echo "updated $SSH_CONFIG Host firecuda-probe"
+fi
+
+if [ "$DO_PRINT" -eq 1 ] || [ "$DO_KEYGEN" -eq 1 ]; then
+  if [ ! -f "${KEY}.pub" ]; then
+    echo "missing ${KEY}.pub — run with --keygen first" >&2
+    exit 1
+  fi
+  pub="$(awk '{print $1, $2}' "${KEY}.pub")"
+  comment="$(awk '{print $3}' "${KEY}.pub")"
+  echo ""
+  echo "=== paste ONE line into firecuda:~max/.ssh/authorized_keys ==="
+  printf 'from="192.168.100.95",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,command="/bin/cat /usr/local/var/lib/firecuda/heartbeat.last" %s %s\n' \
+    "$pub" "${comment:-koopa-firecuda-probe}"
+  echo "=== end ==="
+fi
+
+if [ "$DO_UNITS" -eq 1 ]; then
+  for u in koopa-firecuda-probe.service koopa-firecuda-probe.timer; do
+    if [ ! -f "$UNIT_SRC_DIR/$u" ]; then
+      echo "missing unit $UNIT_SRC_DIR/$u" >&2
+      exit 1
+    fi
+    install -m 0644 "$UNIT_SRC_DIR/$u" "$UNIT_DIR/$u"
+    echo "installed $UNIT_DIR/$u"
+  done
+  systemctl --user daemon-reload
+  systemctl --user enable --now koopa-firecuda-probe.timer
+  systemctl --user start koopa-firecuda-probe.service || true
+  echo "timer: $(systemctl --user is-active koopa-firecuda-probe.timer 2>/dev/null || echo unknown)"
+  echo "probe once: $DEST once"
+  "$DEST" once || true
+  "$DEST" status || true
+fi
+
+echo "done. log: ~/.local/state/firecuda-probe/firecuda-probe.log"
--- a/scripts/firecuda-probe/koopa-firecuda-probe.sh
+++ b/scripts/firecuda-probe/koopa-firecuda-probe.sh
@ -0,0 +1,136 @@
+#!/usr/bin/env bash
+# koopa-firecuda-probe — external ICMP+SSH last-alive for firecuda crashes/hangs.
+# Runs as hernani on koopa (~15s user timer). Log is hernani-readable on koopa.
+# SSH uses dedicated ed25519 → max only (forced command: cat heartbeat.last).
+#
+# Usage:
+#   koopa-firecuda-probe           # one probe cycle (default; timer path)
+#   koopa-firecuda-probe once      # same
+#   koopa-firecuda-probe show      # tail last lines
+#   koopa-firecuda-probe status    # last line + timer hint
+#
+# Env:
+#   FIRECUDA_PROBE_HOST   default 192.168.100.84
+#   FIRECUDA_PROBE_USER   default max
+#   FIRECUDA_PROBE_KEY    default ~/.ssh/id_ed25519_firecuda_probe
+#   FIRECUDA_PROBE_STATE  default ~/.local/state/firecuda-probe
+set +e
+set +u
+
+CMD="${1:-once}"
+HOST="${FIRECUDA_PROBE_HOST:-192.168.100.84}"
+USER_R="${FIRECUDA_PROBE_USER:-max}"
+KEY="${FIRECUDA_PROBE_KEY:-${HOME}/.ssh/id_ed25519_firecuda_probe}"
+STATE="${FIRECUDA_PROBE_STATE:-${XDG_STATE_HOME:-${HOME}/.local/state}/firecuda-probe}"
+LOG="${STATE}/firecuda-probe.log"
+LAST="${STATE}/firecuda-probe.last"
+META="${STATE}/firecuda-probe.meta"
+SSH_TO="${FIRECUDA_PROBE_SSH_TIMEOUT:-5}"
+PING_TO="${FIRECUDA_PROBE_PING_TIMEOUT:-2}"
+
+mkdir -p "${STATE}" 2>/dev/null || true
+
+do_show() {
+  if [ ! -f "${LOG}" ]; then
+    echo "firecuda-probe: no log yet (${LOG})"
+    echo "  start: systemctl --user start koopa-firecuda-probe.service"
+    return 0
+  fi
+  tail -n "${1:-20}" "${LOG}"
+}
+
+do_status() {
+  if [ -f "${LAST}" ]; then
+    echo "last: $(cat "${LAST}")"
+  else
+    echo "last: (none)"
+  fi
+  if [ -f "${META}" ]; then
+    cat "${META}"
+  fi
+  systemctl --user is-active koopa-firecuda-probe.timer 2>/dev/null \
+    | awk '{print "timer=" $0}'
+}
+
+do_once() {
+  local ts icmp ssh hb gap prev_ts prev_epoch now_epoch
+  ts="$(date -Iseconds)"
+  now_epoch="$(date +%s)"
+  icmp=fail
+  ssh=fail
+  hb="-"
+
+  if ping -c1 -W "${PING_TO}" "${HOST}" >/dev/null 2>&1; then
+    icmp=ok
+  fi
+
+  if [ ! -f "${KEY}" ]; then
+    ssh=nokey
+  else
+    # Forced command on firecuda returns heartbeat.last; ignore remote argv.
+    hb="$(
+      ssh -i "${KEY}" -o IdentitiesOnly=yes -o BatchMode=yes \
+        -o StrictHostKeyChecking=accept-new \
+        -o ConnectTimeout="${SSH_TO}" -o ConnectionAttempts=1 \
+        -o PreferredAuthentications=publickey \
+        "${USER_R}@${HOST}" true 2>/dev/null \
+        | tr '\n' ' ' | tr -s ' ' | head -c 400
+    )"
+    if [ -n "${hb}" ]; then
+      ssh=ok
+    else
+      if [ "${icmp}" = "ok" ]; then
+        ssh=fail
+      else
+        ssh=down
+      fi
+      hb="-"
+    fi
+  fi
+
+  gap="?"
+  if [ -f "${LAST}" ]; then
+    prev_ts="$(awk '{print $1}' "${LAST}" 2>/dev/null)"
+    if prev_epoch="$(date -d "${prev_ts}" +%s 2>/dev/null)"; then
+      gap=$((now_epoch - prev_epoch))
+    fi
+  fi
+
+  local line
+  line="$(printf '%s icmp=%s ssh=%s gap_s=%s host=%s user=%s hb=%s' \
+    "${ts}" "${icmp}" "${ssh}" "${gap}" "${HOST}" "${USER_R}" "${hb}")"
+
+  printf '%s\n' "${line}" >>"${LOG}"
+  printf '%s\n' "${line}" >"${LAST}"
+  {
+    echo "updated=${ts}"
+    echo "log=${LOG}"
+    echo "icmp=${icmp}"
+    echo "ssh=${ssh}"
+    echo "gap_s=${gap}"
+  } >"${META}"
+
+  # Keep log bounded (~14d at 15s ≈ 80k lines; rotate soft at 200k lines)
+  if [ -f "${LOG}" ]; then
+    local lines
+    lines="$(wc -l <"${LOG}" 2>/dev/null | tr -d ' ')"
+    if [ "${lines:-0}" -gt 200000 ] 2>/dev/null; then
+      tail -n 100000 "${LOG}" >"${LOG}.tmp" && mv "${LOG}.tmp" "${LOG}"
+    fi
+  fi
+
+  return 0
+}
+
+case "${CMD}" in
+  once|probe|"") do_once ;;
+  show) do_show "${2:-20}" ;;
+  status) do_status ;;
+  -h|--help)
+    sed -n '2,20p' "$0" | sed 's/^# \?//'
+    ;;
+  *)
+    echo "unknown: ${CMD} (once|show|status)" >&2
+    exit 2
+    ;;
+esac