diff --git a/2026/2026-09-18--amanita-external-probe.md b/2026/2026-09-18--amanita-external-probe.md new file mode 100644 index 0000000..954f18d --- /dev/null +++ b/2026/2026-09-18--amanita-external-probe.md @@ -0,0 +1,39 @@ +# 2026-09-18 — koopa as external amanita probe logger + +## Why + +Amanita hard hangs leave no panic; internal heartbeat stops with the box. +External ICMP+SSH from **koopa** records last-alive independently so hang vs +power/PSU can be separated after the next freeze. + +## Live (hernani@koopa) + +| Piece | Detail | +|-------|--------| +| Key | `~/.ssh/id_ed25519_amanita_probe` — fingerprint `SHA256:R2i2nZfI1svR6Ts3DaSw5eZj2fvnGgm1R9PoMFS4Jj0` | +| SSH Host | `amanita-probe` → `vx@192.168.100.6`, IdentityFile that key only | +| Script | `~/.local/bin/koopa-amanita-probe` (SoT: `scripts/amanita-probe/`) | +| Timer | user `koopa-amanita-probe.timer` ~15 s, linger yes | +| Log | `~/.local/state/amanita-probe/amanita-probe.{log,last,meta}` — owned by hernani, mode 644 | + +## amanita vx (only) + +Restricted `authorized_keys` line (not on root): + +```text +from="192.168.100.95",no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-pty,command="/bin/cat /var/lib/amanita/heartbeat.last" ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBeMenvp0D9EarfAEUtu49Q7bCbGhw+R/E1nJPQSSmZc koopa-amanita-probe@hernani-koopa +``` + +Forced command only cats the world-readable HB mirror. No shell, no agent/X11/port forward. + +## Verify (2026-09-18 ~15:09 CEST) + +- `systemctl --user is-active koopa-amanita-probe.timer` → active +- `ssh amanita-probe` → EXIT 0, one HB line +- meta: `icmp=ok ssh=ok gap_s≈15` +- hernani can read the probe log dir on koopa (owner) + +## Docs / install + +`scripts/amanita-probe/README.md` + `install-amanita-probe.sh`. +Do **not** put this key on root AK; do **not** bare `modprobe ramoops`. diff --git a/configs/systemd/user/koopa-amanita-probe.service b/configs/systemd/user/koopa-amanita-probe.service new file mode 100644 index 0000000..508003f --- /dev/null +++ b/configs/systemd/user/koopa-amanita-probe.service @@ -0,0 +1,16 @@ +[Unit] +Description=External amanita freeze probe (ICMP+SSH last-alive) +Documentation=file:%h/src/koopa/koopa-admin-log/scripts/amanita-probe/README.md +After=default.target + +[Service] +Type=oneshot +ExecStart=%h/.local/bin/koopa-amanita-probe once +# Probe outcomes are data in the log; never fail the timer +SuccessExitStatus=0 1 +Nice=10 +IOSchedulingClass=best-effort +IOSchedulingPriority=6 + +[Install] +WantedBy=default.target diff --git a/configs/systemd/user/koopa-amanita-probe.timer b/configs/systemd/user/koopa-amanita-probe.timer new file mode 100644 index 0000000..049fee7 --- /dev/null +++ b/configs/systemd/user/koopa-amanita-probe.timer @@ -0,0 +1,13 @@ +[Unit] +Description=Timer · external amanita probe (~15s) +Documentation=file:%h/src/koopa/koopa-admin-log/scripts/amanita-probe/README.md + +[Timer] +OnBootSec=30s +OnUnitActiveSec=15s +AccuracySec=5s +Persistent=false +Unit=koopa-amanita-probe.service + +[Install] +WantedBy=timers.target diff --git a/scripts/amanita-probe/README.md b/scripts/amanita-probe/README.md new file mode 100644 index 0000000..c825d93 --- /dev/null +++ b/scripts/amanita-probe/README.md @@ -0,0 +1,79 @@ +# amanita-probe — external last-alive from koopa + +**Why:** amanita hard hangs leave no panic; internal HB stops with the box. +External ICMP+SSH from **koopa** distinguishes: + +| Pattern | Likely meaning | +|---------|----------------| +| ICMP fail + SSH fail, long gap | hang, power cut, or **PSU** (shared wall outlet with rabbithole ≠ whole-circuit outage) | +| ICMP ok + SSH fail | kernel/userspace hang, sshd stuck, or vx lock | +| both ok, HB stamp stale vs wall | rare clock skew / forced-cmd path oddity | +| both ok, HB fresh | alive | + +Log is **hernani-readable** on koopa (`~/.local/state/amanita-probe/`). +SSH key is **vx-only** on amanita, restricted to koopa LAN IP + forced `cat` of HB mirror. + +## Pieces + +| Piece | Path | +|-------|------| +| Script | `scripts/amanita-probe/koopa-amanita-probe.sh` → `~/.local/bin/koopa-amanita-probe` | +| Timer | `configs/systemd/user/koopa-amanita-probe.{service,timer}` (~15 s) | +| Key | `~/.ssh/id_ed25519_amanita_probe` (hernani on koopa) | +| Log | `~/.local/state/amanita-probe/amanita-probe.log` (+ `.last` / `.meta`) | +| SSH Host | `amanita-probe` → `vx@192.168.100.6` with that IdentityFile | + +## Install (koopa as hernani) + +Laptop SoT: `$HOME/git/admin-logs/local/koopa-admin-log`. Live mirror: `~/src/koopa/koopa-admin-log` (no git commit on koopa). + +```bash +# on koopa after syncing the tree: +cd ~/src/koopa/koopa-admin-log +./scripts/amanita-probe/install-amanita-probe.sh --keygen +# copy the printed authorized_keys line → amanita vx (see below) +./scripts/amanita-probe/install-amanita-probe.sh # units + first probe +``` + +Linger already yes for hernani → timer survives logout. + +## amanita vx authorized_keys (one line) + +```text +from="192.168.100.95",no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-pty,command="/bin/cat /var/lib/amanita/heartbeat.last" ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBeMenvp0D9EarfAEUtu49Q7bCbGhw+R/E1nJPQSSmZc koopa-amanita-probe@hernani-koopa +``` + +- **User:** `vx` only (not root). +- **from=:** only koopa LAN `192.168.100.95`. +- **forced command:** read-only HB mirror (world-readable on amanita). +- **Fingerprint:** `SHA256:R2i2nZfI1svR6Ts3DaSw5eZj2fvnGgm1R9PoMFS4Jj0` +- hernani on koopa owns the private key and the probe log (live since 2026-09-18). +- Daylog: `2026/2026-09-18--amanita-external-probe.md`. + +## Verify + +```bash +# koopa +systemctl --user list-timers koopa-amanita-probe.timer +koopa-amanita-probe status +koopa-amanita-probe show +ssh amanita-probe # should print one HB line, no shell + +# gaps after a freeze: look for icmp=fail/ssh=fail with large gap_s= +``` + +## Uninstall + +```bash +systemctl --user disable --now koopa-amanita-probe.timer +rm -f ~/.local/bin/koopa-amanita-probe \ + ~/.config/systemd/user/koopa-amanita-probe.{service,timer} +systemctl --user daemon-reload +# optional: remove key + Host block + vx authorized_keys line +``` + +## Do not + +- Put this key on **root** authorized_keys. +- `modprobe ramoops` without reserved mem (known hard reset 2026-09-16). +- Expect this to replace internal HB — both are needed. diff --git a/scripts/amanita-probe/install-amanita-probe.sh b/scripts/amanita-probe/install-amanita-probe.sh new file mode 100644 index 0000000..4f6257c --- /dev/null +++ b/scripts/amanita-probe/install-amanita-probe.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# Install koopa-amanita-probe (hernani on koopa): +# - ~/.local/bin/koopa-amanita-probe +# - user systemd timer ~15s → ICMP+SSH last-alive log +# - optional: generate ~/.ssh/id_ed25519_amanita_probe + Host amanita-probe +# - optional: print restricted pubkey line for amanita vx authorized_keys +# +# On koopa as hernani (after laptop push + pull into ~/src/koopa/koopa-admin-log): +# ./scripts/amanita-probe/install-amanita-probe.sh +# ./scripts/amanita-probe/install-amanita-probe.sh --keygen # if key missing +# ./scripts/amanita-probe/install-amanita-probe.sh --print-pubkey +# +# Key install on amanita is separate (vx authorized_keys) — see README. + +set -euo pipefail + +ROOT=$(cd "$(dirname "$0")" && pwd) +REPO_ROOT=$(cd "$ROOT/../.." && pwd) +SRC="$ROOT/koopa-amanita-probe.sh" +UNIT_SRC_DIR="$REPO_ROOT/configs/systemd/user" +BIN_DIR="${HOME}/.local/bin" +DEST="$BIN_DIR/koopa-amanita-probe" +UNIT_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/systemd/user" +KEY="${AMANITA_PROBE_KEY:-${HOME}/.ssh/id_ed25519_amanita_probe}" +SSH_CONFIG="${HOME}/.ssh/config" +HOST_MARKER_BEGIN="# >>> amanita-probe (koopa-admin-log) >>>" +HOST_MARKER_END="# <<< amanita-probe (koopa-admin-log) <<<" +DO_KEYGEN=0 +DO_PRINT=0 +DO_UNITS=1 + +for arg in "$@"; do + case "$arg" in + --keygen) DO_KEYGEN=1 ;; + --print-pubkey) DO_PRINT=1 ;; + --no-units) DO_UNITS=0 ;; + -h|--help) + sed -n '2,16p' "$0" | sed 's/^# \?//' + exit 0 + ;; + *) + echo "unknown arg: $arg" >&2 + exit 2 + ;; + esac +done + +if [ ! -f "$SRC" ]; then + echo "missing $SRC" >&2 + exit 1 +fi + +mkdir -p "$BIN_DIR" "$UNIT_DIR" "${HOME}/.ssh" +chmod 700 "${HOME}/.ssh" 2>/dev/null || true + +install -m 0755 "$SRC" "$DEST" +echo "installed $DEST" + +if [ "$DO_KEYGEN" -eq 1 ]; then + if [ -f "$KEY" ]; then + echo "key exists: $KEY (skip --keygen)" + else + ssh-keygen -t ed25519 -a 64 -f "$KEY" -N "" \ + -C "koopa-amanita-probe@hernani-$(hostname -s 2>/dev/null || echo koopa)" + chmod 600 "$KEY" "${KEY}.pub" + echo "generated $KEY" + fi +fi + +if [ -f "$KEY" ]; then + # Ensure Host block for manual/debug ssh (timer uses -i directly) + touch "$SSH_CONFIG" + chmod 600 "$SSH_CONFIG" 2>/dev/null || true + if grep -qF "$HOST_MARKER_BEGIN" "$SSH_CONFIG" 2>/dev/null; then + tmp=$(mktemp) + awk -v b="$HOST_MARKER_BEGIN" -v e="$HOST_MARKER_END" ' + $0 == b {skip=1; next} + $0 == e {skip=0; next} + !skip {print} + ' "$SSH_CONFIG" >"$tmp" + mv "$tmp" "$SSH_CONFIG" + fi + { + echo "" + echo "$HOST_MARKER_BEGIN" + echo "Host amanita-probe" + echo " Hostname 192.168.100.6" + echo " User vx" + echo " IdentityFile $KEY" + echo " IdentitiesOnly yes" + echo " BatchMode yes" + echo " PreferredAuthentications publickey" + echo " ConnectTimeout 5" + echo "$HOST_MARKER_END" + } >>"$SSH_CONFIG" + echo "updated $SSH_CONFIG Host amanita-probe" +fi + +if [ "$DO_PRINT" -eq 1 ] || [ "$DO_KEYGEN" -eq 1 ]; then + if [ ! -f "${KEY}.pub" ]; then + echo "missing ${KEY}.pub — run with --keygen first" >&2 + exit 1 + fi + pub="$(awk '{print $1, $2}' "${KEY}.pub")" + comment="$(awk '{print $3}' "${KEY}.pub")" + echo "" + echo "=== paste ONE line into amanita:~vx/.ssh/authorized_keys ===" + printf 'from="192.168.100.95",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,command="/bin/cat /var/lib/amanita/heartbeat.last" %s %s\n' \ + "$pub" "${comment:-koopa-amanita-probe}" + echo "=== end ===" +fi + +if [ "$DO_UNITS" -eq 1 ]; then + for u in koopa-amanita-probe.service koopa-amanita-probe.timer; do + if [ ! -f "$UNIT_SRC_DIR/$u" ]; then + echo "missing unit $UNIT_SRC_DIR/$u" >&2 + exit 1 + fi + install -m 0644 "$UNIT_SRC_DIR/$u" "$UNIT_DIR/$u" + echo "installed $UNIT_DIR/$u" + done + systemctl --user daemon-reload + systemctl --user enable --now koopa-amanita-probe.timer + systemctl --user start koopa-amanita-probe.service || true + echo "timer: $(systemctl --user is-active koopa-amanita-probe.timer 2>/dev/null || echo unknown)" + echo "probe once: $DEST once" + "$DEST" once || true + "$DEST" status || true +fi + +echo "done. log: ~/.local/state/amanita-probe/amanita-probe.log" diff --git a/scripts/amanita-probe/koopa-amanita-probe.sh b/scripts/amanita-probe/koopa-amanita-probe.sh new file mode 100644 index 0000000..7156036 --- /dev/null +++ b/scripts/amanita-probe/koopa-amanita-probe.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# koopa-amanita-probe — external ICMP+SSH last-alive for amanita freezes. +# Runs as hernani on koopa (~15s user timer). Log is hernani-readable on koopa. +# SSH uses dedicated ed25519 → vx only (forced command: cat heartbeat.last). +# +# Usage: +# koopa-amanita-probe # one probe cycle (default; timer path) +# koopa-amanita-probe once # same +# koopa-amanita-probe show # tail last lines +# koopa-amanita-probe status # last line + timer hint +# +# Env: +# AMANITA_PROBE_HOST default 192.168.100.6 +# AMANITA_PROBE_USER default vx +# AMANITA_PROBE_KEY default ~/.ssh/id_ed25519_amanita_probe +# AMANITA_PROBE_STATE default ~/.local/state/amanita-probe +set +e +set +u + +CMD="${1:-once}" +HOST="${AMANITA_PROBE_HOST:-192.168.100.6}" +USER_R="${AMANITA_PROBE_USER:-vx}" +KEY="${AMANITA_PROBE_KEY:-${HOME}/.ssh/id_ed25519_amanita_probe}" +STATE="${AMANITA_PROBE_STATE:-${XDG_STATE_HOME:-${HOME}/.local/state}/amanita-probe}" +LOG="${STATE}/amanita-probe.log" +LAST="${STATE}/amanita-probe.last" +META="${STATE}/amanita-probe.meta" +SSH_TO="${AMANITA_PROBE_SSH_TIMEOUT:-5}" +PING_TO="${AMANITA_PROBE_PING_TIMEOUT:-2}" + +mkdir -p "${STATE}" 2>/dev/null || true + +do_show() { + if [ ! -f "${LOG}" ]; then + echo "amanita-probe: no log yet (${LOG})" + echo " start: systemctl --user start koopa-amanita-probe.service" + return 0 + fi + tail -n "${1:-20}" "${LOG}" +} + +do_status() { + if [ -f "${LAST}" ]; then + echo "last: $(cat "${LAST}")" + else + echo "last: (none)" + fi + if [ -f "${META}" ]; then + cat "${META}" + fi + systemctl --user is-active koopa-amanita-probe.timer 2>/dev/null \ + | awk '{print "timer=" $0}' +} + +do_once() { + local ts icmp ssh hb gap prev_ts prev_epoch now_epoch + ts="$(date -Iseconds)" + now_epoch="$(date +%s)" + icmp=fail + ssh=fail + hb="-" + + if ping -c1 -W "${PING_TO}" "${HOST}" >/dev/null 2>&1; then + icmp=ok + fi + + if [ ! -f "${KEY}" ]; then + ssh=nokey + else + # Forced command on amanita returns heartbeat.last; ignore remote argv. + hb="$( + ssh -i "${KEY}" -o IdentitiesOnly=yes -o BatchMode=yes \ + -o StrictHostKeyChecking=accept-new \ + -o ConnectTimeout="${SSH_TO}" -o ConnectionAttempts=1 \ + -o PreferredAuthentications=publickey \ + "${USER_R}@${HOST}" true 2>/dev/null \ + | tr '\n' ' ' | tr -s ' ' | head -c 400 + )" + if [ -n "${hb}" ]; then + ssh=ok + else + # Distinguish timeout/refuse when ICMP ok + if [ "${icmp}" = "ok" ]; then + ssh=fail + else + ssh=down + fi + hb="-" + fi + fi + + gap="?" + if [ -f "${LAST}" ]; then + prev_ts="$(awk '{print $1}' "${LAST}" 2>/dev/null)" + if prev_epoch="$(date -d "${prev_ts}" +%s 2>/dev/null)"; then + gap=$((now_epoch - prev_epoch)) + fi + fi + + local line + line="$(printf '%s icmp=%s ssh=%s gap_s=%s host=%s user=%s hb=%s' \ + "${ts}" "${icmp}" "${ssh}" "${gap}" "${HOST}" "${USER_R}" "${hb}")" + + printf '%s\n' "${line}" >>"${LOG}" + printf '%s\n' "${line}" >"${LAST}" + { + echo "updated=${ts}" + echo "log=${LOG}" + echo "icmp=${icmp}" + echo "ssh=${ssh}" + echo "gap_s=${gap}" + } >"${META}" + + # Keep log bounded (~14d at 15s ≈ 80k lines; rotate soft at 200k lines) + if [ -f "${LOG}" ]; then + local lines + lines="$(wc -l <"${LOG}" 2>/dev/null | tr -d ' ')" + if [ "${lines:-0}" -gt 200000 ] 2>/dev/null; then + tail -n 100000 "${LOG}" >"${LOG}.tmp" && mv "${LOG}.tmp" "${LOG}" + fi + fi + + # Exit 0 always for timer; FAIL is data, not unit failure + return 0 +} + +case "${CMD}" in + once|probe|"") do_once ;; + show) do_show "${2:-20}" ;; + status) do_status ;; + -h|--help) + sed -n '2,20p' "$0" | sed 's/^# \?//' + ;; + *) + echo "unknown: ${CMD} (once|show|status)" >&2 + exit 2 + ;; +esac