fix(monitoring): fall back SSH to koopa-external when LAN is down

scripts/taler-monitoring/lib.sh

@@ -11,6 +11,8 @@
 : "${MERCHANT_LOCAL:=https://127.0.0.1:9010}"
 : "${LANDING_LOCAL:=http://127.0.0.1:9013}"
 : "${KOOPA_SSH:=koopa}"
+# When LAN Host "koopa" is unreachable, try WAN DNAT (see ~/.ssh/config Host koopa-external).
+: "${KOOPA_SSH_FALLBACKS:=koopa-external}"
 : "${MERCHANT_INSTANCE:=goa-demo-cp4zqk}"
 : "${WITHDRAW_AMT:=GOA:20}"   # single-shot fallback; e2e ladder uses ATM notes
 : "${PAY_AMT:=GOA:0.01}"
@@ -220,11 +222,42 @@ with_timeout() {
   ' "$secs" "$@"
 }
 
-# Probe: 0 if koopa SSH works quickly
+# Pick a working SSH host: KOOPA_SSH first, then KOOPA_SSH_FALLBACKS (koopa-external).
+# Sets KOOPA_SSH to the first host that answers. 0 = ok, 1 = none.
+KOOPA_SSH_RESOLVED=0
+resolve_koopa_ssh() {
+  [ "${SKIP_SSH:-0}" = "1" ] && return 1
+  if [ "${KOOPA_SSH_RESOLVED}" = "1" ]; then
+    return 0
+  fi
+  local cands=() c f seen=" "
+  cands+=("${KOOPA_SSH}")
+  # shellcheck disable=SC2086
+  for f in ${KOOPA_SSH_FALLBACKS}; do
+    case "$seen" in *" $f "*) continue ;; esac
+    cands+=("$f")
+    seen="$seen$f "
+  done
+  for c in "${cands[@]}"; do
+    if with_timeout $((SSH_CONNECT_TIMEOUT + 5)) \
+      ssh "${SSH_BASE_OPTS[@]}" "$c" 'echo ok' >/dev/null 2>&1; then
+      if [ "$c" != "${KOOPA_SSH}" ]; then
+        # surface once so operators know we used WAN jump
+        printf '[INFO]  SSH host %s unreachable — using %s\n' "${KOOPA_SSH}" "$c" >&2 || true
+      fi
+      KOOPA_SSH="$c"
+      KOOPA_SSH_RESOLVED=1
+      export KOOPA_SSH
+      return 0
+    fi
+  done
+  return 1
+}
+
+# Probe: 0 if any koopa SSH host works quickly
 koopa_ssh_ok() {
   [ "${SKIP_SSH}" = "1" ] && return 1
-  with_timeout $((SSH_CONNECT_TIMEOUT + 3)) \
-    ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" 'echo ok' >/dev/null 2>&1
+  resolve_koopa_ssh
 }
 
 # Run remote bash -s with optional stdin script; hard-capped
@@ -233,14 +266,23 @@ koopa_ssh_ok() {
 koopa_ssh_run() {
   local t="${1:-$SSH_CMD_TIMEOUT}"
   shift
+  resolve_koopa_ssh || return 1
   with_timeout "$t" ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" "$@"
 }
 
 koopa_ssh_bash() {
   local t="${1:-$SSH_CMD_TIMEOUT}"
+  resolve_koopa_ssh || return 1
   with_timeout "$t" ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" 'bash -s'
 }
 
+# stdin → remote python3 - (for metrics load probe)
+koopa_ssh_python() {
+  local t="${1:-60}"
+  resolve_koopa_ssh || return 1
+  with_timeout "$t" ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" python3 -
+}
+
 if [ "${NO_COLOR:-0}" = "1" ] || [ ! -t 1 ]; then
   G= R= Y= C= N= B=
 else