monitoring/merge: origin ladder+metrics + stage domains/profiles

scripts/taler-monitoring/lib.sh

@@ -11,6 +11,8 @@
 : "${MERCHANT_LOCAL:=https://127.0.0.1:9010}"
 : "${LANDING_LOCAL:=http://127.0.0.1:9013}"
 : "${KOOPA_SSH:=koopa}"
+# When LAN Host "koopa" is unreachable, try WAN DNAT (see ~/.ssh/config Host koopa-external).
+: "${KOOPA_SSH_FALLBACKS:=koopa-external}"
 : "${MERCHANT_INSTANCE:=goa-demo-cp4zqk}"
 : "${WITHDRAW_AMT:=GOA:20}"   # single-shot fallback; e2e ladder uses ATM notes
 : "${PAY_AMT:=GOA:0.01}"
@@ -289,11 +291,42 @@ with_timeout() {
   ' "$secs" "$@"
 }
 
-# Probe: 0 if koopa SSH works quickly
+# Pick a working SSH host: KOOPA_SSH first, then KOOPA_SSH_FALLBACKS (koopa-external).
+# Sets KOOPA_SSH to the first host that answers. 0 = ok, 1 = none.
+KOOPA_SSH_RESOLVED=0
+resolve_koopa_ssh() {
+  [ "${SKIP_SSH:-0}" = "1" ] && return 1
+  if [ "${KOOPA_SSH_RESOLVED}" = "1" ]; then
+    return 0
+  fi
+  local cands=() c f seen=" "
+  cands+=("${KOOPA_SSH}")
+  # shellcheck disable=SC2086
+  for f in ${KOOPA_SSH_FALLBACKS}; do
+    case "$seen" in *" $f "*) continue ;; esac
+    cands+=("$f")
+    seen="$seen$f "
+  done
+  for c in "${cands[@]}"; do
+    if with_timeout $((SSH_CONNECT_TIMEOUT + 5)) \
+      ssh "${SSH_BASE_OPTS[@]}" "$c" 'echo ok' >/dev/null 2>&1; then
+      if [ "$c" != "${KOOPA_SSH}" ]; then
+        # surface once so operators know we used WAN jump
+        printf '[INFO]  SSH host %s unreachable — using %s\n' "${KOOPA_SSH}" "$c" >&2 || true
+      fi
+      KOOPA_SSH="$c"
+      KOOPA_SSH_RESOLVED=1
+      export KOOPA_SSH
+      return 0
+    fi
+  done
+  return 1
+}
+
+# Probe: 0 if any koopa SSH host works quickly
 koopa_ssh_ok() {
   [ "${SKIP_SSH}" = "1" ] && return 1
-  with_timeout $((SSH_CONNECT_TIMEOUT + 3)) \
-    ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" 'echo ok' >/dev/null 2>&1
+  resolve_koopa_ssh
 }
 
 # Run remote bash -s with optional stdin script; hard-capped
@@ -302,17 +335,25 @@ koopa_ssh_ok() {
 koopa_ssh_run() {
   local t="${1:-$SSH_CMD_TIMEOUT}"
   shift
+  resolve_koopa_ssh || return 1
   with_timeout "$t" ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" "$@"
 }
 
 koopa_ssh_bash() {
   local t="${1:-$SSH_CMD_TIMEOUT}"
+  resolve_koopa_ssh || return 1
   with_timeout "$t" ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" 'bash -s'
 }
 
+# stdin → remote python3 - (for metrics load probe)
+koopa_ssh_python() {
+  local t="${1:-60}"
+  resolve_koopa_ssh || return 1
+  with_timeout "$t" ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" python3 -
+}
+
 # ANSI colours for tags (green OK / yellow WARN / red ERROR·BLOCKER / cyan INFO).
-# Default: always on so pasted ```bash``` logs keep visible tags when the terminal
-# supports colour. Disable: NO_COLOR=1. Force off for dumb pipes: CLICOLOR=0.
+# Default: always on so pasted logs keep visible tags. Disable: NO_COLOR=1 or CLICOLOR=0.
 if [ "${NO_COLOR:-0}" = "1" ] || [ "${CLICOLOR:-1}" = "0" ]; then
   G= R= Y= C= N= B=
 else
@@ -359,6 +400,9 @@ _fmt_tid() {
 }
 
 ok() {
+  # Forms (same idea as info/warn):
+  #   ok "what passed"
+  #   ok "what passed" "detail / ms / bytes / …"
   local label="$1" detail="${2:-}"
   _take_tid
   if [ -n "$detail" ]; then
@@ -385,9 +429,28 @@ fail() {
   ERRORS+=("${LAST_TID:+$LAST_TID }$label${detail:+ — $detail}")
 }
 warn() {
-  local label="$1" detail="${2:-}"
+  # Forms (same idea as err; always try to state the problem):
+  #   warn "problem"
+  #   warn "problem" "why / context"
+  #   warn component "problem" "why / context"
+  local a1="${1:-}" a2="${2:-}" a3="${3:-}"
+  local head detail
   _take_tid
-  printf '%s[WARN]%s  %s%s%s\n' "$Y" "$N" "$(_fmt_tid)" "$label" "${detail:+ — $detail}"
+  if [ -n "$a3" ]; then
+    head="${a1}: ${a2}"
+    detail="$a3"
+  elif [ -n "$a2" ]; then
+    head="$a1"
+    detail="$a2"
+  else
+    head="$a1"
+    detail=""
+  fi
+  if [ -n "$detail" ]; then
+    printf '%s[WARN]%s  %s%s — %s\n' "$Y" "$N" "$(_fmt_tid)" "$head" "$detail"
+  else
+    printf '%s[WARN]%s  %s%s\n' "$Y" "$N" "$(_fmt_tid)" "$head"
+  fi
   WARN_N=$((WARN_N + 1))
 }
 info() {