#!/usr/bin/env bash
# taler-monitoring — public URL / stack checks for a Taler domain
#
#   ./taler-monitoring.sh                          # local GOA (urls + inside + e2e)
#   ./taler-monitoring.sh -d taler.net urls         # public demo, no SSH
#   ./taler-monitoring.sh --domain taler-ops.ch     # public ops, no SSH
#   TALER_DOMAIN=demo.taler.net ./taler-monitoring.sh urls
#
# Tags: [OK] [INFO] [WARN] [ERROR] [BLOCKER]
# Exit 0 only if every selected phase exits 0.

set -euo pipefail
ROOT=$(cd "$(dirname "$0")" && pwd)

# --- suite identity (also: --ver / --version / -V) ---
SOURCE_REPO_WEB="${SOURCE_REPO_WEB:-https://git.hacktivism.ch/hernani/taler-monitoring}"
SOURCE_REPO_GIT="${SOURCE_REPO_GIT:-https://git.hacktivism.ch/hernani/taler-monitoring.git}"

print_version() {
  local ver tag commit commit_full dirty branch
  ver="unknown"
  if [ -f "$ROOT/VERSION" ]; then
    ver=$(tr -d "[:space:]" <"$ROOT/VERSION" | sed "s/^v//")
  fi
  tag=""
  commit="unknown"
  commit_full=""
  branch=""
  dirty=""
  if git -C "$ROOT" rev-parse --git-dir >/dev/null 2>&1; then
    tag=$(git -C "$ROOT" describe --tags --exact-match 2>/dev/null || true)
    if [ -z "$tag" ]; then
      tag=$(git -C "$ROOT" describe --tags --abbrev=0 2>/dev/null || true)
    fi
    commit=$(git -C "$ROOT" rev-parse --short=12 HEAD 2>/dev/null || echo unknown)
    commit_full=$(git -C "$ROOT" rev-parse HEAD 2>/dev/null || true)
    branch=$(git -C "$ROOT" rev-parse --abbrev-ref HEAD 2>/dev/null || true)
    if [ -n "$(git -C "$ROOT" status --porcelain 2>/dev/null)" ]; then
      dirty=" (dirty)"
    fi
  fi
  if [ -n "$tag" ] && [ "$tag" = "v$ver" ]; then
    printf "taler-monitoring %s\n" "$tag"
  else
    printf "taler-monitoring v%s\n" "$ver"
  fi
  printf "version:   %s\n" "$ver"
  if [ -n "$tag" ]; then
    if [ "$tag" = "v$ver" ]; then
      printf "tag:       %s\n" "$tag"
    else
      printf "git-tag:   %s (tree; VERSION file is %s)\n" "$tag" "$ver"
    fi
  fi
  printf "commit:    %s%s\n" "$commit" "$dirty"
  [ -n "$branch" ] && [ "$branch" != "HEAD" ] && printf "branch:    %s\n" "$branch"
  printf "repo:      %s\n" "$SOURCE_REPO_WEB"
  printf "clone:     %s\n" "$SOURCE_REPO_GIT"
  if [ -n "$commit_full" ]; then
    printf "commit-url: %s/src/commit/%s\n" "$SOURCE_REPO_WEB" "$commit_full"
  fi
  if [ -n "$tag" ]; then
    printf "tag-url:   %s/src/tag/%s\n" "$SOURCE_REPO_WEB" "$tag"
  fi
}

# Fast path: version before loading lib.sh / phases
case "${1:-}" in
  --ver|--version|-V)
    print_version
    exit 0
    ;;
esac

# Line-buffered stdout/stderr so redirected logs (tee/host-agent) flush each line.
if [ "${_TALER_MON_STDBUF:-0}" != "1" ] && command -v stdbuf >/dev/null 2>&1; then
  export _TALER_MON_STDBUF=1
  exec stdbuf -oL -eL bash "$0" "$@"
fi

usage() {
  cat <<'EOF'
taler-monitoring — bank / exchange / merchant checks

Usage:
  ./taler-monitoring.sh [options] [phases...]

Phases:
  urls     public HTTPS (no SSH)          ← default with --domain
           (includes merchant /webui/ SPA fingerprints: version.txt, overlay, assets)
  inside   container status via SSH       (local stack only)
  versions taler packages vs deb.taler.net trixie + repo availability
  sanity   public + optional server
  server   server-side only (SSH)
  e2e      withdraw + pay (small amounts; remote aborts on login/KYC)
  ladder   withdraw/pay amount ladder (GOA ceiling or stage TESTPAYSAN max_wire)
  auth401  merchant Basic-auth / case matrix (HTTP 401 paths; may create throwaway instance)
  aptdeploy  koopa podman apt-src smoke: taler-merchant in
             koopa-taler-deploy-test-apt-src-trixie{,-testing}
  surface  REMOTE-ONLY public inventory (NOT in default/all/full):
             ecosystem hosts (taler.net, gnunet.org, taler-systems.com, mattermost, …)
             or -d DOMAIN → that domain’s surface; port/protocol/TLS/CVE (OSV)
  mattermost  Mattermost chat health (default mattermost.taler.net; SPA + /api/v4/system/ping)
  mail     MX/SMTP/IMAP for Taler mail (firefly, pixel/TSA, catalogued domains)
  monpages public monitoring HTML via FQDN (obligatory ERROR; GOA full inventory, FP only FP)
             uses MON_HOSTS + HTML_URL_OK (same as host-agent); v1.3.1+
  devtesting  fake-franken CHF via rusty.taler-ops.ch (taler-devtesting)
             geniban + fake-incoming; needs SSH Host DEVTESTING_SSH (key to devtesting@rusty)
  franken|fake-franken  aliases for devtesting
  all      urls + inside + versions + sanity + e2e  (SSH phases only on koopa)
  full     all + server + ladder + auth401  (maximum; long-running, needs secrets)
           NOTE: surface is never included in all/full — pass it explicitly

Options:
  -d, --domain DOMAIN   load profile from domains.conf (bank/exchange/merchant)
                        presets: koopa | hacktivism.ch | taler.net | taler-ops.ch
                                 | my.taler-ops.ch | stage.taler-ops.ch
                        unknown domain → heuristic hosts (see README)
                        local (koopa) allows SSH; others public-only
  --bank URL            bank base (hostname or https://…) — overrides profile
  --exchange URL        exchange base — overrides profile
  --merchant URL        merchant-backend base — overrides profile
  --currency CODE       expected currency (GOA, KUDOS, CHF, …); empty = report only
  --no-probe            do not probe alternate hosts (unknown domains only)
  --lang en|fr          UI/console language (sticky bar + badges)
                        overrides auto (FrancPaysan→fr, else en)
  --ver, --version, -V print suite version, git commit, and repo URLs
  -h, --help

Add a stack: edit domains.conf (name + bank + exchange + merchant + currency).

Examples:
  ./taler-monitoring.sh -d taler.net
  ./taler-monitoring.sh -d taler-ops.ch urls
  ./taler-monitoring.sh -d my.taler-ops.ch urls
  ./taler-monitoring.sh -d taler-ops.ch --merchant https://my.taler-ops.ch urls
  ./taler-monitoring.sh -d demo.taler.net --currency KUDOS
  ./taler-monitoring.sh --exchange https://exchange.taler-ops.ch \
      --merchant https://my.taler-ops.ch --bank https://bank.taler-ops.ch urls

Env (same meaning):
  TALER_DOMAIN  BANK_PUBLIC  EXCHANGE_PUBLIC  MERCHANT_PUBLIC  EXPECT_CURRENCY
  TALER_DOMAINS_CONF  SKIP_SSH=1
  NO_COLOR=1 / CLICOLOR=0   disable green/yellow/red tags (default: coloured)
  SKIP_SSH=1    NO_COLOR=1
  RUN_TIMEOUT=600   whole-run wall clock seconds (default 600; 0=unlimited)
                    long phases (ladder/full/e2e) need a higher value or 0
  DISK_WARN_USED_PCT=85  disk free: WARN when used ≥ this %
  DISK_ERR_USED_PCT=95   disk free: ERROR when used ≥ this % (or avail=0 / 100%)
  PERF_WARN_MS  PERF_FAIL_MS   (urls latency; default 8000 / 20000)
  QR_CHECK=0    skip QR form + qrencode/zbarimg (urls phase)
  QR_ECC=M      qrencode ECC level (default M)
  KOOPA_SSH  KOOPA_SSH_FALLBACKS   (default koopa → koopa-external)
  METRICS_LOAD=0   skip host/container RAM/CPU probes (e2e/ladder/inside)
  EXPECT_WEBUI_VERSION=1.6.11   pin merchant /webui/version.txt (urls)
  EXPECT_WEBUI_OVERLAY=selfbuild  require substr in version-overlay.txt
  WEBUI_OVERLAY_DENY='master-11340'  fail if overlay matches regex
  CHECK_WEBUI_SPA=0             skip SPA fingerprint block in urls
  AUTH401_*                     see check_auth401.sh / secrets.env.example
  AUTH401_CONTINUE=1 / CONTINUE_ON_ERROR=1
                        auth401: run all groups, collect every ERROR (no mid-run halt)
  APT_DEPLOY_SKIP=1     skip aptdeploy phase
  APT_DEPLOY_CONTAINERS="name:suite …"  default trixie + trixie-testing deploy-test pods
  SURFACE_CVE=0         disable CVE queries in surface phase
  SURFACE_CVE_LEVEL=warn|error  bare Server-header versions default warn
                        (Debian package versions default error)
  SURFACE_CATALOG=path  override surface-catalog.conf
  TALER_MON_LANG=en|fr  same as --lang (default en; auto fr for *lefrancpaysan*)

Full load (GOA / hacktivism):
  ./taler-monitoring.sh -d hacktivism.ch full
  # or explicit:
  ./taler-monitoring.sh -d hacktivism.ch urls inside versions sanity server e2e ladder auth401
  # hacktivism host-agent also runs aptdeploy (apt-src merchant containers on koopa)

Remote surface / ecosystem (explicit only — never default):
  ./taler-monitoring.sh surface
  ./taler-monitoring.sh -d hacktivism.ch surface
  ./taler-monitoring.sh -d lefrancpaysan.ch surface
  ./taler-monitoring.sh mattermost          # mattermost.taler.net chat health
  ./taler-monitoring.sh mail                # firefly + pixel MX/SMTP/IMAP

SPA pin after selfbuild:
  EXPECT_WEBUI_VERSION=1.6.11 EXPECT_WEBUI_OVERLAY=selfbuild-v1.6.11 \\
    WEBUI_OVERLAY_DENY='c778af04|master-11340' \\
    ./taler-monitoring.sh -d hacktivism.ch urls
EOF
}

# shellcheck source=lib.sh
source "$ROOT/lib.sh"
# ensure i18n after domain may change later
i18n_init 2>/dev/null || true
# Optional gitignored secrets.env next to this suite (or MONITORING_SECRETS_ENV=)
load_monitoring_secrets_env 2>/dev/null || true

# Persist #NNN + progress across check_*.sh child processes
TALER_MON_STATE=$(mktemp "${TMPDIR:-/tmp}/taler-mon-state.XXXXXX")
export TALER_MON_STATE
trap 'rm -f "${TALER_MON_STATE:-}"' EXIT

PHASES=()
DOMAIN_SET=0
BANK_OVERRIDE=""
EXCHANGE_OVERRIDE=""
MERCHANT_OVERRIDE=""
CURRENCY_OVERRIDE=""
NO_PROBE=0

while [ $# -gt 0 ]; do
  case "$1" in
    --lang)
      [ $# -ge 2 ] || { echo "error: --lang needs en|fr" >&2; exit 2; }
      _lang="$2"
      case "$_lang" in
        fr|FR|fra|french) TALER_MON_LANG=fr ;;
        en|EN|eng|english) TALER_MON_LANG=en ;;
        *) echo "error: unknown language '$_lang' (use en or fr)" >&2; exit 2 ;;
      esac
      TALER_MON_LANG_SET=1
      export TALER_MON_LANG TALER_MON_LANG_SET
      shift 2
      i18n_init 2>/dev/null || true
      ;;
    -h|--help) usage; exit 0 ;;
    --ver|--version|-V) print_version; exit 0 ;;
    -d|--domain)
      [ $# -ge 2 ] || { echo "missing arg for $1" >&2; exit 2; }
      apply_taler_domain "$2"
      DOMAIN_SET=1
      TALER_DOMAIN_FROM_CLI=1
      shift 2
      ;;
    --bank)
      [ $# -ge 2 ] || { echo "missing arg for $1" >&2; exit 2; }
      BANK_OVERRIDE="${2%/}"; shift 2
      ;;
    --exchange)
      [ $# -ge 2 ] || { echo "missing arg for $1" >&2; exit 2; }
      EXCHANGE_OVERRIDE="${2%/}"; shift 2
      ;;
    --merchant)
      [ $# -ge 2 ] || { echo "missing arg for $1" >&2; exit 2; }
      MERCHANT_OVERRIDE="${2%/}"; shift 2
      ;;
    --currency)
      [ $# -ge 2 ] || { echo "missing arg for $1" >&2; exit 2; }
      CURRENCY_OVERRIDE="$2"; shift 2
      ;;
    --no-probe) NO_PROBE=1; shift ;;
    urls|inside|versions|sanity|server|e2e|ladder|goa-ladder|auth401|aptdeploy|apt-deploy|apt_src|surface|ecosystem|mattermost|mail|monpages|pages|devtesting|franken|fake-franken|fake_franken|all|full) PHASES+=("$1"); shift ;;
    *)
      # bare domain shorthand: ./taler-monitoring.sh taler.net
      if [[ "$1" == *.* && "$1" != *://* && "$1" != -* ]]; then
        apply_taler_domain "$1"
        DOMAIN_SET=1
        TALER_DOMAIN_FROM_CLI=1
        shift
      else
        echo "Unknown: $1" >&2; usage >&2; exit 2
      fi
      ;;
  esac
done

if [ "$NO_PROBE" = "1" ]; then
  TALER_DOMAIN_PROBE=0
fi
if [ -n "$CURRENCY_OVERRIDE" ]; then
  EXPECT_CURRENCY="$CURRENCY_OVERRIDE"
fi
if [ -n "$BANK_OVERRIDE" ]; then BANK_PUBLIC="$BANK_OVERRIDE"; fi
if [ -n "$EXCHANGE_OVERRIDE" ]; then EXCHANGE_PUBLIC="$EXCHANGE_OVERRIDE"; fi
if [ -n "$MERCHANT_OVERRIDE" ]; then MERCHANT_PUBLIC="$MERCHANT_OVERRIDE"; fi

# Only koopa may use SSH. Remote domains: public + optional e2e (no SSH).
if [ "${LOCAL_STACK:-1}" != "1" ]; then
  SKIP_SSH=1
fi

# Remote: no SSH; ATM withdraw ladder set in check_e2e (smaller notes)
if [ "${LOCAL_STACK}" != "1" ]; then
  E2E_FAKE_INCOMING=0
  E2E_REMOTE=1
  : "${E2E_WITHDRAW_VALUES:=10 20 50}"
  : "${E2E_PAY_VALUES:=0.01 0.05 0.1 1}"
fi

# Export so check_*.sh (re-source lib) see the same targets via env
export TALER_DOMAIN BANK_PUBLIC EXCHANGE_PUBLIC MERCHANT_PUBLIC
export EXPECT_CURRENCY SKIP_SSH LOCAL_STACK TALER_DOMAIN_PROBE CHECK_LANDING
export TALER_DOMAIN_FROM_CLI="${TALER_DOMAIN_FROM_CLI:-0}"
export DOMAIN_SET="${DOMAIN_SET:-0}"
export WITHDRAW_AMT PAY_AMT CREDIT_AMT MERCHANT_INSTANCE
export E2E_FAKE_INCOMING E2E_REMOTE E2E_VARIABLE E2E_ATM_MAX
export E2E_WITHDRAW_VALUES E2E_PAY_VALUES E2E_USE_TEMPLATES E2E_TEMPLATE_MAP
export PAIVANA_PUBLIC E2E_PAIVANA E2E_PAIVANA_TEMPLATE E2E_PAIVANA_AMOUNT E2E_PAIVANA_INSTANCE
export INSIDE_PROFILE INSIDE_SSH
export INSIDE_BANK_CTR INSIDE_EXCHANGE_CTR INSIDE_MERCHANT_CTR
export INSIDE_BANK_PORT INSIDE_EXCHANGE_PORT INSIDE_MERCHANT_PORT
export INSIDE_DNS_BANK INSIDE_DNS_EXCHANGE INSIDE_DNS_MERCHANT
# Ladder: withdraw then pay — 0 + random mids + max-1 + max (see check_goa_ladder.sh).
# Defaults so set -u export is safe when vars were never set by caller.
: "${LADDER_STEPS:=23}"
: "${LADDER_MIN_AMOUNT:=0.000001}"
: "${LADDER_CONFIRM_POLLS:=40}"
: "${LADDER_MAX_RUNGS:=99}"
: "${LADDER_TIMEOUT_S:=3600}"
: "${LADDER_REPORT_DIR:=}"
: "${LADDER_SETTLE_ROUNDS:=18}"
: "${LADDER_SETTLE_SLEEP:=2}"
: "${LADDER_MAX_AMOUNT:=4503599627370496}"
: "${LADDER_INCLUDE_ZERO:=1}"
: "${LADDER_INCLUDE_MAX:=1}"
: "${LADDER_HIGH_FROM:=1000000}"
: "${LADDER_HIGH_RUNGS:=12}"
: "${LADDER_LOAD:=1}"
: "${LADDER_PAY:=1}"
: "${LADDER_WITHDRAW_SCALE:=1.5}"
: "${LADDER_PAY_SETTLE_ROUNDS:=6}"
export LADDER_STEPS LADDER_MIN_AMOUNT LADDER_CONFIRM_POLLS LADDER_MAX_RUNGS LADDER_TIMEOUT_S LADDER_REPORT_DIR
export LADDER_SETTLE_ROUNDS LADDER_SETTLE_SLEEP EXP_PW_FILE EXP_USER
export LADDER_MAX_AMOUNT LADDER_INCLUDE_ZERO LADDER_INCLUDE_MAX
export LADDER_HIGH_FROM LADDER_HIGH_RUNGS LADDER_LOAD
export LADDER_PAY LADDER_WITHDRAW_SCALE LADDER_PAY_SETTLE_ROUNDS
export TALER_DOMAIN_APPLIED=1
  i18n_init 2>/dev/null || true
# Optional: auth401 and other phases may honor CONTINUE_ON_ERROR
export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-}"
export AUTH401_CONTINUE="${AUTH401_CONTINUE:-${CONTINUE_ON_ERROR:-}}"

# Default phases
# - local GOA (koopa): urls + inside + versions + e2e
# - FrancPaysan STAGE: urls + inside (stagepaysan) + versions + e2e
# - other remote: urls only (pass e2e/inside explicitly)
if [ "${#PHASES[@]}" -eq 0 ]; then
  if [ "${LOCAL_STACK}" = "1" ]; then
    PHASES=(urls inside versions e2e)
  elif [ "${EXPECT_CURRENCY:-}" = "TESTPAYSAN" ] \
    || [[ "${TALER_DOMAIN:-}" == stage.*lefrancpaysan* ]] \
    || [[ "${TALER_DOMAIN:-}" == *stage.lefrancpaysan* ]]; then
    # Like hacktivism breadth: public urls + inside (stagepaysan) + versions + e2e
    PHASES=(urls inside versions e2e)
  else
    PHASES=(urls)
  fi
fi

OUT_PHASES=()
for p in "${PHASES[@]}"; do
  if [ "$p" = "all" ]; then
    if [ "${LOCAL_STACK}" = "1" ]; then
      OUT_PHASES+=(urls inside versions sanity e2e)
    elif [ -n "${INSIDE_SSH:-}" ] || [ "${INSIDE_PROFILE:-}" = "stage-lfp" ]; then
      OUT_PHASES+=(urls inside versions e2e)
    else
      # remote without inside SSH: public + optional e2e
      OUT_PHASES+=(urls versions e2e)
    fi
  elif [ "$p" = "full" ]; then
    # Maximum coverage. Long. Needs secrets for e2e/ladder/auth401.
    if [ "${LOCAL_STACK}" = "1" ]; then
      OUT_PHASES+=(urls inside versions sanity server e2e ladder auth401)
    elif [ -n "${INSIDE_SSH:-}" ] || [ "${INSIDE_PROFILE:-}" = "stage-lfp" ]; then
      OUT_PHASES+=(urls inside versions e2e ladder auth401)
    else
      OUT_PHASES+=(urls versions e2e auth401)
    fi
  else
    OUT_PHASES+=("$p")
  fi
done
PHASES=()
seen=" "
for p in "${OUT_PHASES[@]}"; do
  # server = koopa-only. inside allowed on koopa OR stagepaysan (INSIDE_SSH).
  case "$p" in
    server)
      if [ "${LOCAL_STACK}" != "1" ]; then
        echo "[INFO]  skip phase 'server' (koopa only)" >&2
        continue
      fi
      ;;
    inside)
      if [ "${LOCAL_STACK}" != "1" ] && [ -z "${INSIDE_SSH:-}" ] && [ "${INSIDE_PROFILE:-}" != "stage-lfp" ]; then
        echo "[INFO]  skip phase 'inside' (no INSIDE_SSH / not local)" >&2
        continue
      fi
      ;;
  esac
  case "$seen" in *" $p "*) ;; *) PHASES+=("$p"); seen="$seen$p " ;; esac
done

if [ "${#PHASES[@]}" -eq 0 ]; then
  PHASES=(urls)
fi

# Global expected check count for progress bar done/total (override: PROGRESS_TOTAL=N).
# One estimate for the whole run — phase summary() must NOT reset this mid-run.
# Prefer slightly high; lib.sh only grows if short (never shrinks until progress_finish).
if [ "${PROGRESS_TOTAL:-0}" = "0" ] || [ -z "${PROGRESS_TOTAL:-}" ]; then
  _pt=0
  for p in "${PHASES[@]}"; do
    case "$p" in
      urls)     _pt=$((_pt + 95)) ;;   # +qr + webui SPA fingerprints
      inside)   _pt=$((_pt + 30)) ;;
      versions) _pt=$((_pt + 35)) ;;   # + disk probes can be many
      sanity)   _pt=$((_pt + 35)) ;;
      server)   _pt=$((_pt + 20)) ;;
      # e2e: ATM ladder emits many INFO lines (coins before/after each note)
      e2e)      _pt=$((_pt + 240)) ;;
      ladder|goa-ladder) _pt=$((_pt + 120)) ;;
      auth401)  _pt=$((_pt + 70)) ;;
      aptdeploy) _pt=$((_pt + 40)) ;;
      surface|ecosystem) _pt=$((_pt + 120)) ;;
      monpages|pages) _pt=$((_pt + 20)) ;;
      mattermost) _pt=$((_pt + 25)) ;;
      mail) _pt=$((_pt + 40)) ;;
      devtesting|franken|fake-franken|fake_franken) _pt=$((_pt + 12)) ;;
    esac
  done
  # monpages post-check after htmlify can add another inventory pass
  case " ${PHASES[*]} " in
    *" monpages "*|*" pages "*) _pt=$((_pt + 12)) ;;
  esac
  set_progress_total "$_pt"
  unset _pt
fi

# Whole-run wall clock (default 10 min). Override: RUN_TIMEOUT=0 (unlimited).
: "${RUN_TIMEOUT:=600}"
export RUN_TIMEOUT
MON_T0=$(date +%s)
export MON_T0
RUN_TIMED_OUT=0
RUN_TIMEOUT_AT_PHASE=""
RUN_SKIPPED_PHASES=()

mon_seconds_left() {
  if [ "${RUN_TIMEOUT:-0}" -eq 0 ]; then
    printf '%s' "999999"
    return 0
  fi
  local now left
  now=$(date +%s)
  left=$((RUN_TIMEOUT - (now - MON_T0)))
  [ "$left" -lt 0 ] && left=0
  printf '%s' "$left"
}

# Run one phase script under remaining RUN_TIMEOUT budget.
# Exit 124 (timeout utility) → mark RUN_TIMED_OUT.
run_phase() {
  local phase="$1" script="$2" left rc
  left=$(mon_seconds_left)
  if [ "${RUN_TIMEOUT:-0}" -gt 0 ] && [ "$left" -le 0 ]; then
    RUN_TIMED_OUT=1
    RUN_TIMEOUT_AT_PHASE="${RUN_TIMEOUT_AT_PHASE:-$phase}"
    return 1
  fi
  if [ "${RUN_TIMEOUT:-0}" -eq 0 ]; then
    "$script"
    return $?
  fi
  # Keep at least 2s so timeout(1) can start the child.
  [ "$left" -lt 2 ] && left=2
  set +e
  with_timeout "$left" "$script"
  rc=$?
  set -e
  if [ "$rc" -eq 124 ]; then
    RUN_TIMED_OUT=1
    RUN_TIMEOUT_AT_PHASE="$phase"
    return 1
  fi
  return "$rc"
}

# How inside/versions will reach containers (host-podman vs ssh)
_INSIDE_ACCESS_HINT=ssh
if [ "${INSIDE_PODMAN:-0}" = "1" ] || [ "${INSIDE_MODE:-}" = "local-podman" ]; then
  _INSIDE_ACCESS_HINT=host-podman
elif [ "${SKIP_SSH:-0}" = "1" ]; then
  _INSIDE_ACCESS_HINT=skipped
elif [ "${LOCAL_STACK:-0}" = "1" ] && command -v podman >/dev/null 2>&1 \
  && podman ps --format '{{.Names}}' 2>/dev/null | grep -qE 'taler-hacktivism'; then
  _INSIDE_ACCESS_HINT=host-podman
fi

printf 'target domain=%s\n' "${TALER_DOMAIN}"
printf '  bank     %s\n' "$BANK_PUBLIC"
printf '  exchange %s\n' "$EXCHANGE_PUBLIC"
printf '  merchant %s\n' "$MERCHANT_PUBLIC"
printf '  currency expect=%s\n' "${EXPECT_CURRENCY:-any}"
printf '  phases   %s\n' "${PHASES[*]}"
printf '  flags    LOCAL_STACK=%s SKIP_SSH=%s INSIDE_PODMAN=%s INSIDE_MODE=%s\n' \
  "${LOCAL_STACK:-}" "${SKIP_SSH:-0}" "${INSIDE_PODMAN:-0}" "${INSIDE_MODE:-}"
printf '  flags    KOOPA_SSH=%s INSIDE_SSH=%s INSIDE_PROFILE=%s\n' \
  "${KOOPA_SSH:-}" "${INSIDE_SSH:-}" "${INSIDE_PROFILE:-}"
printf '  access   inside/versions → %s' "$_INSIDE_ACCESS_HINT"
case "$_INSIDE_ACCESS_HINT" in
  host-podman) printf ' (podman exec on this host; IDs inside.host-*)\n' ;;
  ssh)         printf ' (SSH then podman; IDs inside.ssh-*)\n' ;;
  skipped)     printf ' (SKIP_SSH=1)\n' ;;
  *)           printf '\n' ;;
esac
if [ "${RUN_TIMEOUT:-0}" -eq 0 ]; then
  printf '  run_timeout unlimited (RUN_TIMEOUT=0)\n'
else
  printf '  run_timeout %ss wall clock (RUN_TIMEOUT=; 0=unlimited)\n' "$RUN_TIMEOUT"
fi
printf '  progress %s/%s (global done/total · set PROGRESS_TOTAL= to override; PROGRESS_OFF=1 to hide)\n' \
  "${PROGRESS_DONE:-0}" "${PROGRESS_TOTAL:-0}"
unset _INSIDE_ACCESS_HINT

chmod +x "$ROOT"/check_*.sh 2>/dev/null || true

ec=0
for p in "${PHASES[@]}"; do
  if [ "$RUN_TIMED_OUT" = "1" ]; then
    RUN_SKIPPED_PHASES+=("$p")
    continue
  fi
  left=$(mon_seconds_left)
  if [ "${RUN_TIMEOUT:-0}" -gt 0 ] && [ "$left" -le 0 ]; then
    RUN_TIMED_OUT=1
    RUN_TIMEOUT_AT_PHASE="${RUN_TIMEOUT_AT_PHASE:-$p}"
    RUN_SKIPPED_PHASES+=("$p")
    continue
  fi
  case "$p" in
    urls)     run_phase urls     "$ROOT/check_urls.sh"     || ec=1 ;;
    inside)   run_phase inside   "$ROOT/check_inside.sh"   || ec=1 ;;
    versions) run_phase versions "$ROOT/check_versions.sh" || ec=1 ;;
    sanity)   run_phase sanity   "$ROOT/check_sanity.sh"   || ec=1 ;;
    server)   run_phase server   "$ROOT/check_server.sh"   || ec=1 ;;
    e2e)      run_phase e2e      "$ROOT/check_e2e.sh"      || ec=1 ;;
    ladder|goa-ladder) run_phase ladder "$ROOT/check_goa_ladder.sh" || ec=1 ;;
    auth401)  run_phase auth401  "$ROOT/check_auth401.sh"  || ec=1 ;;
    aptdeploy|apt-deploy|apt_src) run_phase aptdeploy "$ROOT/check_apt_deploy.sh" || ec=1 ;;
    surface|ecosystem) run_phase surface "$ROOT/check_surface.sh" || ec=1 ;;
    mattermost) run_phase mattermost "$ROOT/check_mattermost.sh" || ec=1 ;;
    mail) run_phase mail "$ROOT/check_mail.sh" || ec=1 ;;
    monpages|pages) run_phase monpages "$ROOT/check_monitoring_pages.sh" || ec=1 ;;
    devtesting|franken|fake-franken|fake_franken)
      run_phase devtesting "$ROOT/check_devtesting.sh" || ec=1
      ;;
  esac
done

# Extraordinary run-budget failure: always report at end; HTML links top → here.
if [ "$RUN_TIMED_OUT" = "1" ]; then
  ec=1
  elapsed=$(( $(date +%s) - MON_T0 ))
  skipped="${RUN_SKIPPED_PHASES[*]:-}"
  # Stable id for jump links (console HTML + err-top banner)
  printf '\n'
  printf '╔══════════════════════════════════════════════════════════╗\n'
  printf '║ RUN TIMEOUT · extraordinary (see jump target below)     ║\n'
  printf '╚══════════════════════════════════════════════════════════╝\n'
  printf '┌ ERROR  ┐ #run.timeout-01  RUN_TIMEOUT exceeded · budget %ss · elapsed %ss\n' \
    "$RUN_TIMEOUT" "$elapsed"
  printf '  detail: wall-clock limit hit'
  if [ -n "${RUN_TIMEOUT_AT_PHASE:-}" ]; then
    printf ' during/after phase "%s"' "$RUN_TIMEOUT_AT_PHASE"
  fi
  if [ -n "$skipped" ]; then
    printf ' · skipped: %s' "$skipped"
  fi
  printf '\n'
  printf '  id=run.timeout-01\n'
  printf '  hint: raise RUN_TIMEOUT= (seconds) or set RUN_TIMEOUT=0 for unlimited\n'
  printf -- '┌ RUN TIMEOUT · extraordinary ┐\n'
  printf -- '  • run.timeout-01 [run] RUN_TIMEOUT=%ss exceeded (elapsed %ss)%s%s\n' \
    "$RUN_TIMEOUT" "$elapsed" \
    "${RUN_TIMEOUT_AT_PHASE:+ · phase $RUN_TIMEOUT_AT_PHASE}" \
    "${skipped:+ · skipped $skipped}"
  printf -- '--- ERRORS (failed checks) ---\n'
  printf -- '  • run.timeout-01 [run] RUN_TIMEOUT=%ss exceeded (elapsed %ss)\n' \
    "$RUN_TIMEOUT" "$elapsed"
fi

# Snap progress bar to real global count (done/total) once for the whole run
if type progress_finish >/dev/null 2>&1; then
  printf '\n'
  progress_finish
fi

exit "$ec"