d7a0ceb90b
Stage jobs estimated 139 checks (urls 95 + monpages 20 + phantom post-check 12 + devtesting) but only ~46 ran, so bars sat at 42/139 then snapped to 46/46. Estimate from CHECK_BANK/LANDING and mon hosts; after each phase set total = done + remaining; no suite credit for host-agent monpages post-check outside the process.
649 lines
25 KiB
Bash
Executable file
649 lines
25 KiB
Bash
Executable file
#!/usr/bin/env bash
|
||
# taler-monitoring — public URL / stack checks for a Taler domain
|
||
#
|
||
# ./taler-monitoring.sh # local GOA (urls + inside + e2e)
|
||
# ./taler-monitoring.sh -d taler.net urls # public demo, no SSH
|
||
# ./taler-monitoring.sh --domain taler-ops.ch # public ops, no SSH
|
||
# TALER_DOMAIN=demo.taler.net ./taler-monitoring.sh urls
|
||
#
|
||
# Tags: [OK] [INFO] [WARN] [ERROR] [BLOCKER]
|
||
# Exit 0 only if every selected phase exits 0.
|
||
|
||
set -euo pipefail
|
||
ROOT=$(cd "$(dirname "$0")" && pwd)
|
||
|
||
# --- suite identity (also: --ver / --version / -V) ---
|
||
SOURCE_REPO_WEB="${SOURCE_REPO_WEB:-https://git.hacktivism.ch/hernani/taler-monitoring}"
|
||
SOURCE_REPO_GIT="${SOURCE_REPO_GIT:-https://git.hacktivism.ch/hernani/taler-monitoring.git}"
|
||
|
||
print_version() {
|
||
local ver tag commit commit_full dirty branch
|
||
ver="unknown"
|
||
if [ -f "$ROOT/VERSION" ]; then
|
||
ver=$(tr -d "[:space:]" <"$ROOT/VERSION" | sed "s/^v//")
|
||
fi
|
||
tag=""
|
||
commit="unknown"
|
||
commit_full=""
|
||
branch=""
|
||
dirty=""
|
||
if git -C "$ROOT" rev-parse --git-dir >/dev/null 2>&1; then
|
||
tag=$(git -C "$ROOT" describe --tags --exact-match 2>/dev/null || true)
|
||
if [ -z "$tag" ]; then
|
||
tag=$(git -C "$ROOT" describe --tags --abbrev=0 2>/dev/null || true)
|
||
fi
|
||
commit=$(git -C "$ROOT" rev-parse --short=12 HEAD 2>/dev/null || echo unknown)
|
||
commit_full=$(git -C "$ROOT" rev-parse HEAD 2>/dev/null || true)
|
||
branch=$(git -C "$ROOT" rev-parse --abbrev-ref HEAD 2>/dev/null || true)
|
||
if [ -n "$(git -C "$ROOT" status --porcelain 2>/dev/null)" ]; then
|
||
dirty=" (dirty)"
|
||
fi
|
||
fi
|
||
if [ -n "$tag" ] && [ "$tag" = "v$ver" ]; then
|
||
printf "taler-monitoring %s\n" "$tag"
|
||
else
|
||
printf "taler-monitoring v%s\n" "$ver"
|
||
fi
|
||
printf "version: %s\n" "$ver"
|
||
if [ -n "$tag" ]; then
|
||
if [ "$tag" = "v$ver" ]; then
|
||
printf "tag: %s\n" "$tag"
|
||
else
|
||
printf "git-tag: %s (tree; VERSION file is %s)\n" "$tag" "$ver"
|
||
fi
|
||
fi
|
||
printf "commit: %s%s\n" "$commit" "$dirty"
|
||
[ -n "$branch" ] && [ "$branch" != "HEAD" ] && printf "branch: %s\n" "$branch"
|
||
printf "repo: %s\n" "$SOURCE_REPO_WEB"
|
||
printf "clone: %s\n" "$SOURCE_REPO_GIT"
|
||
if [ -n "$commit_full" ]; then
|
||
printf "commit-url: %s/src/commit/%s\n" "$SOURCE_REPO_WEB" "$commit_full"
|
||
fi
|
||
if [ -n "$tag" ]; then
|
||
printf "tag-url: %s/src/tag/%s\n" "$SOURCE_REPO_WEB" "$tag"
|
||
fi
|
||
}
|
||
|
||
# Fast path: version before loading lib.sh / phases
|
||
case "${1:-}" in
|
||
--ver|--version|-V)
|
||
print_version
|
||
exit 0
|
||
;;
|
||
esac
|
||
|
||
# Line-buffered stdout/stderr so redirected logs (tee/host-agent) flush each line.
|
||
if [ "${_TALER_MON_STDBUF:-0}" != "1" ] && command -v stdbuf >/dev/null 2>&1; then
|
||
export _TALER_MON_STDBUF=1
|
||
exec stdbuf -oL -eL bash "$0" "$@"
|
||
fi
|
||
|
||
usage() {
|
||
cat <<'EOF'
|
||
taler-monitoring — bank / exchange / merchant checks
|
||
|
||
Usage:
|
||
./taler-monitoring.sh [options] [phases...]
|
||
|
||
Phases:
|
||
urls public HTTPS (no SSH) ← default with --domain
|
||
(includes merchant /webui/ SPA fingerprints: version.txt, overlay, assets)
|
||
inside container status via SSH (local stack only)
|
||
versions taler packages vs deb.taler.net trixie + repo availability
|
||
sanity public + optional server
|
||
server server-side only (SSH)
|
||
e2e withdraw + pay (small amounts; remote aborts on login/KYC)
|
||
ladder withdraw/pay amount ladder (GOA ceiling or stage TESTPAYSAN max_wire)
|
||
auth401 merchant Basic-auth / case matrix (HTTP 401 paths; may create throwaway instance)
|
||
aptdeploy koopa podman apt-src smoke: taler-merchant in
|
||
koopa-taler-deploy-test-apt-src-trixie{,-testing}
|
||
surface REMOTE-ONLY public inventory (NOT in default/all/full):
|
||
ecosystem hosts (taler.net, gnunet.org, taler-systems.com, mattermost, …)
|
||
or -d DOMAIN → that domain’s surface; port/protocol/TLS/CVE (OSV)
|
||
mattermost Mattermost chat health (default mattermost.taler.net; SPA + /api/v4/system/ping)
|
||
mail MX/SMTP/IMAP for Taler mail (firefly, pixel/TSA, catalogued domains)
|
||
monpages public monitoring HTML via FQDN (obligatory ERROR; GOA full inventory, FP only FP)
|
||
uses MON_HOSTS + HTML_URL_OK (same as host-agent); v1.3.1+
|
||
devtesting fake-franken CHF via rusty.taler-ops.ch (taler-devtesting)
|
||
geniban + fake-incoming; needs SSH Host DEVTESTING_SSH (key to devtesting@rusty)
|
||
franken|fake-franken aliases for devtesting
|
||
all urls + inside + versions + sanity + e2e (SSH phases only on koopa)
|
||
full all + server + ladder + auth401 (maximum; long-running, needs secrets)
|
||
NOTE: surface is never included in all/full — pass it explicitly
|
||
|
||
Options:
|
||
-d, --domain DOMAIN load profile from domains.conf (bank/exchange/merchant)
|
||
presets: koopa | hacktivism.ch | taler.net | taler-ops.ch
|
||
| my.taler-ops.ch | stage.taler-ops.ch
|
||
unknown domain → heuristic hosts (see README)
|
||
local (koopa) allows SSH; others public-only
|
||
--bank URL bank base (hostname or https://…) — overrides profile
|
||
--exchange URL exchange base — overrides profile
|
||
--merchant URL merchant-backend base — overrides profile
|
||
--currency CODE expected currency (GOA, KUDOS, CHF, …); empty = report only
|
||
--no-probe do not probe alternate hosts (unknown domains only)
|
||
--lang en|fr UI/console language (sticky bar + badges)
|
||
overrides auto (FrancPaysan→fr, else en)
|
||
--ver, --version, -V print suite version, git commit, and repo URLs
|
||
-h, --help
|
||
|
||
Add a stack: edit domains.conf (name + bank + exchange + merchant + currency).
|
||
|
||
Examples:
|
||
./taler-monitoring.sh -d taler.net
|
||
./taler-monitoring.sh -d taler-ops.ch urls
|
||
./taler-monitoring.sh -d my.taler-ops.ch urls
|
||
./taler-monitoring.sh -d taler-ops.ch --merchant https://my.taler-ops.ch urls
|
||
./taler-monitoring.sh -d demo.taler.net --currency KUDOS
|
||
./taler-monitoring.sh --exchange https://exchange.taler-ops.ch \
|
||
--merchant https://my.taler-ops.ch --bank https://bank.taler-ops.ch urls
|
||
|
||
Env (same meaning):
|
||
TALER_DOMAIN BANK_PUBLIC EXCHANGE_PUBLIC MERCHANT_PUBLIC EXPECT_CURRENCY
|
||
TALER_DOMAINS_CONF SKIP_SSH=1
|
||
NO_COLOR=1 / CLICOLOR=0 disable green/yellow/red tags (default: coloured)
|
||
SKIP_SSH=1 NO_COLOR=1
|
||
RUN_TIMEOUT=600 whole-run wall clock seconds (default 600; 0=unlimited)
|
||
long phases (ladder/full/e2e) need a higher value or 0
|
||
DISK_WARN_USED_PCT=85 disk free: WARN when used ≥ this %
|
||
DISK_ERR_USED_PCT=95 disk free: ERROR when used ≥ this % (or avail=0 / 100%)
|
||
PERF_WARN_MS PERF_FAIL_MS (urls latency; default 8000 / 20000)
|
||
QR_CHECK=0 skip QR form + qrencode/zbarimg (urls phase)
|
||
QR_ECC=M qrencode ECC level (default M)
|
||
KOOPA_SSH KOOPA_SSH_FALLBACKS (default koopa → koopa-external)
|
||
METRICS_LOAD=0 skip host/container RAM/CPU probes (e2e/ladder/inside)
|
||
EXPECT_WEBUI_VERSION=1.6.11 pin merchant /webui/version.txt (urls)
|
||
EXPECT_WEBUI_OVERLAY=selfbuild require substr in version-overlay.txt
|
||
WEBUI_OVERLAY_DENY='master-11340' fail if overlay matches regex
|
||
CHECK_WEBUI_SPA=0 skip SPA fingerprint block in urls
|
||
AUTH401_* see check_auth401.sh / secrets.env.example
|
||
AUTH401_CONTINUE=1 / CONTINUE_ON_ERROR=1
|
||
auth401: run all groups, collect every ERROR (no mid-run halt)
|
||
APT_DEPLOY_SKIP=1 skip aptdeploy phase
|
||
APT_DEPLOY_CONTAINERS="name:suite …" default trixie + trixie-testing deploy-test pods
|
||
SURFACE_CVE=0 disable CVE queries in surface phase
|
||
SURFACE_CVE_LEVEL=warn|error bare Server-header versions default warn
|
||
(Debian package versions default error)
|
||
SURFACE_CATALOG=path override surface-catalog.conf
|
||
TALER_MON_LANG=en|fr same as --lang (default en; auto fr for *lefrancpaysan*)
|
||
|
||
Full load (GOA / hacktivism):
|
||
./taler-monitoring.sh -d hacktivism.ch full
|
||
# or explicit:
|
||
./taler-monitoring.sh -d hacktivism.ch urls inside versions sanity server e2e ladder auth401
|
||
# hacktivism host-agent also runs aptdeploy (apt-src merchant containers on koopa)
|
||
|
||
Remote surface / ecosystem (explicit only — never default):
|
||
./taler-monitoring.sh surface
|
||
./taler-monitoring.sh -d hacktivism.ch surface
|
||
./taler-monitoring.sh -d lefrancpaysan.ch surface
|
||
./taler-monitoring.sh mattermost # mattermost.taler.net chat health
|
||
./taler-monitoring.sh mail # firefly + pixel MX/SMTP/IMAP
|
||
|
||
SPA pin after selfbuild:
|
||
EXPECT_WEBUI_VERSION=1.6.11 EXPECT_WEBUI_OVERLAY=selfbuild-v1.6.11 \\
|
||
WEBUI_OVERLAY_DENY='c778af04|master-11340' \\
|
||
./taler-monitoring.sh -d hacktivism.ch urls
|
||
EOF
|
||
}
|
||
|
||
# shellcheck source=lib.sh
|
||
source "$ROOT/lib.sh"
|
||
# ensure i18n after domain may change later
|
||
i18n_init 2>/dev/null || true
|
||
# Optional gitignored secrets.env next to this suite (or MONITORING_SECRETS_ENV=)
|
||
load_monitoring_secrets_env 2>/dev/null || true
|
||
|
||
# Persist #NNN + progress across check_*.sh child processes
|
||
TALER_MON_STATE=$(mktemp "${TMPDIR:-/tmp}/taler-mon-state.XXXXXX")
|
||
export TALER_MON_STATE
|
||
trap 'rm -f "${TALER_MON_STATE:-}"' EXIT
|
||
|
||
PHASES=()
|
||
DOMAIN_SET=0
|
||
BANK_OVERRIDE=""
|
||
EXCHANGE_OVERRIDE=""
|
||
MERCHANT_OVERRIDE=""
|
||
CURRENCY_OVERRIDE=""
|
||
NO_PROBE=0
|
||
|
||
while [ $# -gt 0 ]; do
|
||
case "$1" in
|
||
--lang)
|
||
[ $# -ge 2 ] || { echo "error: --lang needs en|fr" >&2; exit 2; }
|
||
_lang="$2"
|
||
case "$_lang" in
|
||
fr|FR|fra|french) TALER_MON_LANG=fr ;;
|
||
en|EN|eng|english) TALER_MON_LANG=en ;;
|
||
*) echo "error: unknown language '$_lang' (use en or fr)" >&2; exit 2 ;;
|
||
esac
|
||
TALER_MON_LANG_SET=1
|
||
export TALER_MON_LANG TALER_MON_LANG_SET
|
||
shift 2
|
||
i18n_init 2>/dev/null || true
|
||
;;
|
||
-h|--help) usage; exit 0 ;;
|
||
--ver|--version|-V) print_version; exit 0 ;;
|
||
-d|--domain)
|
||
[ $# -ge 2 ] || { echo "missing arg for $1" >&2; exit 2; }
|
||
apply_taler_domain "$2"
|
||
DOMAIN_SET=1
|
||
TALER_DOMAIN_FROM_CLI=1
|
||
shift 2
|
||
;;
|
||
--bank)
|
||
[ $# -ge 2 ] || { echo "missing arg for $1" >&2; exit 2; }
|
||
BANK_OVERRIDE="${2%/}"; shift 2
|
||
;;
|
||
--exchange)
|
||
[ $# -ge 2 ] || { echo "missing arg for $1" >&2; exit 2; }
|
||
EXCHANGE_OVERRIDE="${2%/}"; shift 2
|
||
;;
|
||
--merchant)
|
||
[ $# -ge 2 ] || { echo "missing arg for $1" >&2; exit 2; }
|
||
MERCHANT_OVERRIDE="${2%/}"; shift 2
|
||
;;
|
||
--currency)
|
||
[ $# -ge 2 ] || { echo "missing arg for $1" >&2; exit 2; }
|
||
CURRENCY_OVERRIDE="$2"; shift 2
|
||
;;
|
||
--no-probe) NO_PROBE=1; shift ;;
|
||
urls|inside|versions|sanity|server|e2e|ladder|goa-ladder|auth401|aptdeploy|apt-deploy|apt_src|surface|ecosystem|mattermost|mail|monpages|pages|devtesting|franken|fake-franken|fake_franken|all|full) PHASES+=("$1"); shift ;;
|
||
*)
|
||
# bare domain shorthand: ./taler-monitoring.sh taler.net
|
||
if [[ "$1" == *.* && "$1" != *://* && "$1" != -* ]]; then
|
||
apply_taler_domain "$1"
|
||
DOMAIN_SET=1
|
||
TALER_DOMAIN_FROM_CLI=1
|
||
shift
|
||
else
|
||
echo "Unknown: $1" >&2; usage >&2; exit 2
|
||
fi
|
||
;;
|
||
esac
|
||
done
|
||
|
||
if [ "$NO_PROBE" = "1" ]; then
|
||
TALER_DOMAIN_PROBE=0
|
||
fi
|
||
if [ -n "$CURRENCY_OVERRIDE" ]; then
|
||
EXPECT_CURRENCY="$CURRENCY_OVERRIDE"
|
||
fi
|
||
if [ -n "$BANK_OVERRIDE" ]; then BANK_PUBLIC="$BANK_OVERRIDE"; fi
|
||
if [ -n "$EXCHANGE_OVERRIDE" ]; then EXCHANGE_PUBLIC="$EXCHANGE_OVERRIDE"; fi
|
||
if [ -n "$MERCHANT_OVERRIDE" ]; then MERCHANT_PUBLIC="$MERCHANT_OVERRIDE"; fi
|
||
|
||
# Only koopa may use SSH. Remote domains: public + optional e2e (no SSH).
|
||
if [ "${LOCAL_STACK:-1}" != "1" ]; then
|
||
SKIP_SSH=1
|
||
fi
|
||
|
||
# Remote: no SSH; ATM withdraw ladder set in check_e2e (smaller notes)
|
||
if [ "${LOCAL_STACK}" != "1" ]; then
|
||
E2E_FAKE_INCOMING=0
|
||
E2E_REMOTE=1
|
||
: "${E2E_WITHDRAW_VALUES:=10 20 50}"
|
||
: "${E2E_PAY_VALUES:=0.01 0.05 0.1 1}"
|
||
fi
|
||
|
||
# Export so check_*.sh (re-source lib) see the same targets via env
|
||
export TALER_DOMAIN BANK_PUBLIC EXCHANGE_PUBLIC MERCHANT_PUBLIC
|
||
export EXPECT_CURRENCY SKIP_SSH LOCAL_STACK TALER_DOMAIN_PROBE CHECK_LANDING
|
||
export TALER_DOMAIN_FROM_CLI="${TALER_DOMAIN_FROM_CLI:-0}"
|
||
export DOMAIN_SET="${DOMAIN_SET:-0}"
|
||
export WITHDRAW_AMT PAY_AMT CREDIT_AMT MERCHANT_INSTANCE
|
||
export E2E_FAKE_INCOMING E2E_REMOTE E2E_VARIABLE E2E_ATM_MAX
|
||
export E2E_WITHDRAW_VALUES E2E_PAY_VALUES E2E_USE_TEMPLATES E2E_TEMPLATE_MAP
|
||
export PAIVANA_PUBLIC E2E_PAIVANA E2E_PAIVANA_TEMPLATE E2E_PAIVANA_AMOUNT E2E_PAIVANA_INSTANCE
|
||
export INSIDE_PROFILE INSIDE_SSH
|
||
export INSIDE_BANK_CTR INSIDE_EXCHANGE_CTR INSIDE_MERCHANT_CTR
|
||
export INSIDE_BANK_PORT INSIDE_EXCHANGE_PORT INSIDE_MERCHANT_PORT
|
||
export INSIDE_DNS_BANK INSIDE_DNS_EXCHANGE INSIDE_DNS_MERCHANT
|
||
# Ladder: withdraw then pay — 0 + random mids + max-1 + max (see check_goa_ladder.sh).
|
||
# Defaults so set -u export is safe when vars were never set by caller.
|
||
: "${LADDER_STEPS:=23}"
|
||
: "${LADDER_MIN_AMOUNT:=0.000001}"
|
||
: "${LADDER_CONFIRM_POLLS:=40}"
|
||
: "${LADDER_MAX_RUNGS:=99}"
|
||
: "${LADDER_TIMEOUT_S:=3600}"
|
||
: "${LADDER_REPORT_DIR:=}"
|
||
: "${LADDER_SETTLE_ROUNDS:=18}"
|
||
: "${LADDER_SETTLE_SLEEP:=2}"
|
||
: "${LADDER_MAX_AMOUNT:=4503599627370496}"
|
||
: "${LADDER_INCLUDE_ZERO:=1}"
|
||
: "${LADDER_INCLUDE_MAX:=1}"
|
||
: "${LADDER_HIGH_FROM:=1000000}"
|
||
: "${LADDER_HIGH_RUNGS:=12}"
|
||
: "${LADDER_LOAD:=1}"
|
||
: "${LADDER_PAY:=1}"
|
||
: "${LADDER_WITHDRAW_SCALE:=1.5}"
|
||
: "${LADDER_PAY_SETTLE_ROUNDS:=6}"
|
||
export LADDER_STEPS LADDER_MIN_AMOUNT LADDER_CONFIRM_POLLS LADDER_MAX_RUNGS LADDER_TIMEOUT_S LADDER_REPORT_DIR
|
||
export LADDER_SETTLE_ROUNDS LADDER_SETTLE_SLEEP EXP_PW_FILE EXP_USER
|
||
export LADDER_MAX_AMOUNT LADDER_INCLUDE_ZERO LADDER_INCLUDE_MAX
|
||
export LADDER_HIGH_FROM LADDER_HIGH_RUNGS LADDER_LOAD
|
||
export LADDER_PAY LADDER_WITHDRAW_SCALE LADDER_PAY_SETTLE_ROUNDS
|
||
export TALER_DOMAIN_APPLIED=1
|
||
i18n_init 2>/dev/null || true
|
||
# Optional: auth401 and other phases may honor CONTINUE_ON_ERROR
|
||
export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-}"
|
||
export AUTH401_CONTINUE="${AUTH401_CONTINUE:-${CONTINUE_ON_ERROR:-}}"
|
||
|
||
# Default phases
|
||
# - local GOA (koopa): urls + inside + versions + e2e
|
||
# - FrancPaysan STAGE: urls + inside (stagepaysan) + versions + e2e
|
||
# - other remote: urls only (pass e2e/inside explicitly)
|
||
if [ "${#PHASES[@]}" -eq 0 ]; then
|
||
if [ "${LOCAL_STACK}" = "1" ]; then
|
||
PHASES=(urls inside versions e2e)
|
||
elif [ "${EXPECT_CURRENCY:-}" = "TESTPAYSAN" ] \
|
||
|| [[ "${TALER_DOMAIN:-}" == stage.*lefrancpaysan* ]] \
|
||
|| [[ "${TALER_DOMAIN:-}" == *stage.lefrancpaysan* ]]; then
|
||
# Like hacktivism breadth: public urls + inside (stagepaysan) + versions + e2e
|
||
PHASES=(urls inside versions e2e)
|
||
else
|
||
PHASES=(urls)
|
||
fi
|
||
fi
|
||
|
||
OUT_PHASES=()
|
||
for p in "${PHASES[@]}"; do
|
||
if [ "$p" = "all" ]; then
|
||
if [ "${LOCAL_STACK}" = "1" ]; then
|
||
OUT_PHASES+=(urls inside versions sanity e2e)
|
||
elif [ -n "${INSIDE_SSH:-}" ] || [ "${INSIDE_PROFILE:-}" = "stage-lfp" ]; then
|
||
OUT_PHASES+=(urls inside versions e2e)
|
||
else
|
||
# remote without inside SSH: public + optional e2e
|
||
OUT_PHASES+=(urls versions e2e)
|
||
fi
|
||
elif [ "$p" = "full" ]; then
|
||
# Maximum coverage. Long. Needs secrets for e2e/ladder/auth401.
|
||
if [ "${LOCAL_STACK}" = "1" ]; then
|
||
OUT_PHASES+=(urls inside versions sanity server e2e ladder auth401)
|
||
elif [ -n "${INSIDE_SSH:-}" ] || [ "${INSIDE_PROFILE:-}" = "stage-lfp" ]; then
|
||
OUT_PHASES+=(urls inside versions e2e ladder auth401)
|
||
else
|
||
OUT_PHASES+=(urls versions e2e auth401)
|
||
fi
|
||
else
|
||
OUT_PHASES+=("$p")
|
||
fi
|
||
done
|
||
PHASES=()
|
||
seen=" "
|
||
for p in "${OUT_PHASES[@]}"; do
|
||
# server = koopa-only. inside allowed on koopa OR stagepaysan (INSIDE_SSH).
|
||
case "$p" in
|
||
server)
|
||
if [ "${LOCAL_STACK}" != "1" ]; then
|
||
echo "[INFO] skip phase 'server' (koopa only)" >&2
|
||
continue
|
||
fi
|
||
;;
|
||
inside)
|
||
if [ "${LOCAL_STACK}" != "1" ] && [ -z "${INSIDE_SSH:-}" ] && [ "${INSIDE_PROFILE:-}" != "stage-lfp" ]; then
|
||
echo "[INFO] skip phase 'inside' (no INSIDE_SSH / not local)" >&2
|
||
continue
|
||
fi
|
||
;;
|
||
esac
|
||
case "$seen" in *" $p "*) ;; *) PHASES+=("$p"); seen="$seen$p " ;; esac
|
||
done
|
||
|
||
if [ "${#PHASES[@]}" -eq 0 ]; then
|
||
PHASES=(urls)
|
||
fi
|
||
|
||
# Global expected check count for progress bar done/total (override: PROGRESS_TOTAL=N).
|
||
# Auto estimate is env-aware (CHECK_BANK=0 etc.) and re-fit after each phase so stage
|
||
# jobs do not sit at 42/139 then snap to 46/46. Phase summary() must NOT snap mid-run.
|
||
# Host-agent monpages post-check is outside this process — never counted here.
|
||
_progress_estimate_phase() {
|
||
local p="$1" n=0 hosts=1
|
||
hosts=$(printf '%s' "${MON_HOSTS:-x}" | wc -w)
|
||
[ "$hosts" -lt 1 ] && hosts=1
|
||
case "$p" in
|
||
urls)
|
||
# Core: exchange + merchant + webui + perf + terms (~28–40). Bank/landing optional.
|
||
# Stage (CHECK_BANK=0 CHECK_LANDING=0) lands near ~34 numbered lines — keep close.
|
||
n=28
|
||
[ "${CHECK_BANK:-1}" != "0" ] && n=$((n + 24))
|
||
[ "${CHECK_LANDING:-1}" != "0" ] && n=$((n + 18)) # intro + stats.json + qr
|
||
n=$((n + 6)) # headroom (SPA / terms / privacy / alt paths)
|
||
;;
|
||
inside) n=30 ;;
|
||
versions) n=35 ;;
|
||
sanity) n=35 ;;
|
||
server) n=20 ;;
|
||
# e2e: ATM ladder emits many INFO lines (coins before/after each note)
|
||
e2e) n=240 ;;
|
||
ladder|goa-ladder) n=120 ;;
|
||
auth401) n=70 ;;
|
||
aptdeploy) n=40 ;;
|
||
surface|ecosystem) n=120 ;;
|
||
monpages|pages)
|
||
# slash + bare (+ optional _err) per inventory host; GOA full suite ≈20 URLs
|
||
case "${TALER_DOMAIN:-}" in
|
||
*hacktivism*) n=24 ;;
|
||
*lefrancpaysan*|*francpaysan*) n=$((hosts * 2 + 6)); [ "$n" -lt 12 ] && n=12 ;;
|
||
*) n=$((hosts * 2 + 4)); [ "$n" -lt 8 ] && n=8 ;;
|
||
esac
|
||
[ "$n" -gt 48 ] && n=48
|
||
;;
|
||
mattermost) n=25 ;;
|
||
mail) n=40 ;;
|
||
devtesting|franken|fake-franken|fake_franken) n=10 ;;
|
||
*) n=15 ;;
|
||
esac
|
||
printf '%s' "$n"
|
||
}
|
||
|
||
# Recompute PROGRESS_TOTAL = done + estimate(remaining phases). Allows shrink after short phases.
|
||
_progress_refit_remaining() {
|
||
[ "${PROGRESS_TOTAL_AUTO:-0}" = "1" ] || return 0
|
||
local rem=0 rp
|
||
for rp in "$@"; do
|
||
rem=$((rem + $(_progress_estimate_phase "$rp")))
|
||
done
|
||
if type _mon_state_load >/dev/null 2>&1; then
|
||
_mon_state_load
|
||
fi
|
||
set_progress_total "$(( ${PROGRESS_DONE:-0} + rem ))"
|
||
}
|
||
|
||
PROGRESS_TOTAL_AUTO=0
|
||
if [ "${PROGRESS_TOTAL:-0}" = "0" ] || [ -z "${PROGRESS_TOTAL:-}" ]; then
|
||
PROGRESS_TOTAL_AUTO=1
|
||
_pt=0
|
||
for p in "${PHASES[@]}"; do
|
||
_pt=$((_pt + $(_progress_estimate_phase "$p")))
|
||
done
|
||
set_progress_total "$_pt"
|
||
unset _pt
|
||
fi
|
||
export PROGRESS_TOTAL_AUTO
|
||
|
||
# Whole-run wall clock (default 10 min). Override: RUN_TIMEOUT=0 (unlimited).
|
||
: "${RUN_TIMEOUT:=600}"
|
||
export RUN_TIMEOUT
|
||
MON_T0=$(date +%s)
|
||
export MON_T0
|
||
RUN_TIMED_OUT=0
|
||
RUN_TIMEOUT_AT_PHASE=""
|
||
RUN_SKIPPED_PHASES=()
|
||
|
||
mon_seconds_left() {
|
||
if [ "${RUN_TIMEOUT:-0}" -eq 0 ]; then
|
||
printf '%s' "999999"
|
||
return 0
|
||
fi
|
||
local now left
|
||
now=$(date +%s)
|
||
left=$((RUN_TIMEOUT - (now - MON_T0)))
|
||
[ "$left" -lt 0 ] && left=0
|
||
printf '%s' "$left"
|
||
}
|
||
|
||
# Run one phase script under remaining RUN_TIMEOUT budget.
|
||
# Exit 124 (timeout utility) → mark RUN_TIMED_OUT.
|
||
run_phase() {
|
||
local phase="$1" script="$2" left rc
|
||
left=$(mon_seconds_left)
|
||
if [ "${RUN_TIMEOUT:-0}" -gt 0 ] && [ "$left" -le 0 ]; then
|
||
RUN_TIMED_OUT=1
|
||
RUN_TIMEOUT_AT_PHASE="${RUN_TIMEOUT_AT_PHASE:-$phase}"
|
||
return 1
|
||
fi
|
||
if [ "${RUN_TIMEOUT:-0}" -eq 0 ]; then
|
||
"$script"
|
||
return $?
|
||
fi
|
||
# Keep at least 2s so timeout(1) can start the child.
|
||
[ "$left" -lt 2 ] && left=2
|
||
set +e
|
||
with_timeout "$left" "$script"
|
||
rc=$?
|
||
set -e
|
||
if [ "$rc" -eq 124 ]; then
|
||
RUN_TIMED_OUT=1
|
||
RUN_TIMEOUT_AT_PHASE="$phase"
|
||
return 1
|
||
fi
|
||
return "$rc"
|
||
}
|
||
|
||
# How inside/versions will reach containers (host-podman vs ssh)
|
||
_INSIDE_ACCESS_HINT=ssh
|
||
if [ "${INSIDE_PODMAN:-0}" = "1" ] || [ "${INSIDE_MODE:-}" = "local-podman" ]; then
|
||
_INSIDE_ACCESS_HINT=host-podman
|
||
elif [ "${SKIP_SSH:-0}" = "1" ]; then
|
||
_INSIDE_ACCESS_HINT=skipped
|
||
elif [ "${LOCAL_STACK:-0}" = "1" ] && command -v podman >/dev/null 2>&1 \
|
||
&& podman ps --format '{{.Names}}' 2>/dev/null | grep -qE 'taler-hacktivism'; then
|
||
_INSIDE_ACCESS_HINT=host-podman
|
||
fi
|
||
|
||
printf 'target domain=%s\n' "${TALER_DOMAIN}"
|
||
printf ' bank %s\n' "$BANK_PUBLIC"
|
||
printf ' exchange %s\n' "$EXCHANGE_PUBLIC"
|
||
printf ' merchant %s\n' "$MERCHANT_PUBLIC"
|
||
printf ' currency expect=%s\n' "${EXPECT_CURRENCY:-any}"
|
||
printf ' phases %s\n' "${PHASES[*]}"
|
||
printf ' flags LOCAL_STACK=%s SKIP_SSH=%s INSIDE_PODMAN=%s INSIDE_MODE=%s\n' \
|
||
"${LOCAL_STACK:-}" "${SKIP_SSH:-0}" "${INSIDE_PODMAN:-0}" "${INSIDE_MODE:-}"
|
||
printf ' flags KOOPA_SSH=%s INSIDE_SSH=%s INSIDE_PROFILE=%s\n' \
|
||
"${KOOPA_SSH:-}" "${INSIDE_SSH:-}" "${INSIDE_PROFILE:-}"
|
||
printf ' access inside/versions → %s' "$_INSIDE_ACCESS_HINT"
|
||
case "$_INSIDE_ACCESS_HINT" in
|
||
host-podman) printf ' (podman exec on this host; IDs inside.host-*)\n' ;;
|
||
ssh) printf ' (SSH then podman; IDs inside.ssh-*)\n' ;;
|
||
skipped) printf ' (SKIP_SSH=1)\n' ;;
|
||
*) printf '\n' ;;
|
||
esac
|
||
if [ "${RUN_TIMEOUT:-0}" -eq 0 ]; then
|
||
printf ' run_timeout unlimited (RUN_TIMEOUT=0)\n'
|
||
else
|
||
printf ' run_timeout %ss wall clock (RUN_TIMEOUT=; 0=unlimited)\n' "$RUN_TIMEOUT"
|
||
fi
|
||
printf ' progress %s/%s (global done/total · set PROGRESS_TOTAL= to override; PROGRESS_OFF=1 to hide)\n' \
|
||
"${PROGRESS_DONE:-0}" "${PROGRESS_TOTAL:-0}"
|
||
unset _INSIDE_ACCESS_HINT
|
||
|
||
chmod +x "$ROOT"/check_*.sh 2>/dev/null || true
|
||
|
||
ec=0
|
||
_phase_idx=0
|
||
_n_phases=${#PHASES[@]}
|
||
while [ "$_phase_idx" -lt "$_n_phases" ]; do
|
||
p="${PHASES[$_phase_idx]}"
|
||
if [ "$RUN_TIMED_OUT" = "1" ]; then
|
||
RUN_SKIPPED_PHASES+=("$p")
|
||
_phase_idx=$((_phase_idx + 1))
|
||
continue
|
||
fi
|
||
left=$(mon_seconds_left)
|
||
if [ "${RUN_TIMEOUT:-0}" -gt 0 ] && [ "$left" -le 0 ]; then
|
||
RUN_TIMED_OUT=1
|
||
RUN_TIMEOUT_AT_PHASE="${RUN_TIMEOUT_AT_PHASE:-$p}"
|
||
RUN_SKIPPED_PHASES+=("$p")
|
||
_phase_idx=$((_phase_idx + 1))
|
||
continue
|
||
fi
|
||
case "$p" in
|
||
urls) run_phase urls "$ROOT/check_urls.sh" || ec=1 ;;
|
||
inside) run_phase inside "$ROOT/check_inside.sh" || ec=1 ;;
|
||
versions) run_phase versions "$ROOT/check_versions.sh" || ec=1 ;;
|
||
sanity) run_phase sanity "$ROOT/check_sanity.sh" || ec=1 ;;
|
||
server) run_phase server "$ROOT/check_server.sh" || ec=1 ;;
|
||
e2e) run_phase e2e "$ROOT/check_e2e.sh" || ec=1 ;;
|
||
ladder|goa-ladder) run_phase ladder "$ROOT/check_goa_ladder.sh" || ec=1 ;;
|
||
auth401) run_phase auth401 "$ROOT/check_auth401.sh" || ec=1 ;;
|
||
aptdeploy|apt-deploy|apt_src) run_phase aptdeploy "$ROOT/check_apt_deploy.sh" || ec=1 ;;
|
||
surface|ecosystem) run_phase surface "$ROOT/check_surface.sh" || ec=1 ;;
|
||
mattermost) run_phase mattermost "$ROOT/check_mattermost.sh" || ec=1 ;;
|
||
mail) run_phase mail "$ROOT/check_mail.sh" || ec=1 ;;
|
||
monpages|pages) run_phase monpages "$ROOT/check_monitoring_pages.sh" || ec=1 ;;
|
||
devtesting|franken|fake-franken|fake_franken)
|
||
run_phase devtesting "$ROOT/check_devtesting.sh" || ec=1
|
||
;;
|
||
esac
|
||
# After each phase: refit total = actual done + estimate(remaining). Fixes
|
||
# stage over-estimate (e.g. 42/139 while only ~4 checks left).
|
||
if [ "${PROGRESS_TOTAL_AUTO:-0}" = "1" ]; then
|
||
_rem_args=()
|
||
_j=$((_phase_idx + 1))
|
||
while [ "$_j" -lt "$_n_phases" ]; do
|
||
_rem_args+=("${PHASES[$_j]}")
|
||
_j=$((_j + 1))
|
||
done
|
||
_progress_refit_remaining "${_rem_args[@]+"${_rem_args[@]}"}"
|
||
unset _rem_args _j
|
||
fi
|
||
_phase_idx=$((_phase_idx + 1))
|
||
done
|
||
unset _phase_idx _n_phases
|
||
|
||
# Extraordinary run-budget failure: always report at end; HTML links top → here.
|
||
if [ "$RUN_TIMED_OUT" = "1" ]; then
|
||
ec=1
|
||
elapsed=$(( $(date +%s) - MON_T0 ))
|
||
skipped="${RUN_SKIPPED_PHASES[*]:-}"
|
||
# Stable id for jump links (console HTML + err-top banner)
|
||
printf '\n'
|
||
printf '╔══════════════════════════════════════════════════════════╗\n'
|
||
printf '║ RUN TIMEOUT · extraordinary (see jump target below) ║\n'
|
||
printf '╚══════════════════════════════════════════════════════════╝\n'
|
||
printf '┌ ERROR ┐ #run.timeout-01 RUN_TIMEOUT exceeded · budget %ss · elapsed %ss\n' \
|
||
"$RUN_TIMEOUT" "$elapsed"
|
||
printf ' detail: wall-clock limit hit'
|
||
if [ -n "${RUN_TIMEOUT_AT_PHASE:-}" ]; then
|
||
printf ' during/after phase "%s"' "$RUN_TIMEOUT_AT_PHASE"
|
||
fi
|
||
if [ -n "$skipped" ]; then
|
||
printf ' · skipped: %s' "$skipped"
|
||
fi
|
||
printf '\n'
|
||
printf ' id=run.timeout-01\n'
|
||
printf ' hint: raise RUN_TIMEOUT= (seconds) or set RUN_TIMEOUT=0 for unlimited\n'
|
||
printf -- '┌ RUN TIMEOUT · extraordinary ┐\n'
|
||
printf -- ' • run.timeout-01 [run] RUN_TIMEOUT=%ss exceeded (elapsed %ss)%s%s\n' \
|
||
"$RUN_TIMEOUT" "$elapsed" \
|
||
"${RUN_TIMEOUT_AT_PHASE:+ · phase $RUN_TIMEOUT_AT_PHASE}" \
|
||
"${skipped:+ · skipped $skipped}"
|
||
printf -- '--- ERRORS (failed checks) ---\n'
|
||
printf -- ' • run.timeout-01 [run] RUN_TIMEOUT=%ss exceeded (elapsed %ss)\n' \
|
||
"$RUN_TIMEOUT" "$elapsed"
|
||
fi
|
||
|
||
# Snap progress bar to real global count (done/total) once for the whole run.
|
||
# Reload TALER_MON_STATE — check_*.sh update done/total in a separate process.
|
||
if type _mon_state_load >/dev/null 2>&1; then
|
||
_mon_state_load
|
||
fi
|
||
if type progress_finish >/dev/null 2>&1; then
|
||
printf '\n'
|
||
progress_finish
|
||
fi
|
||
|
||
exit "$ec"
|