taler-monitoring v1.13.12
Multi-phase global SUMMARY, warn-filter dimmed context, SUMMARY chrome as meta; FP stage host-agent via francpaysan-stage-user (stagepaysan); monpages GOA + FP stage.
This commit is contained in:
Hernâni Marques
commit
66df0a7b8f
88 changed files with 21996 additions and 0 deletions
679
taler-monitoring.sh
Executable file
679
taler-monitoring.sh
Executable file
|
|
@ -0,0 +1,679 @@
|
|||
#!/usr/bin/env bash
|
||||
# taler-monitoring — public URL / stack checks for a Taler domain
|
||||
#
|
||||
# ./taler-monitoring.sh # local GOA (urls + inside + e2e)
|
||||
# ./taler-monitoring.sh -d taler.net urls # public demo, no SSH
|
||||
# ./taler-monitoring.sh --domain taler-ops.ch # public ops, no SSH
|
||||
# TALER_DOMAIN=demo.taler.net ./taler-monitoring.sh urls
|
||||
#
|
||||
# Tags: [OK] [INFO] [WARN] [ERROR] [BLOCKER]
|
||||
# Exit 0 only if every selected phase exits 0.
|
||||
|
||||
set -euo pipefail
|
||||
ROOT=$(cd "$(dirname "$0")" && pwd)
|
||||
|
||||
# --- suite identity (also: --ver / --version / -V) ---
|
||||
SOURCE_REPO_WEB="${SOURCE_REPO_WEB:-https://git.hacktivism.ch/hernani/taler-monitoring}"
|
||||
SOURCE_REPO_GIT="${SOURCE_REPO_GIT:-https://git.hacktivism.ch/hernani/taler-monitoring.git}"
|
||||
|
||||
print_version() {
|
||||
local ver tag commit commit_full dirty branch
|
||||
ver="unknown"
|
||||
if [ -f "$ROOT/VERSION" ]; then
|
||||
ver=$(tr -d "[:space:]" <"$ROOT/VERSION" | sed "s/^v//")
|
||||
fi
|
||||
tag=""
|
||||
commit="unknown"
|
||||
commit_full=""
|
||||
branch=""
|
||||
dirty=""
|
||||
if git -C "$ROOT" rev-parse --git-dir >/dev/null 2>&1; then
|
||||
tag=$(git -C "$ROOT" describe --tags --exact-match 2>/dev/null || true)
|
||||
if [ -z "$tag" ]; then
|
||||
tag=$(git -C "$ROOT" describe --tags --abbrev=0 2>/dev/null || true)
|
||||
fi
|
||||
commit=$(git -C "$ROOT" rev-parse --short=12 HEAD 2>/dev/null || echo unknown)
|
||||
commit_full=$(git -C "$ROOT" rev-parse HEAD 2>/dev/null || true)
|
||||
branch=$(git -C "$ROOT" rev-parse --abbrev-ref HEAD 2>/dev/null || true)
|
||||
if [ -n "$(git -C "$ROOT" status --porcelain 2>/dev/null)" ]; then
|
||||
dirty=" (dirty)"
|
||||
fi
|
||||
fi
|
||||
if [ -n "$tag" ] && [ "$tag" = "v$ver" ]; then
|
||||
printf "taler-monitoring %s\n" "$tag"
|
||||
else
|
||||
printf "taler-monitoring v%s\n" "$ver"
|
||||
fi
|
||||
printf "version: %s\n" "$ver"
|
||||
if [ -n "$tag" ]; then
|
||||
if [ "$tag" = "v$ver" ]; then
|
||||
printf "tag: %s\n" "$tag"
|
||||
else
|
||||
printf "git-tag: %s (tree; VERSION file is %s)\n" "$tag" "$ver"
|
||||
fi
|
||||
fi
|
||||
printf "commit: %s%s\n" "$commit" "$dirty"
|
||||
[ -n "$branch" ] && [ "$branch" != "HEAD" ] && printf "branch: %s\n" "$branch"
|
||||
printf "repo: %s\n" "$SOURCE_REPO_WEB"
|
||||
printf "clone: %s\n" "$SOURCE_REPO_GIT"
|
||||
if [ -n "$commit_full" ]; then
|
||||
printf "commit-url: %s/src/commit/%s\n" "$SOURCE_REPO_WEB" "$commit_full"
|
||||
fi
|
||||
if [ -n "$tag" ]; then
|
||||
printf "tag-url: %s/src/tag/%s\n" "$SOURCE_REPO_WEB" "$tag"
|
||||
fi
|
||||
}
|
||||
|
||||
# Fast path: version before loading lib.sh / phases
|
||||
case "${1:-}" in
|
||||
--ver|--version|-V)
|
||||
print_version
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
|
||||
# Line-buffered stdout/stderr so redirected logs (tee/host-agent) flush each line.
|
||||
if [ "${_TALER_MON_STDBUF:-0}" != "1" ] && command -v stdbuf >/dev/null 2>&1; then
|
||||
export _TALER_MON_STDBUF=1
|
||||
exec stdbuf -oL -eL bash "$0" "$@"
|
||||
fi
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
taler-monitoring — bank / exchange / merchant checks
|
||||
|
||||
Usage:
|
||||
./taler-monitoring.sh [options] [phases...]
|
||||
|
||||
Phases:
|
||||
urls public HTTPS (no SSH) ← default with --domain
|
||||
(includes merchant /webui/ SPA fingerprints: version.txt, overlay, assets)
|
||||
inside container status via SSH (local stack only)
|
||||
versions taler packages vs deb.taler.net trixie + repo availability
|
||||
sanity public + optional server
|
||||
server server-side only (SSH)
|
||||
e2e withdraw + pay (small amounts; remote aborts on login/KYC)
|
||||
ladder withdraw/pay amount ladder (GOA ceiling or stage TESTPAYSAN max_wire)
|
||||
auth401 merchant Basic-auth / case matrix (HTTP 401 paths; may create throwaway instance)
|
||||
aptdeploy koopa podman apt-src smoke: taler-merchant in
|
||||
koopa-taler-deploy-test-apt-src-trixie{,-testing}
|
||||
surface REMOTE-ONLY public inventory (NOT in default/all/full):
|
||||
ecosystem hosts (taler.net, gnunet.org, taler-systems.com, mattermost, …)
|
||||
or -d DOMAIN → that domain’s surface; port/protocol/TLS/CVE (OSV)
|
||||
mattermost Mattermost chat health (default mattermost.taler.net; SPA + /api/v4/system/ping)
|
||||
mail MX/SMTP/IMAP for Taler mail (firefly, pixel/TSA, catalogued domains)
|
||||
monpages public monitoring HTML via FQDN (obligatory ERROR; GOA full inventory, FP only FP)
|
||||
uses MON_HOSTS + HTML_URL_OK (same as host-agent); v1.3.1+
|
||||
devtesting fake-franken CHF via rusty.taler-ops.ch (taler-devtesting)
|
||||
geniban + fake-incoming; needs SSH Host DEVTESTING_SSH (key to devtesting@rusty)
|
||||
franken|fake-franken aliases for devtesting
|
||||
all urls + inside + versions + sanity + e2e (SSH phases only on koopa)
|
||||
full all + server + ladder + auth401 (maximum; long-running, needs secrets)
|
||||
NOTE: surface is never included in all/full — pass it explicitly
|
||||
|
||||
Options:
|
||||
-d, --domain DOMAIN load profile from domains.conf (bank/exchange/merchant)
|
||||
presets: koopa | hacktivism.ch | taler.net | taler-ops.ch
|
||||
| my.taler-ops.ch | stage.taler-ops.ch
|
||||
unknown domain → heuristic hosts (see README)
|
||||
local (koopa) allows SSH; others public-only
|
||||
--bank URL bank base (hostname or https://…) — overrides profile
|
||||
--exchange URL exchange base — overrides profile
|
||||
--merchant URL merchant-backend base — overrides profile
|
||||
--currency CODE expected currency (GOA, KUDOS, CHF, …); empty = report only
|
||||
--no-probe do not probe alternate hosts (unknown domains only)
|
||||
--lang en|fr UI/console language (sticky bar + badges)
|
||||
overrides auto (FrancPaysan→fr, else en)
|
||||
--ver, --version, -V print suite version, git commit, and repo URLs
|
||||
-h, --help
|
||||
|
||||
Add a stack: edit domains.conf (name + bank + exchange + merchant + currency).
|
||||
|
||||
Examples:
|
||||
./taler-monitoring.sh -d taler.net
|
||||
./taler-monitoring.sh -d taler-ops.ch urls
|
||||
./taler-monitoring.sh -d my.taler-ops.ch urls
|
||||
./taler-monitoring.sh -d taler-ops.ch --merchant https://my.taler-ops.ch urls
|
||||
./taler-monitoring.sh -d demo.taler.net --currency KUDOS
|
||||
./taler-monitoring.sh --exchange https://exchange.taler-ops.ch \
|
||||
--merchant https://my.taler-ops.ch --bank https://bank.taler-ops.ch urls
|
||||
|
||||
Env (same meaning):
|
||||
TALER_DOMAIN BANK_PUBLIC EXCHANGE_PUBLIC MERCHANT_PUBLIC EXPECT_CURRENCY
|
||||
TALER_DOMAINS_CONF SKIP_SSH=1
|
||||
NO_COLOR=1 / CLICOLOR=0 disable green/yellow/red tags (default: coloured)
|
||||
SKIP_SSH=1 NO_COLOR=1
|
||||
RUN_TIMEOUT=600 whole-run wall clock seconds (default 600; 0=unlimited)
|
||||
long phases (ladder/full/e2e) need a higher value or 0
|
||||
DISK_WARN_USED_PCT=85 disk free: WARN when used ≥ this %
|
||||
DISK_ERR_USED_PCT=95 disk free: ERROR when used ≥ this % (or avail=0 / 100%)
|
||||
PERF_WARN_MS PERF_FAIL_MS (urls latency; default 8000 / 20000)
|
||||
QR_CHECK=0 skip QR form + qrencode/zbarimg (urls phase)
|
||||
QR_ECC=M qrencode ECC level (default M)
|
||||
KOOPA_SSH KOOPA_SSH_FALLBACKS (set KOOPA_SSH / KOOPA_SSH_FALLBACKS in env)
|
||||
METRICS_LOAD=0 skip host/container RAM/CPU probes (e2e/ladder/inside)
|
||||
EXPECT_WEBUI_VERSION=1.6.11 pin merchant /webui/version.txt (urls)
|
||||
EXPECT_WEBUI_OVERLAY=selfbuild require substr in version-overlay.txt
|
||||
WEBUI_OVERLAY_DENY='master-11340' fail if overlay matches regex
|
||||
CHECK_WEBUI_SPA=0 skip SPA fingerprint block in urls
|
||||
AUTH401_* see check_auth401.sh / secrets.env.example
|
||||
AUTH401_CONTINUE=1 / CONTINUE_ON_ERROR=1
|
||||
auth401: run all groups, collect every ERROR (no mid-run halt)
|
||||
APT_DEPLOY_SKIP=1 skip aptdeploy phase
|
||||
APT_DEPLOY_CONTAINERS="name:suite …" default trixie + trixie-testing deploy-test pods
|
||||
SURFACE_CVE=0 disable CVE queries in surface phase
|
||||
SURFACE_CVE_LEVEL=warn|error bare Server-header versions default warn
|
||||
(Debian package versions default error)
|
||||
SURFACE_CATALOG=path override surface-catalog.conf
|
||||
TALER_MON_LANG=en|fr same as --lang (default en; auto fr for *lefrancpaysan*)
|
||||
|
||||
Full load (GOA / hacktivism):
|
||||
./taler-monitoring.sh -d hacktivism.ch full
|
||||
# or explicit:
|
||||
./taler-monitoring.sh -d hacktivism.ch urls inside versions sanity server e2e ladder auth401
|
||||
# hacktivism host-agent also runs aptdeploy (apt-src merchant containers on koopa)
|
||||
|
||||
Remote surface / ecosystem (explicit only — never default):
|
||||
./taler-monitoring.sh surface
|
||||
./taler-monitoring.sh -d hacktivism.ch surface
|
||||
./taler-monitoring.sh -d lefrancpaysan.ch surface
|
||||
./taler-monitoring.sh mattermost # mattermost.taler.net chat health
|
||||
./taler-monitoring.sh mail # firefly + pixel MX/SMTP/IMAP
|
||||
|
||||
SPA pin after selfbuild:
|
||||
EXPECT_WEBUI_VERSION=1.6.11 EXPECT_WEBUI_OVERLAY=selfbuild-v1.6.11 \\
|
||||
WEBUI_OVERLAY_DENY='c778af04|master-11340' \\
|
||||
./taler-monitoring.sh -d hacktivism.ch urls
|
||||
EOF
|
||||
}
|
||||
|
||||
# Machine-local paths/hosts from taler-monitoring-env install (no hardcodes in suite).
|
||||
# Install: ~/src/taler-monitoring-env/bin/install-env.sh <machine>
|
||||
if [ "${_TALER_MON_ENV_LOADED:-0}" != "1" ]; then
|
||||
_mon_cfg="${TALER_MONITORING_ENV:-${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring/env}"
|
||||
if [ -f "$_mon_cfg" ]; then
|
||||
# shellcheck disable=SC1090
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
. "$_mon_cfg"
|
||||
set +a
|
||||
fi
|
||||
unset _mon_cfg
|
||||
export _TALER_MON_ENV_LOADED=1
|
||||
fi
|
||||
|
||||
# shellcheck source=lib.sh
|
||||
source "$ROOT/lib.sh"
|
||||
# ensure i18n after domain may change later
|
||||
i18n_init 2>/dev/null || true
|
||||
# Optional gitignored secrets.env next to this suite (or MONITORING_SECRETS_ENV=)
|
||||
load_monitoring_secrets_env 2>/dev/null || true
|
||||
|
||||
# Persist #NNN + progress across check_*.sh child processes
|
||||
TALER_MON_STATE=$(mktemp "${TMPDIR:-/tmp}/taler-mon-state.XXXXXX")
|
||||
export TALER_MON_STATE
|
||||
trap 'rm -f "${TALER_MON_STATE:-}"' EXIT
|
||||
|
||||
PHASES=()
|
||||
DOMAIN_SET=0
|
||||
BANK_OVERRIDE=""
|
||||
EXCHANGE_OVERRIDE=""
|
||||
MERCHANT_OVERRIDE=""
|
||||
CURRENCY_OVERRIDE=""
|
||||
NO_PROBE=0
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--lang)
|
||||
[ $# -ge 2 ] || { echo "error: --lang needs en|fr" >&2; exit 2; }
|
||||
_lang="$2"
|
||||
case "$_lang" in
|
||||
fr|FR|fra|french) TALER_MON_LANG=fr ;;
|
||||
en|EN|eng|english) TALER_MON_LANG=en ;;
|
||||
*) echo "error: unknown language '$_lang' (use en or fr)" >&2; exit 2 ;;
|
||||
esac
|
||||
TALER_MON_LANG_SET=1
|
||||
export TALER_MON_LANG TALER_MON_LANG_SET
|
||||
shift 2
|
||||
i18n_init 2>/dev/null || true
|
||||
;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
--ver|--version|-V) print_version; exit 0 ;;
|
||||
-d|--domain)
|
||||
[ $# -ge 2 ] || { echo "missing arg for $1" >&2; exit 2; }
|
||||
apply_taler_domain "$2"
|
||||
DOMAIN_SET=1
|
||||
TALER_DOMAIN_FROM_CLI=1
|
||||
shift 2
|
||||
;;
|
||||
--bank)
|
||||
[ $# -ge 2 ] || { echo "missing arg for $1" >&2; exit 2; }
|
||||
BANK_OVERRIDE="${2%/}"; shift 2
|
||||
;;
|
||||
--exchange)
|
||||
[ $# -ge 2 ] || { echo "missing arg for $1" >&2; exit 2; }
|
||||
EXCHANGE_OVERRIDE="${2%/}"; shift 2
|
||||
;;
|
||||
--merchant)
|
||||
[ $# -ge 2 ] || { echo "missing arg for $1" >&2; exit 2; }
|
||||
MERCHANT_OVERRIDE="${2%/}"; shift 2
|
||||
;;
|
||||
--currency)
|
||||
[ $# -ge 2 ] || { echo "missing arg for $1" >&2; exit 2; }
|
||||
CURRENCY_OVERRIDE="$2"; shift 2
|
||||
;;
|
||||
--no-probe) NO_PROBE=1; shift ;;
|
||||
urls|inside|versions|sanity|server|e2e|ladder|goa-ladder|auth401|aptdeploy|apt-deploy|apt_src|surface|ecosystem|mattermost|mail|monpages|pages|devtesting|franken|fake-franken|fake_franken|all|full) PHASES+=("$1"); shift ;;
|
||||
*)
|
||||
# bare domain shorthand: ./taler-monitoring.sh taler.net
|
||||
if [[ "$1" == *.* && "$1" != *://* && "$1" != -* ]]; then
|
||||
apply_taler_domain "$1"
|
||||
DOMAIN_SET=1
|
||||
TALER_DOMAIN_FROM_CLI=1
|
||||
shift
|
||||
else
|
||||
echo "Unknown: $1" >&2; usage >&2; exit 2
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "$NO_PROBE" = "1" ]; then
|
||||
TALER_DOMAIN_PROBE=0
|
||||
fi
|
||||
if [ -n "$CURRENCY_OVERRIDE" ]; then
|
||||
EXPECT_CURRENCY="$CURRENCY_OVERRIDE"
|
||||
fi
|
||||
if [ -n "$BANK_OVERRIDE" ]; then BANK_PUBLIC="$BANK_OVERRIDE"; fi
|
||||
if [ -n "$EXCHANGE_OVERRIDE" ]; then EXCHANGE_PUBLIC="$EXCHANGE_OVERRIDE"; fi
|
||||
if [ -n "$MERCHANT_OVERRIDE" ]; then MERCHANT_PUBLIC="$MERCHANT_OVERRIDE"; fi
|
||||
|
||||
# Only koopa may use SSH. Remote domains: public + optional e2e (no SSH).
|
||||
if [ "${LOCAL_STACK:-1}" != "1" ]; then
|
||||
SKIP_SSH=1
|
||||
fi
|
||||
|
||||
# Remote: no SSH; ATM withdraw ladder set in check_e2e (smaller notes)
|
||||
if [ "${LOCAL_STACK}" != "1" ]; then
|
||||
E2E_FAKE_INCOMING=0
|
||||
E2E_REMOTE=1
|
||||
: "${E2E_WITHDRAW_VALUES:=10 20 50}"
|
||||
: "${E2E_PAY_VALUES:=0.01 0.05 0.1 1}"
|
||||
fi
|
||||
|
||||
# Export so check_*.sh (re-source lib) see the same targets via env
|
||||
export TALER_DOMAIN BANK_PUBLIC EXCHANGE_PUBLIC MERCHANT_PUBLIC
|
||||
export EXPECT_CURRENCY SKIP_SSH LOCAL_STACK TALER_DOMAIN_PROBE CHECK_LANDING
|
||||
export TALER_DOMAIN_FROM_CLI="${TALER_DOMAIN_FROM_CLI:-0}"
|
||||
export DOMAIN_SET="${DOMAIN_SET:-0}"
|
||||
export WITHDRAW_AMT PAY_AMT CREDIT_AMT MERCHANT_INSTANCE
|
||||
export E2E_FAKE_INCOMING E2E_REMOTE E2E_VARIABLE E2E_ATM_MAX
|
||||
export E2E_WITHDRAW_VALUES E2E_PAY_VALUES E2E_USE_TEMPLATES E2E_TEMPLATE_MAP
|
||||
export PAIVANA_PUBLIC E2E_PAIVANA E2E_PAIVANA_TEMPLATE E2E_PAIVANA_AMOUNT E2E_PAIVANA_INSTANCE
|
||||
export INSIDE_PROFILE INSIDE_SSH
|
||||
export INSIDE_BANK_CTR INSIDE_EXCHANGE_CTR INSIDE_MERCHANT_CTR
|
||||
export INSIDE_BANK_PORT INSIDE_EXCHANGE_PORT INSIDE_MERCHANT_PORT
|
||||
export INSIDE_DNS_BANK INSIDE_DNS_EXCHANGE INSIDE_DNS_MERCHANT
|
||||
# Ladder: withdraw then pay — 0 + random mids + max-1 + max (see check_goa_ladder.sh).
|
||||
# Defaults so set -u export is safe when vars were never set by caller.
|
||||
: "${LADDER_STEPS:=23}"
|
||||
: "${LADDER_MIN_AMOUNT:=0.000001}"
|
||||
: "${LADDER_CONFIRM_POLLS:=40}"
|
||||
: "${LADDER_MAX_RUNGS:=99}"
|
||||
: "${LADDER_TIMEOUT_S:=3600}"
|
||||
: "${LADDER_REPORT_DIR:=}"
|
||||
: "${LADDER_SETTLE_ROUNDS:=18}"
|
||||
: "${LADDER_SETTLE_SLEEP:=2}"
|
||||
: "${LADDER_MAX_AMOUNT:=4503599627370496}"
|
||||
: "${LADDER_INCLUDE_ZERO:=1}"
|
||||
: "${LADDER_INCLUDE_MAX:=1}"
|
||||
: "${LADDER_HIGH_FROM:=1000000}"
|
||||
: "${LADDER_HIGH_RUNGS:=12}"
|
||||
: "${LADDER_LOAD:=1}"
|
||||
: "${LADDER_PAY:=1}"
|
||||
: "${LADDER_WITHDRAW_SCALE:=1.5}"
|
||||
: "${LADDER_PAY_SETTLE_ROUNDS:=6}"
|
||||
export LADDER_STEPS LADDER_MIN_AMOUNT LADDER_CONFIRM_POLLS LADDER_MAX_RUNGS LADDER_TIMEOUT_S LADDER_REPORT_DIR
|
||||
export LADDER_SETTLE_ROUNDS LADDER_SETTLE_SLEEP EXP_PW_FILE EXP_USER
|
||||
export LADDER_MAX_AMOUNT LADDER_INCLUDE_ZERO LADDER_INCLUDE_MAX
|
||||
export LADDER_HIGH_FROM LADDER_HIGH_RUNGS LADDER_LOAD
|
||||
export LADDER_PAY LADDER_WITHDRAW_SCALE LADDER_PAY_SETTLE_ROUNDS
|
||||
export TALER_DOMAIN_APPLIED=1
|
||||
i18n_init 2>/dev/null || true
|
||||
# Optional: auth401 and other phases may honor CONTINUE_ON_ERROR
|
||||
export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-}"
|
||||
export AUTH401_CONTINUE="${AUTH401_CONTINUE:-${CONTINUE_ON_ERROR:-}}"
|
||||
|
||||
# Default phases
|
||||
# - local GOA (koopa): urls + inside + versions + e2e
|
||||
# - FrancPaysan STAGE: urls + inside (stagepaysan) + versions + e2e
|
||||
# - other remote: urls only (pass e2e/inside explicitly)
|
||||
if [ "${#PHASES[@]}" -eq 0 ]; then
|
||||
if [ "${LOCAL_STACK}" = "1" ]; then
|
||||
PHASES=(urls inside versions e2e)
|
||||
elif [ "${EXPECT_CURRENCY:-}" = "TESTPAYSAN" ] \
|
||||
|| [[ "${TALER_DOMAIN:-}" == stage.*lefrancpaysan* ]] \
|
||||
|| [[ "${TALER_DOMAIN:-}" == *stage.lefrancpaysan* ]]; then
|
||||
# Like hacktivism breadth: public urls + inside (stagepaysan) + versions + e2e
|
||||
PHASES=(urls inside versions e2e)
|
||||
else
|
||||
PHASES=(urls)
|
||||
fi
|
||||
fi
|
||||
|
||||
OUT_PHASES=()
|
||||
for p in "${PHASES[@]}"; do
|
||||
if [ "$p" = "all" ]; then
|
||||
if [ "${LOCAL_STACK}" = "1" ]; then
|
||||
OUT_PHASES+=(urls inside versions sanity e2e)
|
||||
elif [ -n "${INSIDE_SSH:-}" ] || [ "${INSIDE_PROFILE:-}" = "stage-lfp" ]; then
|
||||
OUT_PHASES+=(urls inside versions e2e)
|
||||
else
|
||||
# remote without inside SSH: public + optional e2e
|
||||
OUT_PHASES+=(urls versions e2e)
|
||||
fi
|
||||
elif [ "$p" = "full" ]; then
|
||||
# Maximum coverage. Long. Needs secrets for e2e/ladder/auth401.
|
||||
if [ "${LOCAL_STACK}" = "1" ]; then
|
||||
OUT_PHASES+=(urls inside versions sanity server e2e ladder auth401)
|
||||
elif [ -n "${INSIDE_SSH:-}" ] || [ "${INSIDE_PROFILE:-}" = "stage-lfp" ]; then
|
||||
OUT_PHASES+=(urls inside versions e2e ladder auth401)
|
||||
else
|
||||
OUT_PHASES+=(urls versions e2e auth401)
|
||||
fi
|
||||
else
|
||||
OUT_PHASES+=("$p")
|
||||
fi
|
||||
done
|
||||
PHASES=()
|
||||
seen=" "
|
||||
for p in "${OUT_PHASES[@]}"; do
|
||||
# server = koopa-only. inside allowed on koopa OR stagepaysan (INSIDE_SSH).
|
||||
case "$p" in
|
||||
server)
|
||||
if [ "${LOCAL_STACK}" != "1" ]; then
|
||||
echo "[INFO] skip phase 'server' (koopa only)" >&2
|
||||
continue
|
||||
fi
|
||||
;;
|
||||
inside)
|
||||
if [ "${LOCAL_STACK}" != "1" ] && [ -z "${INSIDE_SSH:-}" ] && [ "${INSIDE_PROFILE:-}" != "stage-lfp" ]; then
|
||||
echo "[INFO] skip phase 'inside' (no INSIDE_SSH / not local)" >&2
|
||||
continue
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
case "$seen" in *" $p "*) ;; *) PHASES+=("$p"); seen="$seen$p " ;; esac
|
||||
done
|
||||
|
||||
if [ "${#PHASES[@]}" -eq 0 ]; then
|
||||
PHASES=(urls)
|
||||
fi
|
||||
|
||||
# Global expected check count for progress bar done/total (override: PROGRESS_TOTAL=N).
|
||||
# Auto estimate is env-aware (CHECK_BANK=0 etc.) and re-fit after each phase so stage
|
||||
# jobs do not sit at 42/139 then snap to 46/46. Phase summary() must NOT snap mid-run.
|
||||
# Host-agent monpages post-check is outside this process — never counted here.
|
||||
_progress_estimate_phase() {
|
||||
local p="$1" n=0 hosts=1
|
||||
hosts=$(printf '%s' "${MON_HOSTS:-x}" | wc -w)
|
||||
[ "$hosts" -lt 1 ] && hosts=1
|
||||
case "$p" in
|
||||
urls)
|
||||
# Core: exchange + merchant + webui + perf + terms (~28–40). Bank/landing optional.
|
||||
# Stage (CHECK_BANK=0 CHECK_LANDING=0) lands near ~34 numbered lines — keep close.
|
||||
n=28
|
||||
[ "${CHECK_BANK:-1}" != "0" ] && n=$((n + 24))
|
||||
[ "${CHECK_LANDING:-1}" != "0" ] && n=$((n + 18)) # intro + stats.json + qr
|
||||
n=$((n + 6)) # headroom (SPA / terms / privacy / alt paths)
|
||||
;;
|
||||
inside) n=30 ;;
|
||||
versions) n=35 ;;
|
||||
sanity) n=35 ;;
|
||||
server) n=20 ;;
|
||||
# e2e: ATM ladder emits many INFO lines (coins before/after each note)
|
||||
e2e) n=240 ;;
|
||||
ladder|goa-ladder) n=120 ;;
|
||||
auth401) n=70 ;;
|
||||
aptdeploy) n=40 ;;
|
||||
surface|ecosystem) n=120 ;;
|
||||
monpages|pages)
|
||||
# slash + bare (+ optional _err) per inventory host; GOA full suite ≈20 URLs
|
||||
case "${TALER_DOMAIN:-}" in
|
||||
*hacktivism*) n=24 ;;
|
||||
*lefrancpaysan*|*francpaysan*) n=$((hosts * 2 + 6)); [ "$n" -lt 12 ] && n=12 ;;
|
||||
*) n=$((hosts * 2 + 4)); [ "$n" -lt 8 ] && n=8 ;;
|
||||
esac
|
||||
[ "$n" -gt 48 ] && n=48
|
||||
;;
|
||||
mattermost) n=25 ;;
|
||||
mail) n=40 ;;
|
||||
devtesting|franken|fake-franken|fake_franken) n=10 ;;
|
||||
*) n=15 ;;
|
||||
esac
|
||||
printf '%s' "$n"
|
||||
}
|
||||
|
||||
# Recompute PROGRESS_TOTAL = done + estimate(remaining phases). Allows shrink after short phases.
|
||||
_progress_refit_remaining() {
|
||||
[ "${PROGRESS_TOTAL_AUTO:-0}" = "1" ] || return 0
|
||||
local rem=0 rp
|
||||
for rp in "$@"; do
|
||||
rem=$((rem + $(_progress_estimate_phase "$rp")))
|
||||
done
|
||||
if type _mon_state_load >/dev/null 2>&1; then
|
||||
_mon_state_load
|
||||
fi
|
||||
set_progress_total "$(( ${PROGRESS_DONE:-0} + rem ))"
|
||||
}
|
||||
|
||||
PROGRESS_TOTAL_AUTO=0
|
||||
if [ "${PROGRESS_TOTAL:-0}" = "0" ] || [ -z "${PROGRESS_TOTAL:-}" ]; then
|
||||
PROGRESS_TOTAL_AUTO=1
|
||||
_pt=0
|
||||
for p in "${PHASES[@]}"; do
|
||||
_pt=$((_pt + $(_progress_estimate_phase "$p")))
|
||||
done
|
||||
set_progress_total "$_pt"
|
||||
unset _pt
|
||||
fi
|
||||
export PROGRESS_TOTAL_AUTO
|
||||
|
||||
# Whole-run wall clock (default 10 min). Override: RUN_TIMEOUT=0 (unlimited).
|
||||
: "${RUN_TIMEOUT:=600}"
|
||||
export RUN_TIMEOUT
|
||||
MON_T0=$(date +%s)
|
||||
export MON_T0
|
||||
RUN_TIMED_OUT=0
|
||||
RUN_TIMEOUT_AT_PHASE=""
|
||||
RUN_SKIPPED_PHASES=()
|
||||
|
||||
mon_seconds_left() {
|
||||
if [ "${RUN_TIMEOUT:-0}" -eq 0 ]; then
|
||||
printf '%s' "999999"
|
||||
return 0
|
||||
fi
|
||||
local now left
|
||||
now=$(date +%s)
|
||||
left=$((RUN_TIMEOUT - (now - MON_T0)))
|
||||
[ "$left" -lt 0 ] && left=0
|
||||
printf '%s' "$left"
|
||||
}
|
||||
|
||||
# Run one phase script under remaining RUN_TIMEOUT budget.
|
||||
# Exit 124 (timeout utility) → mark RUN_TIMED_OUT.
|
||||
run_phase() {
|
||||
local phase="$1" script="$2" left rc
|
||||
left=$(mon_seconds_left)
|
||||
if [ "${RUN_TIMEOUT:-0}" -gt 0 ] && [ "$left" -le 0 ]; then
|
||||
RUN_TIMED_OUT=1
|
||||
RUN_TIMEOUT_AT_PHASE="${RUN_TIMEOUT_AT_PHASE:-$phase}"
|
||||
return 1
|
||||
fi
|
||||
if [ "${RUN_TIMEOUT:-0}" -eq 0 ]; then
|
||||
"$script"
|
||||
return $?
|
||||
fi
|
||||
# Keep at least 2s so timeout(1) can start the child.
|
||||
[ "$left" -lt 2 ] && left=2
|
||||
set +e
|
||||
with_timeout "$left" "$script"
|
||||
rc=$?
|
||||
set -e
|
||||
if [ "$rc" -eq 124 ]; then
|
||||
RUN_TIMED_OUT=1
|
||||
RUN_TIMEOUT_AT_PHASE="$phase"
|
||||
return 1
|
||||
fi
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
# How inside/versions will reach containers (host-podman vs ssh)
|
||||
_INSIDE_ACCESS_HINT=ssh
|
||||
if [ "${INSIDE_PODMAN:-0}" = "1" ] || [ "${INSIDE_MODE:-}" = "local-podman" ]; then
|
||||
_INSIDE_ACCESS_HINT=host-podman
|
||||
elif [ "${SKIP_SSH:-0}" = "1" ]; then
|
||||
_INSIDE_ACCESS_HINT=skipped
|
||||
elif [ "${LOCAL_STACK:-0}" = "1" ] && command -v podman >/dev/null 2>&1 \
|
||||
&& podman ps --format '{{.Names}}' 2>/dev/null | grep -qE 'taler-hacktivism'; then
|
||||
_INSIDE_ACCESS_HINT=host-podman
|
||||
fi
|
||||
|
||||
printf 'target domain=%s\n' "${TALER_DOMAIN}"
|
||||
printf ' bank %s\n' "$BANK_PUBLIC"
|
||||
printf ' exchange %s\n' "$EXCHANGE_PUBLIC"
|
||||
printf ' merchant %s\n' "$MERCHANT_PUBLIC"
|
||||
printf ' currency expect=%s\n' "${EXPECT_CURRENCY:-any}"
|
||||
printf ' phases %s\n' "${PHASES[*]}"
|
||||
printf ' flags LOCAL_STACK=%s SKIP_SSH=%s INSIDE_PODMAN=%s INSIDE_MODE=%s\n' \
|
||||
"${LOCAL_STACK:-}" "${SKIP_SSH:-0}" "${INSIDE_PODMAN:-0}" "${INSIDE_MODE:-}"
|
||||
printf ' flags KOOPA_SSH=%s INSIDE_SSH=%s INSIDE_PROFILE=%s\n' \
|
||||
"${KOOPA_SSH:-}" "${INSIDE_SSH:-}" "${INSIDE_PROFILE:-}"
|
||||
printf ' access inside/versions → %s' "$_INSIDE_ACCESS_HINT"
|
||||
case "$_INSIDE_ACCESS_HINT" in
|
||||
host-podman) printf ' (podman exec on this host; IDs inside.host-*)\n' ;;
|
||||
ssh) printf ' (SSH then podman; IDs inside.ssh-*)\n' ;;
|
||||
skipped) printf ' (SKIP_SSH=1)\n' ;;
|
||||
*) printf '\n' ;;
|
||||
esac
|
||||
if [ "${RUN_TIMEOUT:-0}" -eq 0 ]; then
|
||||
printf ' run_timeout unlimited (RUN_TIMEOUT=0)\n'
|
||||
else
|
||||
printf ' run_timeout %ss wall clock (RUN_TIMEOUT=; 0=unlimited)\n' "$RUN_TIMEOUT"
|
||||
fi
|
||||
printf ' progress %s/%s (global done/total · set PROGRESS_TOTAL= to override; PROGRESS_OFF=1 to hide)\n' \
|
||||
"${PROGRESS_DONE:-0}" "${PROGRESS_TOTAL:-0}"
|
||||
unset _INSIDE_ACCESS_HINT
|
||||
|
||||
chmod +x "$ROOT"/check_*.sh 2>/dev/null || true
|
||||
|
||||
ec=0
|
||||
_phase_idx=0
|
||||
_n_phases=${#PHASES[@]}
|
||||
# Suppress all phase SUMMARY / progress standings when more than one phase;
|
||||
# parent prints one global SUMMARY after progress_finish (v1.13.10+).
|
||||
if [ "$_n_phases" -gt 1 ]; then
|
||||
export TALER_MON_MULTI_PHASE=1
|
||||
else
|
||||
export TALER_MON_MULTI_PHASE=0
|
||||
fi
|
||||
_MON_WAS_MULTI="${TALER_MON_MULTI_PHASE}"
|
||||
while [ "$_phase_idx" -lt "$_n_phases" ]; do
|
||||
p="${PHASES[$_phase_idx]}"
|
||||
if [ "$RUN_TIMED_OUT" = "1" ]; then
|
||||
RUN_SKIPPED_PHASES+=("$p")
|
||||
_phase_idx=$((_phase_idx + 1))
|
||||
continue
|
||||
fi
|
||||
left=$(mon_seconds_left)
|
||||
if [ "${RUN_TIMEOUT:-0}" -gt 0 ] && [ "$left" -le 0 ]; then
|
||||
RUN_TIMED_OUT=1
|
||||
RUN_TIMEOUT_AT_PHASE="${RUN_TIMEOUT_AT_PHASE:-$p}"
|
||||
RUN_SKIPPED_PHASES+=("$p")
|
||||
_phase_idx=$((_phase_idx + 1))
|
||||
continue
|
||||
fi
|
||||
case "$p" in
|
||||
urls) run_phase urls "$ROOT/check_urls.sh" || ec=1 ;;
|
||||
inside) run_phase inside "$ROOT/check_inside.sh" || ec=1 ;;
|
||||
versions) run_phase versions "$ROOT/check_versions.sh" || ec=1 ;;
|
||||
sanity) run_phase sanity "$ROOT/check_sanity.sh" || ec=1 ;;
|
||||
server) run_phase server "$ROOT/check_server.sh" || ec=1 ;;
|
||||
e2e) run_phase e2e "$ROOT/check_e2e.sh" || ec=1 ;;
|
||||
ladder|goa-ladder) run_phase ladder "$ROOT/check_goa_ladder.sh" || ec=1 ;;
|
||||
auth401) run_phase auth401 "$ROOT/check_auth401.sh" || ec=1 ;;
|
||||
aptdeploy|apt-deploy|apt_src) run_phase aptdeploy "$ROOT/check_apt_deploy.sh" || ec=1 ;;
|
||||
surface|ecosystem) run_phase surface "$ROOT/check_surface.sh" || ec=1 ;;
|
||||
mattermost) run_phase mattermost "$ROOT/check_mattermost.sh" || ec=1 ;;
|
||||
mail) run_phase mail "$ROOT/check_mail.sh" || ec=1 ;;
|
||||
monpages|pages) run_phase monpages "$ROOT/check_monitoring_pages.sh" || ec=1 ;;
|
||||
devtesting|franken|fake-franken|fake_franken)
|
||||
run_phase devtesting "$ROOT/check_devtesting.sh" || ec=1
|
||||
;;
|
||||
esac
|
||||
# After each phase: refit total = actual done + estimate(remaining). Fixes
|
||||
# stage over-estimate (e.g. 42/139 while only ~4 checks left).
|
||||
if [ "${PROGRESS_TOTAL_AUTO:-0}" = "1" ]; then
|
||||
_rem_args=()
|
||||
_j=$((_phase_idx + 1))
|
||||
while [ "$_j" -lt "$_n_phases" ]; do
|
||||
_rem_args+=("${PHASES[$_j]}")
|
||||
_j=$((_j + 1))
|
||||
done
|
||||
_progress_refit_remaining "${_rem_args[@]+"${_rem_args[@]}"}"
|
||||
unset _rem_args _j
|
||||
fi
|
||||
_phase_idx=$((_phase_idx + 1))
|
||||
done
|
||||
unset _phase_idx _n_phases
|
||||
unset PROGRESS_FINAL TALER_MON_MULTI_PHASE 2>/dev/null || true
|
||||
|
||||
# Extraordinary run-budget failure: always report at end; HTML links top → here.
|
||||
if [ "$RUN_TIMED_OUT" = "1" ]; then
|
||||
ec=1
|
||||
elapsed=$(( $(date +%s) - MON_T0 ))
|
||||
skipped="${RUN_SKIPPED_PHASES[*]:-}"
|
||||
# Stable id for jump links (console HTML + err-top banner)
|
||||
printf '\n'
|
||||
printf '╔══════════════════════════════════════════════════════════╗\n'
|
||||
printf '║ RUN TIMEOUT · extraordinary (see jump target below) ║\n'
|
||||
printf '╚══════════════════════════════════════════════════════════╝\n'
|
||||
printf '┌ ERROR ┐ #run.timeout-01 RUN_TIMEOUT exceeded · budget %ss · elapsed %ss\n' \
|
||||
"$RUN_TIMEOUT" "$elapsed"
|
||||
printf ' detail: wall-clock limit hit'
|
||||
if [ -n "${RUN_TIMEOUT_AT_PHASE:-}" ]; then
|
||||
printf ' during/after phase "%s"' "$RUN_TIMEOUT_AT_PHASE"
|
||||
fi
|
||||
if [ -n "$skipped" ]; then
|
||||
printf ' · skipped: %s' "$skipped"
|
||||
fi
|
||||
printf '\n'
|
||||
printf ' id=run.timeout-01\n'
|
||||
printf ' hint: raise RUN_TIMEOUT= (seconds) or set RUN_TIMEOUT=0 for unlimited\n'
|
||||
printf -- '┌ RUN TIMEOUT · extraordinary ┐\n'
|
||||
printf -- ' • run.timeout-01 [run] RUN_TIMEOUT=%ss exceeded (elapsed %ss)%s%s\n' \
|
||||
"$RUN_TIMEOUT" "$elapsed" \
|
||||
"${RUN_TIMEOUT_AT_PHASE:+ · phase $RUN_TIMEOUT_AT_PHASE}" \
|
||||
"${skipped:+ · skipped $skipped}"
|
||||
printf -- '--- ERRORS (failed checks) ---\n'
|
||||
printf -- ' • run.timeout-01 [run] RUN_TIMEOUT=%ss exceeded (elapsed %ss)\n' \
|
||||
"$RUN_TIMEOUT" "$elapsed"
|
||||
fi
|
||||
|
||||
# Snap progress bar to real global count (done/total) once for the whole run.
|
||||
# Reload TALER_MON_STATE — check_*.sh update done/total + GLOBAL_*_N in a separate process.
|
||||
if type _mon_state_load >/dev/null 2>&1; then
|
||||
_mon_state_load
|
||||
fi
|
||||
if type progress_finish >/dev/null 2>&1; then
|
||||
printf '\n'
|
||||
progress_finish
|
||||
fi
|
||||
# Multi-phase: one global SUMMARY (OK/ERROR/WARN/INFO/BLOCK across all phases).
|
||||
# Single-phase already printed phase-local SUMMARY via check_*.sh.
|
||||
if [ "${_MON_WAS_MULTI:-0}" = "1" ] && type summary_global >/dev/null 2>&1; then
|
||||
summary_global || true
|
||||
fi
|
||||
unset _MON_WAS_MULTI 2>/dev/null || true
|
||||
|
||||
exit "$ec"
|
||||
Loading…
Add table
Add a link
Reference in a new issue