taler-monitoring/lib.sh
Hernâni Marques d7831b2374
release 1.13.10: global SUMMARY at end of multi-phase runs
Persist GLOBAL_PASS/FAIL/WARN/INFO/BLOCK in TALER_MON_STATE across
check_*.sh processes. Multi-phase never prints a phase SUMMARY box
(only ERRORS/BLOCKERS mid-run); parent progress_finish + summary_global
prints one whole-run totals stand after all phases.
2026-07-19 04:09:35 +02:00

1413 lines
48 KiB
Bash
Executable file
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

# shellcheck shell=bash
# Shared helpers for taler-monitoring (laptop or koopa).
#
# Machine paths/SSH hosts: only from env (~/.config/taler-monitoring/env or
# TALER_MONITORING_ENV=…), installed via taler-monitoring-env — never hardcode.
# Load host env once (before any := defaults)
if [ "${_TALER_MON_ENV_LOADED:-0}" != "1" ]; then
_mon_cfg="${TALER_MONITORING_ENV:-${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring/env}"
if [ -f "$_mon_cfg" ]; then
# shellcheck disable=SC1090
set -a
# shellcheck disable=SC1091
. "$_mon_cfg"
set +a
fi
unset _mon_cfg
export _TALER_MON_ENV_LOADED=1
fi
# Default stack = GOA / hacktivism (overridden by TALER_DOMAIN / --domain)
# i18n (en default; fr for FrancPaysan — see i18n.sh / TALER_MON_LANG)
_I18N_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
if [ -f "$_I18N_DIR/i18n.sh" ]; then
# shellcheck source=i18n.sh
source "$_I18N_DIR/i18n.sh"
i18n_init
else
i18n_tag() { printf '%s' "$1"; }
i18n_text() { printf '%s' "$*"; }
i18n_init() { :; }
fi
: "${TALER_DOMAIN:=hacktivism.ch}"
: "${BANK_PUBLIC:=https://bank.hacktivism.ch}"
: "${EXCHANGE_PUBLIC:=https://exchange.hacktivism.ch}"
: "${MERCHANT_PUBLIC:=https://taler.hacktivism.ch}"
: "${BANK_LOCAL:=http://127.0.0.1:9012}"
: "${EXCHANGE_LOCAL:=http://127.0.0.1:9011}"
: "${MERCHANT_LOCAL:=https://127.0.0.1:9010}"
: "${LANDING_LOCAL:=http://127.0.0.1:9013}"
# Hostnames/paths: set via ~/.config/taler-monitoring/env (no hardcoded machine values).
: "${KOOPA_SSH:=}"
: "${KOOPA_SSH_FALLBACKS:=}"
: "${INSIDE_SSH:=}"
: "${FRANCPAYSAN_SSH:=}"
: "${DEVTESTING_SSH:=}"
: "${DEPLOY_SSH:=}"
: "${FIRECUDA_SSH:=}"
: "${HTML_OUT:=${HOME}/monitoring-sites-staging}"
: "${DEPLOY_STAGING:=${HTML_OUT}}"
: "${DEPLOY_WWW_ROOT:=}"
: "${APPLY_MONITORING_LIVE:=}"
: "${KOOPA_CADDY_DIR:=}"
: "${FRANCPAYSAN_SECRETS:=}"
: "${FRANCPAYSAN_REMOTE_SECRETS_ROOT:=}"
: "${FRANCPAYSAN_REMOTE_BANK_ADMIN_SECRET:=}"
: "${FRANCPAYSAN_REMOTE_BANK_EXPLORER_SECRET:=}"
: "${FRANCPAYSAN_REMOTE_MERCHANT_TOKEN:=}"
: "${WALLET_CLI:=}"
: "${MERCHANT_INSTANCE:=goa-demo-cp4zqk}"
# Merchant portal /private API: instance ids must be lowercase or you get odd 401s.
# (Display names may use capitals; path segment /instances/{id}/ must not.)
MERCHANT_INSTANCE=$(printf '%s' "${MERCHANT_INSTANCE}" | tr '[:upper:]' '[:lower:]')
: "${WITHDRAW_AMT:=GOA:20}" # single-shot fallback; e2e ladder uses ATM notes
: "${PAY_AMT:=GOA:0.01}"
: "${CREDIT_AMT:=GOA:4700}" # covers ATM ladder 20+50+100+200+4200 (paivana)
: "${TIMEOUT:=12}"
: "${E2E_TIMEOUT:=55}" # whole e2e budget; skip rest when exceeded (e2e raises as needed)
: "${E2E_PAY_SECS:=22}" # dedicated seconds for pay handle-uri (avoid Alarm clock)
# Whole-run wall clock for taler-monitoring.sh (seconds). 0 = unlimited.
# Host-agent default phases (urls/inside/versions) should finish under this.
: "${RUN_TIMEOUT:=600}"
# Local GOA: public paivana paywall (https://paivana.hacktivism.ch · template GOA:4200)
: "${PAIVANA_PUBLIC:=https://paivana.hacktivism.ch}"
: "${E2E_PAIVANA:=1}" # 0 = skip paivana section in e2e
# Devtest: inject reserve credit via wire-gateway admin/add-incoming (optional).
# Default off once wirewatch DNS works; set E2E_FAKE_INCOMING=1 to force.
: "${E2E_FAKE_INCOMING:=0}"
# SSH must never hang the monitoring run
: "${SSH_CONNECT_TIMEOUT:=3}"
: "${SSH_CMD_TIMEOUT:=12}" # hard cap for whole remote script (seconds)
: "${SKIP_SSH:=0}"
# Expected currency for public /config checks (empty = report only, don't fail)
# Only if unset (not if intentionally empty = pre-launch / report-only)
: "${EXPECT_CURRENCY=GOA}"
# 1 = this is the local koopa/hacktivism stack (inside/e2e/SSH make sense)
: "${LOCAL_STACK:=1}"
# Probe merchant host candidates when applying a generic domain (0=off)
: "${TALER_DOMAIN_PROBE:=1}"
BANK_PUBLIC=${BANK_PUBLIC%/}
EXCHANGE_PUBLIC=${EXCHANGE_PUBLIC%/}
MERCHANT_PUBLIC=${MERCHANT_PUBLIC%/}
# ---------------------------------------------------------------------------
# Domain profiles → bank / exchange / merchant base URLs
#
# Single place to declare a stack: domains.conf (suite root)
# (or TALER_DOMAINS_CONF). Each profile names the three public endpoints.
#
# CLI still wins after profile load:
# --bank URL --exchange URL --merchant URL --currency CODE
# Env: BANK_PUBLIC EXCHANGE_PUBLIC MERCHANT_PUBLIC EXPECT_CURRENCY
#
# Unknown domains fall back to heuristics (see apply_taler_domain).
# ---------------------------------------------------------------------------
_MONITOR_LIB_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
: "${TALER_DOMAINS_CONF:=${_MONITOR_LIB_DIR}/domains.conf}"
# host_or_url → https://… (no trailing slash)
_to_https_base() {
local x="${1%/}"
case "$x" in
https://*|http://*) printf '%s' "$x" ;;
"") printf '' ;;
*) printf 'https://%s' "$x" ;;
esac
}
# 1 = check GOA-style /intro landing pages (assets, link crawl, demo-withdraw).
# 0 = skip (e.g. taler-ops.ch / mytops — no public landings).
: "${CHECK_LANDING:=1}"
# Declare bank + exchange + merchant (+ currency, local stack, landing, domain id).
# Usage:
# set_taler_stack BANK EXCHANGE MERCHANT [CURRENCY] [LOCAL 0|1] [LANDING 0|1] [TALER_DOMAIN]
# BANK/EXCHANGE/MERCHANT: hostname or full URL
set_taler_stack() {
local bank="${1:-}" exchange="${2:-}" merchant="${3:-}"
local currency="${4:-}" local_stack="${5:-0}" landing="${6:-}" domain_id="${7:-}"
BANK_PUBLIC=$(_to_https_base "$bank")
EXCHANGE_PUBLIC=$(_to_https_base "$exchange")
MERCHANT_PUBLIC=$(_to_https_base "$merchant")
# currency "-" or empty → report-only (do not force GOA)
if [ -n "$currency" ] && [ "$currency" != "-" ]; then
EXPECT_CURRENCY="$currency"
elif [ "$currency" = "-" ]; then
EXPECT_CURRENCY=""
fi
LOCAL_STACK="$local_stack"
if [ "$LOCAL_STACK" = "1" ]; then
SKIP_SSH=0
else
SKIP_SSH=1
fi
# default: landings only matter on local GOA stack
if [ -n "$landing" ]; then
CHECK_LANDING="$landing"
elif [ "$LOCAL_STACK" = "1" ]; then
CHECK_LANDING=1
else
CHECK_LANDING=0
fi
[ -n "$domain_id" ] && TALER_DOMAIN="$domain_id"
BANK_PUBLIC=${BANK_PUBLIC%/}
EXCHANGE_PUBLIC=${EXCHANGE_PUBLIC%/}
MERCHANT_PUBLIC=${MERCHANT_PUBLIC%/}
# re-evaluate language after domain profile (FrancPaysan → fr)
if [ "${TALER_MON_LANG_SET:-0}" != "1" ]; then
TALER_MON_LANG=
i18n_init
fi
}
# Load first matching profile from domains.conf.
# Fields (whitespace-separated; # comments; blank lines ignored):
# name bank exchange merchant currency local[0|1] landing[0|1] [canonical_domain]
# Returns 0 if found, 1 if not.
load_domain_profile() {
local want="$1" conf="${TALER_DOMAINS_CONF:-}"
local line name bank exchange merchant currency local_stack landing canon
[ -n "$want" ] || return 1
[ -n "$conf" ] && [ -f "$conf" ] || return 1
while IFS= read -r line || [ -n "$line" ]; do
line="${line%%#*}"
# shellcheck disable=SC2086
set -- $line
[ $# -ge 5 ] || continue
name="$1"
[ "$name" = "$want" ] || continue
bank="$2"
exchange="$3"
merchant="$4"
currency="$5"
local_stack="${6:-0}"
landing="${7:-}"
canon="${8:-$name}"
# Backward compat: old 7th field was canonical domain (contains a dot)
if [ -n "$landing" ] && [[ "$landing" == *.* && "$landing" != "0" && "$landing" != "1" ]]; then
canon="$landing"
landing=""
fi
set_taler_stack "$bank" "$exchange" "$merchant" "$currency" "$local_stack" "$landing" "$canon"
return 0
done <"$conf"
return 1
}
_normalize_domain() {
local d="$1"
d="${d#https://}"
d="${d#http://}"
d="${d%%/*}"
d="${d%%:*}"
# Strip service host prefix only if a real domain remains (has a dot).
# e.g. bank.demo.taler.net → demo.taler.net, taler.hacktivism.ch → hacktivism.ch
# but NOT taler.net → net
case "$d" in
bank.*|exchange.*|taler.*|backend.*|merchant.*|shop.*|libeufin.*|my.*|map.*|monnaie.*)
rest="${d#*.}"
if [[ "$rest" == *.* ]]; then
d="$rest"
fi
;;
esac
printf '%s' "$d"
}
_probe_https_config() {
# 0 if https://$1/config returns 200
local host="$1" code
code=$(curl -skS --max-redirs 0 -m 4 -o /dev/null -w '%{http_code}' "https://${host}/config" 2>/dev/null || echo 000)
[ "$code" = "200" ]
}
apply_taler_domain() {
local raw="${1:-}"
local d
[ -n "$raw" ] || return 0
d=$(_normalize_domain "$raw")
TALER_DOMAIN="$d"
# 1) Explicit profile (domains.conf) — preferred way to add stacks
if load_domain_profile "$d" || load_domain_profile "$raw"; then
# Optional tiny defaults for e2e ladders by currency
case "${EXPECT_CURRENCY:-}" in
GOA)
: "${WITHDRAW_AMT:=GOA:20}"
: "${PAY_AMT:=GOA:0.01}"
: "${CREDIT_AMT:=GOA:400}"
;;
KUDOS|TESTKUDOS)
MERCHANT_INSTANCE="${MERCHANT_INSTANCE:-sandbox}"
WITHDRAW_AMT="${WITHDRAW_AMT:-${EXPECT_CURRENCY}:20}"
PAY_AMT="${PAY_AMT:-${EXPECT_CURRENCY}:0.01}"
CREDIT_AMT="${CREDIT_AMT:-${EXPECT_CURRENCY}:100}"
;;
TESTPAYSAN)
# Stage FrancPaysan: farmer shops (override GOA default goa-demo-cp4zqk)
case "${MERCHANT_INSTANCE:-}" in
""|goa-demo*|goa-shop)
MERCHANT_INSTANCE=fermes-des-collines
;;
esac
MERCHANT_INSTANCE=$(printf '%s' "${MERCHANT_INSTANCE}" | tr '[:upper:]' '[:lower:]')
WITHDRAW_AMT="${WITHDRAW_AMT:-TESTPAYSAN:50}"
PAY_AMT="${PAY_AMT:-TESTPAYSAN:5}"
# Cover oeufs(5)+fromage(8.5)+jus(6)+shop pick(~512) + fees
CREDIT_AMT="${CREDIT_AMT:-TESTPAYSAN:200}"
# Public template pays on by default (shops landings)
: "${E2E_USE_TEMPLATES:=1}"
# Map pay-ladder amounts → public shop templates (instance set in e2e)
: "${E2E_TEMPLATE_MAP:=5=oeufs-6 4.5=pain-seigle 6=jus-pomme 8.5=fromage-chevre 12=miel-printemps 25=panier-legumes}"
: "${E2E_PAY_VALUES:=5 8.5 6}"
: "${E2E_WITHDRAW_VALUES:=20 50}"
# Inside checks as low-priv stagepaysan (podman, no sudo) — not koopa.
# INSIDE_SSH / container names: set in host env (no hardcoded SSH host).
INSIDE_PROFILE="${INSIDE_PROFILE:-stage-lfp}"
INSIDE_BANK_CTR="${INSIDE_BANK_CTR:-stage-lfp-bank}"
INSIDE_EXCHANGE_CTR="${INSIDE_EXCHANGE_CTR:-stage-lfp-exchange-ansible}"
INSIDE_MERCHANT_CTR="${INSIDE_MERCHANT_CTR:-stage-lfp-merchant}"
# Host pasta ports (127.0.0.1 on the FP host (INSIDE_SSH))
INSIDE_BANK_PORT="${INSIDE_BANK_PORT:-9032}"
INSIDE_EXCHANGE_PORT="${INSIDE_EXCHANGE_PORT:-9031}"
INSIDE_MERCHANT_PORT="${INSIDE_MERCHANT_PORT:-9030}"
INSIDE_DNS_BANK="${INSIDE_DNS_BANK:-stage.bank.lefrancpaysan.ch}"
INSIDE_DNS_EXCHANGE="${INSIDE_DNS_EXCHANGE:-stage.exchange.lefrancpaysan.ch}"
INSIDE_DNS_MERCHANT="${INSIDE_DNS_MERCHANT:-stage.monnaie.lefrancpaysan.ch}"
;;
CHF)
WITHDRAW_AMT="${WITHDRAW_AMT:-CHF:20}"
PAY_AMT="${PAY_AMT:-CHF:0.01}"
CREDIT_AMT="${CREDIT_AMT:-CHF:100}"
;;
esac
else
# 2) Unknown domain — heuristics only (prefer profile in domains.conf)
# Override: --bank / --exchange / --merchant / --currency
# Do not keep default GOA: empty currency = report only, no hard fail.
BANK_PUBLIC="https://bank.${d}"
EXCHANGE_PUBLIC="https://exchange.${d}"
# TOPS-style multi-tenant merchant first, then legacy names
MERCHANT_PUBLIC="https://my.${d}"
EXPECT_CURRENCY=""
LOCAL_STACK=0
SKIP_SSH=1
CHECK_LANDING=0
if [ "${TALER_DOMAIN_PROBE}" = "1" ]; then
local h
for h in "monnaie.${d}" "backend.${d}" "my.${d}" "taler.${d}" "merchant.${d}" "shop.${d}"; do
_probe_https_config "$h" && { MERCHANT_PUBLIC="https://$h"; break; }
done
for h in "bank.${d}" "libeufin.${d}"; do
_probe_https_config "$h" && { BANK_PUBLIC="https://$h"; break; }
done
_probe_https_config "exchange.${d}" || true
fi
fi
BANK_PUBLIC=${BANK_PUBLIC%/}
EXCHANGE_PUBLIC=${EXCHANGE_PUBLIC%/}
MERCHANT_PUBLIC=${MERCHANT_PUBLIC%/}
# re-evaluate language after domain profile (FrancPaysan → fr)
if [ "${TALER_MON_LANG_SET:-0}" != "1" ]; then
TALER_MON_LANG=
i18n_init
fi
# Koopa SSH only for LOCAL_STACK=1. Stage LFP uses INSIDE_SSH (stagepaysan) separately.
if [ "${LOCAL_STACK}" != "1" ]; then
SKIP_SSH=1
fi
# Default inside profile for local GOA
if [ "${LOCAL_STACK}" = "1" ]; then
INSIDE_PROFILE="${INSIDE_PROFILE:-koopa}"
fi
}
# Apply TALER_DOMAIN from env once (CLI exports TALER_DOMAIN_APPLIED=1 after overrides).
if [ "${TALER_DOMAIN_APPLIED:-0}" != "1" ] \
&& [ -n "${TALER_DOMAIN:-}" ] && [ "${TALER_DOMAIN}" != "hacktivism.ch" ]; then
apply_taler_domain "$TALER_DOMAIN"
TALER_DOMAIN_APPLIED=1
fi
# Safe SSH: publickey only, short connect, overall alarm so we never block forever.
SSH_BASE_OPTS=(
-o BatchMode=yes
-o ConnectTimeout="${SSH_CONNECT_TIMEOUT}"
-o ConnectionAttempts=1
-o ServerAliveInterval=3
-o ServerAliveCountMax=2
-o StrictHostKeyChecking=accept-new
-o PreferredAuthentications=publickey
-o PasswordAuthentication=no
-o KbdInteractiveAuthentication=no
-o GSSAPIAuthentication=no
-o NumberOfPasswordPrompts=0
)
# Hard wall-clock timeout so ssh/curl never block the monitoring run forever.
with_timeout() {
local secs="$1"; shift
local rc
if command -v gtimeout >/dev/null 2>&1; then
gtimeout --kill-after=2 "$secs" "$@"
return $?
fi
if command -v timeout >/dev/null 2>&1; then
# Prefer GNU -k; if that flag form is unsupported, fall back once.
# Do NOT use `cmd || cmd` on the child — a phase exit 1 would re-run it.
set +e
timeout -k 2 "$secs" "$@" 2>/dev/null
rc=$?
set -e
# 125 = timeout(1) itself failed (bad options); 126/127 = not executable
if [ "$rc" -eq 125 ] || [ "$rc" -eq 126 ] || [ "$rc" -eq 127 ]; then
timeout --kill-after=2 "$secs" "$@"
return $?
fi
return "$rc"
fi
# Portable: perl alarm + process group kill
perl -e '
use strict; use warnings;
my $secs = shift @ARGV;
my $pid = fork();
die "fork: $!" unless defined $pid;
if ($pid == 0) {
setpgrp(0, 0);
exec @ARGV;
exit 127;
}
$SIG{ALRM} = sub {
kill "TERM", -$pid;
select(undef, undef, undef, 1.0);
kill "KILL", -$pid;
exit 124;
};
alarm $secs;
waitpid($pid, 0);
my $code = $? >> 8;
alarm 0;
exit $code;
' "$secs" "$@"
}
# Generic short SSH to a named host (low-priv stagepaysan or koopa).
# usage: mon_ssh_bash HOST [timeout] <<'EOF' ... EOF
mon_ssh_bash() {
local host="$1" t="${2:-$SSH_CMD_TIMEOUT}"
[ -n "$host" ] || return 1
with_timeout "$t" ssh "${SSH_BASE_OPTS[@]}" "$host" bash -s
}
mon_ssh_ok() {
local host="$1"
[ -n "$host" ] || return 1
with_timeout $((SSH_CONNECT_TIMEOUT + 5)) \
ssh "${SSH_BASE_OPTS[@]}" "$host" true >/dev/null 2>&1
}
# Pick a working SSH host: KOOPA_SSH first, then KOOPA_SSH_FALLBACKS (from env).
# Sets KOOPA_SSH to the first host that answers. 0 = ok, 1 = none.
KOOPA_SSH_RESOLVED=0
resolve_koopa_ssh() {
[ "${SKIP_SSH:-0}" = "1" ] && return 1
if [ "${KOOPA_SSH_RESOLVED}" = "1" ]; then
return 0
fi
local cands=() c f seen=" "
cands+=("${KOOPA_SSH}")
# shellcheck disable=SC2086
for f in ${KOOPA_SSH_FALLBACKS}; do
case "$seen" in *" $f "*) continue ;; esac
cands+=("$f")
seen="$seen$f "
done
for c in "${cands[@]}"; do
if with_timeout $((SSH_CONNECT_TIMEOUT + 5)) \
ssh "${SSH_BASE_OPTS[@]}" "$c" 'echo ok' >/dev/null 2>&1; then
if [ "$c" != "${KOOPA_SSH}" ]; then
# surface once so operators know we used WAN jump
printf '[INFO] SSH host %s unreachable — using %s\n' "${KOOPA_SSH}" "$c" >&2 || true
fi
KOOPA_SSH="$c"
KOOPA_SSH_RESOLVED=1
export KOOPA_SSH
return 0
fi
done
return 1
}
# Probe: 0 if any koopa SSH host works quickly
koopa_ssh_ok() {
[ "${SKIP_SSH}" = "1" ] && return 1
resolve_koopa_ssh
}
# Run remote bash -s with optional stdin script; hard-capped
# usage: koopa_ssh_bash [timeout_secs] <<'EOF' ... EOF
# or: koopa_ssh_run timeout_secs 'remote command'
koopa_ssh_run() {
local t="${1:-$SSH_CMD_TIMEOUT}"
shift
resolve_koopa_ssh || return 1
with_timeout "$t" ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" "$@"
}
koopa_ssh_bash() {
local t="${1:-$SSH_CMD_TIMEOUT}"
resolve_koopa_ssh || return 1
with_timeout "$t" ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" 'bash -s'
}
# stdin → remote python3 - (for metrics load probe)
koopa_ssh_python() {
local t="${1:-60}"
resolve_koopa_ssh || return 1
with_timeout "$t" ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" python3 -
}
# ANSI colours — left badge (clear severity) + body label colour + dim detail.
# Off: NO_COLOR=1 or CLICOLOR=0.
# OK green badge · green label
# INFO blue badge · cyan label
# WARN yellow badge · yellow label (never red)
# ERROR red badge · red label
# BLOCKER magenta badge· magenta label
# progress cyan badge = " 42%" + bar fills left→right
if [ "${NO_COLOR:-0}" = "1" ] || [ "${CLICOLOR:-1}" = "0" ]; then
G= R= Y= C= M= D= W= B= N=
BG_OK= BG_INFO= BG_WARN= BG_ERR= BG_BLK= BG_SEC= BG_PROG=
BOX=0
else
G=$'\e[1;32m' # green
R=$'\e[1;31m' # red
Y=$'\e[1;33m' # yellow
C=$'\e[1;36m' # cyan
M=$'\e[1;35m' # magenta
D=$'\e[2m' # dim tid / detail
W=$'\e[1;37m' # bold white
B=$'\e[1m' # bold
N=$'\e[0m' # reset
# High-contrast left badges (bright white / black on strong bg)
BG_OK=$'\e[1;97;42m' # white on green
BG_INFO=$'\e[1;97;44m' # white on blue
BG_WARN=$'\e[1;30;103m' # black on bright yellow — not red
BG_ERR=$'\e[1;97;41m' # white on red — errors only
BG_BLK=$'\e[1;97;45m' # white on magenta — blockers
BG_SEC=$'\e[1;97;44m' # white on blue (section)
BG_PROG=$'\e[1;30;106m' # black on bright cyan (progress)
BOX=1
fi
PASS_N=0
FAIL_N=0
WARN_N=0
INFO_N=0
BLOCKERS=() # human-readable payment/withdraw blockers
ERRORS=() # all ERROR lines (component scope)
# Grouped test IDs: area.group-NN (e.g. www.exchange-01, e2e.pay-03)
# plus global run number: #042 (monotonic for the whole ./taler-monitoring.sh run)
# set_area www # phase: www | e2e | inside | versions | …
# set_group exchange # → www.exchange-01
# set_group bank # → www.bank-01
# Without set_group: area-01, area-02, … (flat within area)
# Line shape: ┌ OK ┐ #003 www.exchange-02 label · detail
TEST_AREA=""
TEST_GROUP=""
TEST_N=0
LAST_TID=""
# Global / progress: may continue across check_*.sh processes via TALER_MON_STATE
# GLOBAL_*_N = whole-run status totals (multi-phase summary at parent end)
GLOBAL_N="${GLOBAL_N:-0}"
GLOBAL_PASS_N="${GLOBAL_PASS_N:-0}"
GLOBAL_FAIL_N="${GLOBAL_FAIL_N:-0}"
GLOBAL_WARN_N="${GLOBAL_WARN_N:-0}"
GLOBAL_INFO_N="${GLOBAL_INFO_N:-0}"
GLOBAL_BLOCK_N="${GLOBAL_BLOCK_N:-0}"
LAST_GLOBAL=""
PROGRESS_DONE="${PROGRESS_DONE:-0}"
PROGRESS_TOTAL="${PROGRESS_TOTAL:-0}"
PROGRESS_SHOW_EVERY="${PROGRESS_SHOW_EVERY:-8}"
PROGRESS_LAST_SHOWN="${PROGRESS_LAST_SHOWN:-0}"
_mon_state_load() {
[ -n "${TALER_MON_STATE:-}" ] && [ -f "${TALER_MON_STATE}" ] || return 0
# shellcheck disable=SC1090
. "${TALER_MON_STATE}"
: "${GLOBAL_N:=0}"
: "${GLOBAL_PASS_N:=0}"
: "${GLOBAL_FAIL_N:=0}"
: "${GLOBAL_WARN_N:=0}"
: "${GLOBAL_INFO_N:=0}"
: "${GLOBAL_BLOCK_N:=0}"
: "${PROGRESS_DONE:=0}"
: "${PROGRESS_TOTAL:=0}"
: "${PROGRESS_LAST_SHOWN:=0}"
}
_mon_state_save() {
[ -n "${TALER_MON_STATE:-}" ] || return 0
{
printf 'GLOBAL_N=%s\n' "${GLOBAL_N:-0}"
printf 'GLOBAL_PASS_N=%s\n' "${GLOBAL_PASS_N:-0}"
printf 'GLOBAL_FAIL_N=%s\n' "${GLOBAL_FAIL_N:-0}"
printf 'GLOBAL_WARN_N=%s\n' "${GLOBAL_WARN_N:-0}"
printf 'GLOBAL_INFO_N=%s\n' "${GLOBAL_INFO_N:-0}"
printf 'GLOBAL_BLOCK_N=%s\n' "${GLOBAL_BLOCK_N:-0}"
printf 'PROGRESS_DONE=%s\n' "${PROGRESS_DONE:-0}"
printf 'PROGRESS_TOTAL=%s\n' "${PROGRESS_TOTAL:-0}"
printf 'PROGRESS_LAST_SHOWN=%s\n' "${PROGRESS_LAST_SHOWN:-0}"
} >"${TALER_MON_STATE}"
}
_mon_state_load
# Per-group counters so re-entering www.exchange after perf continues NN
# (shell vars TEST_CNT_<area>_<group>).
_tid_key() {
# $1=area $2=group → safe identifier
printf 'TEST_CNT_%s_%s' "$1" "$2" | tr -c 'A-Za-z0-9_' '_'
}
_tid_save() {
[ -z "${TEST_AREA:-}" ] || [ -z "${TEST_GROUP:-}" ] && return 0
local k
k=$(_tid_key "$TEST_AREA" "$TEST_GROUP")
eval "$k=\$TEST_N"
}
_tid_load() {
local k
k=$(_tid_key "$TEST_AREA" "$1")
eval "TEST_N=\${$k:-0}"
}
set_area() {
# leave previous group counter saved
_tid_save
TEST_AREA="$1"
TEST_GROUP=""
TEST_N=0
LAST_TID=""
}
# Start a logical sub-group. Short stable names:
# exchange bank merchant perf stats landing paivana
# prereq wallet atm settle pay shop dig load report
# ssh caddy outside inside compare withdraw
# Counter continues if the same group is re-entered later in the area.
# Prints a compact group chip so log + issue text map cleanly (www.bank-03).
set_group() {
local g="$1"
# no-op if same group already active (e.g. inside emit loop)
if [ "$g" = "${TEST_GROUP:-}" ] && [ -n "$g" ]; then
return 0
fi
_tid_save
TEST_GROUP="$g"
LAST_TID=""
if [ -z "${TEST_AREA:-}" ] || [ -z "$g" ]; then
TEST_N=0
return 0
fi
_tid_load "$g"
local tag="${TEST_AREA}.${g}"
if [ "${BOX:-0}" = "1" ]; then
# cyan-outline group chip: ┌ www.exchange ┐
printf '%s%s┌ %s ┐%s\n' "$D" "$C" "$tag" "$N"
else
printf -- '-- %s --\n' "$tag"
fi
# show progress when entering a new group (if totals known)
_progress_maybe_show 1
}
set_progress_total() {
# Optional: expected number of numbered checks in this run.
# PROGRESS_TOTAL=0 → no percent, only "done=N".
# Does not reset GLOBAL_N / PROGRESS_DONE if already mid-run (state file).
# May shrink when parent re-fits after a short phase (total = done + remaining est),
# but never below PROGRESS_DONE.
local n="${1:-0}"
if [ "${PROGRESS_DONE:-0}" -gt 0 ] && [ "$n" -gt 0 ] && [ "$n" -lt "$PROGRESS_DONE" ]; then
n=$PROGRESS_DONE
fi
PROGRESS_TOTAL="$n"
if [ "${PROGRESS_DONE:-0}" -eq 0 ]; then
PROGRESS_LAST_SHOWN=0
fi
_mon_state_save
}
add_progress_total() {
local n="${1:-0}"
PROGRESS_TOTAL=$((PROGRESS_TOTAL + n))
_mon_state_save
}
# When the global estimate was short, grow total just enough so done≤total.
# Never shrink. Mid-run headroom is small and monotonic (stable N in done/N).
_progress_rebalance() {
[ "${PROGRESS_TOTAL:-0}" -gt 0 ] || return 0
[ "${PROGRESS_DONE:-0}" -gt 0 ] || return 0
if [ "$PROGRESS_DONE" -gt "$PROGRESS_TOTAL" ]; then
# + max(8, 5% of done) so we do not snap to 100% then jump again next check
local head=$((PROGRESS_DONE / 20))
[ "$head" -lt 8 ] && head=8
PROGRESS_TOTAL=$((PROGRESS_DONE + head))
_mon_state_save
fi
}
# Snap total to actual done — only at whole-run end (not after each phase).
progress_finish() {
[ "${PROGRESS_OFF:-0}" = "1" ] && return 0
[ "${PROGRESS_DONE:-0}" -gt 0 ] || return 0
if [ "${PROGRESS_TOTAL:-0}" -ne "$PROGRESS_DONE" ]; then
PROGRESS_TOTAL=$PROGRESS_DONE
_mon_state_save
fi
_progress_bar_line "$PROGRESS_DONE" "$PROGRESS_TOTAL"
if [ "${GLOBAL_N:-0}" -gt 0 ]; then
printf -- '%s %s %d/%d (#001…#%03d)%s\n' \
"$D" "$(i18n_text 'global checks:')" \
"$PROGRESS_DONE" "$PROGRESS_TOTAL" "$GLOBAL_N" "$N"
fi
}
_progress_bar_line() {
# Format (grows left → right; badge = percent):
# ┌ 42% ┐ ████████░░░░░░░░░░░░░░░░ 31/74
# done/total = global numbered checks so far / global expected total
local done="$1" total="$2" width=28 pct=0 filled empty i bar pct_lab
# Defensive: never display done > total
if [ "$total" -gt 0 ] && [ "$done" -gt "$total" ]; then
total=$done
fi
if [ "$total" -gt 0 ]; then
# integer percent; while mid-run past a short estimate, cap display < 100%
pct=$((done * 100 / total))
[ "$pct" -gt 100 ] && pct=100
if [ "$done" -lt "$total" ] && [ "$pct" -ge 100 ]; then
pct=99
fi
# fill cells from the left (progress moves rightward as filled grows)
filled=$((done * width / total))
[ "$filled" -gt "$width" ] && filled=$width
# show at least 1 block once any work done (unless 0%)
if [ "$done" -gt 0 ] && [ "$filled" -eq 0 ]; then
filled=1
fi
# at 100% fill complete bar
if [ "$done" -ge "$total" ]; then
filled=$width
pct=100
fi
empty=$((width - filled))
bar=""
i=0
while [ "$i" -lt "$filled" ]; do bar="${bar}"; i=$((i + 1)); done
i=0
while [ "$i" -lt "$empty" ]; do bar="${bar}"; i=$((i + 1)); done
pct_lab=$(printf '%3d%%' "$pct")
if [ "${BOX:-0}" = "1" ]; then
# badge = percent; bar grows ░→█ left to right; N/M = global check count
printf '%s┌%s┐%s %s%s%s %s%d/%d%s\n' \
"$BG_PROG" "$pct_lab" "$N" \
"$C" "$bar" "$N" \
"$D" "$done" "$total" "$N"
else
printf -- '%s%4s%s %s %d/%d\n' \
"$C" "$pct_lab" "$N" "$bar" "$done" "$total"
fi
else
if [ "${BOX:-0}" = "1" ]; then
printf '%s┌ … ┐%s %s%d/?%s (set PROGRESS_TOTAL= for global %%)\n' \
"$BG_PROG" "$N" "$D" "$done" "$N"
else
printf -- '%s%d/?%s (set PROGRESS_TOTAL= for global %%)\n' "$D" "$done" "$N"
fi
fi
}
_progress_maybe_show() {
local force="${1:-0}"
[ "${PROGRESS_OFF:-0}" = "1" ] && return 0
[ "$PROGRESS_DONE" -le 0 ] && [ "$force" != "1" ] && return 0
_progress_rebalance
if [ "$force" = "1" ] || \
[ $((PROGRESS_DONE - PROGRESS_LAST_SHOWN)) -ge "$PROGRESS_SHOW_EVERY" ] || \
{ [ "$PROGRESS_TOTAL" -gt 0 ] && [ "$PROGRESS_DONE" -ge "$PROGRESS_TOTAL" ]; }; then
_progress_bar_line "$PROGRESS_DONE" "$PROGRESS_TOTAL"
PROGRESS_LAST_SHOWN=$PROGRESS_DONE
fi
}
# Assign next id into LAST_TID (must not run in a subshell).
_take_tid() {
LAST_TID=""
LAST_GLOBAL=""
if [ -z "${TEST_AREA:-}" ]; then
return
fi
# Global monotonic number for the whole monitoring run (#001 …)
GLOBAL_N=$((GLOBAL_N + 1))
LAST_GLOBAL=$(printf '#%03d' "$GLOBAL_N")
TEST_N=$((TEST_N + 1))
if [ -n "${TEST_GROUP:-}" ]; then
LAST_TID=$(printf '%s.%s-%02d' "$TEST_AREA" "$TEST_GROUP" "$TEST_N")
_tid_save
else
LAST_TID=$(printf '%s-%02d' "$TEST_AREA" "$TEST_N")
fi
PROGRESS_DONE=$((PROGRESS_DONE + 1))
_progress_rebalance
_mon_state_save
_progress_maybe_show 0
}
# Dim ids: "#003 www.exchange-01 " or empty
_fmt_tid() {
if [ -n "${LAST_GLOBAL:-}" ]; then
printf '%s%s%s ' "$D" "$LAST_GLOBAL" "$N"
fi
if [ -n "${LAST_TID:-}" ]; then
printf '%s%s%s ' "$D" "$LAST_TID" "$N"
fi
}
# Boxed badge cell: ┌ OK ┐ with filled bg (tag left-padded, width 7 for BLOCKER)
# $1=badge style $2=tag text
_fmt_badge() {
local badge="$1" tag="$2" inner
inner=$(printf -- '%-7s' "$tag")
if [ "${BOX:-0}" = "1" ]; then
printf '%s┌ %s┐%s' "$badge" "$inner" "$N"
else
printf '%s[ %s]%s' "$badge" "$inner" "$N"
fi
}
# One concrete line: ┌ OK ┐ tid label · detail
# $1=badge style $2=tag text $3=label colour $4=label $5=detail
_msg_line() {
local badge="$1" tag="$2" lcol="$3" label="$4" detail="${5:-}"
# i18n: tags + free text (en default; fr for FrancPaysan)
tag=$(i18n_tag "$tag")
label=$(i18n_text "$label")
[ -n "$detail" ] && detail=$(i18n_text "$detail")
if [ -n "$detail" ]; then
printf -- '%s %s%s%s%s %s·%s %s%s%s\n' \
"$(_fmt_badge "$badge" "$tag")" "$(_fmt_tid)" "$lcol" "$label" "$N" "$D" "$N" "$D" "$detail" "$N"
else
printf -- '%s %s%s%s%s\n' \
"$(_fmt_badge "$badge" "$tag")" "$(_fmt_tid)" "$lcol" "$label" "$N"
fi
}
ok() {
# ok "what is good" ["concrete evidence: HTTP 200 · 12ms · …"]
local label="$1" detail="${2:-}"
_take_tid
_msg_line "$BG_OK" "OK" "$G" "$label" "$detail"
PASS_N=$((PASS_N + 1))
GLOBAL_PASS_N=$((GLOBAL_PASS_N + 1))
_mon_state_save
}
# component-scoped error: err bank "what failed" "why / HTTP / path"
err() {
local comp="$1" msg="$2" detail="${3:-}"
_take_tid
_msg_line "$BG_ERR" "ERROR" "$R" "${comp}: ${msg}" "$detail"
FAIL_N=$((FAIL_N + 1))
GLOBAL_FAIL_N=$((GLOBAL_FAIL_N + 1))
ERRORS+=("${LAST_TID:+$LAST_TID }[$comp] $msg${detail:+ · $detail}")
_mon_state_save
}
# fail "what failed" ["why / HTTP code / path"]
fail() {
local label="$1" detail="${2:-}"
_take_tid
_msg_line "$BG_ERR" "ERROR" "$R" "$label" "$detail"
FAIL_N=$((FAIL_N + 1))
GLOBAL_FAIL_N=$((GLOBAL_FAIL_N + 1))
ERRORS+=("${LAST_TID:+$LAST_TID }$label${detail:+ · $detail}")
_mon_state_save
}
warn() {
# warn "what is soft-bad" ["why still ok to continue"]
# warn component "what" "why"
# Yellow only — never red (red = ERROR / BLOCKER path)
local a1="${1:-}" a2="${2:-}" a3="${3:-}"
local head detail
_take_tid
if [ -n "$a3" ]; then
head="${a1}: ${a2}"
detail="$a3"
elif [ -n "$a2" ]; then
head="$a1"
detail="$a2"
else
head="$a1"
detail=""
fi
_msg_line "$BG_WARN" "WARN" "$Y" "$head" "$detail"
WARN_N=$((WARN_N + 1))
GLOBAL_WARN_N=$((GLOBAL_WARN_N + 1))
_mon_state_save
}
info() {
# info "topic" ["concrete fact / value / next step"]
local label="$1" detail="${2:-}"
_take_tid
_msg_line "$BG_INFO" "INFO" "$C" "$label" "$detail"
INFO_N=$((INFO_N + 1))
GLOBAL_INFO_N=$((GLOBAL_INFO_N + 1))
_mon_state_save
}
blocker() {
# Hard stop on pay/withdraw path: blocker "step" "why it cannot continue"
local step="$1" msg="$2"
_take_tid
_msg_line "$BG_BLK" "BLOCKER" "$M" "${step}" "$msg"
BLOCKERS+=("${LAST_TID:+$LAST_TID }[$step] $msg")
FAIL_N=$((FAIL_N + 1))
GLOBAL_FAIL_N=$((GLOBAL_FAIL_N + 1))
GLOBAL_BLOCK_N=$((GLOBAL_BLOCK_N + 1))
ERRORS+=("BLOCKER ${LAST_TID:+$LAST_TID }[$step] $msg")
_mon_state_save
}
section() {
# Boxed section header (3 lines when colour on)
local title
title=$(i18n_text "$*")
local w=${#title}
[ "$w" -lt 24 ] && w=24
[ "$w" -gt 56 ] && w=56
local pad line i
pad=$(printf -- "%-${w}s" "$title")
if [ "${BOX:-0}" = "1" ]; then
line=""
i=0
while [ "$i" -lt $((w + 2)) ]; do
line="${line}"
i=$((i + 1))
done
printf -- '\n%s╔%s╗%s\n' "$BG_SEC" "$line" "$N"
printf -- '%s║%s %s%s%s %s║%s\n' "$BG_SEC" "$N" "$B" "$pad" "$N" "$BG_SEC" "$N"
printf -- '%s╚%s╝%s\n' "$BG_SEC" "$line" "$N"
else
printf -- '\n== %s ==\n' "$title"
fi
}
# Print SUMMARY totals box for given counts (phase-local or global).
# $1=pass $2=fail $3=warn $4=info $5=block
_summary_totals_box() {
local pass_n="$1" fail_n="$2" warn_n="$3" info_n="$4" blk_n="$5"
if [ "${BOX:-0}" = "1" ]; then
printf -- '\n%s┌ %s ┐%s\n' "$BG_SEC" "$(i18n_tag SUMMARY)" "$N"
_sum_badge() { # $1=bg $2=tag $3=fg $4=n
local _tg
_tg=$(i18n_tag "$2")
printf -- ' %s┌ %-5s┐%s %s%4d%s\n' "$1" "$_tg" "$N" "$3" "$4" "$N"
}
_sum_badge "$BG_OK" "OK" "$G" "$pass_n"
[ "$fail_n" -gt 0 ] && _sum_badge "$BG_ERR" "ERROR" "$R" "$fail_n"
[ "$warn_n" -gt 0 ] && _sum_badge "$BG_WARN" "WARN" "$Y" "$warn_n"
[ "$info_n" -gt 0 ] && _sum_badge "$BG_INFO" "INFO" "$C" "$info_n"
[ "$blk_n" -gt 0 ] && _sum_badge "$BG_BLK" "BLOCK" "$M" "$blk_n"
else
printf -- '\n[ %s ]\n' "$(i18n_tag SUMMARY)"
printf -- ' [ %-5s ] %s%4d%s\n' "$(i18n_tag OK)" "$G" "$pass_n" "$N"
[ "$fail_n" -gt 0 ] && printf -- ' [ %-5s ] %s%4d%s\n' "$(i18n_tag ERROR)" "$R" "$fail_n" "$N"
[ "$warn_n" -gt 0 ] && printf -- ' [ %-5s ] %s%4d%s\n' "$(i18n_tag WARN)" "$Y" "$warn_n" "$N"
[ "$info_n" -gt 0 ] && printf -- ' [ %-5s ] %s%4d%s\n' "$(i18n_tag INFO)" "$C" "$info_n" "$N"
[ "$blk_n" -gt 0 ] && printf -- ' [ %-5s ] %s%4d%s\n' "$(i18n_tag BLOCK)" "$M" "$blk_n" "$N"
fi
# one-line rollup (bold label, severity-coloured counts)
printf -- ' %stotals:%s %s%d OK%s' "$B" "$N" "$G" "$pass_n" "$N"
[ "$fail_n" -gt 0 ] && printf -- ' · %s%d ERROR%s' "$R" "$fail_n" "$N"
[ "$warn_n" -gt 0 ] && printf -- ' · %s%d WARN%s' "$Y" "$warn_n" "$N"
[ "$info_n" -gt 0 ] && printf -- ' · %s%d INFO%s' "$C" "$info_n" "$N"
[ "$blk_n" -gt 0 ] && printf -- ' · %s%d BLOCKER%s' "$M" "$blk_n" "$N"
printf '\n'
# overall verdict
if [ "$fail_n" -eq 0 ] && [ "$blk_n" -eq 0 ] && [ "$warn_n" -eq 0 ]; then
if [ "${BOX:-0}" = "1" ]; then
printf -- ' %s┌ OK ┐%s %sall clear%s\n' "$BG_OK" "$N" "$G" "$N"
else
printf -- ' %s[ OK ] all clear%s\n' "$G" "$N"
fi
elif [ "$fail_n" -eq 0 ] && [ "$blk_n" -eq 0 ]; then
if [ "${BOX:-0}" = "1" ]; then
printf -- ' %s┌ WARN ┐%s %swarnings only (no hard fail)%s\n' "$BG_WARN" "$N" "$Y" "$N"
else
printf -- ' %s[ WARN ] warnings only (no hard fail)%s\n' "$Y" "$N"
fi
else
local _fail_msg
_fail_msg=$(i18n_text 'failed — see list above')
if [ "${BOX:-0}" = "1" ]; then
printf -- ' %s┌ ERROR ┐%s %s%s%s\n' "$BG_ERR" "$N" "$R" "$_fail_msg" "$N"
else
printf -- ' %s[ ERROR ] %s%s\n' "$R" "$_fail_msg" "$N"
fi
fi
}
# Whole-run SUMMARY from TALER_MON_STATE (parent after all phases).
# Multi-phase: phases never print SUMMARY totals; only this does.
summary_global() {
_mon_state_load
local pass_n="${GLOBAL_PASS_N:-0}"
local fail_n="${GLOBAL_FAIL_N:-0}"
local warn_n="${GLOBAL_WARN_N:-0}"
local info_n="${GLOBAL_INFO_N:-0}"
local blk_n="${GLOBAL_BLOCK_N:-0}"
# Progress bar + numbered checks already printed by progress_finish when present.
if [ "${GLOBAL_N:-0}" -gt 0 ]; then
if [ "${PROGRESS_TOTAL:-0}" -gt 0 ]; then
printf -- '%s %s #001…#%03d · %d/%d %s%s\n' \
"$D" "$(i18n_text 'numbered checks:')" "$GLOBAL_N" \
"${PROGRESS_DONE:-0}" "$PROGRESS_TOTAL" \
"$(i18n_text '(global done/total)')" "$N"
else
printf -- '%s %s #001…#%03d%s\n' \
"$D" "$(i18n_text 'numbered checks this run:')" "$GLOBAL_N" "$N"
fi
fi
_summary_totals_box "$pass_n" "$fail_n" "$warn_n" "$info_n" "$blk_n"
[ "$fail_n" -eq 0 ]
}
summary() {
local blk_n=${#BLOCKERS[@]}
# Multi-phase runs: never print phase SUMMARY / progress standings (v1.13.10+).
# Parent prints one global SUMMARY via summary_global after progress_finish.
# Solo check_*.sh (TALER_MON_MULTI_PHASE unset/0) still full summary.
# Phase still lists ERRORS/BLOCKERS so mid-run failures stay visible.
local _full=1
if [ "${TALER_MON_MULTI_PHASE:-0}" = "1" ]; then
_full=0
fi
echo ""
if [ "$_full" = "1" ]; then
# Progress snapshot — whole-run snap also in progress_finish.
if [ "${PROGRESS_OFF:-0}" != "1" ] && [ "${PROGRESS_DONE:-0}" -gt 0 ]; then
_progress_rebalance
_progress_bar_line "$PROGRESS_DONE" "$PROGRESS_TOTAL"
fi
if [ "$GLOBAL_N" -gt 0 ]; then
if [ "${PROGRESS_TOTAL:-0}" -gt 0 ]; then
printf -- '%s %s #001…#%03d · %d/%d %s%s\n' \
"$D" "$(i18n_text 'numbered checks:')" "$GLOBAL_N" \
"$PROGRESS_DONE" "$PROGRESS_TOTAL" \
"$(i18n_text '(global done/total)')" "$N"
else
printf -- '%s %s #001…#%03d%s\n' "$D" "$(i18n_text 'numbered checks this run:')" "$GLOBAL_N" "$N"
fi
fi
fi
if [ "$blk_n" -gt 0 ]; then
if [ "${BOX:-0}" = "1" ]; then
printf -- '%s┌ %s ┐%s\n' "$BG_BLK" "$(i18n_text 'BLOCKERS · pay/withdraw cannot finish')" "$N"
else
printf -- '%s--- %s ---%s\n' "$M" "$(i18n_text 'BLOCKERS (pay/withdraw cannot finish)')" "$N"
fi
local b
for b in "${BLOCKERS[@]}"; do
printf -- ' %s•%s %s%s%s\n' "$M" "$N" "$W" "$b" "$N"
done
fi
if [ "${#ERRORS[@]}" -gt 0 ] && [ "$blk_n" -lt "${#ERRORS[@]}" ]; then
if [ "${BOX:-0}" = "1" ]; then
printf -- '%s┌ %s ┐%s\n' "$BG_ERR" "$(i18n_text 'ERRORS · failed checks')" "$N"
else
printf -- '%s--- %s ---%s\n' "$R" "$(i18n_text 'ERRORS (failed checks)')" "$N"
fi
local e
for e in "${ERRORS[@]}"; do
case "$e" in BLOCKER*) continue ;; esac
printf -- ' %s•%s %s%s%s\n' "$R" "$N" "$W" "$e" "$N"
done
fi
# Coloured totals — only at full phase summary (solo script / single-phase)
if [ "$_full" != "1" ]; then
# Still fail the phase exit when this phase had errors (set -e + trailing summary)
[ "$FAIL_N" -eq 0 ]
return
fi
_summary_totals_box "$PASS_N" "$FAIL_N" "$WARN_N" "$INFO_N" "$blk_n"
[ "$FAIL_N" -eq 0 ]
}
# ---------------------------------------------------------------------------
# Disk space on hosts / containers under test
# WARN when tight, ERROR when full / critically low
# DISK_WARN_USED_PCT=85 # free below 15% → WARN
# DISK_ERR_USED_PCT=95 # free below 5% → ERROR
# DISK_ERR_FREE_BYTES=0 # free == 0 always ERROR
# ---------------------------------------------------------------------------
: "${DISK_WARN_USED_PCT:=85}"
: "${DISK_ERR_USED_PCT:=95}"
# Parse one POSIX `df -P` body line (no header): fs size used avail capacity mount
# size/used/avail in 1K-blocks when using df -Pk
_mon_disk_eval_line() {
local where="$1" fs="$2" size="$3" used="$4" avail="$5" cap="$6" mnt="$7"
local pct label detail
pct=${cap%%%}
# non-numeric capacity → skip
case "$pct" in
''|*[!0-9]*) return 0 ;;
esac
label="disk ${where} ${mnt}"
detail="fs=$fs size_1k=$size used_1k=$used avail_1k=$avail use=${pct}%"
if [ "$avail" = "0" ] || [ "$pct" -ge 100 ]; then
err "disk" "${where} ${mnt} FULL / overflow" "$detail"
return 1
fi
if [ "$pct" -ge "${DISK_ERR_USED_PCT}" ]; then
err "disk" "${where} ${mnt} critically low free space (≥${DISK_ERR_USED_PCT}% used)" "$detail"
return 1
fi
if [ "$pct" -ge "${DISK_WARN_USED_PCT}" ]; then
warn "disk" "${where} ${mnt} free space tight (≥${DISK_WARN_USED_PCT}% used)" "$detail"
return 0
fi
ok "$label" "$detail"
return 0
}
# Report all filesystems from `df -P` / `df -Pk` output (with header).
# $1 = where label (host, container:name, ssh:host)
# $2 = full df text
mon_disk_report_df() {
local where="$1" text="$2" line fs size used avail cap mnt rest ec=0
[ -n "$text" ] || {
warn "disk" "${where}: no df output"
return 0
}
while IFS= read -r line || [ -n "$line" ]; do
[ -n "$line" ] || continue
case "$line" in
Filesystem*|Filesystem*) continue ;;
esac
# df -P: Filesystem 1024-blocks Used Available Capacity Mounted on
# shellcheck disable=SC2086
set -- $line
[ "$#" -ge 6 ] || continue
fs=$1
size=$2
used=$3
avail=$4
cap=$5
shift 5
mnt=$*
# skip special/pseudo if tiny and not root-ish
case "$fs" in
tmpfs|devtmpfs|overlay|shm|nsfs|proc|sysfs|cgroup*)
# still check root-like mounts that fill (overlay / in containers)
case "$mnt" in
/|/var|/var/*|/home|/home/*|/data|/mnt/*) ;;
*) continue ;;
esac
;;
esac
if ! _mon_disk_eval_line "$where" "$fs" "$size" "$used" "$avail" "$cap" "$mnt"; then
ec=1
fi
done <<<"$text"
return "$ec"
}
# Local host: important mounts
mon_disk_check_host() {
local where text
where="${1:-host}"
text=""
text=$(df -Pk / /var /home /tmp 2>/dev/null | awk 'NR==1 || !seen[$1,$6]++' || true)
# fallback whole table
if [ -z "$text" ]; then
text=$(df -Pk 2>/dev/null || true)
fi
mon_disk_report_df "$where" "$text"
}
# Run df inside podman container
mon_disk_check_podman() {
local cname="$1" text
local bin="${2:-podman}"
text=$("$bin" exec "$cname" df -Pk / /var /tmp 2>/dev/null || "$bin" exec "$cname" df -Pk 2>/dev/null || true)
mon_disk_report_df "ctr:${cname}" "$text"
}
# df text already collected remotely
mon_disk_check_remote_text() {
local where="$1" text="$2"
mon_disk_report_df "$where" "$text"
}
http_code() {
local url="$1"; shift
curl -skS --max-redirs 0 -m "${TIMEOUT}" -o /dev/null -w '%{http_code}' "$@" "$url" 2>/dev/null || echo 000
}
http_body() {
local url="$1" out="$2"; shift 2
curl -skS --max-redirs 0 -m "${TIMEOUT}" -o "$out" -w '%{http_code}' "$@" "$url" 2>/dev/null || echo 000
}
# ---------------------------------------------------------------------------
# Currency unit map checks (wallet codec: alt_unit_names must include "0")
# ---------------------------------------------------------------------------
# Returns 0 if JSON body at $1 has usable alt_unit_names.
# Supports:
# - exchange/bank: currency_specification.alt_unit_names
# - merchant: currencies.<CODE>.alt_unit_names for each code
# Optional $2 = required currency code for currency_specification.currency
json_has_alt_unit_names() {
local file="$1" want_cur="${2:-}"
python3 - "$file" "$want_cur" <<'PY'
import json, sys
path, want = sys.argv[1], sys.argv[2]
try:
d = json.load(open(path))
except Exception as e:
print(f"json-error: {e}")
sys.exit(2)
def check_au(au, label):
if not isinstance(au, dict) or not au:
print(f"{label}: missing/empty alt_unit_names")
return False
if "0" not in au or not str(au.get("0") or "").strip():
print(f"{label}: alt_unit_names missing non-empty key \"0\" (have {sorted(au.keys())})")
return False
print(f"{label}: alt_unit_names ok (0={au.get('0')!r}, n={len(au)})")
return True
ok = True
cs = d.get("currency_specification")
if isinstance(cs, dict):
if want and cs.get("currency") and cs.get("currency") != want:
print(f"currency_specification.currency={cs.get('currency')!r} want {want!r}")
ok = False
if not check_au(cs.get("alt_unit_names"), "currency_specification"):
ok = False
elif "currency_specification" in d:
print("currency_specification: not an object")
ok = False
curs = d.get("currencies")
if isinstance(curs, dict) and curs:
for code, spec in curs.items():
if not isinstance(spec, dict):
print(f"currencies.{code}: not an object")
ok = False
continue
if not check_au(spec.get("alt_unit_names"), f"currencies.{code}"):
ok = False
if not isinstance(cs, dict) and not (isinstance(curs, dict) and curs):
# neither shape — fail
print("no currency_specification or currencies map")
ok = False
sys.exit(0 if ok else 1)
PY
}
# Check one exchange base URL's /config for alt_unit_names.
# $1=label $2=base_url $3=expected currency (optional) $4=strict(1) or soft(0)
check_exchange_alt_units() {
local label="$1" base="$2" want_cur="${3:-}" strict="${4:-1}"
local f code
base="${base%/}"
f=$(mktemp)
code=$(http_body "${base}/config" "$f")
if [ "$code" != "200" ]; then
rm -f "$f"
if [ "$strict" = "1" ]; then
fail "$label /config" "HTTP $code ($base)"
else
warn "$label /config" "HTTP $code ($base)"
fi
return
fi
local out ec
set +e
out=$(json_has_alt_unit_names "$f" "$want_cur" 2>&1)
ec=$?
set -e
rm -f "$f"
if [ "$ec" -eq 0 ]; then
ok "$label alt_unit_names" "$(echo "$out" | tr '\n' '; ' | sed 's/; $//')"
else
if [ "$strict" = "1" ]; then
fail "$label alt_unit_names" "$(echo "$out" | tr '\n' '; ' | sed 's/; $//')"
else
warn "$label alt_unit_names" "$(echo "$out" | tr '\n' '; ' | sed 's/; $//')"
fi
fi
}
# From merchant /config JSON file: walk exchanges[] and check each base_url/config.
# Local stack hosts (hacktivism.ch or $EXCHANGE_PUBLIC host) are strict; others soft.
check_merchant_listed_exchanges_alt_units() {
local mer_json="$1"
local list
list=$(python3 - "$mer_json" <<'PY'
import json, sys
d = json.load(open(sys.argv[1]))
for e in d.get("exchanges") or []:
if not isinstance(e, dict):
continue
u = (e.get("base_url") or e.get("url") or "").rstrip("/")
c = e.get("currency") or ""
if u:
print(f"{c}\t{u}")
PY
)
if [ -z "$list" ]; then
fail "merchant exchanges[]" "empty — no exchanges to check for alt_unit_names"
return
fi
local line cur url strict host
while IFS=$'\t' read -r cur url; do
[ -n "$url" ] || continue
host="${url#https://}"; host="${host#http://}"; host="${host%%/*}"
strict=1
case "$host" in
*hacktivism.ch) strict=1 ;;
*)
# foreign exchange (e.g. taler-ops) — soft unless it is our configured EXCHANGE_PUBLIC
if [ "$url" = "${EXCHANGE_PUBLIC}" ] || [ "$url" = "${EXCHANGE_PUBLIC}/" ]; then
strict=1
else
strict=0
fi
;;
esac
check_exchange_alt_units "exchange ${cur:-?} ${host}" "$url" "$cur" "$strict"
done <<<"$list"
}
# Live bank/merchant passwords: sibling **koopa-admin-secrets** (never in admin-log).
# Override: SECRETS_ROOT=/path/to/koopa-admin-secrets/koopa/host-root
# or KOOPA_ADMIN_SECRETS=/path/to/koopa-admin-secrets
SECRETS_ROOT="${SECRETS_ROOT:-}"
_secrets_search_log() { :; } # placeholder if we later want debug
if [ -z "$SECRETS_ROOT" ]; then
_mon_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
_admin_log="$(cd "${_mon_dir}/../.." && pwd)"
# Prefer explicit env pointing at the secrets *repo* root
if [ -n "${KOOPA_ADMIN_SECRETS:-}" ] && [ -d "${KOOPA_ADMIN_SECRETS}/koopa/host-root/taler-bank" ]; then
SECRETS_ROOT="${KOOPA_ADMIN_SECRETS}/koopa/host-root"
fi
fi
# No further home-path probes — set SECRETS_ROOT or KOOPA_ADMIN_SECRETS in env.
read_secret() {
local rel="$1" base val=""
base=$(basename "$rel")
if [ -n "${SECRETS_ROOT:-}" ]; then
if [ -f "${SECRETS_ROOT}/${rel}" ]; then
tr -d '\n\r' <"${SECRETS_ROOT}/${rel}"
return 0
fi
# also accept flat layout under host-root/
if [ -f "${SECRETS_ROOT}/${base}" ]; then
tr -d '\n\r' <"${SECRETS_ROOT}/${base}"
return 0
fi
fi
# Optional: copy under ~/.config/taler-landing/ (landing-stats style)
if [ -f "${HOME}/.config/taler-landing/${base}" ]; then
tr -d '\n\r' <"${HOME}/.config/taler-landing/${base}"
return 0
fi
# Live on koopa host /root (same basenames as deploy)
if koopa_ssh_ok; then
val=$(koopa_ssh_run 12 "tr -d '\\n\\r' </root/${base} 2>/dev/null || true" 2>/dev/null || true)
if [ -n "$val" ]; then
printf '%s' "$val"
return 0
fi
# container path used by some installs
val=$(koopa_ssh_run 12 \
"podman exec taler-hacktivism-bank tr -d '\\n\\r' </root/${base} 2>/dev/null || true" 2>/dev/null || true)
if [ -n "$val" ]; then
printf '%s' "$val"
return 0
fi
fi
return 1
}
# Human hint when secrets missing (e2e prereq)
secrets_hint() {
cat <<EOF
Prefer a local secrets file (not committed):
cp secrets.env.example secrets.env
\$EDITOR secrets.env
Or set SECRETS_ROOT / KOOPA_ADMIN_SECRETS in ~/.config/taler-monitoring/env
Or export E2E_BANK_ADMIN_PASS / E2E_MERCHANT_TOKEN for one shot.
EOF
}
# Load optional secrets.env next to this suite (gitignored). Safe KEY=value only.
# Call once from taler-monitoring.sh after source lib.sh (or any check_*.sh entry).
load_monitoring_secrets_env() {
local mon_dir f
mon_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
f="${MONITORING_SECRETS_ENV:-${mon_dir}/secrets.env}"
if [ ! -f "$f" ]; then
return 0
fi
# shellcheck disable=SC1090
set -a
# Only simple KEY=VALUE lines; ignore comments / blanks
# shellcheck disable=SC1091
. "$f"
set +a
# Map common aliases → e2e vars if not already set
: "${E2E_BANK_ADMIN_PASS:=${BANK_ADMIN_PASS:-${BANK_ADMIN_PASSWORD:-}}}"
: "${E2E_MERCHANT_TOKEN:=${MERCHANT_TOKEN:-${MERCHANT_INSTANCE_PASSWORD:-}}}"
: "${SECRETS_ROOT:=${KOOPA_SECRETS_HOST_ROOT:-}}"
if [ -z "${SECRETS_ROOT:-}" ] && [ -n "${KOOPA_ADMIN_SECRETS:-}" ]; then
if [ -d "${KOOPA_ADMIN_SECRETS}/koopa/host-root/taler-bank" ]; then
SECRETS_ROOT="${KOOPA_ADMIN_SECRETS}/koopa/host-root"
fi
fi
export E2E_BANK_ADMIN_PASS E2E_MERCHANT_TOKEN SECRETS_ROOT KOOPA_ADMIN_SECRETS
}
find_wallet_cli() {
# Must return a path suitable for: node "$WALLET_CLI" …
# Prefer *.mjs / bundled entry; plain /usr/bin/taler-wallet-cli is often a
# shell wrapper and cannot be passed to node.
if [ -n "${WALLET_CLI:-}" ] && [ -f "$WALLET_CLI" ]; then
echo "$WALLET_CLI"; return 0
fi
local c cand
# WALLET_CLI / WALLET_CLI_CANDIDATES from env; then portable package paths only.
# shellcheck disable=SC2086
for c in ${WALLET_CLI_CANDIDATES:-} \
"${HOME}/.local/bin/taler-wallet-cli.mjs" \
/usr/lib/taler-wallet-cli/node_modules/taler-wallet-cli/bin/taler-wallet-cli.mjs \
/usr/share/taler-wallet-cli/bin/taler-wallet-cli.mjs
do
[ -n "$c" ] && [ -f "$c" ] && { echo "$c"; return 0; }
done
# Debian package wrapper → resolve real .mjs
cand=$(command -v taler-wallet-cli 2>/dev/null || true)
if [ -n "$cand" ] && [ -f "$cand" ]; then
if head -1 "$cand" 2>/dev/null | grep -q 'node\|mjs'; then
# shebang node script or .mjs
echo "$cand"; return 0
fi
# follow symlink into package tree
if [ -L "$cand" ]; then
c=$(readlink -f "$cand" 2>/dev/null || true)
[ -n "$c" ] && [ -f "$c" ] && { echo "$c"; return 0; }
fi
fi
return 1
}