monitoring: taler-monitoring www/inside/sanity/server/e2e scaffold

This commit is contained in:
Hernâni Marques 2026-07-09 19:54:42 +02:00
parent fed083d0d5
commit 95499a7f59
No known key found for this signature in database
11 changed files with 2828 additions and 0 deletions

536
scripts/taler-monitoring/lib.sh Executable file
View file

@ -0,0 +1,536 @@
# shellcheck shell=bash
# Shared helpers for taler-monitoring (laptop or koopa).
# Default stack = GOA / hacktivism (overridden by TALER_DOMAIN / --domain)
: "${TALER_DOMAIN:=hacktivism.ch}"
: "${BANK_PUBLIC:=https://bank.hacktivism.ch}"
: "${EXCHANGE_PUBLIC:=https://exchange.hacktivism.ch}"
: "${MERCHANT_PUBLIC:=https://taler.hacktivism.ch}"
: "${BANK_LOCAL:=http://127.0.0.1:9012}"
: "${EXCHANGE_LOCAL:=http://127.0.0.1:9011}"
: "${MERCHANT_LOCAL:=https://127.0.0.1:9010}"
: "${LANDING_LOCAL:=http://127.0.0.1:9013}"
: "${KOOPA_SSH:=koopa}"
: "${MERCHANT_INSTANCE:=goa-demo-cp4zqk}"
: "${WITHDRAW_AMT:=GOA:20}" # single-shot fallback; e2e ladder uses ATM notes
: "${PAY_AMT:=GOA:0.01}"
: "${CREDIT_AMT:=GOA:400}" # covers ATM ladder 20+50+100+200
: "${TIMEOUT:=12}"
: "${E2E_TIMEOUT:=55}" # whole e2e budget; skip rest when exceeded
: "${E2E_PAY_SECS:=22}" # dedicated seconds for pay handle-uri (avoid Alarm clock)
# Devtest: inject reserve credit via wire-gateway admin/add-incoming (optional).
# Default off once wirewatch DNS works; set E2E_FAKE_INCOMING=1 to force.
: "${E2E_FAKE_INCOMING:=0}"
# SSH must never hang the monitoring run
: "${SSH_CONNECT_TIMEOUT:=3}"
: "${SSH_CMD_TIMEOUT:=12}" # hard cap for whole remote script (seconds)
: "${SKIP_SSH:=0}"
# Expected currency for public /config checks (empty = report only, don't fail)
: "${EXPECT_CURRENCY:=GOA}"
# 1 = this is the local koopa/hacktivism stack (inside/e2e/SSH make sense)
: "${LOCAL_STACK:=1}"
# Probe merchant host candidates when applying a generic domain (0=off)
: "${TALER_DOMAIN_PROBE:=1}"
BANK_PUBLIC=${BANK_PUBLIC%/}
EXCHANGE_PUBLIC=${EXCHANGE_PUBLIC%/}
MERCHANT_PUBLIC=${MERCHANT_PUBLIC%/}
# ---------------------------------------------------------------------------
# Domain presets → public bank / exchange / merchant base URLs
#
# TALER_DOMAIN=hacktivism.ch (default, GOA, local stack)
# TALER_DOMAIN=taler.net → demo.taler.net (KUDOS)
# TALER_DOMAIN=demo.taler.net
# TALER_DOMAIN=taler-ops.ch → exchange.taler-ops.ch (CHF; bank/merchant if up)
# TALER_DOMAIN=example.org → bank/exchange/backend|taler|merchant.example.org
#
# Explicit BANK_PUBLIC / EXCHANGE_PUBLIC / MERCHANT_PUBLIC still win if set
# *after* apply_taler_domain, or pass full URLs via --bank/--exchange/--merchant.
# ---------------------------------------------------------------------------
_normalize_domain() {
local d="$1"
d="${d#https://}"
d="${d#http://}"
d="${d%%/*}"
d="${d%%:*}"
# Strip service host prefix only if a real domain remains (has a dot).
# e.g. bank.demo.taler.net → demo.taler.net, taler.hacktivism.ch → hacktivism.ch
# but NOT taler.net → net
case "$d" in
bank.*|exchange.*|taler.*|backend.*|merchant.*|shop.*|libeufin.*)
rest="${d#*.}"
if [[ "$rest" == *.* ]]; then
d="$rest"
fi
;;
esac
printf '%s' "$d"
}
_probe_https_config() {
# 0 if https://$1/config returns 200
local host="$1" code
code=$(curl -skS --max-redirs 0 -m 4 -o /dev/null -w '%{http_code}' "https://${host}/config" 2>/dev/null || echo 000)
[ "$code" = "200" ]
}
apply_taler_domain() {
local raw="${1:-}"
local d
[ -n "$raw" ] || return 0
d=$(_normalize_domain "$raw")
TALER_DOMAIN="$d"
case "$d" in
# Local koopa stack only — SSH / inside / e2e allowed
koopa|hacktivism.ch|hacktivism)
BANK_PUBLIC="https://bank.hacktivism.ch"
EXCHANGE_PUBLIC="https://exchange.hacktivism.ch"
MERCHANT_PUBLIC="https://taler.hacktivism.ch"
EXPECT_CURRENCY="GOA"
LOCAL_STACK=1
SKIP_SSH=0
TALER_DOMAIN="hacktivism.ch"
: "${WITHDRAW_AMT:=GOA:20}"
: "${PAY_AMT:=GOA:0.01}"
: "${CREDIT_AMT:=GOA:400}"
;;
taler.net|demo.taler.net)
# Official public demo (KUDOS) — public only, never SSH; tiny e2e amounts
BANK_PUBLIC="https://bank.demo.taler.net"
EXCHANGE_PUBLIC="https://exchange.demo.taler.net"
MERCHANT_PUBLIC="https://backend.demo.taler.net"
EXPECT_CURRENCY="KUDOS"
LOCAL_STACK=0
SKIP_SSH=1
MERCHANT_INSTANCE="${MERCHANT_INSTANCE:-sandbox}"
WITHDRAW_AMT="${WITHDRAW_AMT:-KUDOS:20}"
PAY_AMT="${PAY_AMT:-KUDOS:0.01}"
CREDIT_AMT="${CREDIT_AMT:-KUDOS:100}"
TALER_DOMAIN="demo.taler.net"
;;
taler-ops.ch)
# Public CHF exchange; bank/merchant hosts vary — probe common names
EXCHANGE_PUBLIC="https://exchange.taler-ops.ch"
BANK_PUBLIC="https://bank.taler-ops.ch"
MERCHANT_PUBLIC="https://backend.taler-ops.ch"
EXPECT_CURRENCY="CHF"
LOCAL_STACK=0
SKIP_SSH=1
WITHDRAW_AMT="${WITHDRAW_AMT:-CHF:20}"
PAY_AMT="${PAY_AMT:-CHF:0.01}"
CREDIT_AMT="${CREDIT_AMT:-CHF:100}"
if [ "${TALER_DOMAIN_PROBE}" = "1" ]; then
local h
for h in bank.taler-ops.ch bank.demo.taler-ops.ch; do
_probe_https_config "$h" && { BANK_PUBLIC="https://$h"; break; }
done
for h in backend.taler-ops.ch merchant.taler-ops.ch taler.taler-ops.ch shop.taler-ops.ch; do
_probe_https_config "$h" && { MERCHANT_PUBLIC="https://$h"; break; }
done
fi
;;
*)
# Any other domain — public HTTPS only, never SSH to koopa
BANK_PUBLIC="https://bank.${d}"
EXCHANGE_PUBLIC="https://exchange.${d}"
MERCHANT_PUBLIC="https://backend.${d}"
EXPECT_CURRENCY="${EXPECT_CURRENCY:-}" # unknown — don't hard-fail currency
LOCAL_STACK=0
SKIP_SSH=1
if [ "${TALER_DOMAIN_PROBE}" = "1" ]; then
local h
for h in "backend.${d}" "taler.${d}" "merchant.${d}" "shop.${d}"; do
_probe_https_config "$h" && { MERCHANT_PUBLIC="https://$h"; break; }
done
for h in "bank.${d}" "libeufin.${d}"; do
_probe_https_config "$h" && { BANK_PUBLIC="https://$h"; break; }
done
_probe_https_config "exchange.${d}" || true
fi
;;
esac
BANK_PUBLIC=${BANK_PUBLIC%/}
EXCHANGE_PUBLIC=${EXCHANGE_PUBLIC%/}
MERCHANT_PUBLIC=${MERCHANT_PUBLIC%/}
# Hard rule: only the local koopa/hacktivism stack may use SSH
if [ "${LOCAL_STACK}" != "1" ]; then
SKIP_SSH=1
fi
}
# Apply TALER_DOMAIN from env once (CLI exports TALER_DOMAIN_APPLIED=1 after overrides).
if [ "${TALER_DOMAIN_APPLIED:-0}" != "1" ] \
&& [ -n "${TALER_DOMAIN:-}" ] && [ "${TALER_DOMAIN}" != "hacktivism.ch" ]; then
apply_taler_domain "$TALER_DOMAIN"
TALER_DOMAIN_APPLIED=1
fi
# Safe SSH: publickey only, short connect, overall alarm so we never block forever.
SSH_BASE_OPTS=(
-o BatchMode=yes
-o ConnectTimeout="${SSH_CONNECT_TIMEOUT}"
-o ConnectionAttempts=1
-o ServerAliveInterval=3
-o ServerAliveCountMax=2
-o StrictHostKeyChecking=accept-new
-o PreferredAuthentications=publickey
-o PasswordAuthentication=no
-o KbdInteractiveAuthentication=no
-o GSSAPIAuthentication=no
-o NumberOfPasswordPrompts=0
)
# Hard wall-clock timeout so ssh/curl never block the monitoring run forever.
with_timeout() {
local secs="$1"; shift
if command -v gtimeout >/dev/null 2>&1; then
gtimeout --kill-after=2 "$secs" "$@"
return $?
fi
if command -v timeout >/dev/null 2>&1; then
timeout -k 2 "$secs" "$@" 2>/dev/null || timeout --kill-after=2 "$secs" "$@"
return $?
fi
# Portable: perl alarm + process group kill
perl -e '
use strict; use warnings;
my $secs = shift @ARGV;
my $pid = fork();
die "fork: $!" unless defined $pid;
if ($pid == 0) {
setpgrp(0, 0);
exec @ARGV;
exit 127;
}
$SIG{ALRM} = sub {
kill "TERM", -$pid;
select(undef, undef, undef, 1.0);
kill "KILL", -$pid;
exit 124;
};
alarm $secs;
waitpid($pid, 0);
my $code = $? >> 8;
alarm 0;
exit $code;
' "$secs" "$@"
}
# Probe: 0 if koopa SSH works quickly
koopa_ssh_ok() {
[ "${SKIP_SSH}" = "1" ] && return 1
with_timeout $((SSH_CONNECT_TIMEOUT + 3)) \
ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" 'echo ok' >/dev/null 2>&1
}
# Run remote bash -s with optional stdin script; hard-capped
# usage: koopa_ssh_bash [timeout_secs] <<'EOF' ... EOF
# or: koopa_ssh_run timeout_secs 'remote command'
koopa_ssh_run() {
local t="${1:-$SSH_CMD_TIMEOUT}"
shift
with_timeout "$t" ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" "$@"
}
koopa_ssh_bash() {
local t="${1:-$SSH_CMD_TIMEOUT}"
with_timeout "$t" ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" 'bash -s'
}
if [ "${NO_COLOR:-0}" = "1" ] || [ ! -t 1 ]; then
G= R= Y= C= N= B=
else
G=$'\e[32m'; R=$'\e[31m'; Y=$'\e[33m'; C=$'\e[36m'; B=$'\e[1m'; N=$'\e[0m'
fi
PASS_N=0
FAIL_N=0
WARN_N=0
INFO_N=0
BLOCKERS=() # human-readable payment/withdraw blockers
ERRORS=() # all ERROR lines (component scope)
# Test IDs by area: www-001, e2e-001, inside-001, …
# Usage: set_area www then each ok/fail/warn/info/blocker/err auto-numbers.
TEST_AREA=""
TEST_N=0
# Last issued id (www-001); set by _take_tid — not via $(…) so TEST_N persists.
LAST_TID=""
set_area() {
TEST_AREA="$1"
TEST_N=0
LAST_TID=""
}
# Assign next id into LAST_TID (must not run in a subshell).
_take_tid() {
LAST_TID=""
if [ -z "${TEST_AREA:-}" ]; then
return
fi
TEST_N=$((TEST_N + 1))
LAST_TID=$(printf '%s-%03d' "$TEST_AREA" "$TEST_N")
}
_fmt_tid() {
# prefix "www-001 " or empty
if [ -n "${LAST_TID:-}" ]; then
printf '%s ' "$LAST_TID"
fi
}
ok() {
local label="$1"
_take_tid
printf '%s[OK]%s %s%s\n' "$G" "$N" "$(_fmt_tid)" "$label"
PASS_N=$((PASS_N + 1))
}
# component-scoped error: err bank "libeufin down" "detail"
err() {
local comp="$1" msg="$2" detail="${3:-}"
_take_tid
printf '%s[ERROR]%s %s%s: %s%s\n' "$R" "$N" "$(_fmt_tid)" "$comp" "$msg" "${detail:+ — $detail}"
FAIL_N=$((FAIL_N + 1))
ERRORS+=("${LAST_TID:+$LAST_TID }[$comp] $msg${detail:+ — $detail}")
}
# legacy fail label ...
fail() {
local label="$1" detail="${2:-}"
_take_tid
printf '%s[ERROR]%s %s%s%s\n' "$R" "$N" "$(_fmt_tid)" "$label" "${detail:+ — $detail}"
FAIL_N=$((FAIL_N + 1))
ERRORS+=("${LAST_TID:+$LAST_TID }$label${detail:+ — $detail}")
}
warn() {
local label="$1" detail="${2:-}"
_take_tid
printf '%s[WARN]%s %s%s%s\n' "$Y" "$N" "$(_fmt_tid)" "$label" "${detail:+ — $detail}"
WARN_N=$((WARN_N + 1))
}
info() {
local label="$1" detail="${2:-}"
_take_tid
if [ -n "$detail" ]; then
printf '%s[INFO]%s %s%s — %s\n' "$C" "$N" "$(_fmt_tid)" "$label" "$detail"
else
printf '%s[INFO]%s %s%s\n' "$C" "$N" "$(_fmt_tid)" "$label"
fi
INFO_N=$((INFO_N + 1))
}
blocker() {
# Payment/withdraw path cannot proceed because of this
local step="$1" msg="$2"
_take_tid
printf '%s[BLOCKER]%s %s%s: %s\n' "$R$B" "$N" "$(_fmt_tid)" "$step" "$msg"
BLOCKERS+=("${LAST_TID:+$LAST_TID }[$step] $msg")
FAIL_N=$((FAIL_N + 1))
ERRORS+=("BLOCKER ${LAST_TID:+$LAST_TID }[$step] $msg")
}
section() { printf '\n%s== %s ==%s\n' "$B" "$*" "$N"; }
summary() {
echo ""
if [ "${#BLOCKERS[@]}" -gt 0 ]; then
printf '%s--- BLOCKERS (fix/withdraw path) ---%s\n' "$R$B" "$N"
local b
for b in "${BLOCKERS[@]}"; do
printf '%s • %s%s\n' "$R" "$b" "$N"
done
fi
if [ "${#ERRORS[@]}" -gt 0 ] && [ "${#BLOCKERS[@]}" -lt "${#ERRORS[@]}" ]; then
printf '%s--- ERRORS ---%s\n' "$R" "$N"
local e
for e in "${ERRORS[@]}"; do
case "$e" in BLOCKER*) continue ;; esac
printf '%s • %s%s\n' "$R" "$e" "$N"
done
fi
printf 'totals: %s%d OK%s' "$G" "$PASS_N" "$N"
[ "$FAIL_N" -gt 0 ] && printf ', %s%d ERROR%s' "$R" "$FAIL_N" "$N"
[ "$WARN_N" -gt 0 ] && printf ', %s%d WARN%s' "$Y" "$WARN_N" "$N"
[ "$INFO_N" -gt 0 ] && printf ', %d INFO' "$INFO_N"
[ "${#BLOCKERS[@]}" -gt 0 ] && printf ', %s%d BLOCKER%s' "$R" "${#BLOCKERS[@]}" "$N"
printf '\n'
[ "$FAIL_N" -eq 0 ]
}
http_code() {
local url="$1"; shift
curl -skS --max-redirs 0 -m "${TIMEOUT}" -o /dev/null -w '%{http_code}' "$@" "$url" 2>/dev/null || echo 000
}
http_body() {
local url="$1" out="$2"; shift 2
curl -skS --max-redirs 0 -m "${TIMEOUT}" -o "$out" -w '%{http_code}' "$@" "$url" 2>/dev/null || echo 000
}
# ---------------------------------------------------------------------------
# Currency unit map checks (wallet codec: alt_unit_names must include "0")
# ---------------------------------------------------------------------------
# Returns 0 if JSON body at $1 has usable alt_unit_names.
# Supports:
# - exchange/bank: currency_specification.alt_unit_names
# - merchant: currencies.<CODE>.alt_unit_names for each code
# Optional $2 = required currency code for currency_specification.currency
json_has_alt_unit_names() {
local file="$1" want_cur="${2:-}"
python3 - "$file" "$want_cur" <<'PY'
import json, sys
path, want = sys.argv[1], sys.argv[2]
try:
d = json.load(open(path))
except Exception as e:
print(f"json-error: {e}")
sys.exit(2)
def check_au(au, label):
if not isinstance(au, dict) or not au:
print(f"{label}: missing/empty alt_unit_names")
return False
if "0" not in au or not str(au.get("0") or "").strip():
print(f"{label}: alt_unit_names missing non-empty key \"0\" (have {sorted(au.keys())})")
return False
print(f"{label}: alt_unit_names ok (0={au.get('0')!r}, n={len(au)})")
return True
ok = True
cs = d.get("currency_specification")
if isinstance(cs, dict):
if want and cs.get("currency") and cs.get("currency") != want:
print(f"currency_specification.currency={cs.get('currency')!r} want {want!r}")
ok = False
if not check_au(cs.get("alt_unit_names"), "currency_specification"):
ok = False
elif "currency_specification" in d:
print("currency_specification: not an object")
ok = False
curs = d.get("currencies")
if isinstance(curs, dict) and curs:
for code, spec in curs.items():
if not isinstance(spec, dict):
print(f"currencies.{code}: not an object")
ok = False
continue
if not check_au(spec.get("alt_unit_names"), f"currencies.{code}"):
ok = False
if not isinstance(cs, dict) and not (isinstance(curs, dict) and curs):
# neither shape — fail
print("no currency_specification or currencies map")
ok = False
sys.exit(0 if ok else 1)
PY
}
# Check one exchange base URL's /config for alt_unit_names.
# $1=label $2=base_url $3=expected currency (optional) $4=strict(1) or soft(0)
check_exchange_alt_units() {
local label="$1" base="$2" want_cur="${3:-}" strict="${4:-1}"
local f code
base="${base%/}"
f=$(mktemp)
code=$(http_body "${base}/config" "$f")
if [ "$code" != "200" ]; then
rm -f "$f"
if [ "$strict" = "1" ]; then
fail "$label /config" "HTTP $code ($base)"
else
warn "$label /config" "HTTP $code ($base)"
fi
return
fi
local out ec
set +e
out=$(json_has_alt_unit_names "$f" "$want_cur" 2>&1)
ec=$?
set -e
rm -f "$f"
if [ "$ec" -eq 0 ]; then
ok "$label alt_unit_names" "$(echo "$out" | tr '\n' '; ' | sed 's/; $//')"
else
if [ "$strict" = "1" ]; then
fail "$label alt_unit_names" "$(echo "$out" | tr '\n' '; ' | sed 's/; $//')"
else
warn "$label alt_unit_names" "$(echo "$out" | tr '\n' '; ' | sed 's/; $//')"
fi
fi
}
# From merchant /config JSON file: walk exchanges[] and check each base_url/config.
# Local stack hosts (hacktivism.ch or $EXCHANGE_PUBLIC host) are strict; others soft.
check_merchant_listed_exchanges_alt_units() {
local mer_json="$1"
local list
list=$(python3 - "$mer_json" <<'PY'
import json, sys
d = json.load(open(sys.argv[1]))
for e in d.get("exchanges") or []:
if not isinstance(e, dict):
continue
u = (e.get("base_url") or e.get("url") or "").rstrip("/")
c = e.get("currency") or ""
if u:
print(f"{c}\t{u}")
PY
)
if [ -z "$list" ]; then
fail "merchant exchanges[]" "empty — no exchanges to check for alt_unit_names"
return
fi
local line cur url strict host
while IFS=$'\t' read -r cur url; do
[ -n "$url" ] || continue
host="${url#https://}"; host="${host#http://}"; host="${host%%/*}"
strict=1
case "$host" in
*hacktivism.ch) strict=1 ;;
*)
# foreign exchange (e.g. taler-ops) — soft unless it is our configured EXCHANGE_PUBLIC
if [ "$url" = "${EXCHANGE_PUBLIC}" ] || [ "$url" = "${EXCHANGE_PUBLIC}/" ]; then
strict=1
else
strict=0
fi
;;
esac
check_exchange_alt_units "exchange ${cur:-?} ${host}" "$url" "$cur" "$strict"
done <<<"$list"
}
SECRETS_ROOT="${SECRETS_ROOT:-}"
if [ -z "$SECRETS_ROOT" ]; then
for d in \
"$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)/koopa-admin-secrets/koopa/host-root" \
"/Users/newkamek/src/koopa/koopa-admin-secrets/koopa/host-root" \
"$HOME/src/koopa/koopa-admin-secrets/koopa/host-root"
do
if [ -d "$d/taler-bank" ]; then SECRETS_ROOT=$d; break; fi
done
fi
read_secret() {
local rel="$1"
if [ -n "${SECRETS_ROOT:-}" ] && [ -f "${SECRETS_ROOT}/${rel}" ]; then
tr -d '\n' <"${SECRETS_ROOT}/${rel}"
return 0
fi
koopa_ssh_ok || return 1
koopa_ssh_run 10 "tr -d '\\n' </root/$(basename "$rel") 2>/dev/null || true" 2>/dev/null
}
find_wallet_cli() {
if [ -n "${WALLET_CLI:-}" ] && [ -f "$WALLET_CLI" ]; then
echo "$WALLET_CLI"; return 0
fi
for c in \
/Users/newkamek/src/taler/taler-typescript-core/packages/taler-wallet-cli/bin/taler-wallet-cli.mjs \
"$(command -v taler-wallet-cli 2>/dev/null || true)"
do
[ -n "$c" ] && [ -f "$c" ] && { echo "$c"; return 0; }
done
return 1
}