From 982432f191b1190f6c295f5c83f209ea7c33b413 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Fri, 17 Jul 2026 02:27:20 +0200 Subject: [PATCH] feat(monitoring): surface load and coins in inside and e2e --- scripts/taler-monitoring/check_e2e.sh | 60 ++++++++++++++++++++++-- scripts/taler-monitoring/check_inside.sh | 12 ++++- 2 files changed, 68 insertions(+), 4 deletions(-) diff --git a/scripts/taler-monitoring/check_e2e.sh b/scripts/taler-monitoring/check_e2e.sh index 2cbf8c4..82c8c6c 100755 --- a/scripts/taler-monitoring/check_e2e.sh +++ b/scripts/taler-monitoring/check_e2e.sh @@ -5,6 +5,8 @@ set -euo pipefail ROOT=$(cd "$(dirname "$0")" && pwd) # shellcheck source=lib.sh source "$ROOT/lib.sh" +# shellcheck source=metrics.sh +source "$ROOT/metrics.sh" : "${E2E_TIMEOUT:=180}" # ATM ladder + wirewatch lag needs room (was 55 — too tight) : "${E2E_PAY_SECS:=22}" # hard reserve for handle-uri + settle (do not starve pay) @@ -150,6 +152,19 @@ e2e_finish() { fi E2E_REPORTED=1 print_balances + # Final coin inventory + host/container load + if [ -n "${METRICS_DIR:-}" ]; then + section "metrics · e2e coins final" + metrics_report_coins "e2e-end" || true + fi + if [ -n "${METRICS_DIR:-}" ] && [ "${METRICS_LOAD:-1}" != "0" ]; then + metrics_report_load "${METRICS_DIR}/load-after.json" "e2e-end" || true + if [ -f "${METRICS_DIR}/load-before.json" ] && [ -f "${METRICS_DIR}/load-after.json" ]; then + section "metrics · e2e load delta" + metrics_print_load_delta "${METRICS_DIR}/load-before.json" "${METRICS_DIR}/load-after.json" || true + fi + metrics_print_overall "e2e overall" || true + fi summary || true return "$code" } @@ -276,6 +291,9 @@ BANK_HOST=$(python3 -c 'from urllib.parse import urlparse; print(urlparse("'"$BA SCRATCH=$(mktemp -d) WDB="$SCRATCH/wallet.sqlite3" +METRICS_DIR="${METRICS_DIR:-$SCRATCH/metrics}" +mkdir -p "$METRICS_DIR" +export METRICS_DIR CUR="${CUR:-${EXPECT_CURRENCY:-GOA}}" WDB export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH" # Always dump balances on any exit (timeout, blocker, signal, success) trap 'ec=$?; e2e_finish "$ec"; rm -rf "$SCRATCH"; exit "$ec"' EXIT @@ -312,6 +330,19 @@ info "ATM withdraw ladder" "$WITHDRAW_LIST (credit $CREDIT_AMT)" info "pay ladder" "$PAY_LIST" info "e2e budget" "${E2E_TIMEOUT}s" +# alt_unit_names for human amounts (Kilo-GOA … + base GOA in parentheses) +ALT_UNITS_FILE="${METRICS_DIR}/alt_unit_names.json" +export ALT_UNITS_FILE +if metrics_load_alt_units "${EXCHANGE_PUBLIC}/config"; then + info "alt_unit_names" "from ${EXCHANGE_PUBLIC}/config" +else + warn "alt_unit_names" "using built-in SI fallback" +fi + +# Host + bank/exchange/merchant RAM/CPU/load (SSH koopa or koopa-external) +section "e2e · load snapshot (before withdraw/pay)" +metrics_report_load "${METRICS_DIR}/load-before.json" "e2e-start" || true + # Local stack: koopa secrets. Remote: only explicit env (never leak local passwords to foreign banks). if [ "$E2E_REMOTE" = "1" ]; then ADMIN_PASS="${E2E_BANK_ADMIN_PASS:-}" @@ -626,6 +657,8 @@ else warn "wallet accept ToS" "$(tail -c 80 "$SCRATCH/ex-tos.out" | tr '\n' ' ')" fi fi +# Empty wallet baseline before ATM / pay load +metrics_report_coins "wallet-ready" || true wd_status() { curl -sS -m 8 -o "$SCRATCH/wd-st.json" -w '' \ @@ -668,8 +701,9 @@ e2e_one_withdraw() { WITHDRAW_AMT="$1" local tag tag=$(printf '%s' "$WITHDRAW_AMT" | tr '.:' '__') - section "e2e · ATM withdraw $WITHDRAW_AMT" + section "e2e · ATM withdraw $WITHDRAW_AMT · $(format_amount_alt "$WITHDRAW_AMT")" e2e_over && { warn "ATM withdraw" "budget exhausted — skip $WITHDRAW_AMT"; return 1; } + metrics_report_coins "before-ATM-${tag}" || true curl -sS -m 15 -o "$SCRATCH/wd-$tag.json" -H "Authorization: Bearer $UT" \ -H 'Content-Type: application/json' \ @@ -762,6 +796,7 @@ else: fi sleep 2 done + metrics_report_coins "after-ATM-${tag}" || true if [ "$ok_bal" = "1" ]; then return 0 fi @@ -786,8 +821,9 @@ e2e_one_pay() { fi # shell-safe tag for files tag=$(printf '%s' "$tag" | tr -c 'A-Za-z0-9._-' '_') - section "e2e · pay $PAY_AMT · $PAY_SUM" + section "e2e · pay $PAY_AMT · $(format_amount_alt "$PAY_AMT") · $PAY_SUM" e2e_over && { warn "pay" "budget exhausted — skip $PAY_AMT ($PAY_SUM)"; return 1; } + metrics_report_coins "before-pay-${tag}" || true if [ -z "${MPW:-}" ]; then warn "pay $PAY_AMT" "no merchant token" return 1 @@ -860,9 +896,11 @@ d=json.load(open(sys.argv[1])) sys.exit(0 if d.get("paid") is True or str(d.get("order_status","")).lower()=="paid" else 1) ' "$SCRATCH/ord-paid-$tag.json" 2>/dev/null; then ok "payment settled $PAY_AMT ($PAY_SUM · order $OID)" + metrics_report_coins "after-pay-${tag}" || true return 0 fi done + metrics_report_coins "after-pay-fail-${tag}" || true warn "pay $PAY_AMT ($PAY_SUM)" "not settled for order $OID" return 1 } @@ -899,8 +937,9 @@ e2e_one_pay_public_template() { local pamt="$3" local tag tag=$(printf 'shop_%s' "$pid" | tr -c 'A-Za-z0-9._-' '_') - section "e2e · shop product · $pname ($pid) · $pamt" + section "e2e · shop product · $pname ($pid) · $pamt · $(format_amount_alt "$pamt")" e2e_over && { warn "shop $pname" "budget exhausted — skip"; return 1; } + metrics_report_coins "before-shop-${tag}" || true local MH MH=$(python3 -c 'from urllib.parse import urlparse; print(urlparse("'"$MERCHANT_PUBLIC"'").hostname or "taler.hacktivism.ch")' 2>/dev/null || echo "taler.hacktivism.ch") @@ -977,15 +1016,18 @@ d=json.load(open(sys.argv[1])) sys.exit(0 if d.get("paid") is True or str(d.get("order_status","")).lower()=="paid" else 1) ' "$SCRATCH/ord-paid-$tag.json" 2>/dev/null; then ok "shop $pname" "payment settled ($pamt · order $OID)" + metrics_report_coins "after-shop-${tag}" || true return 0 fi fi if grep -qiE 'payment|paid|Payment' "$SCRATCH/tx-$tag.out" 2>/dev/null \ || grep -qiE 'done|paid|success|Payment' "$SCRATCH/pay-$tag.out" 2>/dev/null; then ok "shop $pname" "payment settled via wallet tx ($pamt · order $OID)" + metrics_report_coins "after-shop-${tag}" || true return 0 fi done + metrics_report_coins "after-shop-fail-${tag}" || true warn "shop $pname ($pid)" "not settled for order $OID" return 1 } @@ -1021,6 +1063,10 @@ for WITHDRAW_AMT in $WITHDRAW_LIST; do esac done info "ATM withdraw summary" "$WITHDRAW_REPORT (ok=$WITHDRAW_OK_N lag=$WITHDRAW_LAG_N fail=$WITHDRAW_FAIL_N)" +section "e2e · load snapshot (after ATM withdraws)" +metrics_report_load "${METRICS_DIR}/load-after-withdraw.json" "after-withdraw" || true +cp -f "${METRICS_DIR}/load-after-withdraw.json" "${METRICS_DIR}/load-after.json" 2>/dev/null || true +metrics_report_coins "after-ATM-ladder" || true # Settlement catch-up: bank may have confirmed while wallet was still empty section "e2e · wallet settlement (timing)" @@ -1037,6 +1083,7 @@ if python3 -c "import sys; sys.exit(0 if float(sys.argv[1]) > 0 else 1)" "$av_no WITHDRAW_REPORT="${WITHDRAW_REPORT} → late-OK avail=${CUR}:${av_now}" fi ok "spendable balance for payments" "${CUR}:${av_now}" + metrics_report_coins "after-settle" || true else if [ "$WITHDRAW_OK" != "1" ]; then warn "withdraw" "still no spendable ${CUR} after settle wait" @@ -1100,6 +1147,10 @@ for PAY_AMT in $PAY_LIST; do fi done info "pay summary" "$PAY_REPORT (ok=$PAY_OK_N fail=$PAY_FAIL_N skip=$PAY_SKIP_N)" +section "e2e · load snapshot (after payments)" +metrics_report_load "${METRICS_DIR}/load-after-pay.json" "after-pay" || true +cp -f "${METRICS_DIR}/load-after-pay.json" "${METRICS_DIR}/load-after.json" 2>/dev/null || true +metrics_report_coins "after-pay-ladder" || true if [ "$PAY_OK" != "1" ]; then av_now=$(wallet_avail_num) if python3 -c "import sys; sys.exit(0 if float(sys.argv[1]) > 0 else 1)" "$av_now" 2>/dev/null; then @@ -1193,6 +1244,9 @@ EOF if [ "$SHOP_OK" != "1" ] && [ "$SHOP_FAIL_N" -gt 0 ]; then warn "goa-shop" "no sample product payment succeeded ($SHOP_REPORT)" fi + section "e2e · load snapshot (after shop pays)" + metrics_report_load "${METRICS_DIR}/load-after-shop.json" "after-shop" || true + cp -f "${METRICS_DIR}/load-after-shop.json" "${METRICS_DIR}/load-after.json" 2>/dev/null || true fi fi diff --git a/scripts/taler-monitoring/check_inside.sh b/scripts/taler-monitoring/check_inside.sh index aba6115..f49b4b6 100755 --- a/scripts/taler-monitoring/check_inside.sh +++ b/scripts/taler-monitoring/check_inside.sh @@ -4,6 +4,8 @@ set -euo pipefail ROOT=$(cd "$(dirname "$0")" && pwd) # shellcheck source=lib.sh source "$ROOT/lib.sh" +# shellcheck source=metrics.sh +source "$ROOT/metrics.sh" # Area inside-### — container / process state on koopa (SSH) set_area inside @@ -35,8 +37,10 @@ hasp() { podman exec "$1" pgrep -f "$2" >/dev/null 2>&1; } BANK=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -iE 'hacktivism-bank|taler-bank' | head -1) [ -z "$BANK" ] && BANK=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -i bank | head -1) EX=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -i exchange | head -1) +# Exact merchant container name first — never fall through to *-bank MER=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -E '^taler-hacktivism$' | head -1) -[ -z "$MER" ] && MER=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -iE 'merchant|hacktivism' | grep -viE 'bank|exchange' | head -1) +[ -z "$MER" ] && MER=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -i merchant | grep -viE 'bank|exchange' | head -1) +[ -z "$MER" ] && MER=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -E 'hacktivism' | grep -viE 'bank|exchange' | head -1) # Domain resolve inside container (wirewatch needs bank.hacktivism.ch → real IP) # emit: comp LEVEL dns "host → ip" or fail @@ -138,4 +142,10 @@ while IFS= read -r line; do esac done <<<"$RAW" +# Host loadavg + RAM + per-container RSS/CPU (same probe as e2e/ladder) +section "inside · load / memory" +METRICS_DIR="${METRICS_DIR:-$(mktemp -d)}" +export METRICS_DIR +metrics_report_load "${METRICS_DIR}/load-inside.json" "inside" || true + summary