From 44d2a57af9eb2be863074d06fc76849d30c591eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Thu, 16 Jul 2026 23:10:19 +0200 Subject: [PATCH] monitoring: GOA ladder phase with 23 steps from zero to max. taler-monitoring ladder: log-uniform random rungs between fixed GOA:0 and bank ceiling. --- scripts/taler-monitoring/README.md | 1 + scripts/taler-monitoring/TESTS.md | 39 ++ scripts/taler-monitoring/check_goa_ladder.sh | 562 +++++++++++++++++++ scripts/taler-monitoring/taler-monitoring.sh | 8 +- 4 files changed, 609 insertions(+), 1 deletion(-) create mode 100755 scripts/taler-monitoring/check_goa_ladder.sh diff --git a/scripts/taler-monitoring/README.md b/scripts/taler-monitoring/README.md index 35edc67..c867ada 100644 --- a/scripts/taler-monitoring/README.md +++ b/scripts/taler-monitoring/README.md @@ -78,6 +78,7 @@ E2E_VARIABLE=0 WITHDRAW_AMT=GOA:50 PAY_AMT=GOA:1 ./taler-monitoring.sh e2e # s | **versions** | `deb.taler.net` reachable (InRelease/Packages/pool `.deb`); containers can reach it + have apt source; installed Taler packages vs **trixie** | | **sanity** | bank · exchange · merchant sections (public + server) | | **e2e** | account → credit → withdraw → wallet → confirm → order → pay | +| **ladder** | GOA withdraw: fixed **0** + low bands + **random high ranges** + fixed **max** | ```bash # package suite (default trixie on deb.taler.net) diff --git a/scripts/taler-monitoring/TESTS.md b/scripts/taler-monitoring/TESTS.md index 51686d4..eeb05c1 100644 --- a/scripts/taler-monitoring/TESTS.md +++ b/scripts/taler-monitoring/TESTS.md @@ -154,6 +154,44 @@ Blockers keep the same ID prefix: `[BLOCKER] e2e-0NN step: message`. --- +## ladder — GOA withdraw ranges (`./taler-monitoring.sh ladder`) + +| Step | What | +|------|------| +| ladder-001 | `GET /intro/auto-account.json` (personal account, GOA:0) | +| ladder-… | Mint explorer pool withdrawals: **zero**, then **random** in ranges (strictly increasing), then **pin at bank max** | +| ladder-… | wallet-cli accept-uri + explorer confirm when `selected` | +| ladder-… | Settle coins; **stop on first hard failure** | +| report | TSV + JSON with **ms_mint / ms_accept / ms_confirm / ms_settle / ms_total** | + +Default path: + +```text +[fixed] GOA:0 +[low] random pick in each LADDER_RANGES band (atomic … ~2e6) +[high] LADDER_HIGH_RUNGS random *ranges* log-uniform in [HIGH_FROM, max) + (range bounds drawn randomly each run — not fixed 1e7/1e8/…) +[fixed] GOA:4503599627370496 (LADDER_MAX_AMOUNT) +``` + +| Env | Default | Meaning | +|-----|---------|---------| +| `LADDER_INCLUDE_ZERO` | `1` | first rung **0** (fixed) | +| `LADDER_INCLUDE_MAX` | `1` | last rung **max** (fixed) | +| `LADDER_MAX_AMOUNT` | `4503599627370496` | absolute pin | +| `LADDER_HIGH_FROM` | `1000000` | high random zone starts | +| `LADDER_HIGH_RUNGS` | `12` | # random high ranges | +| `LADDER_TIMEOUT_S` | `3600` | large rungs need time | + +```bash +./taler-monitoring.sh ladder +LADDER_HIGH_RUNGS=20 ./taler-monitoring.sh ladder +LADDER_INCLUDE_ZERO=0 LADDER_INCLUDE_MAX=0 ./taler-monitoring.sh ladder +LADDER_REPORT_DIR=/tmp/my-ladder ./taler-monitoring.sh ladder +``` + +--- + ## Run one area ```bash @@ -161,5 +199,6 @@ Blockers keep the same ID prefix: `[BLOCKER] e2e-0NN step: message`. ./taler-monitoring.sh inside # inside only ./taler-monitoring.sh versions # deb.taler.net + package drift ./taler-monitoring.sh e2e # e2e only +./taler-monitoring.sh ladder # GOA amount ladder + timings ./taler-monitoring.sh -d taler.net urls ``` diff --git a/scripts/taler-monitoring/check_goa_ladder.sh b/scripts/taler-monitoring/check_goa_ladder.sh new file mode 100755 index 0000000..59fe37a --- /dev/null +++ b/scripts/taler-monitoring/check_goa_ladder.sh @@ -0,0 +1,562 @@ +#!/usr/bin/env bash +# check_goa_ladder.sh — GOA withdraw ladder for taler-monitoring +# +# bank.hacktivism.ch flow (landing): +# 1) GET /intro/auto-account.json → personal goa-account-* (GOA:0) +# 2) Mint pool withdrawals as explorer (shared pool) + confirm when selected +# 3) wallet-cli accept-uri + run-until-done +# +# Amounts: random within defined ranges, strictly increasing. +# On first hard failure: stop, print timing report, exit 1. +# +# Env: +# LADDER_STEPS total rungs (default 23) = 0 + (N-2) random + max +# LADDER_MAX_AMOUNT fixed last pin (libeufin ceiling 4503599627370496) +# LADDER_TIMEOUT_S default 3600 +# LADDER_LOAD=0 skip host load snapshots +# EXP_PW_FILE, LADDER_REPORT_DIR, … +# +# Path: always [0] → strictly increasing random (log-uniform) → [max] +# Only 0 and max are fixed amounts. +# Load: koopa host snapshot BEFORE withdraw ladder and AFTER (loadavg, mem, podman). +# +# Phase: ./taler-monitoring.sh ladder +set -euo pipefail +ROOT=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=lib.sh +source "$ROOT/lib.sh" + +set_area ladder +SECTION_T0=$(date +%s) +now_ms() { python3 -c 'import time; print(int(time.time()*1000))'; } +elapsed_ms() { + # elapsed_ms START_MS + python3 -c 'import sys; print(int(sys.argv[1]) - int(sys.argv[2]))' "$(now_ms)" "$1" +} + +: "${LADDER_TIMEOUT_S:=3600}" +: "${LADDER_SETTLE_ROUNDS:=18}" +: "${LADDER_SETTLE_SLEEP:=2}" +: "${EXP_USER:=explorer}" +: "${EXP_PW_FILE:=/Users/newkamek/src/koopa/koopa-admin-secrets/koopa/host-root/taler-bank/bank-explorer-password.txt}" +: "${CLI_JS:=/Users/newkamek/src/taler-typescript-core/packages/taler-wallet-cli/bin/taler-wallet-cli.mjs}" +: "${LADDER_MAX_AMOUNT:=4503599627370496}" +: "${LADDER_STEPS:=23}" +: "${LADDER_LOAD:=1}" + +CUR="${EXPECT_CURRENCY:-GOA}" +BANK="${BANK_PUBLIC%/}" +EX="${EXCHANGE_PUBLIC%/}/" +SCRATCH=$(mktemp -d) +WDB="$SCRATCH/wallet.sqlite3" +REPORT_DIR="${LADDER_REPORT_DIR:-$SCRATCH}" +mkdir -p "$REPORT_DIR" +TSV="$REPORT_DIR/ladder-results.tsv" +JSON="$REPORT_DIR/ladder-report.json" +LOAD_BEFORE="$REPORT_DIR/load-before.json" +LOAD_AFTER="$REPORT_DIR/load-after.json" +echo -e "rung\trange\tamount\tstatus\tms_mint\tms_accept\tms_confirm\tms_settle\tms_total\twid\tnote" >"$TSV" + +ladder_over() { + local now + now=$(date +%s) + [ $((now - SECTION_T0)) -ge "$LADDER_TIMEOUT_S" ] +} + +wcli() { + if [ -f "$CLI_JS" ]; then + node "$CLI_JS" --wallet-db="$WDB" --no-throttle "$@" + else + taler-wallet-cli --wallet-db="$WDB" --no-throttle "$@" + fi +} + +wallet_avail() { + wcli balance 2>/dev/null | python3 -c ' +import json,sys +t=sys.stdin.read() +i=t.find("{") +if i<0: + print("0"); raise SystemExit +d=json.loads(t[i:t.rfind("}")+1]) +cur=sys.argv[1] +for b in d.get("balances") or []: + a=b.get("available") or "" + if a.startswith(cur+":"): + print(a.split(":",1)[1]); raise SystemExit +print("0") +' "$CUR" 2>/dev/null || echo "0" +} + +# Build exactly LADDER_STEPS: [0] + (N-2) log-uniform random increasing + [max] +build_ladder() { + python3 - <<'PY' "$CUR" "${LADDER_MAX_AMOUNT}" "${LADDER_STEPS}" +import math, random, sys +from decimal import Decimal, ROUND_HALF_UP + +cur = sys.argv[1] +max_amt = Decimal(sys.argv[2]) +steps = max(2, int(sys.argv[3])) + +def fmt(v: Decimal) -> str: + q = v.quantize(Decimal("0.00000001"), rounding=ROUND_HALF_UP) + if q == q.to_integral(): + return format(int(q), "d") + return format(q, "f").rstrip("0").rstrip(".") + +mid = steps - 2 # between 0 and max +out = ["%s:0" % cur] +if mid > 0 and max_amt > 0: + # log-space cut points in (epsilon, max), pick strictly increasing + lo, hi = 1e-8, float(max_amt) * 0.999999 + if hi <= lo: + hi = lo * 10 + cuts = sorted(math.exp(random.uniform(math.log(lo), math.log(hi))) for _ in range(mid)) + # enforce strict increase after quantize + prev = Decimal(0) + for c in cuts: + v = Decimal(str(c)) + if v >= 1: + v = v.quantize(Decimal(1), rounding=ROUND_HALF_UP) + else: + v = v.quantize(Decimal("0.00000001"), rounding=ROUND_HALF_UP) + if v <= prev: + step = max(prev * Decimal("1e-6"), Decimal("0.00000001")) if prev > 0 else Decimal("0.00000001") + v = (prev + step).quantize(Decimal("0.00000001") if prev < 1 else Decimal(1)) + if v >= max_amt: + v = max_amt - (Decimal(1) if max_amt > 1 else Decimal("0.00000001")) + if v <= prev: + continue + out.append("%s:%s" % (cur, fmt(v))) + prev = v +out.append("%s:%s" % (cur, fmt(max_amt))) +# trim/pad to exact steps if quantize collapsed some +while len(out) > steps: + # drop from middle + out.pop(len(out) // 2) +while len(out) < steps and len(out) >= 2: + # insert geometric mean mid + i = len(out) // 2 + a = Decimal(out[i - 1].split(":", 1)[1]) + b = Decimal(out[i].split(":", 1)[1]) + if a <= 0: + m = b / 2 if b > 0 else Decimal("0.000001") + else: + m = (a * b).sqrt() if a * b > 0 else (a + b) / 2 + if m <= a or m >= b: + break + out.insert(i, "%s:%s" % (cur, fmt(m))) +print(" ".join(out[:steps])) +PY +} + +# shellcheck source=metrics.sh +source "$ROOT/metrics.sh" +METRICS_DIR="$REPORT_DIR" +export METRICS_DIR CUR WDB CLI_JS + +section "ladder · GOA withdraw (0 → random → max · ${LADDER_STEPS} steps)" +info "bank" "$BANK" +info "exchange" "$EX" +info "currency" "$CUR" +info "budget" "${LADDER_TIMEOUT_S}s" +info "steps" "${LADDER_STEPS} (fixed 0 + $((LADDER_STEPS - 2)) random + fixed max=${CUR}:${LADDER_MAX_AMOUNT})" + +if [ ! -f "$EXP_PW_FILE" ]; then + err bank "explorer password missing" "$EXP_PW_FILE" + exit 1 +fi +EXP_PW=$(tr -d '\n' <"$EXP_PW_FILE") + +# --- auto-account --- +t0=$(now_ms) +if ! curl -sS -m 30 -o "$SCRATCH/auto-account.json" "${BANK}/intro/auto-account.json"; then + err bank "auto-account.json unreachable" + exit 1 +fi +ms_auto=$(elapsed_ms "$t0") +if ! python3 -c 'import json; d=json.load(open("'"$SCRATCH"'/auto-account.json")); assert d.get("ok") or d.get("username")' 2>/dev/null; then + err bank "auto-account create failed" "$(head -c 120 "$SCRATCH/auto-account.json" | tr '\n' ' ')" + exit 1 +fi +ACCT_USER=$(python3 -c 'import json; print(json.load(open("'"$SCRATCH"'/auto-account.json"))["username"])') +ok "auto-account ${ACCT_USER} (${ms_auto}ms) — personal GOA:0; pool=explorer" +info "auto-account password" "(see $SCRATCH/auto-account.json — not logged)" + +# --- explorer token --- +t0=$(now_ms) +TOK=$(curl -sS -m 20 -u "${EXP_USER}:${EXP_PW}" \ + -H 'Content-Type: application/json' \ + -d '{"scope":"readwrite","duration":{"d_us":3600000000}}' \ + "${BANK}/accounts/${EXP_USER}/token" \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["access_token"])') +ms_tok=$(elapsed_ms "$t0") +[ -n "$TOK" ] || { err bank "explorer token failed"; exit 1; } +ok "explorer token (${ms_tok}ms)" + +# --- wallet exchange + ToS --- +t0=$(now_ms) +wcli exchanges add "$EX" >"$SCRATCH/ex-add.out" 2>&1 || true +wcli exchanges update "$EX" >"$SCRATCH/ex-upd.out" 2>&1 || true +wcli exchanges accept-tos "$EX" >"$SCRATCH/ex-tos.out" 2>&1 || true +ms_tos=$(elapsed_ms "$t0") +ok "wallet exchange + ToS (${ms_tos}ms)" + +LADDER_LIST=$(build_ladder) +: "${LADDER_MAX_RUNGS:=99}" +# shellcheck disable=SC2086 +set -- $LADDER_LIST +if [ "$#" -gt "$LADDER_MAX_RUNGS" ]; then + set -- $(printf '%s\n' "$@" | head -n "$LADDER_MAX_RUNGS") +fi +info "ladder plan" "$*" +printf '%s\n' "$@" >"$SCRATCH/ladder-plan.txt" + +OK_N=0 +FAIL_N_L=0 +STOP_REASON="" +STOP_AMOUNT="" +declare -a RUNG_JSON=() + +rung=0 +for AMT in "$@"; do + rung=$((rung + 1)) + if ladder_over; then + STOP_REASON="timeout budget ${LADDER_TIMEOUT_S}s" + warn "ladder" "budget exhausted after $OK_N ok rungs" + break + fi + + section "ladder · rung $rung $AMT" + tag=$(printf '%s' "$AMT" | tr '.:' '__') + range_note=$(printf '%s' "$LADDER_RANGES" | awk -v n="$rung" '{print $n}') + t_rung=$(now_ms) + ms_mint=0 ms_accept=0 ms_confirm=0 ms_settle=0 + note="" + status="FAIL" + WID="-" + + # mint from explorer pool + t0=$(now_ms) + code=$(curl -sS -m 30 -o "$SCRATCH/wd-$tag.json" -w '%{http_code}' \ + -H "Authorization: Bearer ${TOK}" \ + -H 'Content-Type: application/json' \ + -d "{\"amount\":\"${AMT}\"}" \ + "${BANK}/accounts/${EXP_USER}/withdrawals") + ms_mint=$(elapsed_ms "$t0") + WID=$(python3 -c 'import json;d=json.load(open("'"$SCRATCH"'/wd-'"$tag"'.json"));print(d.get("withdrawal_id") or "")' 2>/dev/null || true) + URI=$(python3 -c 'import json;u=json.load(open("'"$SCRATCH"'/wd-'"$tag"'.json")).get("taler_withdraw_uri") or "";print(u.replace(":443/","/"))' 2>/dev/null || true) + # numeric amount (for zero / settle special-cases) + AMT_NUM=$(python3 -c 'import sys; print(sys.argv[1].split(":",1)[-1])' "$AMT") + IS_ZERO=0 + python3 -c 'import sys; from decimal import Decimal; sys.exit(0 if Decimal(sys.argv[1])==0 else 1)' "$AMT_NUM" 2>/dev/null && IS_ZERO=1 + + if [ "$code" != "200" ] && [ "$code" != "201" ] || [ -z "$WID" ] || [ -z "$URI" ]; then + note="mint HTTP $code $(head -c 100 "$SCRATCH/wd-$tag.json" 2>/dev/null | tr '\n' ' ')" + ms_total=$(elapsed_ms "$t_rung") + if [ "$IS_ZERO" = "1" ]; then + # Probe only: bank may reject GOA:0 — record and continue ladder + status="ZERO_REJECT" + note="zero-withdraw rejected (expected possible): $note" + warn bank "mint $AMT" "$note" + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + OK_N=$((OK_N + 1)) + continue + fi + err bank "mint $AMT failed" "$note" + status="FAIL_MINT" + STOP_REASON="$note" + STOP_AMOUNT="$AMT" + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + FAIL_N_L=$((FAIL_N_L + 1)) + break + fi + ok "mint $AMT ($WID) ${ms_mint}ms" + + before=$(wallet_avail) + + # accept + t0=$(now_ms) + if wcli withdraw accept-uri --exchange "$EX" "$URI" >"$SCRATCH/accept-$tag.out" 2>&1; then + ms_accept=$(elapsed_ms "$t0") + ok "accept-uri $AMT ${ms_accept}ms" + else + ms_accept=$(elapsed_ms "$t0") + note="accept-uri failed: $(tail -c 120 "$SCRATCH/accept-$tag.out" | tr '\n' ' ')" + err wallet "accept $AMT" "$note" + status="FAIL_ACCEPT" + STOP_REASON="$note" + STOP_AMOUNT="$AMT" + ms_total=$(elapsed_ms "$t_rung") + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + FAIL_N_L=$((FAIL_N_L + 1)) + break + fi + + # Confirm ASAP when bank status is selected (do NOT block on long run-until-done first). + # Server-side auto-confirm only watches landing withdraw-watch.ids — ladder must confirm itself. + bank_st() { + curl -sS -m 8 "${BANK}/taler-integration/withdrawal-operation/${WID}" 2>/dev/null \ + | python3 -c 'import json,sys; print(json.load(sys.stdin).get("status",""))' 2>/dev/null || true + } + do_confirm() { + curl -sS -m 15 -o "$SCRATCH/conf-$tag.json" -w '%{http_code}' -X POST \ + -H "Authorization: Bearer ${TOK}" -H 'Content-Type: application/json' -d '{}' \ + "${BANK}/accounts/${EXP_USER}/withdrawals/${WID}/confirm" + } + force_select_if_needed() { + local st_now="$1" + [ "$st_now" = "pending" ] || [ -z "$st_now" ] || return 0 + local rpub epayto + rpub=$(python3 -c ' +import re,sys +paths=sys.argv[1:] +t="" +for p in paths: + try: t+=open(p).read() + except Exception: pass +m=re.search(r"reserve[_ ]?pub[\"=: ]+([A-Z0-9]{40,})", t, re.I) +if not m: m=re.search(r"\"reservePub\"\s*:\s*\"([^\"]+)\"", t) +print(m.group(1) if m else "") +' "$SCRATCH/accept-$tag.out" "$SCRATCH/tx-$tag.json" 2>/dev/null || true) + if [ -z "$rpub" ]; then + wcli transactions >"$SCRATCH/tx-$tag.json" 2>&1 || true + rpub=$(python3 -c ' +import re,sys +t=open(sys.argv[1]).read() +m=re.search(r"reserve[_ ]?pub[\"=: ]+([A-Z0-9]{40,})", t, re.I) +if not m: m=re.search(r"\"reservePub\"\s*:\s*\"([^\"]+)\"", t) +print(m.group(1) if m else "") +' "$SCRATCH/tx-$tag.json" 2>/dev/null || true) + fi + epayto=$(curl -sS -m 10 "${EX%/}/keys" 2>/dev/null | python3 -c ' +import json,sys +d=json.load(sys.stdin) +acc=d.get("accounts") or [] +for a in acc: + p=a.get("payto_uri") or a.get("payto_address") or "" + if "x-taler-bank" in p or "exchange" in p: + print(p); break +else: + if acc: print(acc[0].get("payto_uri") or "") +' 2>/dev/null || true) + if [ -n "$rpub" ] && [ -n "$epayto" ]; then + curl -sS -m 12 -o "$SCRATCH/force-sel-$tag.json" -X POST \ + -H 'Content-Type: application/json' \ + -d "{\"reserve_pub\":\"${rpub}\",\"selected_exchange\":\"${epayto}\"}" \ + "${BANK}/taler-integration/withdrawal-operation/${WID}" >/dev/null || true + info "force-select" "rpub=${rpub:0:12}…" + fi + } + + t0=$(now_ms) + conf_ok=0 + st="" + # Immediate poll: confirm the moment we see selected (no long wallet block first) + for i in $(seq 1 60); do + ladder_over && break + st=$(bank_st) + case "$st" in + selected) + ccode=$(do_confirm) + if [ "$ccode" = "204" ] || [ "$ccode" = "200" ]; then + conf_ok=1 + info "confirm" "immediate HTTP $ccode after selected (poll $i)" + else + note="confirm HTTP $ccode" + fi + break + ;; + confirmed) + conf_ok=1 + break + ;; + aborted) + note="withdrawal aborted by bank" + break + ;; + esac + # short shepherd only — never block tens of seconds on run-until-done + if [ $((i % 3)) -eq 1 ]; then + if command -v timeout >/dev/null 2>&1; then + timeout 4 wcli run-until-done >"$SCRATCH/sel-$tag-$i.out" 2>&1 || true + else + # macOS: background + kill + wcli run-until-done >"$SCRATCH/sel-$tag-$i.out" 2>&1 & + wpid=$! + sleep 4 + kill "$wpid" 2>/dev/null || true + wait "$wpid" 2>/dev/null || true + fi + fi + # if still pending after a few polls, force bank select once + if [ "$i" = "4" ] || [ "$i" = "12" ]; then + force_select_if_needed "$st" + fi + sleep 0.5 + done + ms_confirm=$(elapsed_ms "$t0") + if [ "$conf_ok" != "1" ]; then + note="${note:-confirm timeout last=$st}" + err bank "confirm $AMT" "$note" + status="FAIL_CONFIRM" + STOP_REASON="$note" + STOP_AMOUNT="$AMT" + ms_total=$(elapsed_ms "$t_rung") + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$rung" "$range_note" "$AMT" "$status" "$ms_mint" "$ms_accept" "$ms_confirm" "$ms_settle" "$ms_total" "$WID" "$note" >>"$TSV" + FAIL_N_L=$((FAIL_N_L + 1)) + break + fi + ok "confirm $AMT ${ms_confirm}ms (client, on selected)" + + # settle coins (zero amount: no balance increase expected) + t0=$(now_ms) + settled=0 + if [ "$IS_ZERO" = "1" ]; then + # short wallet run only + wcli run-until-done >"$SCRATCH/rud-$tag-zero.out" 2>&1 || true + settled=1 + note="zero-amount: no coin delta expected" + else + for r in $(seq 1 "$LADDER_SETTLE_ROUNDS"); do + ladder_over && break + wcli run-until-done >"$SCRATCH/rud-$tag-$r.out" 2>&1 || true + after=$(wallet_avail) + if python3 -c " +from decimal import Decimal +import sys +sys.exit(0 if Decimal(sys.argv[1]) > Decimal(sys.argv[2]) else 1) +" "$after" "$before" 2>/dev/null; then + settled=1 + break + fi + sleep "$LADDER_SETTLE_SLEEP" + done + fi + ms_settle=$(elapsed_ms "$t0") + after=$(wallet_avail) + ms_total=$(elapsed_ms "$t_rung") + + if [ "$settled" = "1" ]; then + status="OK" + note="${note:-avail=${CUR}:${after}}" + ok "settle $AMT → ${CUR}:${after} (settle ${ms_settle}ms, rung ${ms_total}ms)" + OK_N=$((OK_N + 1)) + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + else + xfer=$(curl -sS -m 10 "${BANK}/taler-integration/withdrawal-operation/${WID}" \ + | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("transfer_done"), d.get("status"))' 2>/dev/null || echo "?") + if echo "$xfer" | grep -qi True; then + status="OK_BANK_LAG" + note="bank transfer_done avail=${after} $xfer" + warn "settle lag $AMT" "bank confirmed; wallet still ${CUR}:${after} (${ms_settle}ms)" + OK_N=$((OK_N + 1)) + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + else + note="no coins avail=${after} $xfer" + err wallet "settle $AMT" "$note" + status="FAIL_SETTLE" + STOP_REASON="$note" + STOP_AMOUNT="$AMT" + FAIL_N_L=$((FAIL_N_L + 1)) + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + break + fi + fi +done + +ms_phase=$(python3 -c 'import sys,time; print(int((time.time()-float(sys.argv[1]))*1000))' "$SECTION_T0") + +# --- report --- +section "ladder · report" +info "auto-account" "$ACCT_USER" +info "ok_rungs" "$OK_N" +info "fail_rungs" "$FAIL_N_L" +info "phase_ms" "$ms_phase" +info "tsv" "$TSV" + +# speed summary via python +python3 - "$TSV" "$JSON" "$OK_N" "$FAIL_N_L" "${STOP_AMOUNT:-}" "${STOP_REASON:-}" "$ms_phase" "$ACCT_USER" "$CUR" <<'PY' +import csv, json, sys, statistics +tsv, jpath, ok_n, fail_n, stop_amt, stop_reason, phase_ms, acct, cur = sys.argv[1:10] +rows = [] +with open(tsv, newline="") as f: + r = csv.DictReader(f, delimiter="\t") + for row in r: + rows.append(row) + +def nums(key): + out = [] + for row in rows: + try: + out.append(int(row[key])) + except Exception: + pass + return out + +def stats(xs): + if not xs: + return {"n": 0} + return { + "n": len(xs), + "min_ms": min(xs), + "max_ms": max(xs), + "avg_ms": int(sum(xs) / len(xs)), + "p50_ms": int(statistics.median(xs)), + } + +report = { + "currency": cur, + "auto_account": acct, + "ok_rungs": int(ok_n), + "fail_rungs": int(fail_n), + "stopped_at_amount": stop_amt or None, + "stop_reason": stop_reason or None, + "phase_ms": int(phase_ms), + "timing": { + "mint": stats(nums("ms_mint")), + "accept": stats(nums("ms_accept")), + "confirm": stats(nums("ms_confirm")), + "settle": stats(nums("ms_settle")), + "rung_total": stats(nums("ms_total")), + }, + "rungs": rows, +} +json.dump(report, open(jpath, "w"), indent=2) +print("JSON", jpath) +print("--- speed (ms) ---") +for k, v in report["timing"].items(): + if v.get("n"): + print(f" {k:12} n={v['n']} min={v['min_ms']} p50={v['p50_ms']} avg={v['avg_ms']} max={v['max_ms']}") +print("--- rungs ---") +for row in rows: + print(f" {row['rung']:>2} {row['amount']:16} {row['status']:12} total={row['ms_total']}ms mint={row['ms_mint']} accept={row['ms_accept']} conf={row['ms_confirm']} set={row['ms_settle']}") +if stop_amt: + print(f"STOPPED at {stop_amt}: {stop_reason}") +else: + print("Completed without hard failure (or budget stop without fail).") +PY + +wcli balance 2>&1 | tee "$REPORT_DIR/balance-final.out" | tail -20 || true + +# Keep scratch if LADDER_REPORT_DIR set; else copy key files to /tmp +if [ -z "${LADDER_REPORT_DIR:-}" ]; then + KEEP="/tmp/goa-ladder-report-$(date +%Y%m%d-%H%M%S)" + mkdir -p "$KEEP" + cp -a "$TSV" "$JSON" "$SCRATCH/auto-account.json" "$SCRATCH/ladder-plan.txt" \ + "$REPORT_DIR/balance-final.out" "$KEEP/" 2>/dev/null || true + info "report_dir" "$KEEP" + echo "$KEEP" >"$SCRATCH/KEEP_PATH" +fi + +if [ "$FAIL_N_L" -gt 0 ]; then + blocker "ladder" "stopped at ${STOP_AMOUNT:-?} — ${STOP_REASON:-error}" + exit 1 +fi +if [ "$OK_N" -eq 0 ]; then + blocker "ladder" "no successful rungs" + exit 1 +fi +ok "ladder finished ok_rungs=$OK_N phase=${ms_phase}ms" +exit 0 diff --git a/scripts/taler-monitoring/taler-monitoring.sh b/scripts/taler-monitoring/taler-monitoring.sh index 321525b..30b8c30 100755 --- a/scripts/taler-monitoring/taler-monitoring.sh +++ b/scripts/taler-monitoring/taler-monitoring.sh @@ -26,6 +26,7 @@ Phases: sanity public + optional server server server-side only (SSH) e2e withdraw + pay (small amounts; remote aborts on login/KYC) + ladder GOA withdraw ladder (auto-account + explorer pool + timings) all urls + inside + versions + sanity + e2e (SSH phases only on koopa) Options: @@ -90,7 +91,7 @@ while [ $# -gt 0 ]; do CURRENCY_OVERRIDE="$2"; shift 2 ;; --no-probe) NO_PROBE=1; shift ;; - urls|inside|versions|sanity|server|e2e|all) PHASES+=("$1"); shift ;; + urls|inside|versions|sanity|server|e2e|ladder|goa-ladder|all) PHASES+=("$1"); shift ;; *) # bare domain shorthand: ./taler-monitoring.sh taler.net if [[ "$1" == *.* && "$1" != *://* && "$1" != -* ]]; then @@ -133,6 +134,10 @@ export EXPECT_CURRENCY SKIP_SSH LOCAL_STACK TALER_DOMAIN_PROBE export WITHDRAW_AMT PAY_AMT CREDIT_AMT MERCHANT_INSTANCE export E2E_FAKE_INCOMING E2E_REMOTE E2E_VARIABLE E2E_ATM_MAX export E2E_WITHDRAW_VALUES E2E_PAY_VALUES +export LADDER_RANGES LADDER_MAX_RUNGS LADDER_TIMEOUT_S LADDER_REPORT_DIR +export LADDER_SETTLE_ROUNDS LADDER_SETTLE_SLEEP EXP_PW_FILE EXP_USER +export LADDER_MAX_AMOUNT LADDER_INCLUDE_ZERO LADDER_INCLUDE_MAX +export LADDER_HIGH_FROM LADDER_HIGH_RUNGS export TALER_DOMAIN_APPLIED=1 # Default phases @@ -193,6 +198,7 @@ for p in "${PHASES[@]}"; do sanity) "$ROOT/check_sanity.sh" || ec=1 ;; server) "$ROOT/check_server.sh" || ec=1 ;; e2e) "$ROOT/check_e2e.sh" || ec=1 ;; + ladder|goa-ladder) "$ROOT/check_goa_ladder.sh" || ec=1 ;; esac done exit "$ec"