fix(monitoring): drop run-until-done from GOA ladder

This commit is contained in:
Hernâni Marques 2026-07-16 21:17:12 +02:00
parent ab8399d034
commit d2a2caacc8
No known key found for this signature in database

View file

@ -4,16 +4,19 @@
# bank.hacktivism.ch flow (landing): # bank.hacktivism.ch flow (landing):
# 1) GET /intro/auto-account.json → personal goa-account-* (GOA:0) # 1) GET /intro/auto-account.json → personal goa-account-* (GOA:0)
# 2) Mint pool withdrawals as explorer (shared pool) + confirm when selected # 2) Mint pool withdrawals as explorer (shared pool) + confirm when selected
# 3) wallet-cli accept-uri + run-until-done # 3) wallet-cli accept-uri only (no run-until-done — hangs / developer ban)
# 4) bank confirm when selected; settle = poll balance + bank transfer_done
# #
# Amounts: random within defined ranges, strictly increasing. # Amounts: random strictly increasing; 0 and max fixed.
# On first hard failure: stop, print timing report, exit 1. # On first hard failure: stop, print timing report, exit 1.
# Soft: GOA:0 / wallet 7006 (no denoms) → WARN and continue.
# #
# Env: # Env:
# LADDER_STEPS total rungs (default 23) = 0 + (N-2) random + max # LADDER_STEPS total rungs (default 23) = 0 + (N-2) random + max
# LADDER_MAX_AMOUNT fixed last pin (libeufin ceiling 4503599627370496) # LADDER_MAX_AMOUNT fixed last pin (libeufin ceiling 4503599627370496)
# LADDER_TIMEOUT_S default 3600 # LADDER_TIMEOUT_S default 3600
# LADDER_LOAD=0 skip host load snapshots # LADDER_LOAD=0 skip host load snapshots
# LADDER_SETTLE_ROUNDS / LADDER_SETTLE_SLEEP — balance poll only (no shepherd)
# EXP_PW_FILE, LADDER_REPORT_DIR, … # EXP_PW_FILE, LADDER_REPORT_DIR, …
# #
# Path: always [0] → strictly increasing random (log-uniform) → [max] # Path: always [0] → strictly increasing random (log-uniform) → [max]
@ -321,7 +324,7 @@ for AMT in "$@"; do
break break
fi fi
# Confirm ASAP when bank status is selected (do NOT block on long run-until-done first). # Confirm ASAP when bank status is selected. No run-until-done (hangs on macOS/wallet).
# Server-side auto-confirm only watches landing withdraw-watch.ids — ladder must confirm itself. # Server-side auto-confirm only watches landing withdraw-watch.ids — ladder must confirm itself.
bank_st() { bank_st() {
curl -sS -m 8 "${BANK}/taler-integration/withdrawal-operation/${WID}" 2>/dev/null \ curl -sS -m 8 "${BANK}/taler-integration/withdrawal-operation/${WID}" 2>/dev/null \
@ -404,20 +407,7 @@ else:
break break
;; ;;
esac esac
# short shepherd only — never block tens of seconds on run-until-done # if still pending after a few polls, force bank select once (no wallet shepherd)
if [ $((i % 3)) -eq 1 ]; then
if command -v timeout >/dev/null 2>&1; then
timeout 4 wcli run-until-done >"$SCRATCH/sel-$tag-$i.out" 2>&1 || true
else
# macOS: background + kill
wcli run-until-done >"$SCRATCH/sel-$tag-$i.out" 2>&1 &
wpid=$!
sleep 4
kill "$wpid" 2>/dev/null || true
wait "$wpid" 2>/dev/null || true
fi
fi
# if still pending after a few polls, force bank select once
if [ "$i" = "4" ] || [ "$i" = "12" ]; then if [ "$i" = "4" ] || [ "$i" = "12" ]; then
force_select_if_needed "$st" force_select_if_needed "$st"
fi fi
@ -438,18 +428,16 @@ else:
fi fi
ok "confirm $AMT ${ms_confirm}ms (client, on selected)" ok "confirm $AMT ${ms_confirm}ms (client, on selected)"
# settle coins (zero amount: no balance increase expected) # settle: poll wallet balance + bank transfer_done only — never run-until-done
t0=$(now_ms) t0=$(now_ms)
settled=0 settled=0
xfer="?"
if [ "$IS_ZERO" = "1" ]; then if [ "$IS_ZERO" = "1" ]; then
# short wallet run only
wcli run-until-done >"$SCRATCH/rud-$tag-zero.out" 2>&1 || true
settled=1 settled=1
note="zero-amount: no coin delta expected" note="zero-amount: no coin delta expected"
else else
for r in $(seq 1 "$LADDER_SETTLE_ROUNDS"); do for r in $(seq 1 "$LADDER_SETTLE_ROUNDS"); do
ladder_over && break ladder_over && break
wcli run-until-done >"$SCRATCH/rud-$tag-$r.out" 2>&1 || true
after=$(wallet_avail) after=$(wallet_avail)
if python3 -c " if python3 -c "
from decimal import Decimal from decimal import Decimal
@ -459,12 +447,23 @@ sys.exit(0 if Decimal(sys.argv[1]) > Decimal(sys.argv[2]) else 1)
settled=1 settled=1
break break
fi fi
xfer=$(curl -sS -m 10 "${BANK}/taler-integration/withdrawal-operation/${WID}" \
| python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("transfer_done"), d.get("status"))' 2>/dev/null || echo "?")
# bank done is enough to leave settle without hanging on wallet
if echo "$xfer" | grep -qi True; then
note="bank transfer_done (no run-until-done) avail=${after} $xfer"
break
fi
sleep "$LADDER_SETTLE_SLEEP" sleep "$LADDER_SETTLE_SLEEP"
done done
fi fi
ms_settle=$(elapsed_ms "$t0") ms_settle=$(elapsed_ms "$t0")
after=$(wallet_avail) after=$(wallet_avail)
ms_total=$(elapsed_ms "$t_rung") ms_total=$(elapsed_ms "$t_rung")
if [ -z "${xfer:-}" ] || [ "$xfer" = "?" ]; then
xfer=$(curl -sS -m 10 "${BANK}/taler-integration/withdrawal-operation/${WID}" \
| python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("transfer_done"), d.get("status"))' 2>/dev/null || echo "?")
fi
if [ "$settled" = "1" ]; then if [ "$settled" = "1" ]; then
status="OK" status="OK"
@ -472,26 +471,21 @@ sys.exit(0 if Decimal(sys.argv[1]) > Decimal(sys.argv[2]) else 1)
ok "settle $AMT${CUR}:${after} (settle ${ms_settle}ms, rung ${ms_total}ms)" ok "settle $AMT${CUR}:${after} (settle ${ms_settle}ms, rung ${ms_total}ms)"
OK_N=$((OK_N + 1)) OK_N=$((OK_N + 1))
echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV"
elif echo "$xfer" | grep -qi True; then
status="OK_BANK"
note="bank transfer_done avail=${after} $xfer (no run-until-done)"
ok "settle $AMT bank transfer_done (wallet avail=${CUR}:${after}, ${ms_settle}ms)"
OK_N=$((OK_N + 1))
echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV"
else else
xfer=$(curl -sS -m 10 "${BANK}/taler-integration/withdrawal-operation/${WID}" \ note="no coins / no transfer_done avail=${after} $xfer"
| python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("transfer_done"), d.get("status"))' 2>/dev/null || echo "?") err wallet "settle $AMT" "$note"
if echo "$xfer" | grep -qi True; then status="FAIL_SETTLE"
status="OK_BANK_LAG" STOP_REASON="$note"
note="bank transfer_done avail=${after} $xfer" STOP_AMOUNT="$AMT"
warn settle "lag after $AMT" \ FAIL_N_L=$((FAIL_N_L + 1))
"problem: bank transfer_done but wallet balance not increased yet (avail=${CUR}:${after}, settle ${ms_settle}ms) — coins may still be in flight" echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV"
OK_N=$((OK_N + 1)) break
echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV"
else
note="no coins avail=${after} $xfer"
err wallet "settle $AMT" "$note"
status="FAIL_SETTLE"
STOP_REASON="$note"
STOP_AMOUNT="$AMT"
FAIL_N_L=$((FAIL_N_L + 1))
echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV"
break
fi
fi fi
done done