From ba899dd8b528c4bda6d06a54860edcab4d5c92a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sat, 29 Aug 2026 02:40:48 +0200 Subject: [PATCH] fix 1.23.8: ladder harness-style wallet serve for coin settle --- CLI-AUTOMATION-NOTES.md | 12 +++ TESTS.md | 2 +- VERSION | 2 +- VERSIONS.md | 1 + check_amount_ladder.sh | 57 ++++++++++---- ladder/README.md | 13 ++++ ladder/lib_pay.sh | 12 +-- ladder/lib_wallet_serve.sh | 156 +++++++++++++++++++++++++++++++++++++ 8 files changed, 233 insertions(+), 22 deletions(-) create mode 100644 ladder/lib_wallet_serve.sh diff --git a/CLI-AUTOMATION-NOTES.md b/CLI-AUTOMATION-NOTES.md index 22f75e5..b4823b1 100644 --- a/CLI-AUTOMATION-NOTES.md +++ b/CLI-AUTOMATION-NOTES.md @@ -35,6 +35,18 @@ Legend: --- + +## 1a. Harness-style wallet daemon (mon ladder + e2e) + +| | | +|--|--| +| **Seen** | Ladder `OK_BANK` with `avail=0` while mon e2e / harness get full coins; force-select 400 without `selected_exchange` | +| **Area** | **cli** / **core** / **ops** | +| **Problem** | Harness uses a long-lived **WalletService** (unix socket client). One-shot `wallet-cli` starts and stops the shepherd each call — after bank `transfer_done`, coins often stay `pendingIncoming` until something keeps the wallet awake. `run-until-done` is banned (hangs). | +| **Workaround** | Mon: `advanced serve --unix-path $WSOCK` then `--wallet-connection=$WSOCK` (e2e default; ladder `LADDER_WALLET_SERVE=1` via `ladder/lib_wallet_serve.sh`). Force-select body must be `{"reserve_pub":"…","selected_exchange":""}`. Mint may include `"exchange_url":"$EX"`. Poll balance + bank status; never `run-until-done`. | +| **Wanted** | Documented wallet-cli “serve for automation” profile matching harness WalletClient. | + + ## 2. Withdraw completion is not observable from CLI alone | | | diff --git a/TESTS.md b/TESTS.md index 7364968..6e24e73 100644 --- a/TESTS.md +++ b/TESTS.md @@ -25,7 +25,7 @@ Every check line has a **global** run number and a **grouped** id: | **sanity** | `check_sanity.sh` | `bank` `exchange` `merchant` | | **server** | `check_server.sh` | (flat `server-NN` or host groups) | | **e2e** | `check_e2e.sh` | `prereq` `load` `bank` `wallet` `bankwd` `settle` `pay` `shop` `paivana` `dig` `report` | -| **ladder** | `check_amount_ladder.sh` + `ladder/lib_pay.sh` | `plan` `load` `withdraw` `pay` `report` — classic / max-search; phases `ladder` · `max-ladder` · `goa-ladder` (alias) | +| **ladder** | `check_amount_ladder.sh` + `ladder/lib_pay.sh` + `ladder/lib_wallet_serve.sh` | `plan` `load` `withdraw` `pay` `report` — classic / max-search; harness-style `advanced serve` (`LADDER_WALLET_SERVE=1`); phases `ladder` · `max-ladder` · `goa-ladder` (alias) | **Why groups:** flat `www-001`…`www-080` was hard to talk about. `www.bank-04` / `e2e.bankwd-02` pin the failure to a logical block. diff --git a/VERSION b/VERSION index 2d27ccb..82bfa5c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.23.7 +1.23.8 diff --git a/VERSIONS.md b/VERSIONS.md index 58d20c9..af708d6 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| +| **v1.23.8** | 2026-08-29 | **Bugfix / ladder:** harness-style wallet daemon — `ladder/lib_wallet_serve.sh` (`advanced serve` + `--wallet-connection`, default `LADDER_WALLET_SERVE=1`). Mint sends `exchange_url`; settle/pay-wait skip one-shot `run-pending` when serve is up. Aligns mon ladder with taler-harness WalletService + e2e serve path (no run-until-done). CLI-AUTOMATION-NOTES §1a. | | **v1.23.7** | 2026-07-22 | **FP stage host-agent:** ship `taler-monitoring-fp-stage.{service,timer}`; `install-host-agent.sh --fp-stage` (stagepaysan) enables FP mon only, disables GOA timers on that host, enforces suite tip knobs. Same sticky/publish pipeline as GOA. | | **v1.23.6** | 2026-07-22 | **Strict suite tip for mon pages:** every host-agent run must `fetch`+`reset` to `origin/main` and **refuse** if HEAD ≠ remote tip (`SUITE_REQUIRE_REMOTE_TIP=1`). Overlay is refreshed **from** the tip after update (never re-applied over tip by default — was publishing stale site-gen). `git clean` after reset. Mon HTML still embeds that COMMIT in sticky/meta. | | **v1.23.5** | 2026-07-22 | **Bugfix / host-agent:** multi-agent shared `/var/www/monitoring-sites` — stop full-tree `rsync --delete` (wiped sibling pages e.g. `/monitoring-max-ladder/` after main/surface/aptdeploy runs). Publish **only this job’s** `HTML_OK_DIR`/`HTML_ERR_DIR` per host; `apply-monitoring-live.sh` merges without tree-delete. | diff --git a/check_amount_ladder.sh b/check_amount_ladder.sh index 2f5da98..c91528b 100755 --- a/check_amount_ladder.sh +++ b/check_amount_ladder.sh @@ -10,8 +10,8 @@ # Flow (bank landings): # 1) GET /intro/auto-account.json → personal *account-* (balance 0) # 2) Mint pool withdrawals as explorer + confirm when selected -# 3) wallet-cli accept-uri only (no run-until-done) -# 4) bank confirm when selected; settle = balance + transfer_done +# 3) wallet-cli accept-uri via harness-style advanced serve (no run-until-done) +# 4) bank confirm when selected; settle = balance + transfer_done (serve keeps shepherd) # 5) pay via free-amount template (GOA: goa-free) or private orders # # Modes (LADDER_MODE): @@ -89,6 +89,9 @@ elapsed_ms() { : "${LADDER_MAX_SEED:=}" # After withdraw, wait for spendable available>0 before pay (secs total; shows countdown) : "${LADDER_PAY_WAIT_AVAILABLE_S:=180}" +# Harness-style long-lived wallet daemon (advanced serve). 1=on (default). +# Without serve, coins often stay pendingIncoming after bank transfer_done. +: "${LADDER_WALLET_SERVE:=1}" # Historic libeufin-ish absolute ceiling used as default LADDER_MAX_AMOUNT on GOA LADDER_ABS_CEILING="4503599627370496" @@ -225,13 +228,18 @@ ladder_over() { } wcli() { - # taler-helper-sqlite3 is Python ≥3.11; prefer Homebrew/local python on macOS - # so Apple /usr/bin/python3 3.9 does not FATAL the sqlite backend (no reservePub). + # Prefer harness-style serve socket (long-lived shepherd); else one-shot --wallet-db. + # PATH: taler-helper-sqlite3 needs Python ≥3.11 (Homebrew/local on macOS). local _path="${PATH:-}" case ":${_path}:" in *:/opt/homebrew/bin:*) ;; - *) _path="/opt/homebrew/bin:/usr/local/bin:${_path}" ;; + *) _path="/opt/homebrew/bin:/usr/local/bin:${HOME}/.local/bin:${_path}" ;; esac + PATH="$_path" + if command -v wallet_serve_wcli >/dev/null 2>&1; then + wallet_serve_wcli "$@" + return $? + fi if [ -n "${CLI_JS:-}" ] && [ -f "$CLI_JS" ]; then PATH="$_path" node "$CLI_JS" --wallet-db="$WDB" --no-throttle "$@" else @@ -481,6 +489,8 @@ source "$ROOT/metrics.sh" # Payment module (classic + max-search payable) # shellcheck source=ladder/lib_pay.sh source "$ROOT/ladder/lib_pay.sh" +# shellcheck source=ladder/lib_wallet_serve.sh +source "$ROOT/ladder/lib_wallet_serve.sh" METRICS_DIR="$REPORT_DIR" ALT_UNITS_FILE="${REPORT_DIR}/alt_unit_names.json" export METRICS_DIR CUR WDB CLI_JS ALT_UNITS_FILE @@ -578,18 +588,31 @@ wallet_prepare() { local label="${1:-wallet}" WDB="$SCRATCH/wallet-${label}.sqlite3" export WDB - if [ ! -f "$WDB" ]; then - wcli exchanges add "$EX" >"$SCRATCH/ex-add-$label.out" 2>&1 || true - wcli exchanges update "$EX" >"$SCRATCH/ex-upd-$label.out" 2>&1 || true - wcli exchanges accept-tos "$EX" >"$SCRATCH/ex-tos-$label.out" 2>&1 || true + # Start harness-style daemon once per DB (shepherd stays up for wirewatch settle). + if [ ! -f "$WDB" ] || [ -z "${WSERVE_PID:-}" ] || ! kill -0 "${WSERVE_PID:-0}" 2>/dev/null; then + wallet_serve_install_trap + if wallet_serve_start; then + if [ -n "${WSERVE_PID:-}" ] && [ -S "${WSOCK:-}" ]; then + ok "wallet serve" "pid=$WSERVE_PID sock=$WSOCK (harness-style; no run-until-done)" + fi + else + if [ "${LADDER_WALLET_SERVE:-1}" = "1" ]; then + warn wallet "wallet serve unavailable" "problem: advanced serve did not come up — falling back to one-shot CLI (coins may stay pendingIncoming). See $SCRATCH/wallet-serve.log" + fi + fi fi + # ToS / exchange — via serve socket when up + wcli exchanges add "$EX" >"$SCRATCH/ex-add-$label.out" 2>&1 || true + wcli exchanges update "$EX" >"$SCRATCH/ex-upd-$label.out" 2>&1 || true + wcli exchanges accept-tos "$EX" >"$SCRATCH/ex-tos-$label.out" 2>&1 || true } t0=$(now_ms) rm -f "$SCRATCH/wallet-main.sqlite3" +wallet_serve_stop 2>/dev/null || true wallet_prepare "main" ms_tos=$(elapsed_ms "$t0") -ok "wallet exchange + ToS (${ms_tos}ms) — cumulative DB for withdraw+pay (no run-until-done)" +ok "wallet exchange + ToS (${ms_tos}ms) — cumulative DB for withdraw+pay (serve=${LADDER_WALLET_SERVE:-1})" : "${LADDER_MAX_RUNGS:=99}" if [ "${LADDER_MODE}" = "max" ]; then @@ -664,10 +687,11 @@ ladder_withdraw_rung() { # mint from explorer pool t0=$(now_ms) + # e2e/harness: pass exchange_url so bank binds the op to our exchange code=$(curl -sS -m 30 -o "$SCRATCH/wd-$tag.json" -w '%{http_code}' \ -H "Authorization: Bearer ${TOK}" \ -H 'Content-Type: application/json' \ - -d "{\"amount\":\"${AMT}\"}" \ + -d "{\"amount\":\"${AMT}\",\"exchange_url\":\"${EX}\"}" \ "${BANK}/accounts/${EXP_USER}/withdrawals") ms_mint=$(elapsed_ms "$t0") WID=$(python3 -c 'import json;d=json.load(open("'"$SCRATCH"'/wd-'"$tag"'.json"));print(d.get("withdrawal_id") or "")' 2>/dev/null || true) @@ -960,11 +984,14 @@ sys.exit(0 if Decimal(sys.argv[1]) > Decimal(sys.argv[2]) else 1) | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("transfer_done"), d.get("status"))' 2>/dev/null || echo "?") if echo "$xfer" | grep -qi True; then bank_done=1 - # nudge wallet to pick up wire (bounded; not run-until-done) + # With harness-style serve, shepherd is already running — poll balance only. + # One-shot CLI: bounded run-pending (never run-until-done). wcli transactions >"$SCRATCH/tx-settle-$tag.out" 2>&1 || true - if command -v timeout >/dev/null 2>&1 || command -v gtimeout >/dev/null 2>&1; then - _to=$(command -v gtimeout 2>/dev/null || command -v timeout) - "$_to" 8 wcli advanced run-pending >"$SCRATCH/run-pending-settle.out" 2>&1 || true + if [ -z "${WSERVE_PID:-}" ] || [ ! -S "${WSOCK:-}" ]; then + if command -v timeout >/dev/null 2>&1 || command -v gtimeout >/dev/null 2>&1; then + _to=$(command -v gtimeout 2>/dev/null || command -v timeout) + "$_to" 8 wcli advanced run-pending >"$SCRATCH/run-pending-settle.out" 2>&1 || true + fi fi # after bank done, still prefer coins; allow remaining rounds for available note="bank transfer_done waiting coins avail=${after} $xfer" diff --git a/ladder/README.md b/ladder/README.md index 3c6e826..171454f 100644 --- a/ladder/README.md +++ b/ladder/README.md @@ -30,3 +30,16 @@ There is **no** `check_goa_ladder.sh` — removed; use the phases above. | `LADDER_FREE_TEMPLATE` | `goa-free` | template id (hacktivism) | | `LADDER_FREE_TEMPLATE_INSTANCE` | merchant instance | e.g. `goa-demo-cp4zqk` | | `LADDER_PAY_WAIT_AVAILABLE_S` | `90` | after withdraw, wait for spendable **available**>0; logs **withdraw board in start order** (OK/OK_BANK) with live `pendingIncoming`→`done` + timer (same wallet DB) | + + +## Wallet daemon (harness-style) + +taler-harness keeps a **WalletService** up for the whole withdraw/confirm/wirewatch path. +Mon ladder defaults to the same idea: + +| Env | Default | Meaning | +|-----|---------|---------| +| `LADDER_WALLET_SERVE` | `1` | `advanced serve` + `--wallet-connection` (no `run-until-done`) | +| `LADDER_WALLET_SERVE=0` | — | one-shot CLI only (coins may stay `pendingIncoming` / `OK_BANK`) | + +Implementation: `ladder/lib_wallet_serve.sh` (shared with the e2e pattern). diff --git a/ladder/lib_pay.sh b/ladder/lib_pay.sh index 01ba819..c21d4d1 100644 --- a/ladder/lib_pay.sh +++ b/ladder/lib_pay.sh @@ -371,12 +371,14 @@ ladder_wait_available() { info pay "timer 0s left — stop waiting (available still ${CUR}:0)" break fi - # nudge wallet (same DB); do not run-until-done (can hang) + # serve: shepherd already running (harness-style). one-shot: bounded run-pending. wcli transactions >"$SCRATCH/tx-wait-pay.out" 2>&1 || true - if command -v timeout >/dev/null 2>&1 || command -v gtimeout >/dev/null 2>&1; then - local _to - _to=$(command -v gtimeout 2>/dev/null || command -v timeout) - "$_to" 8 wcli advanced run-pending >"$SCRATCH/run-pending.out" 2>&1 || true + if [ -z "${WSERVE_PID:-}" ] || [ ! -S "${WSOCK:-}" ]; then + if command -v timeout >/dev/null 2>&1 || command -v gtimeout >/dev/null 2>&1; then + local _to + _to=$(command -v gtimeout 2>/dev/null || command -v timeout) + "$_to" 8 wcli advanced run-pending >"$SCRATCH/run-pending.out" 2>&1 || true + fi fi # progress board every ~5s (tick every 2s → every 3rd) if [ $((tick % 3)) -eq 0 ]; then diff --git a/ladder/lib_wallet_serve.sh b/ladder/lib_wallet_serve.sh new file mode 100644 index 0000000..e38bfaa --- /dev/null +++ b/ladder/lib_wallet_serve.sh @@ -0,0 +1,156 @@ +# ladder/lib_wallet_serve.sh — long-lived wallet daemon (harness-style) +# +# taler-harness uses WalletService + WalletClient over a unix socket so the +# wallet shepherd stays up while bank confirm + wirewatch complete. Mon e2e +# already mirrors that with `advanced serve`. Ladder historically used one-shot +# CLI only → coins often stayed pendingIncoming (OK_BANK) until pay-wait. +# +# Source after lib.sh. Callers set: SCRATCH, WDB, CLI_JS or WALLET_CLI, NODE_BIN. +# Optional: LADDER_WALLET_SERVE=1 (default), E2E_WALLET_SERVE (e2e alias). +# +# Public: +# wallet_serve_resolve_bins — NODE_BIN + CLI path +# wallet_serve_start — advanced serve --unix-path +# wallet_serve_stop — kill daemon +# wallet_serve_wcli args… — client via socket if up, else --wallet-db +# wallet_serve_install_trap — EXIT cleanup (idempotent) + +: "${LADDER_WALLET_SERVE:=1}" +# e2e reuses same default name +: "${E2E_WALLET_SERVE:=${LADDER_WALLET_SERVE}}" + +WSOCK="${WSOCK:-}" +WSERVE_PID="${WSERVE_PID:-}" +_WALLET_SERVE_TRAP_INSTALLED="${_WALLET_SERVE_TRAP_INSTALLED:-0}" + +wallet_serve_resolve_bins() { + # Prefer explicit WALLET_CLI / CLI_JS (.mjs); else find_wallet_cli from lib.sh + if [ -z "${CLI_JS:-}" ] || [ ! -f "${CLI_JS:-}" ]; then + if [ -n "${WALLET_CLI:-}" ] && [ -f "${WALLET_CLI}" ]; then + CLI_JS="$WALLET_CLI" + elif command -v find_wallet_cli >/dev/null 2>&1; then + CLI_JS=$(find_wallet_cli 2>/dev/null || true) + fi + fi + if [ -n "${CLI_JS:-}" ] && [ -f "$CLI_JS" ]; then + WALLET_CLI="$CLI_JS" + fi + if [ -z "${NODE_BIN:-}" ] || [ ! -x "${NODE_BIN:-}" ]; then + NODE_BIN=$(command -v node 2>/dev/null || true) + fi + # PATH for taler-helper-sqlite3 (harness/e2e preflight) + local _path="${PATH:-}" + case ":${_path}:" in + *:/opt/homebrew/bin:*) ;; + *) _path="/opt/homebrew/bin:/usr/local/bin:${HOME}/.local/bin:${_path}" ;; + esac + PATH="$_path" + export PATH NODE_BIN CLI_JS WALLET_CLI +} + +wallet_serve_stop() { + if [ -n "${WSERVE_PID:-}" ] && kill -0 "$WSERVE_PID" 2>/dev/null; then + kill "$WSERVE_PID" 2>/dev/null || true + wait "$WSERVE_PID" 2>/dev/null || true + fi + WSERVE_PID="" + if [ -n "${WSOCK:-}" ]; then + rm -f "$WSOCK" 2>/dev/null || true + fi +} + +wallet_serve_start() { + # Returns 0 if serve is up (or intentionally disabled); 1 if wanted but failed. + wallet_serve_resolve_bins + local want="${LADDER_WALLET_SERVE:-1}" + # e2e callers may only set E2E_WALLET_SERVE + if [ "${E2E_WALLET_SERVE:-}" = "0" ]; then + want=0 + fi + if [ "$want" != "1" ]; then + return 0 + fi + if [ -z "${NODE_BIN:-}" ] || [ ! -x "${NODE_BIN}" ]; then + return 1 + fi + if [ -z "${WALLET_CLI:-}" ] || [ ! -f "${WALLET_CLI}" ]; then + return 1 + fi + if [ -z "${WDB:-}" ]; then + return 1 + fi + if [ -z "${SCRATCH:-}" ]; then + SCRATCH=$(mktemp -d) + fi + WSOCK="${WSOCK:-$SCRATCH/wallet.sock}" + wallet_serve_stop + rm -f "$WSOCK" + # harness: daemon owns DB; clients use --wallet-connection only (no concurrent --wallet-db) + "$NODE_BIN" "$WALLET_CLI" --wallet-db="$WDB" --no-throttle --skip-defaults \ + advanced serve --unix-path "$WSOCK" \ + >"${SCRATCH}/wallet-serve.log" 2>&1 & + WSERVE_PID=$! + local i + for i in $(seq 1 50); do + if [ -S "$WSOCK" ] && kill -0 "$WSERVE_PID" 2>/dev/null; then + return 0 + fi + # died early + if ! kill -0 "$WSERVE_PID" 2>/dev/null; then + break + fi + sleep 0.1 + done + wallet_serve_stop + return 1 +} + +wallet_serve_install_trap() { + [ "${_WALLET_SERVE_TRAP_INSTALLED}" = "1" ] && return 0 + _WALLET_SERVE_TRAP_INSTALLED=1 + # Append to existing EXIT trap if any — stop serve first + # shellcheck disable=SC2064 + trap 'wallet_serve_stop' EXIT +} + +# Run wallet-cli: prefer live serve socket (harness WalletClient path). +# Optional first arg: max seconds (integer) → wrapped with timeout when available. +wallet_serve_wcli() { + wallet_serve_resolve_bins + local maxc="" + if [[ "${1:-}" =~ ^[0-9]+$ ]]; then + maxc=$1 + shift + fi + local -a cmd + if [ -n "${NODE_BIN:-}" ] && [ -x "${NODE_BIN}" ] && [ -n "${WALLET_CLI:-}" ] && [ -f "${WALLET_CLI}" ]; then + if [ -n "${WSERVE_PID:-}" ] && [ -S "${WSOCK:-}" ] && kill -0 "$WSERVE_PID" 2>/dev/null; then + cmd=( "$NODE_BIN" "$WALLET_CLI" --wallet-connection="$WSOCK" --no-throttle "$@" ) + else + cmd=( "$NODE_BIN" "$WALLET_CLI" --wallet-db="${WDB:?}" --no-throttle --skip-defaults "$@" ) + fi + elif [ -n "${CLI_JS:-}" ] && [ -f "$CLI_JS" ]; then + if [ -n "${WSERVE_PID:-}" ] && [ -S "${WSOCK:-}" ]; then + cmd=( node "$CLI_JS" --wallet-connection="$WSOCK" --no-throttle "$@" ) + else + cmd=( node "$CLI_JS" --wallet-db="${WDB:?}" --no-throttle "$@" ) + fi + else + if [ -n "${WSERVE_PID:-}" ] && [ -S "${WSOCK:-}" ]; then + cmd=( taler-wallet-cli --wallet-connection="$WSOCK" --no-throttle "$@" ) + else + cmd=( taler-wallet-cli --wallet-db="${WDB:?}" --no-throttle "$@" ) + fi + fi + if [ -n "$maxc" ]; then + if command -v timeout >/dev/null 2>&1; then + timeout -k 2 "$maxc" "${cmd[@]}" + elif command -v gtimeout >/dev/null 2>&1; then + gtimeout --kill-after=2 "$maxc" "${cmd[@]}" + else + "${cmd[@]}" + fi + else + "${cmd[@]}" + fi +}