hernani/koopa-admin-log
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

monitoring: final stats dashboard and ladder reserve matching

Browse source 
Print a boxed FINAL STATISTICS block after e2e/ladder (coins, flow,
tendencies, perf, load) and match force-select reserve_pub to the current
withdrawal so cumulative ladder wallets do not reuse stale pubs.
This commit is contained in:
Hernâni Marques 2026-07-17 08:28:00 +02:00
parent 48f77139c5
commit b65018cd0b
No known key found for this signature in database
4 changed files with 609 additions and 105 deletions
Download patch file Download diff file Expand all files Collapse all files

223
scripts/taler-monitoring/check_goa_ladder.sh
View file

@ -483,58 +483,138 @@ for AMT in "$@"; do
-H "Authorization: Bearer ${TOK}" -H 'Content-Type: application/json' -d '{}' \
"${BANK}/accounts/${EXP_USER}/withdrawals/${WID}/confirm"
}
extract_rpub() {
# Prefer *last* match (current withdraw), not first (stale from older accepts/tx).
python3 -c '
import re, sys, json
paths = sys.argv[1:]
found = []
blob = ""
for p in paths:
try:
blob += open(p, errors="replace").read() + "\n"
except Exception:
pass
for pat in (
r"\"reserve_pub\"\s*:\s*\"([A-Z0-9]+)\"",
r"\"reservePub\"\s*:\s*\"([A-Z0-9]+)\"",
r"reserve_pub[\"\s:=]+([A-Z0-9]{40,})",
r"reservePub[\"\s:=]+([A-Z0-9]{40,})",
r"reserve[_ ]?pub[\"=: ]+([A-Z0-9]{40,})",
):
found.extend(m.group(1) for m in re.finditer(pat, blob, re.I))
def walk(o):
# Collect reserve_pub candidates for *this* withdrawal (WID + amount).
# Cumulative wallets re-print old reserves in accept/tx dumps — never trust a single "last" blindly.
# Prints unique pubs one per line, preferred order first.
extract_rpubs_for_wid() {
python3 - "$WID" "$AMT" "$SCRATCH/accept-$tag.out" "$SCRATCH/tx-$tag.json" "$SCRATCH/used-rpubs.txt" <<'PY'
import json, re, sys
from pathlib import Path
wid = sys.argv[1]
amt = sys.argv[2]
paths = sys.argv[3:5]
used_path = sys.argv[5]
used = set()
if Path(used_path).is_file():
used = {ln.strip() for ln in open(used_path) if ln.strip()}
def walk_collect(o, bag, ctx=None):
ctx = dict(ctx or {})
if isinstance(o, dict):
for k, v in o.items():
if k.lower() in ("reserve_pub", "reservepub") and isinstance(v, str) and len(v) >= 40:
found.append(v)
walk(v)
kl = str(k).lower()
if kl in ("withdrawal_id", "withdraw_id", "wopid", "id") and isinstance(v, str):
ctx["id"] = v
if kl in ("amount", "rawamount", "instructedamount") and isinstance(v, str):
ctx["amount"] = v
if kl in ("taler_withdraw_uri", "talerwithdrawuri", "uri") and isinstance(v, str):
ctx["uri"] = v
if kl in ("reserve_pub", "reservepub") and isinstance(v, str) and len(v) >= 40:
bag.append((v, dict(ctx)))
walk_collect(v, bag, ctx)
elif isinstance(o, list):
for i in o:
walk(i)
for line in blob.splitlines():
line = line.strip()
if not line.startswith("{"):
continue
walk_collect(i, bag, ctx)
raw_pubs = [] # ordered as found
scored = [] # (score, pub) higher better
blob_all = ""
for p in paths:
try:
walk(json.loads(line))
blob_all += open(p, errors="replace").read() + "\n"
except Exception:
pass
if found:
print(found[-1])
' "$@" 2>/dev/null || true
# 1) full JSON objects in files
for p in paths:
try:
t = open(p, errors="replace").read()
except Exception:
continue
# whole-file JSON
for m in re.finditer(r"\{", t):
try:
o = json.loads(t[m.start():])
except Exception:
continue
bag = []
walk_collect(o, bag)
for pub, ctx in bag:
raw_pubs.append(pub)
score = 0
cid = str(ctx.get("id") or "")
camt = str(ctx.get("amount") or "")
curi = str(ctx.get("uri") or "")
if wid and wid in cid:
score += 100
if wid and wid in curi:
score += 80
if amt and (camt == amt or camt.endswith(amt.split(":", 1)[-1])):
score += 40
if pub in used:
score -= 200
scored.append((score, pub))
# 2) regex fallback on accept file only (more current)
try:
acc = open(paths[0], errors="replace").read()
except Exception:
acc = ""
for pat in (
r"\"reserve_pub\"\s*:\s*\"([A-Z0-9]{40,})\"",
r"\"reservePub\"\s*:\s*\"([A-Z0-9]{40,})\"",
r"reserve_pub[\"\s:=]+([A-Z0-9]{40,})",
):
for m in re.finditer(pat, acc, re.I):
pub = m.group(1)
raw_pubs.append(pub)
score = 10
# proximity to WID in accept output
window = acc[max(0, m.start() - 400) : m.end() + 400]
if wid and wid in window:
score += 100
if amt and amt in window:
score += 30
if pub in used:
score -= 200
scored.append((score, pub))
# prefer high score, then later occurrence
order = []
seen = set()
for score, pub in sorted(scored, key=lambda x: (-x[0],), reverse=False):
# sort by score desc: use reverse sorted
pass
for score, pub in sorted(scored, key=lambda x: x[0], reverse=True):
if pub in seen or pub in used:
continue
seen.add(pub)
order.append(pub)
# append unused raw in reverse (newest-ish)
for pub in reversed(raw_pubs):
if pub in seen or pub in used:
continue
seen.add(pub)
order.append(pub)
for pub in order:
print(pub)
PY
}
mark_rpub_used() {
local p="$1"
[ -n "$p" ] || return 0
mkdir -p "$SCRATCH" 2>/dev/null || true
grep -qxF "$p" "$SCRATCH/used-rpubs.txt" 2>/dev/null || echo "$p" >>"$SCRATCH/used-rpubs.txt"
}
force_select_if_needed() {
local st_now="$1"
[ "$st_now" = "pending" ] || [ -z "$st_now" ] || return 0
local rpub epayto code_fs
# current accept only first — avoid reusing reserve from previous rungs via tx dump
rpub=$(extract_rpub "$SCRATCH/accept-$tag.out")
if [ -z "$rpub" ]; then
wcli transactions >"$SCRATCH/tx-$tag.json" 2>&1 || true
rpub=$(extract_rpub "$SCRATCH/tx-$tag.json")
fi
local rpub epayto code_fs any=0
# refresh tx dump each try (wallet may attach reserve late)
wcli transactions >"$SCRATCH/tx-$tag.json" 2>&1 || true
epayto=$(curl -sS -m 10 "${EX%/}/keys" 2>/dev/null | python3 -c '
import json,sys
d=json.load(sys.stdin)
@ -546,31 +626,46 @@ for a in acc:
else:
if acc: print(acc[0].get("payto_uri") or "")
' 2>/dev/null || true)
if [ -n "$rpub" ] && [ -n "$epayto" ]; then
if [ -z "$epayto" ]; then
warn bank "force-select skipped" "problem: exchange payto empty from /keys"
return 0
fi
# Try candidates until bank leaves pending (200/204) or we exhaust
while IFS= read -r rpub; do
[ -n "$rpub" ] || continue
any=1
code_fs=$(curl -sS -m 12 -o "$SCRATCH/force-sel-$tag.json" -w '%{http_code}' -X POST \
-H 'Content-Type: application/json' \
-d "{\"reserve_pub\":\"${rpub}\",\"selected_exchange\":\"${epayto}\"}" \
"${BANK}/taler-integration/withdrawal-operation/${WID}" 2>/dev/null || echo "000")
# 409: conflict (wrong/stale reserve, or already bound) — log body once, keep polling
if [ "$code_fs" = "409" ]; then
if [ "${FORCE_SEL_409_LOGGED:-0}" != "1" ]; then
info "force-select" "HTTP 409 rpub=${rpub:0:12}… body=$(tr '\n' ' ' <"$SCRATCH/force-sel-$tag.json" | head -c 160)"
FORCE_SEL_409_LOGGED=1
fi
else
info "force-select" "HTTP ${code_fs} rpub=${rpub:0:12}…"
FORCE_SEL_409_LOGGED=0
if [ "$code_fs" = "200" ] || [ "$code_fs" = "204" ]; then
info "force-select" "HTTP ${code_fs} rpub=${rpub:0:12}… (ok for WID ${WID:0:8})"
mark_rpub_used "$rpub"
return 0
fi
else
_rpub_empty=no
[ -z "$rpub" ] && _rpub_empty=yes
_epayto_empty=no
[ -z "$epayto" ] && _epayto_empty=yes
if [ "$code_fs" = "409" ]; then
# 5114 = this reserve already bound to another op — not "out of money"
info "force-select" "HTTP 409 rpub=${rpub:0:12}… (stale/used reserve — not balance; trying next)"
mark_rpub_used "$rpub"
continue
fi
info "force-select" "HTTP ${code_fs} rpub=${rpub:0:12}… body=$(tr '\n' ' ' <"$SCRATCH/force-sel-$tag.json" 2>/dev/null | head -c 120)"
# other errors: still try next candidate
done < <(extract_rpubs_for_wid)
if [ "$any" != "1" ]; then
warn bank "force-select skipped" \
"problem: cannot move bank pending->selected (reserve_pub empty=${_rpub_empty}, exchange payto empty=${_epayto_empty})"
"problem: no reserve_pub for this withdraw (WID=${WID:0:8}…); wallet may not have selected yet"
fi
}
# Short bounded select assist (≤8s) — not a hang path; helps attach fresh reserve_pub
if command -v perl >/dev/null 2>&1 && [ -f "$CLI_JS" ]; then
perl -e 'alarm shift; exec @ARGV' 8 \
node "$CLI_JS" --wallet-db="$WDB" --no-throttle run-until-done \
>"$SCRATCH/select-$tag.out" 2>&1 || true
fi
wcli transactions >"$SCRATCH/tx-$tag.json" 2>&1 || true
t0=$(now_ms)
conf_ok=0
st=""
@ -598,9 +693,9 @@ else:
break
;;
esac
# force-select while pending (no run-until-done); avoid spam on repeated 409
# force-select while pending — try alternate rpubs on 5114 (not out-of-money)
if [ "$st" = "pending" ] || [ -z "$st" ]; then
if [ "$i" -eq 1 ] || [ "$i" -eq 2 ] || [ $((i % 5)) -eq 0 ]; then
if [ "$i" -eq 1 ] || [ "$i" -eq 2 ] || [ $((i % 3)) -eq 0 ]; then
force_select_if_needed "$st"
fi
fi
@ -610,10 +705,10 @@ else:
st=$(printf '%s' "${st:-}" | tr -d '\r\n' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
if [ "$conf_ok" != "1" ]; then
note="${note:-confirm timeout last=${st:-empty}}"
# Soft: not confirmed after polls — WARN and continue (pending/select lag or force-select issues)
# Soft: not confirmed — usually stale reserve_pub (5114), NOT empty pool balance
status="SKIP_CONFIRM"
warn bank "confirm $AMT skipped" \
"problem: bank status='${st:-empty}' after ${LADDER_CONFIRM_POLLS} polls (want selected/confirmed); ladder continues. detail: $note"
"problem: bank status='${st:-empty}' after ${LADDER_CONFIRM_POLLS} polls (want selected/confirmed). Usually reserve_pub mismatch (5114), not out-of-money — mint/accept already OK. detail: $note"
ms_total=$(elapsed_ms "$t_rung")
echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV"
continue
@ -664,6 +759,7 @@ sys.exit(0 if Decimal(sys.argv[1]) > Decimal(sys.argv[2]) else 1)
OK_N=$((OK_N + 1))
echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV"
metrics_report_coins "ladder-r${rung}-after-${tag}" || true
metrics_record_flow withdrawn "$AMT" || true
elif echo "$xfer" | grep -qi True; then
status="OK_BANK"
note="bank transfer_done avail=${after} $xfer (no run-until-done)"
@ -671,6 +767,7 @@ sys.exit(0 if Decimal(sys.argv[1]) > Decimal(sys.argv[2]) else 1)
OK_N=$((OK_N + 1))
echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV"
metrics_report_coins "ladder-r${rung}-after-${tag}" || true
metrics_record_flow withdrawn "$AMT" || true
else
note="no coins / no transfer_done avail=${after} $xfer"
err wallet "settle $AMT" "$note"
@ -897,6 +994,7 @@ sys.exit(0 if d.get("paid") is True or str(d.get("order_status","")).lower()=="p
PAY_OK_N=$((PAY_OK_N + 1))
echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t${ms_handle}\t${ms_psettle}\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV"
metrics_report_coins "after-pay-${ptag}" || true
metrics_record_flow spent "$PAMT" || true
else
pnote="not settled order=$OID avail=${CUR}:${after}"
if [ "$IS_PMAX" = "1" ]; then
@ -1048,7 +1146,10 @@ for k, v in (rep.get("timing") or {}).items():
json.dump(out, open(sys.argv[2], "w"), indent=2)
PY
fi
metrics_print_overall "ladder overall" || true
export METRICS_WITHDRAW_TSV="$TSV"
export METRICS_PAY_TSV="$PAY_TSV"
metrics_report_coins "ladder-end" || true
metrics_print_overall "ladder final" || true
# Keep scratch if LADDER_REPORT_DIR set; else copy key files to /tmp
if [ -z "${LADDER_REPORT_DIR:-}" ]; then