fix(monitoring): ladder min denom floor and soft confirm timeout
This commit is contained in:
Hernâni Marques
parent
d2a2caacc8
commit
376f15d702
2 changed files with 104 additions and 50 deletions
|
|
@ -46,6 +46,9 @@ elapsed_ms() {
|
|||
: "${LADDER_MAX_AMOUNT:=4503599627370496}"
|
||||
: "${LADDER_STEPS:=23}"
|
||||
: "${LADDER_LOAD:=1}"
|
||||
# Floor for random mids (must be ≥ smallest exchange coin; GOA min denom ≈ 0.000001)
|
||||
: "${LADDER_MIN_AMOUNT:=0.000001}"
|
||||
: "${LADDER_CONFIRM_POLLS:=40}"
|
||||
|
||||
CUR="${EXPECT_CURRENCY:-GOA}"
|
||||
BANK="${BANK_PUBLIC%/}"
|
||||
|
|
@ -92,14 +95,20 @@ print("0")
|
|||
}
|
||||
|
||||
# Build exactly LADDER_STEPS: [0] + (N-2) log-uniform random increasing + [max]
|
||||
# Random mids are always ≥ LADDER_MIN_AMOUNT (smallest viable coin).
|
||||
build_ladder() {
|
||||
python3 - <<'PY' "$CUR" "${LADDER_MAX_AMOUNT}" "${LADDER_STEPS}"
|
||||
python3 - <<'PY' "$CUR" "${LADDER_MAX_AMOUNT}" "${LADDER_STEPS}" "${LADDER_MIN_AMOUNT}"
|
||||
import math, random, sys
|
||||
from decimal import Decimal, ROUND_HALF_UP
|
||||
from decimal import Decimal, ROUND_HALF_UP, ROUND_UP
|
||||
|
||||
cur = sys.argv[1]
|
||||
max_amt = Decimal(sys.argv[2])
|
||||
steps = max(2, int(sys.argv[3]))
|
||||
min_amt = Decimal(sys.argv[4])
|
||||
if min_amt <= 0:
|
||||
min_amt = Decimal("0.000001")
|
||||
if min_amt >= max_amt:
|
||||
min_amt = max_amt / Decimal(1000)
|
||||
|
||||
def fmt(v: Decimal) -> str:
|
||||
q = v.quantize(Decimal("0.00000001"), rounding=ROUND_HALF_UP)
|
||||
|
|
@ -107,45 +116,46 @@ def fmt(v: Decimal) -> str:
|
|||
return format(int(q), "d")
|
||||
return format(q, "f").rstrip("0").rstrip(".")
|
||||
|
||||
def quant(v: Decimal) -> Decimal:
|
||||
if v >= 1:
|
||||
return v.quantize(Decimal(1), rounding=ROUND_HALF_UP)
|
||||
# snap up to min_amt grid for sub-unit amounts
|
||||
if v < min_amt:
|
||||
return min_amt
|
||||
return v.quantize(Decimal("0.00000001"), rounding=ROUND_UP)
|
||||
|
||||
mid = steps - 2 # between 0 and max
|
||||
out = ["%s:0" % cur]
|
||||
if mid > 0 and max_amt > 0:
|
||||
# log-space cut points in (epsilon, max), pick strictly increasing
|
||||
lo, hi = 1e-8, float(max_amt) * 0.999999
|
||||
lo, hi = float(min_amt), float(max_amt) * 0.999999
|
||||
if hi <= lo:
|
||||
hi = lo * 10
|
||||
cuts = sorted(math.exp(random.uniform(math.log(lo), math.log(hi))) for _ in range(mid))
|
||||
# enforce strict increase after quantize
|
||||
prev = Decimal(0)
|
||||
for c in cuts:
|
||||
v = Decimal(str(c))
|
||||
if v >= 1:
|
||||
v = v.quantize(Decimal(1), rounding=ROUND_HALF_UP)
|
||||
else:
|
||||
v = v.quantize(Decimal("0.00000001"), rounding=ROUND_HALF_UP)
|
||||
v = quant(Decimal(str(c)))
|
||||
if v <= prev:
|
||||
step = max(prev * Decimal("1e-6"), Decimal("0.00000001")) if prev > 0 else Decimal("0.00000001")
|
||||
v = (prev + step).quantize(Decimal("0.00000001") if prev < 1 else Decimal(1))
|
||||
step = min_amt if prev < 1 else max(prev * Decimal("1e-6"), Decimal(1))
|
||||
v = quant(prev + step)
|
||||
if v >= max_amt:
|
||||
v = max_amt - (Decimal(1) if max_amt > 1 else Decimal("0.00000001"))
|
||||
if v <= prev:
|
||||
v = max_amt - (Decimal(1) if max_amt > 1 else min_amt)
|
||||
v = quant(v)
|
||||
if v <= prev or v >= max_amt:
|
||||
continue
|
||||
out.append("%s:%s" % (cur, fmt(v)))
|
||||
prev = v
|
||||
out.append("%s:%s" % (cur, fmt(max_amt)))
|
||||
# trim/pad to exact steps if quantize collapsed some
|
||||
while len(out) > steps:
|
||||
# drop from middle
|
||||
out.pop(len(out) // 2)
|
||||
while len(out) < steps and len(out) >= 2:
|
||||
# insert geometric mean mid
|
||||
i = len(out) // 2
|
||||
a = Decimal(out[i - 1].split(":", 1)[1])
|
||||
b = Decimal(out[i].split(":", 1)[1])
|
||||
if a <= 0:
|
||||
m = b / 2 if b > 0 else Decimal("0.000001")
|
||||
m = max(min_amt, b / 2 if b > 0 else min_amt)
|
||||
else:
|
||||
m = (a * b).sqrt() if a * b > 0 else (a + b) / 2
|
||||
m = quant(m)
|
||||
if m <= a or m >= b:
|
||||
break
|
||||
out.insert(i, "%s:%s" % (cur, fmt(m)))
|
||||
|
|
@ -163,7 +173,7 @@ info "bank" "$BANK"
|
|||
info "exchange" "$EX"
|
||||
info "currency" "$CUR"
|
||||
info "budget" "${LADDER_TIMEOUT_S}s"
|
||||
info "steps" "${LADDER_STEPS} (fixed 0 + $((LADDER_STEPS - 2)) random + fixed max=${CUR}:${LADDER_MAX_AMOUNT})"
|
||||
info "steps" "${LADDER_STEPS} (0 + $((LADDER_STEPS - 2)) random≥${LADDER_MIN_AMOUNT} + max=${CUR}:${LADDER_MAX_AMOUNT})"
|
||||
|
||||
if [ ! -f "$EXP_PW_FILE" ]; then
|
||||
err bank "explorer password missing" "$EXP_PW_FILE"
|
||||
|
|
@ -335,30 +345,58 @@ for AMT in "$@"; do
|
|||
-H "Authorization: Bearer ${TOK}" -H 'Content-Type: application/json' -d '{}' \
|
||||
"${BANK}/accounts/${EXP_USER}/withdrawals/${WID}/confirm"
|
||||
}
|
||||
extract_rpub() {
|
||||
python3 -c '
|
||||
import re, sys, json
|
||||
paths = sys.argv[1:]
|
||||
blob = ""
|
||||
for p in paths:
|
||||
try:
|
||||
blob += open(p, errors="replace").read() + "\n"
|
||||
except Exception:
|
||||
pass
|
||||
# JSON fields
|
||||
for pat in (
|
||||
r"\"reserve_pub\"\s*:\s*\"([A-Z0-9]+)\"",
|
||||
r"\"reservePub\"\s*:\s*\"([A-Z0-9]+)\"",
|
||||
r"reserve_pub[\"\s:=]+([A-Z0-9]{40,})",
|
||||
r"reservePub[\"\s:=]+([A-Z0-9]{40,})",
|
||||
r"reserve[_ ]?pub[\"=: ]+([A-Z0-9]{40,})",
|
||||
):
|
||||
m = re.search(pat, blob, re.I)
|
||||
if m:
|
||||
print(m.group(1))
|
||||
raise SystemExit
|
||||
# line-wise JSON
|
||||
for line in blob.splitlines():
|
||||
line = line.strip()
|
||||
if not line.startswith("{"):
|
||||
continue
|
||||
try:
|
||||
d = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
def walk(o):
|
||||
if isinstance(o, dict):
|
||||
for k, v in o.items():
|
||||
if k.lower() in ("reserve_pub", "reservepub") and isinstance(v, str) and len(v) >= 40:
|
||||
print(v); raise SystemExit
|
||||
walk(v)
|
||||
elif isinstance(o, list):
|
||||
for i in o:
|
||||
walk(i)
|
||||
walk(d)
|
||||
' "$@" 2>/dev/null || true
|
||||
}
|
||||
|
||||
force_select_if_needed() {
|
||||
local st_now="$1"
|
||||
[ "$st_now" = "pending" ] || [ -z "$st_now" ] || return 0
|
||||
local rpub epayto
|
||||
rpub=$(python3 -c '
|
||||
import re,sys
|
||||
paths=sys.argv[1:]
|
||||
t=""
|
||||
for p in paths:
|
||||
try: t+=open(p).read()
|
||||
except Exception: pass
|
||||
m=re.search(r"reserve[_ ]?pub[\"=: ]+([A-Z0-9]{40,})", t, re.I)
|
||||
if not m: m=re.search(r"\"reservePub\"\s*:\s*\"([^\"]+)\"", t)
|
||||
print(m.group(1) if m else "")
|
||||
' "$SCRATCH/accept-$tag.out" "$SCRATCH/tx-$tag.json" 2>/dev/null || true)
|
||||
local rpub epayto code_fs
|
||||
rpub=$(extract_rpub "$SCRATCH/accept-$tag.out" "$SCRATCH/tx-$tag.json")
|
||||
if [ -z "$rpub" ]; then
|
||||
wcli transactions >"$SCRATCH/tx-$tag.json" 2>&1 || true
|
||||
rpub=$(python3 -c '
|
||||
import re,sys
|
||||
t=open(sys.argv[1]).read()
|
||||
m=re.search(r"reserve[_ ]?pub[\"=: ]+([A-Z0-9]{40,})", t, re.I)
|
||||
if not m: m=re.search(r"\"reservePub\"\s*:\s*\"([^\"]+)\"", t)
|
||||
print(m.group(1) if m else "")
|
||||
' "$SCRATCH/tx-$tag.json" 2>/dev/null || true)
|
||||
rpub=$(extract_rpub "$SCRATCH/accept-$tag.out" "$SCRATCH/tx-$tag.json")
|
||||
fi
|
||||
epayto=$(curl -sS -m 10 "${EX%/}/keys" 2>/dev/null | python3 -c '
|
||||
import json,sys
|
||||
|
|
@ -372,19 +410,22 @@ else:
|
|||
if acc: print(acc[0].get("payto_uri") or "")
|
||||
' 2>/dev/null || true)
|
||||
if [ -n "$rpub" ] && [ -n "$epayto" ]; then
|
||||
curl -sS -m 12 -o "$SCRATCH/force-sel-$tag.json" -X POST \
|
||||
code_fs=$(curl -sS -m 12 -o "$SCRATCH/force-sel-$tag.json" -w '%{http_code}' -X POST \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"reserve_pub\":\"${rpub}\",\"selected_exchange\":\"${epayto}\"}" \
|
||||
"${BANK}/taler-integration/withdrawal-operation/${WID}" >/dev/null || true
|
||||
info "force-select" "rpub=${rpub:0:12}…"
|
||||
"${BANK}/taler-integration/withdrawal-operation/${WID}" 2>/dev/null || echo "000")
|
||||
info "force-select" "HTTP ${code_fs} rpub=${rpub:0:12}…"
|
||||
else
|
||||
warn bank "force-select skipped" \
|
||||
"problem: missing reserve_pub or exchange payto (rpub_empty=$( [ -z "$rpub" ] && echo yes || echo no ); cannot move bank status pending→selected)"
|
||||
fi
|
||||
}
|
||||
|
||||
t0=$(now_ms)
|
||||
conf_ok=0
|
||||
st=""
|
||||
# Immediate poll: confirm the moment we see selected (no long wallet block first)
|
||||
for i in $(seq 1 60); do
|
||||
# Poll bank status; force-select while pending; confirm as soon as selected
|
||||
for i in $(seq 1 "${LADDER_CONFIRM_POLLS}"); do
|
||||
ladder_over && break
|
||||
st=$(bank_st)
|
||||
case "$st" in
|
||||
|
|
@ -392,7 +433,7 @@ else:
|
|||
ccode=$(do_confirm)
|
||||
if [ "$ccode" = "204" ] || [ "$ccode" = "200" ]; then
|
||||
conf_ok=1
|
||||
info "confirm" "immediate HTTP $ccode after selected (poll $i)"
|
||||
info "confirm" "HTTP $ccode after selected (poll $i)"
|
||||
else
|
||||
note="confirm HTTP $ccode"
|
||||
fi
|
||||
|
|
@ -407,15 +448,26 @@ else:
|
|||
break
|
||||
;;
|
||||
esac
|
||||
# if still pending after a few polls, force bank select once (no wallet shepherd)
|
||||
if [ "$i" = "4" ] || [ "$i" = "12" ]; then
|
||||
force_select_if_needed "$st"
|
||||
# aggressive force-select while pending (no run-until-done)
|
||||
if [ "$st" = "pending" ] || [ -z "$st" ]; then
|
||||
if [ "$i" -eq 1 ] || [ $((i % 3)) -eq 0 ]; then
|
||||
force_select_if_needed "$st"
|
||||
fi
|
||||
fi
|
||||
sleep 0.5
|
||||
sleep 0.4
|
||||
done
|
||||
ms_confirm=$(elapsed_ms "$t0")
|
||||
if [ "$conf_ok" != "1" ]; then
|
||||
note="${note:-confirm timeout last=$st}"
|
||||
note="${note:-confirm timeout last=${st:-empty}}"
|
||||
# Soft: stuck pending is often force-select/rpub or bank lag — do not kill whole ladder
|
||||
if [ "${st:-}" = "pending" ] || [ -z "${st:-}" ]; then
|
||||
status="SKIP_CONFIRM"
|
||||
warn bank "confirm $AMT skipped" \
|
||||
"problem: bank withdrawal stayed '${st:-empty}' (not selected) after ${LADDER_CONFIRM_POLLS} polls — reserve may not have been attached; ladder continues. detail: $note"
|
||||
ms_total=$(elapsed_ms "$t_rung")
|
||||
echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV"
|
||||
continue
|
||||
fi
|
||||
err bank "confirm $AMT" "$note"
|
||||
status="FAIL_CONFIRM"
|
||||
STOP_REASON="$note"
|
||||
|
|
|
|||
|
|
@ -137,6 +137,8 @@ export E2E_WITHDRAW_VALUES E2E_PAY_VALUES
|
|||
# Ladder: 0 + random mids + max (see check_goa_ladder.sh build_ladder). No LADDER_RANGES.
|
||||
# Defaults so set -u export is safe when vars were never set by caller.
|
||||
: "${LADDER_STEPS:=23}"
|
||||
: "${LADDER_MIN_AMOUNT:=0.000001}"
|
||||
: "${LADDER_CONFIRM_POLLS:=40}"
|
||||
: "${LADDER_MAX_RUNGS:=99}"
|
||||
: "${LADDER_TIMEOUT_S:=3600}"
|
||||
: "${LADDER_REPORT_DIR:=}"
|
||||
|
|
@ -148,7 +150,7 @@ export E2E_WITHDRAW_VALUES E2E_PAY_VALUES
|
|||
: "${LADDER_HIGH_FROM:=1000000}"
|
||||
: "${LADDER_HIGH_RUNGS:=12}"
|
||||
: "${LADDER_LOAD:=1}"
|
||||
export LADDER_STEPS LADDER_MAX_RUNGS LADDER_TIMEOUT_S LADDER_REPORT_DIR
|
||||
export LADDER_STEPS LADDER_MIN_AMOUNT LADDER_CONFIRM_POLLS LADDER_MAX_RUNGS LADDER_TIMEOUT_S LADDER_REPORT_DIR
|
||||
export LADDER_SETTLE_ROUNDS LADDER_SETTLE_SLEEP EXP_PW_FILE EXP_USER
|
||||
export LADDER_MAX_AMOUNT LADDER_INCLUDE_ZERO LADDER_INCLUDE_MAX
|
||||
export LADDER_HIGH_FROM LADDER_HIGH_RUNGS LADDER_LOAD
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue