feat(monitoring): surface load and coins in inside and e2e
This commit is contained in:
Hernâni Marques
parent
1e6a050ee7
commit
982432f191
2 changed files with 68 additions and 4 deletions
|
|
@ -5,6 +5,8 @@ set -euo pipefail
|
|||
ROOT=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=lib.sh
|
||||
source "$ROOT/lib.sh"
|
||||
# shellcheck source=metrics.sh
|
||||
source "$ROOT/metrics.sh"
|
||||
|
||||
: "${E2E_TIMEOUT:=180}" # ATM ladder + wirewatch lag needs room (was 55 — too tight)
|
||||
: "${E2E_PAY_SECS:=22}" # hard reserve for handle-uri + settle (do not starve pay)
|
||||
|
|
@ -150,6 +152,19 @@ e2e_finish() {
|
|||
fi
|
||||
E2E_REPORTED=1
|
||||
print_balances
|
||||
# Final coin inventory + host/container load
|
||||
if [ -n "${METRICS_DIR:-}" ]; then
|
||||
section "metrics · e2e coins final"
|
||||
metrics_report_coins "e2e-end" || true
|
||||
fi
|
||||
if [ -n "${METRICS_DIR:-}" ] && [ "${METRICS_LOAD:-1}" != "0" ]; then
|
||||
metrics_report_load "${METRICS_DIR}/load-after.json" "e2e-end" || true
|
||||
if [ -f "${METRICS_DIR}/load-before.json" ] && [ -f "${METRICS_DIR}/load-after.json" ]; then
|
||||
section "metrics · e2e load delta"
|
||||
metrics_print_load_delta "${METRICS_DIR}/load-before.json" "${METRICS_DIR}/load-after.json" || true
|
||||
fi
|
||||
metrics_print_overall "e2e overall" || true
|
||||
fi
|
||||
summary || true
|
||||
return "$code"
|
||||
}
|
||||
|
|
@ -276,6 +291,9 @@ BANK_HOST=$(python3 -c 'from urllib.parse import urlparse; print(urlparse("'"$BA
|
|||
|
||||
SCRATCH=$(mktemp -d)
|
||||
WDB="$SCRATCH/wallet.sqlite3"
|
||||
METRICS_DIR="${METRICS_DIR:-$SCRATCH/metrics}"
|
||||
mkdir -p "$METRICS_DIR"
|
||||
export METRICS_DIR CUR="${CUR:-${EXPECT_CURRENCY:-GOA}}" WDB
|
||||
export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH"
|
||||
# Always dump balances on any exit (timeout, blocker, signal, success)
|
||||
trap 'ec=$?; e2e_finish "$ec"; rm -rf "$SCRATCH"; exit "$ec"' EXIT
|
||||
|
|
@ -312,6 +330,19 @@ info "ATM withdraw ladder" "$WITHDRAW_LIST (credit $CREDIT_AMT)"
|
|||
info "pay ladder" "$PAY_LIST"
|
||||
info "e2e budget" "${E2E_TIMEOUT}s"
|
||||
|
||||
# alt_unit_names for human amounts (Kilo-GOA … + base GOA in parentheses)
|
||||
ALT_UNITS_FILE="${METRICS_DIR}/alt_unit_names.json"
|
||||
export ALT_UNITS_FILE
|
||||
if metrics_load_alt_units "${EXCHANGE_PUBLIC}/config"; then
|
||||
info "alt_unit_names" "from ${EXCHANGE_PUBLIC}/config"
|
||||
else
|
||||
warn "alt_unit_names" "using built-in SI fallback"
|
||||
fi
|
||||
|
||||
# Host + bank/exchange/merchant RAM/CPU/load (SSH koopa or koopa-external)
|
||||
section "e2e · load snapshot (before withdraw/pay)"
|
||||
metrics_report_load "${METRICS_DIR}/load-before.json" "e2e-start" || true
|
||||
|
||||
# Local stack: koopa secrets. Remote: only explicit env (never leak local passwords to foreign banks).
|
||||
if [ "$E2E_REMOTE" = "1" ]; then
|
||||
ADMIN_PASS="${E2E_BANK_ADMIN_PASS:-}"
|
||||
|
|
@ -626,6 +657,8 @@ else
|
|||
warn "wallet accept ToS" "$(tail -c 80 "$SCRATCH/ex-tos.out" | tr '\n' ' ')"
|
||||
fi
|
||||
fi
|
||||
# Empty wallet baseline before ATM / pay load
|
||||
metrics_report_coins "wallet-ready" || true
|
||||
|
||||
wd_status() {
|
||||
curl -sS -m 8 -o "$SCRATCH/wd-st.json" -w '' \
|
||||
|
|
@ -668,8 +701,9 @@ e2e_one_withdraw() {
|
|||
WITHDRAW_AMT="$1"
|
||||
local tag
|
||||
tag=$(printf '%s' "$WITHDRAW_AMT" | tr '.:' '__')
|
||||
section "e2e · ATM withdraw $WITHDRAW_AMT"
|
||||
section "e2e · ATM withdraw $WITHDRAW_AMT · $(format_amount_alt "$WITHDRAW_AMT")"
|
||||
e2e_over && { warn "ATM withdraw" "budget exhausted — skip $WITHDRAW_AMT"; return 1; }
|
||||
metrics_report_coins "before-ATM-${tag}" || true
|
||||
|
||||
curl -sS -m 15 -o "$SCRATCH/wd-$tag.json" -H "Authorization: Bearer $UT" \
|
||||
-H 'Content-Type: application/json' \
|
||||
|
|
@ -762,6 +796,7 @@ else:
|
|||
fi
|
||||
sleep 2
|
||||
done
|
||||
metrics_report_coins "after-ATM-${tag}" || true
|
||||
if [ "$ok_bal" = "1" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
|
@ -786,8 +821,9 @@ e2e_one_pay() {
|
|||
fi
|
||||
# shell-safe tag for files
|
||||
tag=$(printf '%s' "$tag" | tr -c 'A-Za-z0-9._-' '_')
|
||||
section "e2e · pay $PAY_AMT · $PAY_SUM"
|
||||
section "e2e · pay $PAY_AMT · $(format_amount_alt "$PAY_AMT") · $PAY_SUM"
|
||||
e2e_over && { warn "pay" "budget exhausted — skip $PAY_AMT ($PAY_SUM)"; return 1; }
|
||||
metrics_report_coins "before-pay-${tag}" || true
|
||||
if [ -z "${MPW:-}" ]; then
|
||||
warn "pay $PAY_AMT" "no merchant token"
|
||||
return 1
|
||||
|
|
@ -860,9 +896,11 @@ d=json.load(open(sys.argv[1]))
|
|||
sys.exit(0 if d.get("paid") is True or str(d.get("order_status","")).lower()=="paid" else 1)
|
||||
' "$SCRATCH/ord-paid-$tag.json" 2>/dev/null; then
|
||||
ok "payment settled $PAY_AMT ($PAY_SUM · order $OID)"
|
||||
metrics_report_coins "after-pay-${tag}" || true
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
metrics_report_coins "after-pay-fail-${tag}" || true
|
||||
warn "pay $PAY_AMT ($PAY_SUM)" "not settled for order $OID"
|
||||
return 1
|
||||
}
|
||||
|
|
@ -899,8 +937,9 @@ e2e_one_pay_public_template() {
|
|||
local pamt="$3"
|
||||
local tag
|
||||
tag=$(printf 'shop_%s' "$pid" | tr -c 'A-Za-z0-9._-' '_')
|
||||
section "e2e · shop product · $pname ($pid) · $pamt"
|
||||
section "e2e · shop product · $pname ($pid) · $pamt · $(format_amount_alt "$pamt")"
|
||||
e2e_over && { warn "shop $pname" "budget exhausted — skip"; return 1; }
|
||||
metrics_report_coins "before-shop-${tag}" || true
|
||||
|
||||
local MH
|
||||
MH=$(python3 -c 'from urllib.parse import urlparse; print(urlparse("'"$MERCHANT_PUBLIC"'").hostname or "taler.hacktivism.ch")' 2>/dev/null || echo "taler.hacktivism.ch")
|
||||
|
|
@ -977,15 +1016,18 @@ d=json.load(open(sys.argv[1]))
|
|||
sys.exit(0 if d.get("paid") is True or str(d.get("order_status","")).lower()=="paid" else 1)
|
||||
' "$SCRATCH/ord-paid-$tag.json" 2>/dev/null; then
|
||||
ok "shop $pname" "payment settled ($pamt · order $OID)"
|
||||
metrics_report_coins "after-shop-${tag}" || true
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
if grep -qiE 'payment|paid|Payment' "$SCRATCH/tx-$tag.out" 2>/dev/null \
|
||||
|| grep -qiE 'done|paid|success|Payment' "$SCRATCH/pay-$tag.out" 2>/dev/null; then
|
||||
ok "shop $pname" "payment settled via wallet tx ($pamt · order $OID)"
|
||||
metrics_report_coins "after-shop-${tag}" || true
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
metrics_report_coins "after-shop-fail-${tag}" || true
|
||||
warn "shop $pname ($pid)" "not settled for order $OID"
|
||||
return 1
|
||||
}
|
||||
|
|
@ -1021,6 +1063,10 @@ for WITHDRAW_AMT in $WITHDRAW_LIST; do
|
|||
esac
|
||||
done
|
||||
info "ATM withdraw summary" "$WITHDRAW_REPORT (ok=$WITHDRAW_OK_N lag=$WITHDRAW_LAG_N fail=$WITHDRAW_FAIL_N)"
|
||||
section "e2e · load snapshot (after ATM withdraws)"
|
||||
metrics_report_load "${METRICS_DIR}/load-after-withdraw.json" "after-withdraw" || true
|
||||
cp -f "${METRICS_DIR}/load-after-withdraw.json" "${METRICS_DIR}/load-after.json" 2>/dev/null || true
|
||||
metrics_report_coins "after-ATM-ladder" || true
|
||||
|
||||
# Settlement catch-up: bank may have confirmed while wallet was still empty
|
||||
section "e2e · wallet settlement (timing)"
|
||||
|
|
@ -1037,6 +1083,7 @@ if python3 -c "import sys; sys.exit(0 if float(sys.argv[1]) > 0 else 1)" "$av_no
|
|||
WITHDRAW_REPORT="${WITHDRAW_REPORT} → late-OK avail=${CUR}:${av_now}"
|
||||
fi
|
||||
ok "spendable balance for payments" "${CUR}:${av_now}"
|
||||
metrics_report_coins "after-settle" || true
|
||||
else
|
||||
if [ "$WITHDRAW_OK" != "1" ]; then
|
||||
warn "withdraw" "still no spendable ${CUR} after settle wait"
|
||||
|
|
@ -1100,6 +1147,10 @@ for PAY_AMT in $PAY_LIST; do
|
|||
fi
|
||||
done
|
||||
info "pay summary" "$PAY_REPORT (ok=$PAY_OK_N fail=$PAY_FAIL_N skip=$PAY_SKIP_N)"
|
||||
section "e2e · load snapshot (after payments)"
|
||||
metrics_report_load "${METRICS_DIR}/load-after-pay.json" "after-pay" || true
|
||||
cp -f "${METRICS_DIR}/load-after-pay.json" "${METRICS_DIR}/load-after.json" 2>/dev/null || true
|
||||
metrics_report_coins "after-pay-ladder" || true
|
||||
if [ "$PAY_OK" != "1" ]; then
|
||||
av_now=$(wallet_avail_num)
|
||||
if python3 -c "import sys; sys.exit(0 if float(sys.argv[1]) > 0 else 1)" "$av_now" 2>/dev/null; then
|
||||
|
|
@ -1193,6 +1244,9 @@ EOF
|
|||
if [ "$SHOP_OK" != "1" ] && [ "$SHOP_FAIL_N" -gt 0 ]; then
|
||||
warn "goa-shop" "no sample product payment succeeded ($SHOP_REPORT)"
|
||||
fi
|
||||
section "e2e · load snapshot (after shop pays)"
|
||||
metrics_report_load "${METRICS_DIR}/load-after-shop.json" "after-shop" || true
|
||||
cp -f "${METRICS_DIR}/load-after-shop.json" "${METRICS_DIR}/load-after.json" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue