1431 lines
50 KiB
Bash
1431 lines
50 KiB
Bash
# shellcheck shell=bash
|
||
# Shared Taler stack metrics for e2e + ladder (source after lib.sh).
|
||
#
|
||
# - Host/container load (RAM, processes, DB sizes) before/after a phase
|
||
# - Wallet coin inventory (total + by denomination)
|
||
# - Final overall statistics block
|
||
#
|
||
# Env:
|
||
# METRICS_DIR where JSON snapshots go (default $SCRATCH or /tmp)
|
||
# METRICS_LOAD=0 skip remote/host load probes
|
||
# KOOPA_SSH / SKIP_SSH as in lib.sh
|
||
|
||
: "${METRICS_LOAD:=1}"
|
||
: "${METRICS_DIR:=${SCRATCH:-/tmp}}"
|
||
mkdir -p "$METRICS_DIR" 2>/dev/null || true
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# alt_unit_names — human amounts (Kilo-GOA / Mega-GOA / …) with base in parens
|
||
# From exchange/bank currency_specification.alt_unit_names:
|
||
# "0"→GOA, "3"→Kilo-GOA, "6"→Mega-GOA, … "-3"→Milli-GOA, …
|
||
# Example: GOA:5000 → 5 Kilo-GOA (GOA:5000)
|
||
# ---------------------------------------------------------------------------
|
||
: "${ALT_UNITS_FILE:=${METRICS_DIR}/alt_unit_names.json}"
|
||
|
||
# Load map from public /config into ALT_UNITS_FILE. $1=optional config URL.
|
||
metrics_load_alt_units() {
|
||
local url="${1:-${EXCHANGE_PUBLIC:-https://exchange.hacktivism.ch}/config}"
|
||
local code
|
||
mkdir -p "$(dirname "$ALT_UNITS_FILE")" 2>/dev/null || true
|
||
code=$(curl -skS -m 12 -o "${ALT_UNITS_FILE}.raw" -w '%{http_code}' "$url" 2>/dev/null || echo 000)
|
||
if [ "$code" != "200" ]; then
|
||
# soft fallback: SI-style names if live config unreachable
|
||
printf '%s\n' '{"0":"GOA","3":"Kilo-GOA","6":"Mega-GOA","9":"Giga-GOA","12":"Tera-GOA","15":"Peta-GOA","18":"Exa-GOA","21":"Zetta-GOA","24":"Yotta-GOA","-1":"Deci-GOA","-2":"Centi-GOA","-3":"Milli-GOA","-6":"Micro-GOA","-8":"Atomic-GOA"}' \
|
||
>"$ALT_UNITS_FILE"
|
||
return 1
|
||
fi
|
||
python3 - "${ALT_UNITS_FILE}.raw" "$ALT_UNITS_FILE" <<'PY'
|
||
import json, sys
|
||
src, dst = sys.argv[1:3]
|
||
d = json.load(open(src))
|
||
au = None
|
||
cs = d.get("currency_specification")
|
||
if isinstance(cs, dict):
|
||
au = cs.get("alt_unit_names")
|
||
if not au and isinstance(d.get("currencies"), dict):
|
||
# merchant-style: currencies.GOA.alt_unit_names
|
||
for _code, spec in d["currencies"].items():
|
||
if isinstance(spec, dict) and spec.get("alt_unit_names"):
|
||
au = spec["alt_unit_names"]
|
||
break
|
||
if not isinstance(au, dict) or "0" not in au:
|
||
au = {"0": d.get("currency") or "GOA"}
|
||
json.dump(au, open(dst, "w"), indent=2, sort_keys=True)
|
||
PY
|
||
export ALT_UNITS_FILE
|
||
return 0
|
||
}
|
||
|
||
# Format one amount: "GOA:5000" → "5 Kilo-GOA (GOA:5000)"
|
||
# Uses ALT_UNITS_FILE if present. Pure stdout.
|
||
format_amount_alt() {
|
||
local amt="${1:-}"
|
||
[ -n "$amt" ] || return 0
|
||
python3 - "$amt" "${ALT_UNITS_FILE:-}" <<'PY'
|
||
import json, sys
|
||
from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
|
||
|
||
amt = sys.argv[1].strip()
|
||
path = sys.argv[2] if len(sys.argv) > 2 else ""
|
||
alt = {}
|
||
if path:
|
||
try:
|
||
alt = json.load(open(path))
|
||
except Exception:
|
||
alt = {}
|
||
if not alt:
|
||
alt = {"0": "GOA"}
|
||
|
||
def parse(s):
|
||
if ":" in s:
|
||
c, v = s.split(":", 1)
|
||
return c, Decimal(v)
|
||
return "GOA", Decimal(s)
|
||
|
||
def fmt_num(v: Decimal) -> str:
|
||
if v == v.to_integral():
|
||
return format(int(v), "d")
|
||
s = format(v.quantize(Decimal("0.00000001"), rounding=ROUND_HALF_UP), "f")
|
||
return s.rstrip("0").rstrip(".")
|
||
|
||
try:
|
||
cur, val = parse(amt)
|
||
except (InvalidOperation, ValueError):
|
||
print(amt)
|
||
raise SystemExit(0)
|
||
|
||
base_name = alt.get("0") or cur
|
||
# always show canonical base form in parens
|
||
base_s = "%s:%s" % (cur, fmt_num(val))
|
||
if val == 0:
|
||
print("0 %s (%s)" % (base_name, base_s))
|
||
raise SystemExit(0)
|
||
|
||
# scales: power of 10 relative to unit "0"
|
||
scales = []
|
||
for k, name in alt.items():
|
||
try:
|
||
scales.append((int(k), str(name)))
|
||
except Exception:
|
||
pass
|
||
scales.sort(key=lambda x: -x[0]) # largest unit first
|
||
|
||
# pick largest scale where |value| >= 10^scale (for scale>=0),
|
||
# or for fractions the finest unit that makes the coefficient >= 1
|
||
chosen = None # (scale, name, coeff)
|
||
absval = abs(val)
|
||
for sc, name in scales:
|
||
unit = Decimal(10) ** sc
|
||
if unit <= 0:
|
||
continue
|
||
coeff = absval / unit
|
||
if coeff >= 1:
|
||
chosen = (sc, name, coeff if val >= 0 else -coeff)
|
||
break
|
||
if chosen is None:
|
||
# smaller than smallest unit — use base
|
||
print("%s %s (%s)" % (fmt_num(val), base_name, base_s))
|
||
raise SystemExit(0)
|
||
|
||
sc, name, coeff = chosen
|
||
# if base unit, prefer "GOA:12" style still with parens only when alt differs
|
||
if sc == 0:
|
||
print("%s %s" % (fmt_num(val), name))
|
||
raise SystemExit(0)
|
||
print("%s %s (%s)" % (fmt_num(coeff), name, base_s))
|
||
PY
|
||
}
|
||
|
||
# Format list of "CUR:n" amounts for plans / logs (space-separated → multiline or compact)
|
||
format_amount_list_alt() {
|
||
local a out=""
|
||
for a in "$@"; do
|
||
[ -n "$out" ] && out="$out "
|
||
out="${out}$(format_amount_alt "$a")"
|
||
done
|
||
printf '%s\n' "$out"
|
||
}
|
||
|
||
# --- coin inventory from wallet-cli dump-coins ---
|
||
# Writes JSON to $1; prints one-line summary to stdout.
|
||
# Sets COINS_TOTAL, COINS_FRESH, COINS_SPENT, COINS_AMOUNT_CIRC, COINS_SUMMARY.
|
||
metrics_wallet_coins() {
|
||
local out="${1:-$METRICS_DIR/coins.json}"
|
||
local dump="${METRICS_DIR}/dump-coins.raw"
|
||
mkdir -p "$(dirname "$out")" 2>/dev/null || true
|
||
COINS_TOTAL=0
|
||
COINS_FRESH=0
|
||
COINS_SPENT=0
|
||
COINS_AMOUNT_CIRC="0"
|
||
COINS_SUMMARY="(no coins)"
|
||
# Prefer wcli() from caller (e2e/ladder). Do not pass a leading timeout number —
|
||
# ladder wcli() has no optional secs arg (would become a wallet subcommand).
|
||
if type wcli >/dev/null 2>&1; then
|
||
wcli advanced dump-coins >"$dump" 2>/dev/null || true
|
||
elif [ -n "${CLI_JS:-}" ] && [ -f "${CLI_JS}" ]; then
|
||
node "$CLI_JS" --wallet-db="${WDB:-}" --no-throttle advanced dump-coins >"$dump" 2>/dev/null || true
|
||
elif [ -n "${WALLET_CLI:-}" ] && [ -n "${WDB:-}" ]; then
|
||
node "$WALLET_CLI" --wallet-db="${WDB}" --no-throttle --skip-defaults advanced dump-coins >"$dump" 2>/dev/null || true
|
||
else
|
||
echo "$COINS_SUMMARY"
|
||
printf '%s\n' '{"ok":false,"reason":"no-wcli"}' >"$out"
|
||
return 1
|
||
fi
|
||
python3 - "$dump" "$out" "${CUR:-GOA}" "${ALT_UNITS_FILE:-}" <<'PY'
|
||
import json, re, sys
|
||
from collections import Counter, defaultdict
|
||
from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
|
||
|
||
raw_path, out_path, cur = sys.argv[1:4]
|
||
alt_path = sys.argv[4] if len(sys.argv) > 4 else ""
|
||
raw = open(raw_path).read() if raw_path else ""
|
||
d = None
|
||
for m in re.finditer(r"\{", raw):
|
||
try:
|
||
d = json.loads(raw[m.start():])
|
||
if isinstance(d, dict) and ("coins" in d or "coin" in d):
|
||
break
|
||
except Exception:
|
||
d = None
|
||
coins = []
|
||
if isinstance(d, dict):
|
||
coins = d.get("coins") or d.get("coin") or []
|
||
|
||
alt = {}
|
||
if alt_path:
|
||
try:
|
||
alt = json.load(open(alt_path))
|
||
except Exception:
|
||
alt = {}
|
||
if not alt:
|
||
alt = {"0": cur}
|
||
|
||
def parse_amt(s):
|
||
"""'GOA:10' / '10' → (currency, Decimal) or (cur, 0)."""
|
||
s = str(s or "").strip()
|
||
if not s or s == "?":
|
||
return cur, Decimal(0)
|
||
if ":" in s:
|
||
c, v = s.split(":", 1)
|
||
try:
|
||
return c, Decimal(v)
|
||
except InvalidOperation:
|
||
return c, Decimal(0)
|
||
try:
|
||
return cur, Decimal(s)
|
||
except InvalidOperation:
|
||
return cur, Decimal(0)
|
||
|
||
def fmt_num(v: Decimal) -> str:
|
||
if v == v.to_integral():
|
||
return format(int(v), "d")
|
||
s = format(v.quantize(Decimal("0.00000001"), rounding=ROUND_HALF_UP), "f")
|
||
return s.rstrip("0").rstrip(".")
|
||
|
||
def fmt_amt(c, v: Decimal) -> str:
|
||
return "%s:%s" % (c, fmt_num(v))
|
||
|
||
def fmt_amt_alt(c, v: Decimal) -> str:
|
||
"""5 Kilo-GOA (GOA:5000) using alt_unit_names."""
|
||
base_s = fmt_amt(c, v)
|
||
base_name = alt.get("0") or c
|
||
if v == 0:
|
||
return "0 %s (%s)" % (base_name, base_s)
|
||
scales = []
|
||
for k, name in alt.items():
|
||
try:
|
||
scales.append((int(k), str(name)))
|
||
except Exception:
|
||
pass
|
||
scales.sort(key=lambda x: -x[0])
|
||
absval = abs(v)
|
||
chosen = None
|
||
for sc, name in scales:
|
||
unit = Decimal(10) ** sc
|
||
coeff = absval / unit
|
||
if coeff >= 1:
|
||
chosen = (sc, name, coeff if v >= 0 else -coeff)
|
||
break
|
||
if chosen is None:
|
||
return "%s %s (%s)" % (fmt_num(v), base_name, base_s)
|
||
sc, name, coeff = chosen
|
||
if sc == 0:
|
||
return "%s %s" % (fmt_num(v), name)
|
||
return "%s %s (%s)" % (fmt_num(coeff), name, base_s)
|
||
|
||
by_denom_all = Counter() # all coins
|
||
by_denom_circ = Counter() # non-spent
|
||
by_denom_spent = Counter()
|
||
by_status = Counter()
|
||
amt_circ = defaultdict(lambda: Decimal(0)) # currency → amount
|
||
amt_spent = defaultdict(lambda: Decimal(0))
|
||
amt_all = defaultdict(lambda: Decimal(0))
|
||
circ_n = 0
|
||
spent_n = 0
|
||
for c in coins:
|
||
if not isinstance(c, dict):
|
||
continue
|
||
dv = c.get("denomValue") or c.get("value") or c.get("denom_value") or "?"
|
||
st = str(c.get("coinStatus") or c.get("status") or "?")
|
||
by_denom_all[dv] += 1
|
||
by_status[st] += 1
|
||
ac, av = parse_amt(dv)
|
||
amt_all[ac] += av
|
||
st_l = st.lower()
|
||
if "spent" in st_l or "delete" in st_l or "dirty" in st_l:
|
||
spent_n += 1
|
||
by_denom_spent[dv] += 1
|
||
amt_spent[ac] += av
|
||
else:
|
||
circ_n += 1
|
||
by_denom_circ[dv] += 1
|
||
amt_circ[ac] += av
|
||
|
||
total = len(coins)
|
||
|
||
def denom_key(k):
|
||
try:
|
||
return float(str(k).split(":", 1)[-1])
|
||
except Exception:
|
||
return 0.0
|
||
|
||
def denom_list(counter, spent_counter=None):
|
||
out = []
|
||
for k in sorted(counter.keys(), key=denom_key):
|
||
item = {"denom": k, "count": counter[k]}
|
||
ac, av = parse_amt(k)
|
||
item["unit_value"] = str(av)
|
||
item["amount"] = fmt_amt(ac, av * counter[k])
|
||
if spent_counter is not None:
|
||
item["spent_count"] = spent_counter.get(k, 0)
|
||
out.append(item)
|
||
return out
|
||
|
||
# human: "10 GOA×3 (=30 GOA) …" and alt: "3 Kilo-GOA (GOA:3000)×1"
|
||
parts = []
|
||
parts_alt = []
|
||
for item in denom_list(by_denom_circ):
|
||
ac, av = parse_amt(item["denom"])
|
||
total_v = av * item["count"]
|
||
parts.append("%s×%d (=%s)" % (item["denom"], item["count"], item["amount"]))
|
||
parts_alt.append(
|
||
"%s×%d (=%s)"
|
||
% (fmt_amt_alt(ac, av), item["count"], fmt_amt_alt(ac, total_v))
|
||
)
|
||
circ_amt_s = " ".join(fmt_amt(c, amt_circ[c]) for c in sorted(amt_circ))
|
||
circ_amt_alt = " ".join(fmt_amt_alt(c, amt_circ[c]) for c in sorted(amt_circ))
|
||
spent_amt_s = " ".join(fmt_amt(c, amt_spent[c]) for c in sorted(amt_spent)) if amt_spent else "0"
|
||
spent_amt_alt = " ".join(fmt_amt_alt(c, amt_spent[c]) for c in sorted(amt_spent)) if amt_spent else "0"
|
||
if not circ_amt_s:
|
||
circ_amt_s = "%s:0" % cur
|
||
circ_amt_alt = "0 %s (%s:0)" % (alt.get("0") or cur, cur)
|
||
summary = (
|
||
"coins=%d in_circ=%d spent=%d amount_circ=%s | %s"
|
||
% (total, circ_n, spent_n, circ_amt_alt, " ".join(parts_alt) if parts_alt else "(empty)")
|
||
)
|
||
# enrich denom lists with human labels
|
||
def enrich(lst):
|
||
out = []
|
||
for item in lst:
|
||
ac, av = parse_amt(item["denom"])
|
||
total_v = av * int(item["count"])
|
||
item = dict(item)
|
||
item["denom_alt"] = fmt_amt_alt(ac, av)
|
||
item["amount_alt"] = fmt_amt_alt(ac, total_v)
|
||
out.append(item)
|
||
return out
|
||
|
||
report = {
|
||
"ok": True,
|
||
"currency": cur,
|
||
"total_coins": total,
|
||
"in_circulation": circ_n,
|
||
"spent": spent_n,
|
||
"amount_in_circulation": {c: str(amt_circ[c]) for c in amt_circ},
|
||
"amount_in_circulation_s": circ_amt_s,
|
||
"amount_in_circulation_alt": circ_amt_alt,
|
||
"amount_spent": {c: str(amt_spent[c]) for c in amt_spent},
|
||
"amount_spent_s": spent_amt_s,
|
||
"amount_spent_alt": spent_amt_alt,
|
||
"by_status": dict(by_status),
|
||
"by_denom": enrich(denom_list(by_denom_all, by_denom_spent)),
|
||
"by_denom_circulation": enrich(denom_list(by_denom_circ)),
|
||
"by_denom_spent": enrich(denom_list(by_denom_spent)),
|
||
"summary": summary,
|
||
"alt_unit_names": alt,
|
||
}
|
||
json.dump(report, open(out_path, "w"), indent=2)
|
||
print(summary)
|
||
open(out_path + ".total", "w").write(str(total))
|
||
open(out_path + ".circ", "w").write(str(circ_n))
|
||
open(out_path + ".spent", "w").write(str(spent_n))
|
||
open(out_path + ".amt", "w").write(circ_amt_s)
|
||
PY
|
||
COINS_TOTAL=$(cat "${out}.total" 2>/dev/null || echo 0)
|
||
COINS_FRESH=$(cat "${out}.circ" 2>/dev/null || echo 0)
|
||
COINS_SPENT=$(cat "${out}.spent" 2>/dev/null || echo 0)
|
||
COINS_AMOUNT_CIRC=$(cat "${out}.amt" 2>/dev/null || echo "0")
|
||
COINS_SUMMARY=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("summary","?"))' "$out" 2>/dev/null || echo "$COINS_SUMMARY")
|
||
# history TSV for end-of-run table
|
||
if [ -n "${METRICS_DIR:-}" ]; then
|
||
hist="${METRICS_DIR}/coins-history.tsv"
|
||
if [ ! -f "$hist" ]; then
|
||
printf 'ts\tlabel\ttotal\tin_circ\tspent\tamount_circ\tsummary\n' >"$hist"
|
||
fi
|
||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||
"$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date)" \
|
||
"${METRICS_COINS_LABEL:-snap}" \
|
||
"$COINS_TOTAL" "$COINS_FRESH" "$COINS_SPENT" "$COINS_AMOUNT_CIRC" \
|
||
"$(printf '%s' "$COINS_SUMMARY" | tr '\t' ' ')" >>"$hist"
|
||
fi
|
||
echo "$COINS_SUMMARY"
|
||
}
|
||
|
||
# Emit coin inventory as monitoring info lines (label = step marker).
|
||
# $1=label $2=optional json path (default METRICS_DIR/coins-<safe-label>.json)
|
||
metrics_report_coins() {
|
||
local label="$1"
|
||
local safe
|
||
safe=$(printf '%s' "$label" | tr -c 'A-Za-z0-9._-' '_' | head -c 80)
|
||
local out="${2:-${METRICS_DIR}/coins-${safe}.json}"
|
||
local line prev="${METRICS_DIR}/coins-prev.json"
|
||
if [ -z "${WDB:-}" ] && [ -z "${WALLET_CLI:-}" ] && ! type wcli >/dev/null 2>&1; then
|
||
info "coins ${label}" "skipped (no wallet yet)"
|
||
return 0
|
||
fi
|
||
METRICS_COINS_LABEL="$label"
|
||
export METRICS_COINS_LABEL
|
||
if ! metrics_wallet_coins "$out" >/dev/null; then
|
||
warn "coins ${label}" "dump-coins failed / empty"
|
||
return 1
|
||
fi
|
||
# multi-line detail from JSON
|
||
while IFS= read -r line; do
|
||
[ -n "$line" ] || continue
|
||
info "coins ${label}" "$line"
|
||
done < <(python3 - "$out" <<'PY'
|
||
import json, sys
|
||
d = json.load(open(sys.argv[1]))
|
||
if not d.get("ok"):
|
||
print("unavailable: %s" % d.get("reason", "?"))
|
||
raise SystemExit(0)
|
||
amt = d.get("amount_in_circulation_alt") or d.get("amount_in_circulation_s") or "0"
|
||
print(
|
||
"n=%s in_circulation=%s spent=%s amount_circ=%s"
|
||
% (d.get("total_coins"), d.get("in_circulation"), d.get("spent"), amt)
|
||
)
|
||
st = d.get("by_status") or {}
|
||
if st:
|
||
print("status " + " ".join("%s=%s" % (k, st[k]) for k in sorted(st)))
|
||
circ = d.get("by_denom_circulation") or []
|
||
if circ:
|
||
bits = []
|
||
for x in circ:
|
||
dlab = x.get("denom_alt") or x.get("denom")
|
||
alab = x.get("amount_alt") or x.get("amount")
|
||
bits.append("%s×%d (=%s)" % (dlab, x["count"], alab))
|
||
print("denoms_in_circ " + " ".join(bits))
|
||
else:
|
||
print("denoms_in_circ (none)")
|
||
spent = d.get("by_denom_spent") or []
|
||
if spent:
|
||
bits = []
|
||
for x in spent:
|
||
dlab = x.get("denom_alt") or x.get("denom")
|
||
alab = x.get("amount_alt") or x.get("amount")
|
||
bits.append("%s×%d (=%s)" % (dlab, x["count"], alab))
|
||
print("denoms_spent " + " ".join(bits))
|
||
sp = d.get("amount_spent_alt") or d.get("amount_spent_s")
|
||
if sp and sp not in ("0", "0 GOA (GOA:0)"):
|
||
print("amount_spent %s" % sp)
|
||
PY
|
||
)
|
||
# delta vs previous snapshot at this run
|
||
if [ -f "$prev" ]; then
|
||
local dsum
|
||
dsum=$(metrics_coins_delta "$prev" "$out" "${METRICS_DIR}/coins-delta-${safe}.json" 2>/dev/null || true)
|
||
if [ -n "$dsum" ]; then
|
||
info "coins ${label} Δ" "$dsum"
|
||
fi
|
||
fi
|
||
cp -f "$out" "$prev" 2>/dev/null || true
|
||
cp -f "$out" "${METRICS_DIR}/coins-final.json" 2>/dev/null || true
|
||
return 0
|
||
}
|
||
|
||
# Diff two coin JSON snapshots → new coins this step
|
||
metrics_coins_delta() {
|
||
local before="${1:-}" after="${2:-}" out="${3:-$METRICS_DIR/coins-delta.json}"
|
||
python3 - "$before" "$after" "$out" <<'PY'
|
||
import json, sys
|
||
from collections import Counter
|
||
def load(p):
|
||
try:
|
||
d=json.load(open(p))
|
||
return d
|
||
except Exception:
|
||
return {}
|
||
b, a = load(sys.argv[1]), load(sys.argv[2])
|
||
outp = sys.argv[3]
|
||
def bag(d):
|
||
c=Counter()
|
||
for item in d.get("by_denom") or []:
|
||
c[item.get("denom") or "?"] += int(item.get("count") or 0)
|
||
return c
|
||
bb, aa = bag(b), bag(a)
|
||
delta = aa - bb
|
||
parts = ["%s×%+d" % (k, delta[k]) for k in sorted(delta.keys(), key=lambda x: float(str(x).split(":")[-1]) if ":" in str(x) or str(x).replace(".","").isdigit() else 0) if delta[k]]
|
||
new_total = int(a.get("total_coins") or 0) - int(b.get("total_coins") or 0)
|
||
rep = {
|
||
"new_coins": new_total,
|
||
"total_after": int(a.get("total_coins") or 0),
|
||
"circulation_after": int(a.get("in_circulation") or 0),
|
||
"delta_by_denom": {k: delta[k] for k in delta},
|
||
"summary": ("new=%+d total_now=%s | %s" % (
|
||
new_total, a.get("total_coins"), " ".join(parts) if parts else "(no denom change)")),
|
||
}
|
||
json.dump(rep, open(outp, "w"), indent=2)
|
||
print(rep["summary"])
|
||
PY
|
||
}
|
||
|
||
# --- Taler stack load (host + bank/exchange/merchant) ---
|
||
# Writes JSON to $1. RAM, process counts, DB sizes, disk I/O counters.
|
||
# koopa: KOOPA_SSH (+ fallbacks). stage-lfp: INSIDE_SSH (stagepaysan) + stage-lfp-* names.
|
||
metrics_taler_load() {
|
||
local out="${1:-$METRICS_DIR/load.json}"
|
||
local label="${2:-snap}"
|
||
mkdir -p "$(dirname "$out")" 2>/dev/null || true
|
||
if [ "${METRICS_LOAD}" = "0" ]; then
|
||
printf '%s\n' "{\"ok\":false,\"reason\":\"disabled\",\"label\":\"$label\"}" >"$out"
|
||
return 0
|
||
fi
|
||
local raw=""
|
||
local remote_py
|
||
local _m_bank _m_mer _m_ex _m_ssh
|
||
_m_bank="taler-hacktivism-bank"
|
||
_m_mer="taler-hacktivism"
|
||
_m_ex="taler-hacktivism-exchange-ansible"
|
||
_m_ssh=""
|
||
if [ "${INSIDE_PROFILE:-}" = "stage-lfp" ] \
|
||
|| [ "${EXPECT_CURRENCY:-}" = "TESTPAYSAN" ] \
|
||
|| { [ -n "${INSIDE_SSH:-}" ] && [ "${LOCAL_STACK:-0}" != "1" ]; }; then
|
||
_m_bank="${INSIDE_BANK_CTR:-stage-lfp-bank}"
|
||
_m_mer="${INSIDE_MERCHANT_CTR:-stage-lfp-merchant}"
|
||
_m_ex="${INSIDE_EXCHANGE_CTR:-stage-lfp-exchange-ansible}"
|
||
_m_ssh="${INSIDE_SSH:-francpaysan-stage-user}"
|
||
fi
|
||
remote_py=$(cat <<PY
|
||
import base64, json, os, re, subprocess, time
|
||
from collections import defaultdict
|
||
|
||
def sh(cmd, t=15):
|
||
try:
|
||
return subprocess.check_output(cmd, shell=True, text=True, stderr=subprocess.DEVNULL, timeout=t)
|
||
except Exception:
|
||
return ""
|
||
|
||
def podman_sh(name, script, t=25):
|
||
"""Run a shell script inside container without nested-quote hell (base64)."""
|
||
b64 = base64.b64encode(script.encode()).decode()
|
||
return sh(
|
||
f"podman exec {name} sh -c 'echo {b64} | base64 -d | sh'",
|
||
t=t,
|
||
)
|
||
|
||
def loadavg():
|
||
try:
|
||
a,b,c = open("/proc/loadavg").read().split()[:3]
|
||
return [float(a), float(b), float(c)]
|
||
except Exception:
|
||
return []
|
||
|
||
def meminfo():
|
||
d = {}
|
||
try:
|
||
for line in open("/proc/meminfo"):
|
||
k,v = line.split(":",1)
|
||
d[k.strip()] = int(v.strip().split()[0]) * 1024
|
||
except Exception:
|
||
pass
|
||
return {
|
||
"mem_total_b": d.get("MemTotal"),
|
||
"mem_available_b": d.get("MemAvailable"),
|
||
"mem_free_b": d.get("MemFree"),
|
||
"buffers_b": d.get("Buffers"),
|
||
"cached_b": d.get("Cached"),
|
||
}
|
||
|
||
def disk_io():
|
||
r = w = 0
|
||
try:
|
||
for line in open("/proc/diskstats"):
|
||
p = line.split()
|
||
if len(p) < 14: continue
|
||
name = p[2]
|
||
if name.startswith(("loop","ram","dm-")): continue
|
||
r += int(p[5]); w += int(p[9])
|
||
except Exception:
|
||
pass
|
||
return {"sectors_read": r, "sectors_written": w,
|
||
"approx_write_bytes": w*512, "approx_read_bytes": r*512}
|
||
|
||
CTRS = [("bank","${_m_bank}"),("merchant","${_m_mer}"),
|
||
("exchange","${_m_ex}")]
|
||
PY
|
||
)
|
||
# Append the rest of the remote probe (quoted so shell does not expand)
|
||
remote_py+=$(cat <<'PY'
|
||
|
||
def running(name):
|
||
return sh(f"podman inspect -f '{{{{.State.Running}}}}' {name}").strip() == "true"
|
||
|
||
def stats(name):
|
||
# Prefer Go template (stable across podman versions); JSON field names vary.
|
||
o = sh(
|
||
f"podman stats --no-stream --format "
|
||
f"'{{{{.CPUPerc}}}}|{{{{.MemUsage}}}}|{{{{.MemPerc}}}}|{{{{.BlockIO}}}}|{{{{.NetIO}}}}|{{{{.PIDs}}}}' "
|
||
f"{name}",
|
||
t=25,
|
||
).strip()
|
||
if o and "|" in o:
|
||
p = o.split("|")
|
||
while len(p) < 6:
|
||
p.append("")
|
||
return {
|
||
"cpu_pct": p[0] or None,
|
||
"mem_usage": p[1] or None,
|
||
"mem_pct": p[2] or None,
|
||
"block_io": p[3] or None,
|
||
"net_io": p[4] or None,
|
||
"pids": p[5] or None,
|
||
}
|
||
o = sh(f"podman stats --no-stream --format json {name}", t=25)
|
||
try:
|
||
data = json.loads(o)
|
||
if isinstance(data, list) and data:
|
||
data = data[0]
|
||
if not isinstance(data, dict):
|
||
return {}
|
||
return {
|
||
"cpu_pct": data.get("CPU") or data.get("CPUPerc"),
|
||
"mem_usage": data.get("MemUsage"),
|
||
"mem_pct": data.get("MemPerc"),
|
||
"block_io": data.get("BlockIO"),
|
||
"net_io": data.get("NetIO"),
|
||
"pids": data.get("PIDs"),
|
||
}
|
||
except Exception:
|
||
return {}
|
||
|
||
def procs(name):
|
||
# Pure shell /proc scan via base64 (containers often lack python3).
|
||
script = r"""
|
||
n=0; rss=0
|
||
n_postgres=0; rss_postgres=0
|
||
n_libeufin=0; rss_libeufin=0
|
||
n_taler_merchant=0; rss_taler_merchant=0
|
||
n_taler_exchange=0; rss_taler_exchange=0
|
||
n_nginx=0; rss_nginx=0
|
||
n_java=0; rss_java=0
|
||
n_other=0; rss_other=0
|
||
for d in /proc/[0-9]*; do
|
||
[ -r "$d/status" ] || continue
|
||
r=$(sed -n 's/^VmRSS:[[:space:]]*\([0-9][0-9]*\).*/\1/p' "$d/status" | head -1)
|
||
[ -n "$r" ] || continue
|
||
n=$((n+1)); rss=$((rss+r))
|
||
nm=$(sed -n 's/^Name:[[:space:]]*//p' "$d/status" | head -1)
|
||
cmd=$(tr '\0' ' ' <"$d/cmdline" 2>/dev/null | head -c 240)
|
||
blob=$(printf '%s %s' "$nm" "$cmd" | tr 'A-Z' 'a-z')
|
||
case "$blob" in
|
||
*postgres*|*postmaster*) n_postgres=$((n_postgres+1)); rss_postgres=$((rss_postgres+r)) ;;
|
||
*libeufin*|*mainkt*) n_libeufin=$((n_libeufin+1)); rss_libeufin=$((rss_libeufin+r)) ;;
|
||
*taler-merchant*) n_taler_merchant=$((n_taler_merchant+1)); rss_taler_merchant=$((rss_taler_merchant+r)) ;;
|
||
*taler-exchange*) n_taler_exchange=$((n_taler_exchange+1)); rss_taler_exchange=$((rss_taler_exchange+r)) ;;
|
||
*nginx*) n_nginx=$((n_nginx+1)); rss_nginx=$((rss_nginx+r)) ;;
|
||
*java*) n_java=$((n_java+1)); rss_java=$((rss_java+r)) ;;
|
||
*) n_other=$((n_other+1)); rss_other=$((rss_other+r)) ;;
|
||
esac
|
||
done
|
||
printf 'TOTAL %s %s\n' "$n" "$rss"
|
||
[ "$n_postgres" -gt 0 ] && printf 'ROLE postgres %s %s\n' "$n_postgres" "$rss_postgres"
|
||
[ "$n_libeufin" -gt 0 ] && printf 'ROLE libeufin %s %s\n' "$n_libeufin" "$rss_libeufin"
|
||
[ "$n_taler_merchant" -gt 0 ] && printf 'ROLE taler-merchant %s %s\n' "$n_taler_merchant" "$rss_taler_merchant"
|
||
[ "$n_taler_exchange" -gt 0 ] && printf 'ROLE taler-exchange %s %s\n' "$n_taler_exchange" "$rss_taler_exchange"
|
||
[ "$n_nginx" -gt 0 ] && printf 'ROLE nginx %s %s\n' "$n_nginx" "$rss_nginx"
|
||
[ "$n_java" -gt 0 ] && printf 'ROLE java %s %s\n' "$n_java" "$rss_java"
|
||
[ "$n_other" -gt 0 ] && printf 'ROLE other %s %s\n' "$n_other" "$rss_other"
|
||
"""
|
||
o = podman_sh(name, script, t=25)
|
||
by = {}
|
||
n = None
|
||
rss_kb = None
|
||
for line in o.splitlines():
|
||
p = line.split()
|
||
if not p:
|
||
continue
|
||
if p[0] == "TOTAL" and len(p) >= 3:
|
||
try:
|
||
n = int(p[1])
|
||
rss_kb = int(p[2])
|
||
except Exception:
|
||
pass
|
||
elif p[0] == "ROLE" and len(p) >= 4:
|
||
try:
|
||
by[p[1]] = {"n": int(p[2]), "rss_b": int(p[3]) * 1024}
|
||
except Exception:
|
||
pass
|
||
if n is None:
|
||
nn = sh(f"podman exec {name} sh -c 'ps -e --no-headers 2>/dev/null | wc -l'").strip()
|
||
n = int(nn) if nn.isdigit() else None
|
||
return {
|
||
"proc_total": n,
|
||
"rss_total_b": (rss_kb * 1024) if rss_kb is not None else None,
|
||
"by_role": by,
|
||
}
|
||
|
||
def dbs(name):
|
||
o = sh(
|
||
f"podman exec {name} su -s /bin/bash postgres -c "
|
||
+ json.dumps(
|
||
"psql -Atc \"SELECT datname||'|'||pg_database_size(datname)||'|'||pg_size_pretty(pg_database_size(datname)) "
|
||
"FROM pg_database WHERE datistemplate=false ORDER BY 1\""
|
||
)
|
||
)
|
||
out=[]
|
||
for line in o.splitlines():
|
||
p=line.strip().split("|")
|
||
if len(p)>=3:
|
||
try: out.append({"name":p[0],"size_b":int(p[1]),"size_pretty":p[2]})
|
||
except Exception: pass
|
||
du=sh(f"podman exec {name} sh -c 'du -sb /var/lib/postgresql 2>/dev/null | cut -f1'").strip()
|
||
pretty=sh(f"podman exec {name} sh -c 'du -sh /var/lib/postgresql 2>/dev/null | cut -f1'").strip()
|
||
return {"databases": out, "pgdata_bytes": int(du) if du.isdigit() else None, "pgdata_pretty": pretty or None}
|
||
|
||
components={}
|
||
for role,cname in CTRS:
|
||
if not running(cname):
|
||
components[role]={"container":cname,"running":False}
|
||
continue
|
||
components[role]={
|
||
"container":cname,"running":True,
|
||
"podman_stats":stats(cname),
|
||
"processes":procs(cname),
|
||
"databases":dbs(cname),
|
||
}
|
||
|
||
host_ps=sh("ps -eo comm=")
|
||
host_counts={k:sum(1 for line in host_ps.splitlines() if k in line.lower())
|
||
for k in ("postgres","nginx","caddy","podman","conmon","pasta")}
|
||
print(json.dumps({
|
||
"ok": True,
|
||
"ts": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||
"host": {
|
||
"hostname": sh("hostname").strip(),
|
||
"loadavg": loadavg(),
|
||
"nproc": os.cpu_count(),
|
||
"memory": meminfo(),
|
||
"disk_io": disk_io(),
|
||
"process_counts": host_counts,
|
||
},
|
||
"taler": components,
|
||
}))
|
||
PY
|
||
)
|
||
# Prefer stagepaysan (stage-lfp), else koopa SSH, else local podman only on LOCAL_STACK.
|
||
if [ -n "${_m_ssh}" ] && type mon_ssh_ok >/dev/null 2>&1 && mon_ssh_ok "${_m_ssh}"; then
|
||
raw=$(printf '%s' "$remote_py" | with_timeout "${METRICS_LOAD_SSH_TIMEOUT:-90}" \
|
||
ssh "${SSH_BASE_OPTS[@]}" "${_m_ssh}" python3 - 2>/dev/null || true)
|
||
elif [ "${SKIP_SSH:-0}" != "1" ] && type koopa_ssh_ok >/dev/null 2>&1 && koopa_ssh_ok; then
|
||
if type koopa_ssh_python >/dev/null 2>&1; then
|
||
raw=$(printf '%s' "$remote_py" | koopa_ssh_python "${METRICS_LOAD_SSH_TIMEOUT:-90}" 2>/dev/null || true)
|
||
else
|
||
raw=$(printf '%s' "$remote_py" | with_timeout "${METRICS_LOAD_SSH_TIMEOUT:-90}" \
|
||
ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" python3 - 2>/dev/null || true)
|
||
fi
|
||
elif [ "${LOCAL_STACK:-0}" = "1" ] && command -v podman >/dev/null 2>&1; then
|
||
raw=$(printf '%s' "$remote_py" | python3 - 2>/dev/null || true)
|
||
else
|
||
printf '%s\n' "{\"ok\":false,\"reason\":\"no-ssh-for-load\",\"label\":\"$label\"}" >"$out"
|
||
return 0
|
||
fi
|
||
if [ -z "$raw" ]; then
|
||
printf '%s\n' "{\"ok\":false,\"reason\":\"probe-failed\",\"label\":\"$label\",\"ssh\":\"${_m_ssh:-${KOOPA_SSH:-?}}\"}" >"$out"
|
||
return 1
|
||
fi
|
||
printf '%s\n' "$raw" | python3 -c '
|
||
import sys,json,re
|
||
t=sys.stdin.read(); obj=None
|
||
for m in re.finditer(r"\{", t):
|
||
try:
|
||
obj=json.loads(t[m.start():])
|
||
if isinstance(obj, dict) and (obj.get("host") or obj.get("taler") or obj.get("ok") is False):
|
||
break
|
||
except Exception:
|
||
obj=None
|
||
if not obj: obj={"ok":False,"reason":"parse"}
|
||
obj["label"]=sys.argv[1]
|
||
json.dump(obj, open(sys.argv[2],"w"), indent=2)
|
||
' "$label" "$out"
|
||
}
|
||
|
||
# One-line human summaries (no leading indent) for info()/ok() detail fields.
|
||
metrics_load_lines() {
|
||
local f="$1"
|
||
python3 - "$f" <<'PY'
|
||
import json, sys
|
||
try:
|
||
d = json.load(open(sys.argv[1]))
|
||
except Exception as e:
|
||
print(f"unreadable: {e}")
|
||
raise SystemExit(0)
|
||
if not d.get("ok"):
|
||
print(f"unavailable: {d.get('reason', 'n/a')} ssh={d.get('ssh', '')}".strip())
|
||
raise SystemExit(0)
|
||
h = d.get("host") or {}
|
||
mem = h.get("memory") or {}
|
||
la = h.get("loadavg") or []
|
||
|
||
def gi(b):
|
||
if b is None:
|
||
return "?"
|
||
return f"{b/1024/1024/1024:.2f}GiB"
|
||
|
||
def mi(b):
|
||
if b is None:
|
||
return "?"
|
||
return f"{b/1024/1024:.0f}MiB"
|
||
|
||
la_s = " ".join(f"{x:.2f}" for x in la) if la else "?"
|
||
tot = mem.get("mem_total_b")
|
||
avail = mem.get("mem_available_b")
|
||
used = (tot - avail) if (tot is not None and avail is not None) else None
|
||
print(
|
||
f"host loadavg=[{la_s}] nproc={h.get('nproc')} "
|
||
f"mem_used={gi(used)} avail={gi(avail)} total={gi(tot)}"
|
||
)
|
||
pc = h.get("process_counts") or {}
|
||
if pc:
|
||
bits = [f"{k}={v}" for k, v in sorted(pc.items()) if v]
|
||
if bits:
|
||
print("host procs " + " ".join(bits))
|
||
for role in ("bank", "exchange", "merchant"):
|
||
c = (d.get("taler") or {}).get(role) or {}
|
||
if not c:
|
||
continue
|
||
if not c.get("running"):
|
||
print(f"{role} DOWN ({c.get('container')})")
|
||
continue
|
||
pr = c.get("processes") or {}
|
||
st = c.get("podman_stats") or {}
|
||
roles = pr.get("by_role") or {}
|
||
role_bits = []
|
||
for k in ("libeufin", "taler-exchange", "taler-merchant", "postgres", "nginx", "java"):
|
||
if k in roles:
|
||
role_bits.append(f"{k}={mi(roles[k].get('rss_b'))}")
|
||
dbs = c.get("databases") or {}
|
||
db_s = ",".join(
|
||
f"{x.get('name')}={x.get('size_pretty')}"
|
||
for x in (dbs.get("databases") or [])[:5]
|
||
)
|
||
# podman MemUsage often reports 0B under pasta/cgroup — prefer /proc RSS
|
||
mem_u = st.get("mem_usage") or ""
|
||
mem_bit = ""
|
||
if mem_u and not str(mem_u).startswith("0B"):
|
||
mem_bit = f" podman_mem={mem_u}"
|
||
cpu = st.get("cpu_pct") or "?"
|
||
blk = st.get("block_io") or ""
|
||
blk_bit = ""
|
||
if blk and blk not in ("0B / 0B", "0B/0B", "-- / --"):
|
||
blk_bit = f" block={blk}"
|
||
line = (
|
||
f"{role} rss={gi(pr.get('rss_total_b'))} procs={pr.get('proc_total')} "
|
||
f"cpu={cpu}{mem_bit}{blk_bit}"
|
||
)
|
||
if role_bits:
|
||
line += " | " + " ".join(role_bits)
|
||
if db_s:
|
||
line += f" | db:{db_s}"
|
||
if dbs.get("pgdata_pretty"):
|
||
line += f" pgdata={dbs.get('pgdata_pretty')}"
|
||
print(line)
|
||
PY
|
||
}
|
||
|
||
# Snapshot load and emit as monitoring info lines (withdraw/pay phase markers).
|
||
# $1=json path $2=label (e.g. after-withdraw)
|
||
metrics_report_load() {
|
||
local out="$1" label="$2"
|
||
local line rc=0
|
||
if [ "${METRICS_LOAD}" = "0" ]; then
|
||
info "load ${label}" "skipped (METRICS_LOAD=0)"
|
||
return 0
|
||
fi
|
||
if [ "${SKIP_SSH:-0}" = "1" ] && ! command -v podman >/dev/null 2>&1; then
|
||
info "load ${label}" "skipped (no SSH / no local podman)"
|
||
return 0
|
||
fi
|
||
if ! metrics_taler_load "$out" "$label"; then
|
||
warn "load ${label}" "probe failed via ${KOOPA_SSH:-?} — try KOOPA_SSH=koopa-external"
|
||
return 1
|
||
fi
|
||
while IFS= read -r line; do
|
||
[ -n "$line" ] || continue
|
||
case "$line" in
|
||
unavailable:*|unreadable:*)
|
||
warn "load ${label}" "$line"
|
||
rc=1
|
||
;;
|
||
*)
|
||
info "load ${label}" "$line"
|
||
;;
|
||
esac
|
||
done < <(metrics_load_lines "$out")
|
||
return "$rc"
|
||
}
|
||
|
||
# Human one-screen summary of a load JSON
|
||
metrics_print_load() {
|
||
local f="$1" title="${2:-load}"
|
||
python3 - "$f" "$title" <<'PY'
|
||
import json,sys
|
||
try:
|
||
d=json.load(open(sys.argv[1]))
|
||
except Exception as e:
|
||
print(f" ({sys.argv[2]}: unreadable {e})")
|
||
raise SystemExit
|
||
if not d.get("ok"):
|
||
print(f" ({sys.argv[2]}: {d.get('reason','n/a')})")
|
||
raise SystemExit
|
||
h=d.get("host") or {}
|
||
mem=h.get("memory") or {}
|
||
la=h.get("loadavg") or []
|
||
def gi(b):
|
||
if b is None: return "?"
|
||
return f"{b/1024/1024/1024:.2f} GiB"
|
||
print(f" host loadavg {la} nproc={h.get('nproc')} mem_avail={gi(mem.get('mem_available_b'))}/{gi(mem.get('mem_total_b'))}")
|
||
dio=h.get("disk_io") or {}
|
||
if dio:
|
||
print(f" host disk Δsectors read={dio.get('sectors_read')} written={dio.get('sectors_written')} (~write {gi(dio.get('approx_write_bytes'))})")
|
||
for role in ("bank","exchange","merchant"):
|
||
c=(d.get("taler") or {}).get(role) or {}
|
||
if not c:
|
||
continue
|
||
if not c.get("running"):
|
||
print(f" {role:8} DOWN ({c.get('container')})")
|
||
continue
|
||
pr=c.get("processes") or {}
|
||
rss=pr.get("rss_total_b")
|
||
n=pr.get("proc_total")
|
||
roles=pr.get("by_role") or {}
|
||
bits=[]
|
||
for k in ("libeufin","taler-exchange","taler-merchant","postgres","nginx"):
|
||
if k in roles:
|
||
bits.append(f"{k}:n={roles[k].get('n')} rss={gi(roles[k].get('rss_b'))}")
|
||
dbs=c.get("databases") or {}
|
||
db_s=", ".join(f"{x.get('name')}={x.get('size_pretty')}" for x in (dbs.get("databases") or [])[:6])
|
||
pg=dbs.get("pgdata_pretty") or ""
|
||
st=c.get("podman_stats") or {}
|
||
print(f" {role:8} procs={n} rss={gi(rss)} cpu={st.get('cpu_pct','?')} block={st.get('block_io','?')}")
|
||
if bits:
|
||
print(f" " + " ".join(bits))
|
||
if db_s or pg:
|
||
print(f" db: {db_s}" + (f" pgdata={pg}" if pg else ""))
|
||
PY
|
||
}
|
||
|
||
# Diff two load snaps: highlight RAM/proc/DB growth for taler roles
|
||
metrics_print_load_delta() {
|
||
local before="$1" after="$2"
|
||
python3 - "$before" "$after" <<'PY'
|
||
import json,sys
|
||
def load(p):
|
||
try: return json.load(open(p))
|
||
except Exception: return {}
|
||
b,a=load(sys.argv[1]),load(sys.argv[2])
|
||
if not b.get("ok") or not a.get("ok"):
|
||
print(" (load delta unavailable)")
|
||
raise SystemExit
|
||
def gi(x):
|
||
if x is None: return None
|
||
return x/1024/1024/1024
|
||
print(" --- delta (after − before) ---")
|
||
# host load
|
||
bla=(b.get("host") or {}).get("loadavg") or [0,0,0]
|
||
ala=(a.get("host") or {}).get("loadavg") or [0,0,0]
|
||
if bla and ala:
|
||
print(f" loadavg1 {bla[0]:.2f} → {ala[0]:.2f} (Δ {ala[0]-bla[0]:+.2f})")
|
||
bm=(b.get("host") or {}).get("memory") or {}
|
||
am=(a.get("host") or {}).get("memory") or {}
|
||
if bm.get("mem_available_b") is not None and am.get("mem_available_b") is not None:
|
||
print(f" mem_avail {gi(bm['mem_available_b']):.2f} → {gi(am['mem_available_b']):.2f} GiB (Δ {gi(am['mem_available_b'])-gi(bm['mem_available_b']):+.3f} GiB)")
|
||
bd=(b.get("host") or {}).get("disk_io") or {}
|
||
ad=(a.get("host") or {}).get("disk_io") or {}
|
||
if bd.get("sectors_written") is not None and ad.get("sectors_written") is not None:
|
||
dw=(ad["sectors_written"]-bd["sectors_written"])*512
|
||
dr=(ad["sectors_read"]-bd["sectors_read"])*512
|
||
print(f" disk I/O write≈{dw/1024/1024:.1f} MiB read≈{dr/1024/1024:.1f} MiB (during phase)")
|
||
for role in ("bank","exchange","merchant"):
|
||
bc=(b.get("taler") or {}).get(role) or {}
|
||
ac=(a.get("taler") or {}).get(role) or {}
|
||
if not ac.get("running"):
|
||
continue
|
||
br=(bc.get("processes") or {}).get("rss_total_b")
|
||
ar=(ac.get("processes") or {}).get("rss_total_b")
|
||
bn=(bc.get("processes") or {}).get("proc_total")
|
||
an=(ac.get("processes") or {}).get("proc_total")
|
||
line=f" {role:8}"
|
||
if br is not None and ar is not None:
|
||
line+=f" rss {gi(br):.3f}→{gi(ar):.3f} GiB (Δ{gi(ar)-gi(br):+.3f})"
|
||
if bn is not None and an is not None:
|
||
line+=f" procs {bn}→{an} (Δ{an-bn:+d})"
|
||
# DB sizes
|
||
def dbmap(c):
|
||
m={}
|
||
for x in ((c.get("databases") or {}).get("databases") or []):
|
||
m[x.get("name")]=x.get("size_b")
|
||
return m
|
||
bdb,adb=dbmap(bc),dbmap(ac)
|
||
dbits=[]
|
||
for name in sorted(set(bdb)|set(adb)):
|
||
bb,aa=bdb.get(name),adb.get(name)
|
||
if bb is not None and aa is not None and aa!=bb:
|
||
dbits.append(f"{name} {aa-bb:+d}B")
|
||
elif aa is not None and bb is None:
|
||
dbits.append(f"{name}={aa}B")
|
||
if dbits:
|
||
line+=" dbΔ["+", ".join(dbits)+"]"
|
||
print(line)
|
||
PY
|
||
}
|
||
|
||
# Record a successful withdraw/pay amount for end statistics (one amount per line).
|
||
# $1=withdrawn|spent $2=amount (CUR:n)
|
||
metrics_record_flow() {
|
||
local kind="$1" amt="$2"
|
||
[ -n "$amt" ] || return 0
|
||
mkdir -p "${METRICS_DIR}" 2>/dev/null || true
|
||
case "$kind" in
|
||
withdrawn|withdraw) printf '%s\n' "$amt" >>"${METRICS_DIR}/flow-withdrawn.txt" ;;
|
||
spent|pay) printf '%s\n' "$amt" >>"${METRICS_DIR}/flow-spent.txt" ;;
|
||
*) return 1 ;;
|
||
esac
|
||
}
|
||
|
||
# Overall end-of-run statistics block
|
||
# Files under METRICS_DIR:
|
||
# coins-final.json, coins-history.tsv, perf-summary.json,
|
||
# load-before/after.json, flow-withdrawn.txt, flow-spent.txt,
|
||
# optional ladder TSV via METRICS_WITHDRAW_TSV / METRICS_PAY_TSV
|
||
metrics_print_overall() {
|
||
local title="${1:-overall statistics}"
|
||
section "statistics · $title"
|
||
metrics_print_final_stats || true
|
||
[ -n "${METRICS_EXTRA_LINES:-}" ] && printf '%s\n' "$METRICS_EXTRA_LINES"
|
||
}
|
||
|
||
# Rich final dashboard: coins, withdrawn/spent, performance, tendencies
|
||
metrics_print_final_stats() {
|
||
python3 - \
|
||
"${METRICS_DIR:-/tmp}" \
|
||
"${ALT_UNITS_FILE:-}" \
|
||
"${METRICS_WITHDRAW_TSV:-}" \
|
||
"${METRICS_PAY_TSV:-}" \
|
||
"${CUR:-GOA}" \
|
||
<<'PY'
|
||
import csv, json, os, sys
|
||
from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
|
||
from pathlib import Path
|
||
|
||
mdir = Path(sys.argv[1])
|
||
alt_path = sys.argv[2] or ""
|
||
wd_tsv = sys.argv[3] or ""
|
||
pay_tsv = sys.argv[4] or ""
|
||
cur = sys.argv[5] or "GOA"
|
||
|
||
alt = {}
|
||
if alt_path and Path(alt_path).is_file():
|
||
try:
|
||
alt = json.load(open(alt_path))
|
||
except Exception:
|
||
alt = {}
|
||
if not alt:
|
||
alt = {"0": cur}
|
||
|
||
def D(x, default=Decimal(0)):
|
||
try:
|
||
return Decimal(str(x))
|
||
except Exception:
|
||
return default
|
||
|
||
def parse_amt(s):
|
||
s = str(s or "").strip()
|
||
if not s:
|
||
return cur, Decimal(0)
|
||
if ":" in s:
|
||
c, v = s.split(":", 1)
|
||
try:
|
||
return c, Decimal(v)
|
||
except InvalidOperation:
|
||
return c, Decimal(0)
|
||
try:
|
||
return cur, Decimal(s)
|
||
except InvalidOperation:
|
||
return cur, Decimal(0)
|
||
|
||
def fmt_num(v: Decimal) -> str:
|
||
if v == v.to_integral():
|
||
return format(int(v), "d")
|
||
s = format(v.quantize(Decimal("0.00000001"), rounding=ROUND_HALF_UP), "f")
|
||
return s.rstrip("0").rstrip(".")
|
||
|
||
def fmt_amt(c, v: Decimal) -> str:
|
||
return "%s:%s" % (c, fmt_num(v))
|
||
|
||
def fmt_alt(c, v: Decimal) -> str:
|
||
base_s = fmt_amt(c, v)
|
||
base_name = alt.get("0") or c
|
||
if v == 0:
|
||
return "0 %s (%s)" % (base_name, base_s)
|
||
scales = []
|
||
for k, name in alt.items():
|
||
try:
|
||
scales.append((int(k), str(name)))
|
||
except Exception:
|
||
pass
|
||
scales.sort(key=lambda x: -x[0])
|
||
absval = abs(v)
|
||
for sc, name in scales:
|
||
unit = Decimal(10) ** sc
|
||
coeff = absval / unit
|
||
if coeff >= 1:
|
||
if sc == 0:
|
||
return "%s %s" % (fmt_num(v), name)
|
||
return "%s %s (%s)" % (fmt_num(coeff if v >= 0 else -coeff), name, base_s)
|
||
return "%s %s (%s)" % (fmt_num(v), base_name, base_s)
|
||
|
||
def sum_amount_file(path: Path):
|
||
total = Decimal(0)
|
||
ccy = cur
|
||
n = 0
|
||
if not path.is_file():
|
||
return ccy, total, n
|
||
for line in path.read_text().splitlines():
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
c, v = parse_amt(line)
|
||
ccy = c
|
||
total += v
|
||
n += 1
|
||
return ccy, total, n
|
||
|
||
def sum_tsv_amounts(path, amount_col="amount", status_ok=None):
|
||
total = Decimal(0)
|
||
ccy = cur
|
||
n = 0
|
||
if not path or not Path(path).is_file():
|
||
return ccy, total, n
|
||
with open(path, newline="") as f:
|
||
for row in csv.DictReader(f, delimiter="\t"):
|
||
st = (row.get("status") or "").upper()
|
||
if status_ok is not None and st not in status_ok:
|
||
continue
|
||
c, v = parse_amt(row.get(amount_col) or "")
|
||
if v <= 0:
|
||
continue
|
||
ccy = c
|
||
total += v
|
||
n += 1
|
||
return ccy, total, n
|
||
|
||
def trend_label(delta, eps=Decimal("0.01")):
|
||
if delta > eps:
|
||
return "↑ rising"
|
||
if delta < -eps:
|
||
return "↓ falling"
|
||
return "→ stable"
|
||
|
||
def half_trend(values):
|
||
"""Compare avg of second half vs first half of a list of numbers."""
|
||
xs = [float(x) for x in values if x is not None]
|
||
if len(xs) < 4:
|
||
if len(xs) < 2:
|
||
return "n/a (few samples)", None, None
|
||
a, b = xs[0], xs[-1]
|
||
return trend_label(Decimal(str(b - a))), a, b
|
||
mid = len(xs) // 2
|
||
first = sum(xs[:mid]) / mid
|
||
second = sum(xs[mid:]) / (len(xs) - mid)
|
||
d = second - first
|
||
lab = "↑ slowing (worse)" if d > 0.05 * max(abs(first), 1) else (
|
||
"↓ faster (better)" if d < -0.05 * max(abs(first), 1) else "→ steady"
|
||
)
|
||
return lab, first, second
|
||
|
||
print("")
|
||
print("╔══════════════════════════════════════════════════════════════╗")
|
||
print("║ FINAL STATISTICS ║")
|
||
print("╚══════════════════════════════════════════════════════════════╝")
|
||
|
||
# --- coins snapshot ---
|
||
coins = {}
|
||
cf = mdir / "coins-final.json"
|
||
if cf.is_file():
|
||
try:
|
||
coins = json.load(open(cf))
|
||
except Exception:
|
||
coins = {}
|
||
|
||
print("")
|
||
print("── Coins (wallet) ─────────────────────────────────────────────")
|
||
if coins.get("ok"):
|
||
circ_n = coins.get("in_circulation") or 0
|
||
spent_n = coins.get("spent") or 0
|
||
total_n = coins.get("total_coins") or 0
|
||
amt_alt = coins.get("amount_in_circulation_alt") or coins.get("amount_in_circulation_s") or "?"
|
||
spent_alt = coins.get("amount_spent_alt") or coins.get("amount_spent_s") or "0"
|
||
print(" total coins %s" % total_n)
|
||
print(" in circulation %s coins · %s" % (circ_n, amt_alt))
|
||
print(" spent (in dump) %s coins · %s" % (spent_n, spent_alt))
|
||
circ = coins.get("by_denom_circulation") or []
|
||
if circ:
|
||
print(" denoms in circ:")
|
||
for x in circ:
|
||
print(
|
||
" · %s × %s = %s"
|
||
% (
|
||
x.get("denom_alt") or x.get("denom"),
|
||
x.get("count"),
|
||
x.get("amount_alt") or x.get("amount"),
|
||
)
|
||
)
|
||
spent = coins.get("by_denom_spent") or []
|
||
if spent:
|
||
print(" denoms spent:")
|
||
for x in spent:
|
||
print(
|
||
" · %s × %s = %s"
|
||
% (
|
||
x.get("denom_alt") or x.get("denom"),
|
||
x.get("count"),
|
||
x.get("amount_alt") or x.get("amount"),
|
||
)
|
||
)
|
||
else:
|
||
print(" (no coins-final.json — run with wallet snaps)")
|
||
|
||
# --- withdrawn / spent flows ---
|
||
print("")
|
||
print("── Money flow (this run) ──────────────────────────────────────")
|
||
# Prefer ladder TSV if provided; else flow-*.txt from e2e
|
||
wd_ok = {"OK", "OK_BANK", "ZERO_REJECT", "ZERO_SKIP", "SKIP_DENOM", "CEILING_REJECT"}
|
||
pay_ok = {"OK", "ZERO_SKIP", "CEILING_SKIP", "CEILING_REJECT"}
|
||
c1, wd_sum, wd_n = sum_tsv_amounts(wd_tsv, "amount", None)
|
||
# count successful withdraws by status prefix OK or soft
|
||
if wd_tsv and Path(wd_tsv).is_file():
|
||
c1, wd_sum, wd_n = Decimal(0), Decimal(0), 0
|
||
ccy = cur
|
||
with open(wd_tsv, newline="") as f:
|
||
for row in csv.DictReader(f, delimiter="\t"):
|
||
st = (row.get("status") or "").upper()
|
||
# count value when mint/settle produced money in wallet
|
||
if st in ("OK", "OK_BANK"):
|
||
c, v = parse_amt(row.get("amount"))
|
||
ccy, wd_sum, wd_n = c, wd_sum + v, wd_n + 1
|
||
c1 = ccy
|
||
else:
|
||
c1, wd_sum, wd_n = sum_amount_file(mdir / "flow-withdrawn.txt")
|
||
|
||
if pay_tsv and Path(pay_tsv).is_file():
|
||
c2, pay_sum, pay_n = Decimal(0), Decimal(0), 0
|
||
ccy = cur
|
||
with open(pay_tsv, newline="") as f:
|
||
for row in csv.DictReader(f, delimiter="\t"):
|
||
st = (row.get("status") or "").upper()
|
||
if st == "OK":
|
||
c, v = parse_amt(row.get("amount"))
|
||
ccy, pay_sum, pay_n = c, pay_sum + v, pay_n + 1
|
||
c2 = ccy
|
||
else:
|
||
c2, pay_sum, pay_n = sum_amount_file(mdir / "flow-spent.txt")
|
||
|
||
print(" withdrawn %s events · %s" % (wd_n, fmt_alt(c1, wd_sum)))
|
||
print(" spent (paid) %s events · %s" % (pay_n, fmt_alt(c2, pay_sum)))
|
||
net = wd_sum - pay_sum
|
||
print(" net (wd − spent) %s %s" % (fmt_alt(c1, net), trend_label(net)))
|
||
if coins.get("ok"):
|
||
# circulation amount vs net flow
|
||
circ_map = coins.get("amount_in_circulation") or {}
|
||
circ_v = Decimal(0)
|
||
for _c, vs in circ_map.items():
|
||
circ_v += D(vs)
|
||
print(" wallet circ now %s" % fmt_alt(cur, circ_v))
|
||
print(" residual check wallet_circ vs net: Δ %s"
|
||
% fmt_alt(cur, circ_v - net))
|
||
|
||
# --- coin history tendency ---
|
||
print("")
|
||
print("── Tendency · coins over run ─────────────────────────────────")
|
||
hist = mdir / "coins-history.tsv"
|
||
if hist.is_file() and hist.stat().st_size > 0:
|
||
rows = list(csv.DictReader(open(hist), delimiter="\t"))
|
||
if rows:
|
||
def circ_amt(row):
|
||
# amount_circ column may be "GOA:20" or alt form — try numeric from total/in_circ
|
||
a = row.get("amount_circ") or "0"
|
||
if ":" in a and " " not in a:
|
||
return parse_amt(a)[1]
|
||
# try extract GOA: from parentheses
|
||
import re
|
||
m = re.search(r"\(([^)]+:[0-9.]+)\)", a)
|
||
if m:
|
||
return parse_amt(m.group(1))[1]
|
||
return D(row.get("in_circ") or 0)
|
||
|
||
first, last = rows[0], rows[-1]
|
||
c0, c1_ = D(first.get("in_circ") or 0), D(last.get("in_circ") or 0)
|
||
a0, a1 = circ_amt(first), circ_amt(last)
|
||
print(" samples %d snaps (%s → %s)"
|
||
% (len(rows), first.get("label"), last.get("label")))
|
||
print(" coins in circ %s → %s (Δ %+d) %s"
|
||
% (fmt_num(c0), fmt_num(c1_), int(c1_ - c0), trend_label(c1_ - c0)))
|
||
print(" amount in circ %s → %s %s"
|
||
% (fmt_alt(cur, a0), fmt_alt(cur, a1), trend_label(a1 - a0)))
|
||
# simple linear slope on coin count
|
||
if len(rows) >= 3:
|
||
ys = [float(D(r.get("in_circ") or 0)) for r in rows]
|
||
n = len(ys)
|
||
xs = list(range(n))
|
||
xm, ym = sum(xs) / n, sum(ys) / n
|
||
num = sum((x - xm) * (y - ym) for x, y in zip(xs, ys))
|
||
den = sum((x - xm) ** 2 for x in xs) or 1
|
||
slope = num / den
|
||
print(" trend slope %+.3f coins/snap %s"
|
||
% (slope, "↑ accumulating" if slope > 0.05 else ("↓ draining" if slope < -0.05 else "→ flat")))
|
||
print(" history (label · in_circ · amount):")
|
||
for r in rows[-12:]: # last 12
|
||
print(" · %-28s circ=%-4s %s"
|
||
% (r.get("label", "?")[:28], r.get("in_circ"), r.get("amount_circ")))
|
||
if len(rows) > 12:
|
||
print(" · … (%d earlier snaps)" % (len(rows) - 12))
|
||
else:
|
||
print(" (empty history)")
|
||
else:
|
||
print(" (no coins-history.tsv)")
|
||
|
||
# --- performance ---
|
||
print("")
|
||
print("── Performance indicators ───────────────────────────────────")
|
||
perf = {}
|
||
pf = mdir / "perf-summary.json"
|
||
if pf.is_file():
|
||
try:
|
||
perf = json.load(open(pf))
|
||
except Exception:
|
||
perf = {}
|
||
|
||
def print_bucket(name, v):
|
||
if not isinstance(v, dict) or not v.get("n"):
|
||
return
|
||
print(
|
||
" %-14s n=%s min=%sms p50=%sms avg=%sms max=%sms"
|
||
% (name, v.get("n"), v.get("min_ms"), v.get("p50_ms"), v.get("avg_ms"), v.get("max_ms"))
|
||
)
|
||
|
||
if perf:
|
||
for k, v in perf.items():
|
||
print_bucket(k, v)
|
||
else:
|
||
print(" (no perf-summary.json)")
|
||
|
||
# per-rung timing tendency from ladder TSV
|
||
for label, path, cols in (
|
||
("withdraw rungs", wd_tsv, ("ms_total", "ms_mint", "ms_settle")),
|
||
("pay rungs", pay_tsv, ("ms_total", "ms_order", "ms_handle", "ms_settle")),
|
||
):
|
||
if not path or not Path(path).is_file():
|
||
continue
|
||
with open(path, newline="") as f:
|
||
rrows = list(csv.DictReader(f, delimiter="\t"))
|
||
if len(rrows) < 2:
|
||
continue
|
||
print(" tendency · %s:" % label)
|
||
for col in cols:
|
||
vals = []
|
||
for r in rrows:
|
||
try:
|
||
vals.append(int(r.get(col) or 0))
|
||
except Exception:
|
||
pass
|
||
vals = [v for v in vals if v > 0]
|
||
if len(vals) < 2:
|
||
continue
|
||
lab, a, b = half_trend(vals)
|
||
if a is None:
|
||
print(" %-12s %s" % (col, lab))
|
||
else:
|
||
print(" %-12s first-half avg=%.0fms → second-half avg=%.0fms %s"
|
||
% (col, a, b, lab))
|
||
|
||
# --- load ---
|
||
print("")
|
||
print("── Host / container load ────────────────────────────────────")
|
||
lb, la = mdir / "load-before.json", mdir / "load-after.json"
|
||
|
||
def load_brief(path, tag):
|
||
if not path.is_file():
|
||
print(" %s: (missing)" % tag)
|
||
return None
|
||
try:
|
||
d = json.load(open(path))
|
||
except Exception:
|
||
print(" %s: (unreadable)" % tag)
|
||
return None
|
||
if not d.get("ok"):
|
||
print(" %s: %s" % (tag, d.get("reason", "?")))
|
||
return d
|
||
h = d.get("host") or {}
|
||
mem = h.get("memory") or {}
|
||
la_ = h.get("loadavg") or []
|
||
avail = mem.get("mem_available_b")
|
||
tot = mem.get("mem_total_b")
|
||
def gi(b):
|
||
return "?" if b is None else "%.2f GiB" % (b / 1024 / 1024 / 1024)
|
||
used = (tot - avail) if (tot is not None and avail is not None) else None
|
||
print(" %s loadavg=%s mem_used=%s / %s"
|
||
% (tag, la_, gi(used), gi(tot)))
|
||
for role in ("bank", "exchange", "merchant"):
|
||
c = (d.get("taler") or {}).get(role) or {}
|
||
if not c.get("running"):
|
||
continue
|
||
pr = c.get("processes") or {}
|
||
st = c.get("podman_stats") or {}
|
||
rss = pr.get("rss_total_b")
|
||
rss_s = "?" if rss is None else "%.2f GiB" % (rss / 1024 / 1024 / 1024)
|
||
print(" %-8s rss=%s cpu=%s procs=%s"
|
||
% (role, rss_s, st.get("cpu_pct") or "?", pr.get("proc_total")))
|
||
return d
|
||
|
||
db = load_brief(lb, "before")
|
||
da = load_brief(la, "after")
|
||
if db and da and db.get("ok") and da.get("ok"):
|
||
bla = (db.get("host") or {}).get("loadavg") or [0]
|
||
ala = (da.get("host") or {}).get("loadavg") or [0]
|
||
if bla and ala:
|
||
dload = float(ala[0]) - float(bla[0])
|
||
print(" loadavg1 tendency %.2f → %.2f (Δ %+.2f) %s"
|
||
% (float(bla[0]), float(ala[0]), dload,
|
||
"↑ higher load" if dload > 0.1 else ("↓ lower load" if dload < -0.1 else "→ steady")))
|
||
bm = (db.get("host") or {}).get("memory") or {}
|
||
am = (da.get("host") or {}).get("memory") or {}
|
||
if bm.get("mem_available_b") is not None and am.get("mem_available_b") is not None:
|
||
dmem = (am["mem_available_b"] - bm["mem_available_b"]) / 1024 / 1024 / 1024
|
||
print(" mem_avail tendency %.2f → %.2f GiB (Δ %+.3f) %s"
|
||
% (bm["mem_available_b"] / 1024**3, am["mem_available_b"] / 1024**3, dmem,
|
||
"↑ more free" if dmem > 0.05 else ("↓ less free" if dmem < -0.05 else "→ steady")))
|
||
|
||
print("")
|
||
print("──────────────────────────────────────────────────────────────")
|
||
PY
|
||
}
|