From 3945de58316267774c8bd3797a8afd322ba649e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Thu, 16 Jul 2026 23:45:51 +0200 Subject: [PATCH] monitoring: shared metrics for Taler load, DB size, and wallet coins. Host/container RAM and process probes plus dump-coins summaries for e2e and ladder. --- scripts/taler-monitoring/metrics.sh | 484 ++++++++++++++++++++++++++++ 1 file changed, 484 insertions(+) create mode 100644 scripts/taler-monitoring/metrics.sh diff --git a/scripts/taler-monitoring/metrics.sh b/scripts/taler-monitoring/metrics.sh new file mode 100644 index 0000000..e862644 --- /dev/null +++ b/scripts/taler-monitoring/metrics.sh @@ -0,0 +1,484 @@ +# shellcheck shell=bash +# Shared Taler stack metrics for e2e + ladder (source after lib.sh). +# +# - Host/container load (RAM, processes, DB sizes) before/after a phase +# - Wallet coin inventory (total + by denomination) +# - Final overall statistics block +# +# Env: +# METRICS_DIR where JSON snapshots go (default $SCRATCH or /tmp) +# METRICS_LOAD=0 skip remote/host load probes +# KOOPA_SSH / SKIP_SSH as in lib.sh + +: "${METRICS_LOAD:=1}" +: "${METRICS_DIR:=${SCRATCH:-/tmp}}" +mkdir -p "$METRICS_DIR" 2>/dev/null || true + +# --- coin inventory from wallet-cli dump-coins --- +# Writes JSON to $1; prints one-line summary to stdout. +# Sets COINS_TOTAL, COINS_FRESH, COINS_SUMMARY (human one-liner). +metrics_wallet_coins() { + local out="${1:-$METRICS_DIR/coins.json}" + local dump="$METRICS_DIR/dump-coins.raw" + COINS_TOTAL=0 + COINS_FRESH=0 + COINS_SUMMARY="(no coins)" + # Prefer wcli() from caller (bash function in e2e/ladder) + if type wcli >/dev/null 2>&1; then + wcli advanced dump-coins >"$dump" 2>/dev/null || true + elif [ -n "${CLI_JS:-}" ] && [ -f "${CLI_JS}" ]; then + node "$CLI_JS" --wallet-db="${WDB:-}" --no-throttle advanced dump-coins >"$dump" 2>/dev/null || true + elif [ -n "${WALLET_CLI:-}" ]; then + node "$WALLET_CLI" --wallet-db="${WDB:-}" --no-throttle advanced dump-coins >"$dump" 2>/dev/null || true + else + echo "$COINS_SUMMARY" + printf '%s\n' '{"ok":false,"reason":"no-wcli"}' >"$out" + return 1 + fi + python3 - "$dump" "$out" "${CUR:-GOA}" <<'PY' +import json, re, sys +from collections import Counter +raw_path, out_path, cur = sys.argv[1:4] +raw = open(raw_path).read() if raw_path else "" +d = None +for m in re.finditer(r"\{", raw): + try: + d = json.loads(raw[m.start():]) + if isinstance(d, dict) and ("coins" in d or "coin" in d): + break + except Exception: + d = None +coins = [] +if isinstance(d, dict): + coins = d.get("coins") or d.get("coin") or [] +by_denom = Counter() +by_status = Counter() +fresh = 0 +for c in coins: + if not isinstance(c, dict): + continue + dv = c.get("denomValue") or c.get("value") or "?" + st = str(c.get("coinStatus") or c.get("status") or "?") + by_denom[dv] += 1 + by_status[st] += 1 + if st.lower() in ("fresh", "pending", "dormant", "usable", ""): + # count non-spent-looking as "in circulation" for summary + pass + if "spent" not in st.lower() and "delete" not in st.lower(): + fresh += 1 +# prefer explicit status +fresh = sum(n for s, n in by_status.items() if "spent" not in s.lower() and "delete" not in s.lower()) +total = len(coins) +# stable sort denoms by numeric value if CUR:num +def denom_key(k): + try: + return float(str(k).split(":", 1)[-1]) + except Exception: + return 0.0 +denom_list = [ + {"denom": k, "count": by_denom[k]} + for k in sorted(by_denom.keys(), key=denom_key) +] +# human bar: "GOA:10×3 GOA:1×5" +parts = [] +for item in denom_list: + parts.append("%s×%d" % (item["denom"], item["count"])) +summary = ("total=%d in_wallet=%d | %s" % (total, fresh, " ".join(parts))) if parts else ("total=%d" % total) +report = { + "ok": True, + "currency": cur, + "total_coins": total, + "in_circulation": fresh, + "by_status": dict(by_status), + "by_denom": denom_list, + "summary": summary, +} +json.dump(report, open(out_path, "w"), indent=2) +print(summary) +# side channel for bash via file +open(out_path + ".total", "w").write(str(total)) +open(out_path + ".circ", "w").write(str(fresh)) +PY + COINS_TOTAL=$(cat "${out}.total" 2>/dev/null || echo 0) + COINS_FRESH=$(cat "${out}.circ" 2>/dev/null || echo 0) + COINS_SUMMARY=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("summary","?"))' "$out" 2>/dev/null || echo "$COINS_SUMMARY") + echo "$COINS_SUMMARY" +} + +# Diff two coin JSON snapshots → new coins this step +metrics_coins_delta() { + local before="${1:-}" after="${2:-}" out="${3:-$METRICS_DIR/coins-delta.json}" + python3 - "$before" "$after" "$out" <<'PY' +import json, sys +from collections import Counter +def load(p): + try: + d=json.load(open(p)) + return d + except Exception: + return {} +b, a = load(sys.argv[1]), load(sys.argv[2]) +outp = sys.argv[3] +def bag(d): + c=Counter() + for item in d.get("by_denom") or []: + c[item.get("denom") or "?"] += int(item.get("count") or 0) + return c +bb, aa = bag(b), bag(a) +delta = aa - bb +parts = ["%s×%+d" % (k, delta[k]) for k in sorted(delta.keys(), key=lambda x: float(str(x).split(":")[-1]) if ":" in str(x) or str(x).replace(".","").isdigit() else 0) if delta[k]] +new_total = int(a.get("total_coins") or 0) - int(b.get("total_coins") or 0) +rep = { + "new_coins": new_total, + "total_after": int(a.get("total_coins") or 0), + "circulation_after": int(a.get("in_circulation") or 0), + "delta_by_denom": {k: delta[k] for k in delta}, + "summary": ("new=%+d total_now=%s | %s" % ( + new_total, a.get("total_coins"), " ".join(parts) if parts else "(no denom change)")), +} +json.dump(rep, open(outp, "w"), indent=2) +print(rep["summary"]) +PY +} + +# --- Taler stack load on koopa (host + bank/exchange/merchant) --- +# Writes JSON to $1. RAM, process counts, DB sizes, disk I/O counters. +metrics_taler_load() { + local out="${1:-$METRICS_DIR/load.json}" + local label="${2:-snap}" + if [ "${METRICS_LOAD}" = "0" ] || [ "${LADDER_LOAD:-1}" = "0" ]; then + printf '%s\n' "{\"ok\":false,\"reason\":\"disabled\",\"label\":\"$label\"}" >"$out" + return 0 + fi + local raw="" + local remote_py + remote_py=$(cat <<'PY' +import json, os, re, subprocess, time +from collections import defaultdict + +def sh(cmd, t=15): + try: + return subprocess.check_output(cmd, shell=True, text=True, stderr=subprocess.DEVNULL, timeout=t) + except Exception: + return "" + +def loadavg(): + try: + a,b,c = open("/proc/loadavg").read().split()[:3] + return [float(a), float(b), float(c)] + except Exception: + return [] + +def meminfo(): + d = {} + try: + for line in open("/proc/meminfo"): + k,v = line.split(":",1) + d[k.strip()] = int(v.strip().split()[0]) * 1024 + except Exception: + pass + return { + "mem_total_b": d.get("MemTotal"), + "mem_available_b": d.get("MemAvailable"), + "mem_free_b": d.get("MemFree"), + "buffers_b": d.get("Buffers"), + "cached_b": d.get("Cached"), + } + +def disk_io(): + r = w = 0 + try: + for line in open("/proc/diskstats"): + p = line.split() + if len(p) < 14: continue + name = p[2] + if name.startswith(("loop","ram","dm-")): continue + r += int(p[5]); w += int(p[9]) + except Exception: + pass + return {"sectors_read": r, "sectors_written": w, + "approx_write_bytes": w*512, "approx_read_bytes": r*512} + +CTRS = [("bank","taler-hacktivism-bank"),("merchant","taler-hacktivism"), + ("exchange","taler-hacktivism-exchange-ansible")] + +def running(name): + return sh(f"podman inspect -f '{{{{.State.Running}}}}' {name}").strip() == "true" + +def stats(name): + o = sh(f"podman stats --no-stream --format json {name}") + try: + data = json.loads(o) + if isinstance(data, list) and data: data = data[0] + if not isinstance(data, dict): return {} + return {"cpu_pct": data.get("CPU") or data.get("CPUPerc"), + "mem_usage": data.get("MemUsage"), + "mem_pct": data.get("MemPerc"), + "block_io": data.get("BlockIO"), + "net_io": data.get("NetIO"), + "pids": data.get("PIDs")} + except Exception: + return {} + +def procs(name): + # count + RSS by role inside container + code = r''' +import os,re,json +from collections import defaultdict +by=defaultdict(lambda:{"n":0,"rss_b":0}); n=0; rss=0 +for ent in os.listdir("/proc"): + if not ent.isdigit(): continue + try: st=open(f"/proc/{ent}/status").read() + except Exception: continue + m=re.search(r"^VmRSS:\s+(\d+)",st,re.M) + if not m: continue + rb=int(m.group(1))*1024 + nm=(re.search(r"^Name:\s+(\S+)",st,re.M) or type("x",(object,),{"group":lambda s,i: "?"})()).group(1) + try: cmd=open(f"/proc/{ent}/cmdline","rb").read().decode("utf-8","replace").replace("\0"," ") + except Exception: cmd="" + blob=(nm+" "+cmd).lower(); key="other" + if "postgres" in blob or "postmaster" in blob: key="postgres" + elif "libeufin" in blob or "mainkt" in blob: key="libeufin" + elif "java" in blob: key="java" + elif "taler-merchant" in blob: key="taler-merchant" + elif "taler-exchange" in blob: key="taler-exchange" + elif "nginx" in blob: key="nginx" + elif "apache" in blob: key="apache" + by[key]["n"]+=1; by[key]["rss_b"]+=rb; n+=1; rss+=rb +print(json.dumps({"proc_total":n,"rss_total_b":rss,"by_role":dict(by)})) +''' + o = sh(f"podman exec {name} python3 -c {json.dumps(code)}") + try: return json.loads(o) + except Exception: + n = sh(f"podman exec {name} sh -c 'ps -e --no-headers 2>/dev/null | wc -l'").strip() + return {"proc_total": int(n) if n.isdigit() else None, "rss_total_b": None, "by_role": {}} + +def dbs(name): + o = sh( + f"podman exec {name} su -s /bin/bash postgres -c " + + json.dumps( + "psql -Atc \"SELECT datname||'|'||pg_database_size(datname)||'|'||pg_size_pretty(pg_database_size(datname)) " + "FROM pg_database WHERE datistemplate=false ORDER BY 1\"" + ) + ) + out=[] + for line in o.splitlines(): + p=line.strip().split("|") + if len(p)>=3: + try: out.append({"name":p[0],"size_b":int(p[1]),"size_pretty":p[2]}) + except Exception: pass + du=sh(f"podman exec {name} sh -c 'du -sb /var/lib/postgresql 2>/dev/null | cut -f1'").strip() + pretty=sh(f"podman exec {name} sh -c 'du -sh /var/lib/postgresql 2>/dev/null | cut -f1'").strip() + return {"databases": out, "pgdata_bytes": int(du) if du.isdigit() else None, "pgdata_pretty": pretty or None} + +components={} +for role,cname in CTRS: + if not running(cname): + components[role]={"container":cname,"running":False} + continue + components[role]={ + "container":cname,"running":True, + "podman_stats":stats(cname), + "processes":procs(cname), + "databases":dbs(cname), + } + +host_ps=sh("ps -eo comm=") +host_counts={k:sum(1 for line in host_ps.splitlines() if k in line.lower()) + for k in ("postgres","nginx","caddy","podman","conmon","pasta")} +print(json.dumps({ + "ok": True, + "ts": time.strftime("%Y-%m-%dT%H:%M:%S%z"), + "host": { + "hostname": sh("hostname").strip(), + "loadavg": loadavg(), + "nproc": os.cpu_count(), + "memory": meminfo(), + "disk_io": disk_io(), + "process_counts": host_counts, + }, + "taler": components, +})) +PY +) + if [ "${SKIP_SSH:-0}" != "1" ] && type koopa_ssh_ok >/dev/null 2>&1 && koopa_ssh_ok; then + raw=$(printf '%s\n' "$remote_py" | koopa_ssh_bash 50 'python3 -' 2>/dev/null || true) + # koopa_ssh_bash only runs bash -s; pipe python via bash + if [ -z "$raw" ]; then + raw=$(printf 'python3 - <<'"'"'PY'"'"'\n%s\nPY\n' "$remote_py" | koopa_ssh_bash 50 2>/dev/null || true) + fi + elif command -v podman >/dev/null 2>&1; then + raw=$(python3 -c "$remote_py" 2>/dev/null || true) + fi + if [ -z "$raw" ]; then + printf '%s\n' "{\"ok\":false,\"reason\":\"probe-failed\",\"label\":\"$label\"}" >"$out" + return 1 + fi + printf '%s\n' "$raw" | python3 -c ' +import sys,json,re +t=sys.stdin.read(); obj=None +for m in re.finditer(r"\{", t): + try: obj=json.loads(t[m.start():]) + except Exception: pass +if not obj: obj={"ok":False,"reason":"parse"} +obj["label"]=sys.argv[1] +json.dump(obj, open(sys.argv[2],"w"), indent=2) +' "$label" "$out" +} + +# Human one-screen summary of a load JSON +metrics_print_load() { + local f="$1" title="${2:-load}" + python3 - "$f" "$title" <<'PY' +import json,sys +try: + d=json.load(open(sys.argv[1])) +except Exception as e: + print(f" ({sys.argv[2]}: unreadable {e})") + raise SystemExit +if not d.get("ok"): + print(f" ({sys.argv[2]}: {d.get('reason','n/a')})") + raise SystemExit +h=d.get("host") or {} +mem=h.get("memory") or {} +la=h.get("loadavg") or [] +def gi(b): + if b is None: return "?" + return f"{b/1024/1024/1024:.2f} GiB" +print(f" host loadavg {la} nproc={h.get('nproc')} mem_avail={gi(mem.get('mem_available_b'))}/{gi(mem.get('mem_total_b'))}") +dio=h.get("disk_io") or {} +if dio: + print(f" host disk Δsectors read={dio.get('sectors_read')} written={dio.get('sectors_written')} (~write {gi(dio.get('approx_write_bytes'))})") +for role in ("bank","exchange","merchant"): + c=(d.get("taler") or {}).get(role) or {} + if not c: + continue + if not c.get("running"): + print(f" {role:8} DOWN ({c.get('container')})") + continue + pr=c.get("processes") or {} + rss=pr.get("rss_total_b") + n=pr.get("proc_total") + roles=pr.get("by_role") or {} + bits=[] + for k in ("libeufin","taler-exchange","taler-merchant","postgres","nginx"): + if k in roles: + bits.append(f"{k}:n={roles[k].get('n')} rss={gi(roles[k].get('rss_b'))}") + dbs=c.get("databases") or {} + db_s=", ".join(f"{x.get('name')}={x.get('size_pretty')}" for x in (dbs.get("databases") or [])[:6]) + pg=dbs.get("pgdata_pretty") or "" + st=c.get("podman_stats") or {} + print(f" {role:8} procs={n} rss={gi(rss)} cpu={st.get('cpu_pct','?')} block={st.get('block_io','?')}") + if bits: + print(f" " + " ".join(bits)) + if db_s or pg: + print(f" db: {db_s}" + (f" pgdata={pg}" if pg else "")) +PY +} + +# Diff two load snaps: highlight RAM/proc/DB growth for taler roles +metrics_print_load_delta() { + local before="$1" after="$2" + python3 - "$before" "$after" <<'PY' +import json,sys +def load(p): + try: return json.load(open(p)) + except Exception: return {} +b,a=load(sys.argv[1]),load(sys.argv[2]) +if not b.get("ok") or not a.get("ok"): + print(" (load delta unavailable)") + raise SystemExit +def gi(x): + if x is None: return None + return x/1024/1024/1024 +print(" --- delta (after − before) ---") +# host load +bla=(b.get("host") or {}).get("loadavg") or [0,0,0] +ala=(a.get("host") or {}).get("loadavg") or [0,0,0] +if bla and ala: + print(f" loadavg1 {bla[0]:.2f} → {ala[0]:.2f} (Δ {ala[0]-bla[0]:+.2f})") +bm=(b.get("host") or {}).get("memory") or {} +am=(a.get("host") or {}).get("memory") or {} +if bm.get("mem_available_b") is not None and am.get("mem_available_b") is not None: + print(f" mem_avail {gi(bm['mem_available_b']):.2f} → {gi(am['mem_available_b']):.2f} GiB (Δ {gi(am['mem_available_b'])-gi(bm['mem_available_b']):+.3f} GiB)") +bd=(b.get("host") or {}).get("disk_io") or {} +ad=(a.get("host") or {}).get("disk_io") or {} +if bd.get("sectors_written") is not None and ad.get("sectors_written") is not None: + dw=(ad["sectors_written"]-bd["sectors_written"])*512 + dr=(ad["sectors_read"]-bd["sectors_read"])*512 + print(f" disk I/O write≈{dw/1024/1024:.1f} MiB read≈{dr/1024/1024:.1f} MiB (during phase)") +for role in ("bank","exchange","merchant"): + bc=(b.get("taler") or {}).get(role) or {} + ac=(a.get("taler") or {}).get(role) or {} + if not ac.get("running"): + continue + br=(bc.get("processes") or {}).get("rss_total_b") + ar=(ac.get("processes") or {}).get("rss_total_b") + bn=(bc.get("processes") or {}).get("proc_total") + an=(ac.get("processes") or {}).get("proc_total") + line=f" {role:8}" + if br is not None and ar is not None: + line+=f" rss {gi(br):.3f}→{gi(ar):.3f} GiB (Δ{gi(ar)-gi(br):+.3f})" + if bn is not None and an is not None: + line+=f" procs {bn}→{an} (Δ{an-bn:+d})" + # DB sizes + def dbmap(c): + m={} + for x in ((c.get("databases") or {}).get("databases") or []): + m[x.get("name")]=x.get("size_b") + return m + bdb,adb=dbmap(bc),dbmap(ac) + dbits=[] + for name in sorted(set(bdb)|set(adb)): + bb,aa=bdb.get(name),adb.get(name) + if bb is not None and aa is not None and aa!=bb: + dbits.append(f"{name} {aa-bb:+d}B") + elif aa is not None and bb is None: + dbits.append(f"{name}={aa}B") + if dbits: + line+=" dbΔ["+", ".join(dbits)+"]" + print(line) +PY +} + +# Overall end-of-run statistics block +# Args via env / files: +# METRICS_DIR, optional: WITHDRAW_REPORT, PAY_REPORT, phase timings JSON files +metrics_print_overall() { + local title="${1:-overall statistics}" + section "metrics · $title" + if [ -f "${METRICS_DIR}/load-before.json" ]; then + info "load BEFORE" "" + metrics_print_load "${METRICS_DIR}/load-before.json" "before" + fi + if [ -f "${METRICS_DIR}/load-after.json" ]; then + info "load AFTER" "" + metrics_print_load "${METRICS_DIR}/load-after.json" "after" + metrics_print_load_delta "${METRICS_DIR}/load-before.json" "${METRICS_DIR}/load-after.json" + fi + if [ -f "${METRICS_DIR}/coins-final.json" ]; then + info "coins final" "$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("summary","?"))' "${METRICS_DIR}/coins-final.json" 2>/dev/null || echo n/a)" + fi + if [ -f "${METRICS_DIR}/coins-history.tsv" ]; then + echo " --- coins after each withdraw ---" + # header + rows + if [ -s "${METRICS_DIR}/coins-history.tsv" ]; then + column -t -s $'\t' "${METRICS_DIR}/coins-history.tsv" 2>/dev/null \ + || cat "${METRICS_DIR}/coins-history.tsv" + fi + fi + if [ -f "${METRICS_DIR}/perf-summary.json" ]; then + info "performance" "" + python3 - "${METRICS_DIR}/perf-summary.json" <<'PY' +import json,sys +d=json.load(open(sys.argv[1])) +for k,v in d.items(): + if isinstance(v, dict) and "n" in v: + print(f" {k:14} n={v.get('n')} min={v.get('min_ms')}ms p50={v.get('p50_ms')}ms avg={v.get('avg_ms')}ms max={v.get('max_ms')}ms") + else: + print(f" {k}: {v}") +PY + fi + # free-form extras from caller + [ -n "${METRICS_EXTRA_LINES:-}" ] && printf '%s\n' "$METRICS_EXTRA_LINES" +}