monitoring: shared metrics for Taler load, DB size, and wallet coins.

Host/container RAM and process probes plus dump-coins summaries for e2e and ladder.
This commit is contained in:
Hernâni Marques 2026-07-16 23:45:51 +02:00
parent 44d2a57af9
commit 3945de5831

View file

@ -0,0 +1,484 @@
# shellcheck shell=bash
# Shared Taler stack metrics for e2e + ladder (source after lib.sh).
#
# - Host/container load (RAM, processes, DB sizes) before/after a phase
# - Wallet coin inventory (total + by denomination)
# - Final overall statistics block
#
# Env:
# METRICS_DIR where JSON snapshots go (default $SCRATCH or /tmp)
# METRICS_LOAD=0 skip remote/host load probes
# KOOPA_SSH / SKIP_SSH as in lib.sh
: "${METRICS_LOAD:=1}"
: "${METRICS_DIR:=${SCRATCH:-/tmp}}"
mkdir -p "$METRICS_DIR" 2>/dev/null || true
# --- coin inventory from wallet-cli dump-coins ---
# Writes JSON to $1; prints one-line summary to stdout.
# Sets COINS_TOTAL, COINS_FRESH, COINS_SUMMARY (human one-liner).
metrics_wallet_coins() {
local out="${1:-$METRICS_DIR/coins.json}"
local dump="$METRICS_DIR/dump-coins.raw"
COINS_TOTAL=0
COINS_FRESH=0
COINS_SUMMARY="(no coins)"
# Prefer wcli() from caller (bash function in e2e/ladder)
if type wcli >/dev/null 2>&1; then
wcli advanced dump-coins >"$dump" 2>/dev/null || true
elif [ -n "${CLI_JS:-}" ] && [ -f "${CLI_JS}" ]; then
node "$CLI_JS" --wallet-db="${WDB:-}" --no-throttle advanced dump-coins >"$dump" 2>/dev/null || true
elif [ -n "${WALLET_CLI:-}" ]; then
node "$WALLET_CLI" --wallet-db="${WDB:-}" --no-throttle advanced dump-coins >"$dump" 2>/dev/null || true
else
echo "$COINS_SUMMARY"
printf '%s\n' '{"ok":false,"reason":"no-wcli"}' >"$out"
return 1
fi
python3 - "$dump" "$out" "${CUR:-GOA}" <<'PY'
import json, re, sys
from collections import Counter
raw_path, out_path, cur = sys.argv[1:4]
raw = open(raw_path).read() if raw_path else ""
d = None
for m in re.finditer(r"\{", raw):
try:
d = json.loads(raw[m.start():])
if isinstance(d, dict) and ("coins" in d or "coin" in d):
break
except Exception:
d = None
coins = []
if isinstance(d, dict):
coins = d.get("coins") or d.get("coin") or []
by_denom = Counter()
by_status = Counter()
fresh = 0
for c in coins:
if not isinstance(c, dict):
continue
dv = c.get("denomValue") or c.get("value") or "?"
st = str(c.get("coinStatus") or c.get("status") or "?")
by_denom[dv] += 1
by_status[st] += 1
if st.lower() in ("fresh", "pending", "dormant", "usable", ""):
# count non-spent-looking as "in circulation" for summary
pass
if "spent" not in st.lower() and "delete" not in st.lower():
fresh += 1
# prefer explicit status
fresh = sum(n for s, n in by_status.items() if "spent" not in s.lower() and "delete" not in s.lower())
total = len(coins)
# stable sort denoms by numeric value if CUR:num
def denom_key(k):
try:
return float(str(k).split(":", 1)[-1])
except Exception:
return 0.0
denom_list = [
{"denom": k, "count": by_denom[k]}
for k in sorted(by_denom.keys(), key=denom_key)
]
# human bar: "GOA:10×3 GOA:1×5"
parts = []
for item in denom_list:
parts.append("%s×%d" % (item["denom"], item["count"]))
summary = ("total=%d in_wallet=%d | %s" % (total, fresh, " ".join(parts))) if parts else ("total=%d" % total)
report = {
"ok": True,
"currency": cur,
"total_coins": total,
"in_circulation": fresh,
"by_status": dict(by_status),
"by_denom": denom_list,
"summary": summary,
}
json.dump(report, open(out_path, "w"), indent=2)
print(summary)
# side channel for bash via file
open(out_path + ".total", "w").write(str(total))
open(out_path + ".circ", "w").write(str(fresh))
PY
COINS_TOTAL=$(cat "${out}.total" 2>/dev/null || echo 0)
COINS_FRESH=$(cat "${out}.circ" 2>/dev/null || echo 0)
COINS_SUMMARY=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("summary","?"))' "$out" 2>/dev/null || echo "$COINS_SUMMARY")
echo "$COINS_SUMMARY"
}
# Diff two coin JSON snapshots → new coins this step
metrics_coins_delta() {
local before="${1:-}" after="${2:-}" out="${3:-$METRICS_DIR/coins-delta.json}"
python3 - "$before" "$after" "$out" <<'PY'
import json, sys
from collections import Counter
def load(p):
try:
d=json.load(open(p))
return d
except Exception:
return {}
b, a = load(sys.argv[1]), load(sys.argv[2])
outp = sys.argv[3]
def bag(d):
c=Counter()
for item in d.get("by_denom") or []:
c[item.get("denom") or "?"] += int(item.get("count") or 0)
return c
bb, aa = bag(b), bag(a)
delta = aa - bb
parts = ["%s×%+d" % (k, delta[k]) for k in sorted(delta.keys(), key=lambda x: float(str(x).split(":")[-1]) if ":" in str(x) or str(x).replace(".","").isdigit() else 0) if delta[k]]
new_total = int(a.get("total_coins") or 0) - int(b.get("total_coins") or 0)
rep = {
"new_coins": new_total,
"total_after": int(a.get("total_coins") or 0),
"circulation_after": int(a.get("in_circulation") or 0),
"delta_by_denom": {k: delta[k] for k in delta},
"summary": ("new=%+d total_now=%s | %s" % (
new_total, a.get("total_coins"), " ".join(parts) if parts else "(no denom change)")),
}
json.dump(rep, open(outp, "w"), indent=2)
print(rep["summary"])
PY
}
# --- Taler stack load on koopa (host + bank/exchange/merchant) ---
# Writes JSON to $1. RAM, process counts, DB sizes, disk I/O counters.
metrics_taler_load() {
local out="${1:-$METRICS_DIR/load.json}"
local label="${2:-snap}"
if [ "${METRICS_LOAD}" = "0" ] || [ "${LADDER_LOAD:-1}" = "0" ]; then
printf '%s\n' "{\"ok\":false,\"reason\":\"disabled\",\"label\":\"$label\"}" >"$out"
return 0
fi
local raw=""
local remote_py
remote_py=$(cat <<'PY'
import json, os, re, subprocess, time
from collections import defaultdict
def sh(cmd, t=15):
try:
return subprocess.check_output(cmd, shell=True, text=True, stderr=subprocess.DEVNULL, timeout=t)
except Exception:
return ""
def loadavg():
try:
a,b,c = open("/proc/loadavg").read().split()[:3]
return [float(a), float(b), float(c)]
except Exception:
return []
def meminfo():
d = {}
try:
for line in open("/proc/meminfo"):
k,v = line.split(":",1)
d[k.strip()] = int(v.strip().split()[0]) * 1024
except Exception:
pass
return {
"mem_total_b": d.get("MemTotal"),
"mem_available_b": d.get("MemAvailable"),
"mem_free_b": d.get("MemFree"),
"buffers_b": d.get("Buffers"),
"cached_b": d.get("Cached"),
}
def disk_io():
r = w = 0
try:
for line in open("/proc/diskstats"):
p = line.split()
if len(p) < 14: continue
name = p[2]
if name.startswith(("loop","ram","dm-")): continue
r += int(p[5]); w += int(p[9])
except Exception:
pass
return {"sectors_read": r, "sectors_written": w,
"approx_write_bytes": w*512, "approx_read_bytes": r*512}
CTRS = [("bank","taler-hacktivism-bank"),("merchant","taler-hacktivism"),
("exchange","taler-hacktivism-exchange-ansible")]
def running(name):
return sh(f"podman inspect -f '{{{{.State.Running}}}}' {name}").strip() == "true"
def stats(name):
o = sh(f"podman stats --no-stream --format json {name}")
try:
data = json.loads(o)
if isinstance(data, list) and data: data = data[0]
if not isinstance(data, dict): return {}
return {"cpu_pct": data.get("CPU") or data.get("CPUPerc"),
"mem_usage": data.get("MemUsage"),
"mem_pct": data.get("MemPerc"),
"block_io": data.get("BlockIO"),
"net_io": data.get("NetIO"),
"pids": data.get("PIDs")}
except Exception:
return {}
def procs(name):
# count + RSS by role inside container
code = r'''
import os,re,json
from collections import defaultdict
by=defaultdict(lambda:{"n":0,"rss_b":0}); n=0; rss=0
for ent in os.listdir("/proc"):
if not ent.isdigit(): continue
try: st=open(f"/proc/{ent}/status").read()
except Exception: continue
m=re.search(r"^VmRSS:\s+(\d+)",st,re.M)
if not m: continue
rb=int(m.group(1))*1024
nm=(re.search(r"^Name:\s+(\S+)",st,re.M) or type("x",(object,),{"group":lambda s,i: "?"})()).group(1)
try: cmd=open(f"/proc/{ent}/cmdline","rb").read().decode("utf-8","replace").replace("\0"," ")
except Exception: cmd=""
blob=(nm+" "+cmd).lower(); key="other"
if "postgres" in blob or "postmaster" in blob: key="postgres"
elif "libeufin" in blob or "mainkt" in blob: key="libeufin"
elif "java" in blob: key="java"
elif "taler-merchant" in blob: key="taler-merchant"
elif "taler-exchange" in blob: key="taler-exchange"
elif "nginx" in blob: key="nginx"
elif "apache" in blob: key="apache"
by[key]["n"]+=1; by[key]["rss_b"]+=rb; n+=1; rss+=rb
print(json.dumps({"proc_total":n,"rss_total_b":rss,"by_role":dict(by)}))
'''
o = sh(f"podman exec {name} python3 -c {json.dumps(code)}")
try: return json.loads(o)
except Exception:
n = sh(f"podman exec {name} sh -c 'ps -e --no-headers 2>/dev/null | wc -l'").strip()
return {"proc_total": int(n) if n.isdigit() else None, "rss_total_b": None, "by_role": {}}
def dbs(name):
o = sh(
f"podman exec {name} su -s /bin/bash postgres -c "
+ json.dumps(
"psql -Atc \"SELECT datname||'|'||pg_database_size(datname)||'|'||pg_size_pretty(pg_database_size(datname)) "
"FROM pg_database WHERE datistemplate=false ORDER BY 1\""
)
)
out=[]
for line in o.splitlines():
p=line.strip().split("|")
if len(p)>=3:
try: out.append({"name":p[0],"size_b":int(p[1]),"size_pretty":p[2]})
except Exception: pass
du=sh(f"podman exec {name} sh -c 'du -sb /var/lib/postgresql 2>/dev/null | cut -f1'").strip()
pretty=sh(f"podman exec {name} sh -c 'du -sh /var/lib/postgresql 2>/dev/null | cut -f1'").strip()
return {"databases": out, "pgdata_bytes": int(du) if du.isdigit() else None, "pgdata_pretty": pretty or None}
components={}
for role,cname in CTRS:
if not running(cname):
components[role]={"container":cname,"running":False}
continue
components[role]={
"container":cname,"running":True,
"podman_stats":stats(cname),
"processes":procs(cname),
"databases":dbs(cname),
}
host_ps=sh("ps -eo comm=")
host_counts={k:sum(1 for line in host_ps.splitlines() if k in line.lower())
for k in ("postgres","nginx","caddy","podman","conmon","pasta")}
print(json.dumps({
"ok": True,
"ts": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
"host": {
"hostname": sh("hostname").strip(),
"loadavg": loadavg(),
"nproc": os.cpu_count(),
"memory": meminfo(),
"disk_io": disk_io(),
"process_counts": host_counts,
},
"taler": components,
}))
PY
)
if [ "${SKIP_SSH:-0}" != "1" ] && type koopa_ssh_ok >/dev/null 2>&1 && koopa_ssh_ok; then
raw=$(printf '%s\n' "$remote_py" | koopa_ssh_bash 50 'python3 -' 2>/dev/null || true)
# koopa_ssh_bash only runs bash -s; pipe python via bash
if [ -z "$raw" ]; then
raw=$(printf 'python3 - <<'"'"'PY'"'"'\n%s\nPY\n' "$remote_py" | koopa_ssh_bash 50 2>/dev/null || true)
fi
elif command -v podman >/dev/null 2>&1; then
raw=$(python3 -c "$remote_py" 2>/dev/null || true)
fi
if [ -z "$raw" ]; then
printf '%s\n' "{\"ok\":false,\"reason\":\"probe-failed\",\"label\":\"$label\"}" >"$out"
return 1
fi
printf '%s\n' "$raw" | python3 -c '
import sys,json,re
t=sys.stdin.read(); obj=None
for m in re.finditer(r"\{", t):
try: obj=json.loads(t[m.start():])
except Exception: pass
if not obj: obj={"ok":False,"reason":"parse"}
obj["label"]=sys.argv[1]
json.dump(obj, open(sys.argv[2],"w"), indent=2)
' "$label" "$out"
}
# Human one-screen summary of a load JSON
metrics_print_load() {
local f="$1" title="${2:-load}"
python3 - "$f" "$title" <<'PY'
import json,sys
try:
d=json.load(open(sys.argv[1]))
except Exception as e:
print(f" ({sys.argv[2]}: unreadable {e})")
raise SystemExit
if not d.get("ok"):
print(f" ({sys.argv[2]}: {d.get('reason','n/a')})")
raise SystemExit
h=d.get("host") or {}
mem=h.get("memory") or {}
la=h.get("loadavg") or []
def gi(b):
if b is None: return "?"
return f"{b/1024/1024/1024:.2f} GiB"
print(f" host loadavg {la} nproc={h.get('nproc')} mem_avail={gi(mem.get('mem_available_b'))}/{gi(mem.get('mem_total_b'))}")
dio=h.get("disk_io") or {}
if dio:
print(f" host disk Δsectors read={dio.get('sectors_read')} written={dio.get('sectors_written')} (~write {gi(dio.get('approx_write_bytes'))})")
for role in ("bank","exchange","merchant"):
c=(d.get("taler") or {}).get(role) or {}
if not c:
continue
if not c.get("running"):
print(f" {role:8} DOWN ({c.get('container')})")
continue
pr=c.get("processes") or {}
rss=pr.get("rss_total_b")
n=pr.get("proc_total")
roles=pr.get("by_role") or {}
bits=[]
for k in ("libeufin","taler-exchange","taler-merchant","postgres","nginx"):
if k in roles:
bits.append(f"{k}:n={roles[k].get('n')} rss={gi(roles[k].get('rss_b'))}")
dbs=c.get("databases") or {}
db_s=", ".join(f"{x.get('name')}={x.get('size_pretty')}" for x in (dbs.get("databases") or [])[:6])
pg=dbs.get("pgdata_pretty") or ""
st=c.get("podman_stats") or {}
print(f" {role:8} procs={n} rss={gi(rss)} cpu={st.get('cpu_pct','?')} block={st.get('block_io','?')}")
if bits:
print(f" " + " ".join(bits))
if db_s or pg:
print(f" db: {db_s}" + (f" pgdata={pg}" if pg else ""))
PY
}
# Diff two load snaps: highlight RAM/proc/DB growth for taler roles
metrics_print_load_delta() {
local before="$1" after="$2"
python3 - "$before" "$after" <<'PY'
import json,sys
def load(p):
try: return json.load(open(p))
except Exception: return {}
b,a=load(sys.argv[1]),load(sys.argv[2])
if not b.get("ok") or not a.get("ok"):
print(" (load delta unavailable)")
raise SystemExit
def gi(x):
if x is None: return None
return x/1024/1024/1024
print(" --- delta (after before) ---")
# host load
bla=(b.get("host") or {}).get("loadavg") or [0,0,0]
ala=(a.get("host") or {}).get("loadavg") or [0,0,0]
if bla and ala:
print(f" loadavg1 {bla[0]:.2f} → {ala[0]:.2f} (Δ {ala[0]-bla[0]:+.2f})")
bm=(b.get("host") or {}).get("memory") or {}
am=(a.get("host") or {}).get("memory") or {}
if bm.get("mem_available_b") is not None and am.get("mem_available_b") is not None:
print(f" mem_avail {gi(bm['mem_available_b']):.2f} → {gi(am['mem_available_b']):.2f} GiB (Δ {gi(am['mem_available_b'])-gi(bm['mem_available_b']):+.3f} GiB)")
bd=(b.get("host") or {}).get("disk_io") or {}
ad=(a.get("host") or {}).get("disk_io") or {}
if bd.get("sectors_written") is not None and ad.get("sectors_written") is not None:
dw=(ad["sectors_written"]-bd["sectors_written"])*512
dr=(ad["sectors_read"]-bd["sectors_read"])*512
print(f" disk I/O write≈{dw/1024/1024:.1f} MiB read≈{dr/1024/1024:.1f} MiB (during phase)")
for role in ("bank","exchange","merchant"):
bc=(b.get("taler") or {}).get(role) or {}
ac=(a.get("taler") or {}).get(role) or {}
if not ac.get("running"):
continue
br=(bc.get("processes") or {}).get("rss_total_b")
ar=(ac.get("processes") or {}).get("rss_total_b")
bn=(bc.get("processes") or {}).get("proc_total")
an=(ac.get("processes") or {}).get("proc_total")
line=f" {role:8}"
if br is not None and ar is not None:
line+=f" rss {gi(br):.3f}→{gi(ar):.3f} GiB (Δ{gi(ar)-gi(br):+.3f})"
if bn is not None and an is not None:
line+=f" procs {bn}→{an} (Δ{an-bn:+d})"
# DB sizes
def dbmap(c):
m={}
for x in ((c.get("databases") or {}).get("databases") or []):
m[x.get("name")]=x.get("size_b")
return m
bdb,adb=dbmap(bc),dbmap(ac)
dbits=[]
for name in sorted(set(bdb)|set(adb)):
bb,aa=bdb.get(name),adb.get(name)
if bb is not None and aa is not None and aa!=bb:
dbits.append(f"{name} {aa-bb:+d}B")
elif aa is not None and bb is None:
dbits.append(f"{name}={aa}B")
if dbits:
line+=" dbΔ["+", ".join(dbits)+"]"
print(line)
PY
}
# Overall end-of-run statistics block
# Args via env / files:
# METRICS_DIR, optional: WITHDRAW_REPORT, PAY_REPORT, phase timings JSON files
metrics_print_overall() {
local title="${1:-overall statistics}"
section "metrics · $title"
if [ -f "${METRICS_DIR}/load-before.json" ]; then
info "load BEFORE" ""
metrics_print_load "${METRICS_DIR}/load-before.json" "before"
fi
if [ -f "${METRICS_DIR}/load-after.json" ]; then
info "load AFTER" ""
metrics_print_load "${METRICS_DIR}/load-after.json" "after"
metrics_print_load_delta "${METRICS_DIR}/load-before.json" "${METRICS_DIR}/load-after.json"
fi
if [ -f "${METRICS_DIR}/coins-final.json" ]; then
info "coins final" "$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("summary","?"))' "${METRICS_DIR}/coins-final.json" 2>/dev/null || echo n/a)"
fi
if [ -f "${METRICS_DIR}/coins-history.tsv" ]; then
echo " --- coins after each withdraw ---"
# header + rows
if [ -s "${METRICS_DIR}/coins-history.tsv" ]; then
column -t -s $'\t' "${METRICS_DIR}/coins-history.tsv" 2>/dev/null \
|| cat "${METRICS_DIR}/coins-history.tsv"
fi
fi
if [ -f "${METRICS_DIR}/perf-summary.json" ]; then
info "performance" ""
python3 - "${METRICS_DIR}/perf-summary.json" <<'PY'
import json,sys
d=json.load(open(sys.argv[1]))
for k,v in d.items():
if isinstance(v, dict) and "n" in v:
print(f" {k:14} n={v.get('n')} min={v.get('min_ms')}ms p50={v.get('p50_ms')}ms avg={v.get('avg_ms')}ms max={v.get('max_ms')}ms")
else:
print(f" {k}: {v}")
PY
fi
# free-form extras from caller
[ -n "${METRICS_EXTRA_LINES:-}" ] && printf '%s\n' "$METRICS_EXTRA_LINES"
}