# shellcheck shell=bash
# Shared Taler stack metrics for e2e + ladder (source after lib.sh).
#
# - Host/container load (RAM, processes, DB sizes) before/after a phase
# - Wallet coin inventory (total + by denomination)
# - Final overall statistics block
#
# Env:
#   METRICS_DIR          where JSON snapshots go (default $SCRATCH or /tmp)
#   METRICS_LOAD=0       skip remote/host load probes
#   KOOPA_SSH / SKIP_SSH as in lib.sh

: "${METRICS_LOAD:=1}"
: "${METRICS_DIR:=${SCRATCH:-/tmp}}"
mkdir -p "$METRICS_DIR" 2>/dev/null || true

# --- coin inventory from wallet-cli dump-coins ---
# Writes JSON to $1; prints one-line summary to stdout.
# Sets COINS_TOTAL, COINS_FRESH, COINS_SUMMARY (human one-liner).
metrics_wallet_coins() {
  local out="${1:-$METRICS_DIR/coins.json}"
  local dump="$METRICS_DIR/dump-coins.raw"
  COINS_TOTAL=0
  COINS_FRESH=0
  COINS_SUMMARY="(no coins)"
  # Prefer wcli() from caller (bash function in e2e/ladder)
  if type wcli >/dev/null 2>&1; then
    wcli advanced dump-coins >"$dump" 2>/dev/null || true
  elif [ -n "${CLI_JS:-}" ] && [ -f "${CLI_JS}" ]; then
    node "$CLI_JS" --wallet-db="${WDB:-}" --no-throttle advanced dump-coins >"$dump" 2>/dev/null || true
  elif [ -n "${WALLET_CLI:-}" ]; then
    node "$WALLET_CLI" --wallet-db="${WDB:-}" --no-throttle advanced dump-coins >"$dump" 2>/dev/null || true
  else
    echo "$COINS_SUMMARY"
    printf '%s\n' '{"ok":false,"reason":"no-wcli"}' >"$out"
    return 1
  fi
  python3 - "$dump" "$out" "${CUR:-GOA}" <<'PY'
import json, re, sys
from collections import Counter
raw_path, out_path, cur = sys.argv[1:4]
raw = open(raw_path).read() if raw_path else ""
d = None
for m in re.finditer(r"\{", raw):
    try:
        d = json.loads(raw[m.start():])
        if isinstance(d, dict) and ("coins" in d or "coin" in d):
            break
    except Exception:
        d = None
coins = []
if isinstance(d, dict):
    coins = d.get("coins") or d.get("coin") or []
by_denom = Counter()
by_status = Counter()
fresh = 0
for c in coins:
    if not isinstance(c, dict):
        continue
    dv = c.get("denomValue") or c.get("value") or "?"
    st = str(c.get("coinStatus") or c.get("status") or "?")
    by_denom[dv] += 1
    by_status[st] += 1
    if st.lower() in ("fresh", "pending", "dormant", "usable", ""):
        # count non-spent-looking as "in circulation" for summary
        pass
    if "spent" not in st.lower() and "delete" not in st.lower():
        fresh += 1
# prefer explicit status
fresh = sum(n for s, n in by_status.items() if "spent" not in s.lower() and "delete" not in s.lower())
total = len(coins)
# stable sort denoms by numeric value if CUR:num
def denom_key(k):
    try:
        return float(str(k).split(":", 1)[-1])
    except Exception:
        return 0.0
denom_list = [
    {"denom": k, "count": by_denom[k]}
    for k in sorted(by_denom.keys(), key=denom_key)
]
# human bar: "GOA:10×3  GOA:1×5"
parts = []
for item in denom_list:
    parts.append("%s×%d" % (item["denom"], item["count"]))
summary = ("total=%d  in_wallet=%d  |  %s" % (total, fresh, "  ".join(parts))) if parts else ("total=%d" % total)
report = {
    "ok": True,
    "currency": cur,
    "total_coins": total,
    "in_circulation": fresh,
    "by_status": dict(by_status),
    "by_denom": denom_list,
    "summary": summary,
}
json.dump(report, open(out_path, "w"), indent=2)
print(summary)
# side channel for bash via file
open(out_path + ".total", "w").write(str(total))
open(out_path + ".circ", "w").write(str(fresh))
PY
  COINS_TOTAL=$(cat "${out}.total" 2>/dev/null || echo 0)
  COINS_FRESH=$(cat "${out}.circ" 2>/dev/null || echo 0)
  COINS_SUMMARY=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("summary","?"))' "$out" 2>/dev/null || echo "$COINS_SUMMARY")
  echo "$COINS_SUMMARY"
}

# Diff two coin JSON snapshots → new coins this step
metrics_coins_delta() {
  local before="${1:-}" after="${2:-}" out="${3:-$METRICS_DIR/coins-delta.json}"
  python3 - "$before" "$after" "$out" <<'PY'
import json, sys
from collections import Counter
def load(p):
    try:
        d=json.load(open(p))
        return d
    except Exception:
        return {}
b, a = load(sys.argv[1]), load(sys.argv[2])
outp = sys.argv[3]
def bag(d):
    c=Counter()
    for item in d.get("by_denom") or []:
        c[item.get("denom") or "?"] += int(item.get("count") or 0)
    return c
bb, aa = bag(b), bag(a)
delta = aa - bb
parts = ["%s×%+d" % (k, delta[k]) for k in sorted(delta.keys(), key=lambda x: float(str(x).split(":")[-1]) if ":" in str(x) or str(x).replace(".","").isdigit() else 0) if delta[k]]
new_total = int(a.get("total_coins") or 0) - int(b.get("total_coins") or 0)
rep = {
    "new_coins": new_total,
    "total_after": int(a.get("total_coins") or 0),
    "circulation_after": int(a.get("in_circulation") or 0),
    "delta_by_denom": {k: delta[k] for k in delta},
    "summary": ("new=%+d  total_now=%s  |  %s" % (
        new_total, a.get("total_coins"), "  ".join(parts) if parts else "(no denom change)")),
}
json.dump(rep, open(outp, "w"), indent=2)
print(rep["summary"])
PY
}

# --- Taler stack load on koopa (host + bank/exchange/merchant) ---
# Writes JSON to $1. RAM, process counts, DB sizes, disk I/O counters.
metrics_taler_load() {
  local out="${1:-$METRICS_DIR/load.json}"
  local label="${2:-snap}"
  if [ "${METRICS_LOAD}" = "0" ] || [ "${LADDER_LOAD:-1}" = "0" ]; then
    printf '%s\n' "{\"ok\":false,\"reason\":\"disabled\",\"label\":\"$label\"}" >"$out"
    return 0
  fi
  local raw=""
  local remote_py
  remote_py=$(cat <<'PY'
import json, os, re, subprocess, time
from collections import defaultdict

def sh(cmd, t=15):
    try:
        return subprocess.check_output(cmd, shell=True, text=True, stderr=subprocess.DEVNULL, timeout=t)
    except Exception:
        return ""

def loadavg():
    try:
        a,b,c = open("/proc/loadavg").read().split()[:3]
        return [float(a), float(b), float(c)]
    except Exception:
        return []

def meminfo():
    d = {}
    try:
        for line in open("/proc/meminfo"):
            k,v = line.split(":",1)
            d[k.strip()] = int(v.strip().split()[0]) * 1024
    except Exception:
        pass
    return {
        "mem_total_b": d.get("MemTotal"),
        "mem_available_b": d.get("MemAvailable"),
        "mem_free_b": d.get("MemFree"),
        "buffers_b": d.get("Buffers"),
        "cached_b": d.get("Cached"),
    }

def disk_io():
    r = w = 0
    try:
        for line in open("/proc/diskstats"):
            p = line.split()
            if len(p) < 14: continue
            name = p[2]
            if name.startswith(("loop","ram","dm-")): continue
            r += int(p[5]); w += int(p[9])
    except Exception:
        pass
    return {"sectors_read": r, "sectors_written": w,
            "approx_write_bytes": w*512, "approx_read_bytes": r*512}

CTRS = [("bank","taler-hacktivism-bank"),("merchant","taler-hacktivism"),
        ("exchange","taler-hacktivism-exchange-ansible")]

def running(name):
    return sh(f"podman inspect -f '{{{{.State.Running}}}}' {name}").strip() == "true"

def stats(name):
    o = sh(f"podman stats --no-stream --format json {name}")
    try:
        data = json.loads(o)
        if isinstance(data, list) and data: data = data[0]
        if not isinstance(data, dict): return {}
        return {"cpu_pct": data.get("CPU") or data.get("CPUPerc"),
                "mem_usage": data.get("MemUsage"),
                "mem_pct": data.get("MemPerc"),
                "block_io": data.get("BlockIO"),
                "net_io": data.get("NetIO"),
                "pids": data.get("PIDs")}
    except Exception:
        return {}

def procs(name):
    # count + RSS by role inside container
    code = r'''
import os,re,json
from collections import defaultdict
by=defaultdict(lambda:{"n":0,"rss_b":0}); n=0; rss=0
for ent in os.listdir("/proc"):
  if not ent.isdigit(): continue
  try: st=open(f"/proc/{ent}/status").read()
  except Exception: continue
  m=re.search(r"^VmRSS:\s+(\d+)",st,re.M)
  if not m: continue
  rb=int(m.group(1))*1024
  nm=(re.search(r"^Name:\s+(\S+)",st,re.M) or type("x",(object,),{"group":lambda s,i: "?"})()).group(1)
  try: cmd=open(f"/proc/{ent}/cmdline","rb").read().decode("utf-8","replace").replace("\0"," ")
  except Exception: cmd=""
  blob=(nm+" "+cmd).lower(); key="other"
  if "postgres" in blob or "postmaster" in blob: key="postgres"
  elif "libeufin" in blob or "mainkt" in blob: key="libeufin"
  elif "java" in blob: key="java"
  elif "taler-merchant" in blob: key="taler-merchant"
  elif "taler-exchange" in blob: key="taler-exchange"
  elif "nginx" in blob: key="nginx"
  elif "apache" in blob: key="apache"
  by[key]["n"]+=1; by[key]["rss_b"]+=rb; n+=1; rss+=rb
print(json.dumps({"proc_total":n,"rss_total_b":rss,"by_role":dict(by)}))
'''
    o = sh(f"podman exec {name} python3 -c {json.dumps(code)}")
    try: return json.loads(o)
    except Exception:
        n = sh(f"podman exec {name} sh -c 'ps -e --no-headers 2>/dev/null | wc -l'").strip()
        return {"proc_total": int(n) if n.isdigit() else None, "rss_total_b": None, "by_role": {}}

def dbs(name):
    o = sh(
        f"podman exec {name} su -s /bin/bash postgres -c "
        + json.dumps(
            "psql -Atc \"SELECT datname||'|'||pg_database_size(datname)||'|'||pg_size_pretty(pg_database_size(datname)) "
            "FROM pg_database WHERE datistemplate=false ORDER BY 1\""
        )
    )
    out=[]
    for line in o.splitlines():
        p=line.strip().split("|")
        if len(p)>=3:
            try: out.append({"name":p[0],"size_b":int(p[1]),"size_pretty":p[2]})
            except Exception: pass
    du=sh(f"podman exec {name} sh -c 'du -sb /var/lib/postgresql 2>/dev/null | cut -f1'").strip()
    pretty=sh(f"podman exec {name} sh -c 'du -sh /var/lib/postgresql 2>/dev/null | cut -f1'").strip()
    return {"databases": out, "pgdata_bytes": int(du) if du.isdigit() else None, "pgdata_pretty": pretty or None}

components={}
for role,cname in CTRS:
    if not running(cname):
        components[role]={"container":cname,"running":False}
        continue
    components[role]={
        "container":cname,"running":True,
        "podman_stats":stats(cname),
        "processes":procs(cname),
        "databases":dbs(cname),
    }

host_ps=sh("ps -eo comm=")
host_counts={k:sum(1 for line in host_ps.splitlines() if k in line.lower())
             for k in ("postgres","nginx","caddy","podman","conmon","pasta")}
print(json.dumps({
    "ok": True,
    "ts": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
    "host": {
        "hostname": sh("hostname").strip(),
        "loadavg": loadavg(),
        "nproc": os.cpu_count(),
        "memory": meminfo(),
        "disk_io": disk_io(),
        "process_counts": host_counts,
    },
    "taler": components,
}))
PY
)
  if [ "${SKIP_SSH:-0}" != "1" ] && type koopa_ssh_ok >/dev/null 2>&1 && koopa_ssh_ok; then
    raw=$(printf '%s\n' "$remote_py" | koopa_ssh_bash 50 'python3 -' 2>/dev/null || true)
    # koopa_ssh_bash only runs bash -s; pipe python via bash
    if [ -z "$raw" ]; then
      raw=$(printf 'python3 - <<'"'"'PY'"'"'\n%s\nPY\n' "$remote_py" | koopa_ssh_bash 50 2>/dev/null || true)
    fi
  elif command -v podman >/dev/null 2>&1; then
    raw=$(python3 -c "$remote_py" 2>/dev/null || true)
  fi
  if [ -z "$raw" ]; then
    printf '%s\n' "{\"ok\":false,\"reason\":\"probe-failed\",\"label\":\"$label\"}" >"$out"
    return 1
  fi
  printf '%s\n' "$raw" | python3 -c '
import sys,json,re
t=sys.stdin.read(); obj=None
for m in re.finditer(r"\{", t):
  try: obj=json.loads(t[m.start():])
  except Exception: pass
if not obj: obj={"ok":False,"reason":"parse"}
obj["label"]=sys.argv[1]
json.dump(obj, open(sys.argv[2],"w"), indent=2)
' "$label" "$out"
}

# Human one-screen summary of a load JSON
metrics_print_load() {
  local f="$1" title="${2:-load}"
  python3 - "$f" "$title" <<'PY'
import json,sys
try:
  d=json.load(open(sys.argv[1]))
except Exception as e:
  print(f"  ({sys.argv[2]}: unreadable {e})")
  raise SystemExit
if not d.get("ok"):
  print(f"  ({sys.argv[2]}: {d.get('reason','n/a')})")
  raise SystemExit
h=d.get("host") or {}
mem=h.get("memory") or {}
la=h.get("loadavg") or []
def gi(b):
  if b is None: return "?"
  return f"{b/1024/1024/1024:.2f} GiB"
print(f"  host loadavg {la}  nproc={h.get('nproc')}  mem_avail={gi(mem.get('mem_available_b'))}/{gi(mem.get('mem_total_b'))}")
dio=h.get("disk_io") or {}
if dio:
  print(f"  host disk Δsectors read={dio.get('sectors_read')} written={dio.get('sectors_written')} (~write {gi(dio.get('approx_write_bytes'))})")
for role in ("bank","exchange","merchant"):
  c=(d.get("taler") or {}).get(role) or {}
  if not c:
    continue
  if not c.get("running"):
    print(f"  {role:8} DOWN ({c.get('container')})")
    continue
  pr=c.get("processes") or {}
  rss=pr.get("rss_total_b")
  n=pr.get("proc_total")
  roles=pr.get("by_role") or {}
  bits=[]
  for k in ("libeufin","taler-exchange","taler-merchant","postgres","nginx"):
    if k in roles:
      bits.append(f"{k}:n={roles[k].get('n')} rss={gi(roles[k].get('rss_b'))}")
  dbs=c.get("databases") or {}
  db_s=", ".join(f"{x.get('name')}={x.get('size_pretty')}" for x in (dbs.get("databases") or [])[:6])
  pg=dbs.get("pgdata_pretty") or ""
  st=c.get("podman_stats") or {}
  print(f"  {role:8} procs={n} rss={gi(rss)} cpu={st.get('cpu_pct','?')} block={st.get('block_io','?')}")
  if bits:
    print(f"           " + "  ".join(bits))
  if db_s or pg:
    print(f"           db: {db_s}" + (f"  pgdata={pg}" if pg else ""))
PY
}

# Diff two load snaps: highlight RAM/proc/DB growth for taler roles
metrics_print_load_delta() {
  local before="$1" after="$2"
  python3 - "$before" "$after" <<'PY'
import json,sys
def load(p):
  try: return json.load(open(p))
  except Exception: return {}
b,a=load(sys.argv[1]),load(sys.argv[2])
if not b.get("ok") or not a.get("ok"):
  print("  (load delta unavailable)")
  raise SystemExit
def gi(x):
  if x is None: return None
  return x/1024/1024/1024
print("  --- delta (after − before) ---")
# host load
bla=(b.get("host") or {}).get("loadavg") or [0,0,0]
ala=(a.get("host") or {}).get("loadavg") or [0,0,0]
if bla and ala:
  print(f"  loadavg1  {bla[0]:.2f} → {ala[0]:.2f}  (Δ {ala[0]-bla[0]:+.2f})")
bm=(b.get("host") or {}).get("memory") or {}
am=(a.get("host") or {}).get("memory") or {}
if bm.get("mem_available_b") is not None and am.get("mem_available_b") is not None:
  print(f"  mem_avail {gi(bm['mem_available_b']):.2f} → {gi(am['mem_available_b']):.2f} GiB  (Δ {gi(am['mem_available_b'])-gi(bm['mem_available_b']):+.3f} GiB)")
bd=(b.get("host") or {}).get("disk_io") or {}
ad=(a.get("host") or {}).get("disk_io") or {}
if bd.get("sectors_written") is not None and ad.get("sectors_written") is not None:
  dw=(ad["sectors_written"]-bd["sectors_written"])*512
  dr=(ad["sectors_read"]-bd["sectors_read"])*512
  print(f"  disk I/O  write≈{dw/1024/1024:.1f} MiB  read≈{dr/1024/1024:.1f} MiB  (during phase)")
for role in ("bank","exchange","merchant"):
  bc=(b.get("taler") or {}).get(role) or {}
  ac=(a.get("taler") or {}).get(role) or {}
  if not ac.get("running"):
    continue
  br=(bc.get("processes") or {}).get("rss_total_b")
  ar=(ac.get("processes") or {}).get("rss_total_b")
  bn=(bc.get("processes") or {}).get("proc_total")
  an=(ac.get("processes") or {}).get("proc_total")
  line=f"  {role:8}"
  if br is not None and ar is not None:
    line+=f" rss {gi(br):.3f}→{gi(ar):.3f} GiB (Δ{gi(ar)-gi(br):+.3f})"
  if bn is not None and an is not None:
    line+=f"  procs {bn}→{an} (Δ{an-bn:+d})"
  # DB sizes
  def dbmap(c):
    m={}
    for x in ((c.get("databases") or {}).get("databases") or []):
      m[x.get("name")]=x.get("size_b")
    return m
  bdb,adb=dbmap(bc),dbmap(ac)
  dbits=[]
  for name in sorted(set(bdb)|set(adb)):
    bb,aa=bdb.get(name),adb.get(name)
    if bb is not None and aa is not None and aa!=bb:
      dbits.append(f"{name} {aa-bb:+d}B")
    elif aa is not None and bb is None:
      dbits.append(f"{name}={aa}B")
  if dbits:
    line+="  dbΔ["+", ".join(dbits)+"]"
  print(line)
PY
}

# Overall end-of-run statistics block
# Args via env / files:
#   METRICS_DIR, optional: WITHDRAW_REPORT, PAY_REPORT, phase timings JSON files
metrics_print_overall() {
  local title="${1:-overall statistics}"
  section "metrics · $title"
  if [ -f "${METRICS_DIR}/load-before.json" ]; then
    info "load BEFORE" ""
    metrics_print_load "${METRICS_DIR}/load-before.json" "before"
  fi
  if [ -f "${METRICS_DIR}/load-after.json" ]; then
    info "load AFTER" ""
    metrics_print_load "${METRICS_DIR}/load-after.json" "after"
    metrics_print_load_delta "${METRICS_DIR}/load-before.json" "${METRICS_DIR}/load-after.json"
  fi
  if [ -f "${METRICS_DIR}/coins-final.json" ]; then
    info "coins final" "$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("summary","?"))' "${METRICS_DIR}/coins-final.json" 2>/dev/null || echo n/a)"
  fi
  if [ -f "${METRICS_DIR}/coins-history.tsv" ]; then
    echo "  --- coins after each withdraw ---"
    # header + rows
    if [ -s "${METRICS_DIR}/coins-history.tsv" ]; then
      column -t -s $'\t' "${METRICS_DIR}/coins-history.tsv" 2>/dev/null \
        || cat "${METRICS_DIR}/coins-history.tsv"
    fi
  fi
  if [ -f "${METRICS_DIR}/perf-summary.json" ]; then
    info "performance" ""
    python3 - "${METRICS_DIR}/perf-summary.json" <<'PY'
import json,sys
d=json.load(open(sys.argv[1]))
for k,v in d.items():
  if isinstance(v, dict) and "n" in v:
    print(f"  {k:14} n={v.get('n')}  min={v.get('min_ms')}ms  p50={v.get('p50_ms')}ms  avg={v.get('avg_ms')}ms  max={v.get('max_ms')}ms")
  else:
    print(f"  {k}: {v}")
PY
  fi
  # free-form extras from caller
  [ -n "${METRICS_EXTRA_LINES:-}" ] && printf '%s\n' "$METRICS_EXTRA_LINES"
}