# shellcheck shell=bash
# Shared Taler stack metrics for e2e + ladder (source after lib.sh).
#
# - Host/container load (RAM, processes, DB sizes) before/after a phase
# - Wallet coin inventory (total + by denomination)
# - Final overall statistics block
#
# Env:
#   METRICS_DIR          where JSON snapshots go (default $SCRATCH or /tmp)
#   METRICS_LOAD=0       skip remote/host load probes
#   KOOPA_SSH / SKIP_SSH as in lib.sh

: "${METRICS_LOAD:=1}"
: "${METRICS_DIR:=${SCRATCH:-/tmp}}"
mkdir -p "$METRICS_DIR" 2>/dev/null || true

# ---------------------------------------------------------------------------
# alt_unit_names — human amounts (Kilo-GOA / Mega-GOA / …) with base in parens
# From exchange/bank currency_specification.alt_unit_names:
#   "0"→GOA, "3"→Kilo-GOA, "6"→Mega-GOA, …  "-3"→Milli-GOA, …
# Example: GOA:5000  →  5 Kilo-GOA (GOA:5000)
# ---------------------------------------------------------------------------
: "${ALT_UNITS_FILE:=${METRICS_DIR}/alt_unit_names.json}"

# Load map from public /config into ALT_UNITS_FILE. $1=optional config URL.
metrics_load_alt_units() {
  local url="${1:-${EXCHANGE_PUBLIC:-https://exchange.hacktivism.ch}/config}"
  local code
  mkdir -p "$(dirname "$ALT_UNITS_FILE")" 2>/dev/null || true
  code=$(curl -skS -m 12 -o "${ALT_UNITS_FILE}.raw" -w '%{http_code}' "$url" 2>/dev/null || echo 000)
  if [ "$code" != "200" ]; then
    # soft fallback: SI-style names if live config unreachable
    printf '%s\n' '{"0":"GOA","3":"Kilo-GOA","6":"Mega-GOA","9":"Giga-GOA","12":"Tera-GOA","15":"Peta-GOA","18":"Exa-GOA","21":"Zetta-GOA","24":"Yotta-GOA","-1":"Deci-GOA","-2":"Centi-GOA","-3":"Milli-GOA","-6":"Micro-GOA","-8":"Atomic-GOA"}' \
      >"$ALT_UNITS_FILE"
    return 1
  fi
  python3 - "${ALT_UNITS_FILE}.raw" "$ALT_UNITS_FILE" <<'PY'
import json, sys
src, dst = sys.argv[1:3]
d = json.load(open(src))
au = None
cs = d.get("currency_specification")
if isinstance(cs, dict):
    au = cs.get("alt_unit_names")
if not au and isinstance(d.get("currencies"), dict):
    # merchant-style: currencies.GOA.alt_unit_names
    for _code, spec in d["currencies"].items():
        if isinstance(spec, dict) and spec.get("alt_unit_names"):
            au = spec["alt_unit_names"]
            break
if not isinstance(au, dict) or "0" not in au:
    au = {"0": d.get("currency") or "GOA"}
json.dump(au, open(dst, "w"), indent=2, sort_keys=True)
PY
  export ALT_UNITS_FILE
  return 0
}

# Format one amount: "GOA:5000" → "5 Kilo-GOA (GOA:5000)"
# Uses ALT_UNITS_FILE if present. Pure stdout.
format_amount_alt() {
  local amt="${1:-}"
  [ -n "$amt" ] || return 0
  python3 - "$amt" "${ALT_UNITS_FILE:-}" <<'PY'
import json, sys
from decimal import Decimal, InvalidOperation, ROUND_HALF_UP

amt = sys.argv[1].strip()
path = sys.argv[2] if len(sys.argv) > 2 else ""
alt = {}
if path:
    try:
        alt = json.load(open(path))
    except Exception:
        alt = {}
if not alt:
    alt = {"0": "GOA"}

def parse(s):
    if ":" in s:
        c, v = s.split(":", 1)
        return c, Decimal(v)
    return "GOA", Decimal(s)

def fmt_num(v: Decimal) -> str:
    if v == v.to_integral():
        return format(int(v), "d")
    s = format(v.quantize(Decimal("0.00000001"), rounding=ROUND_HALF_UP), "f")
    return s.rstrip("0").rstrip(".")

try:
    cur, val = parse(amt)
except (InvalidOperation, ValueError):
    print(amt)
    raise SystemExit(0)

base_name = alt.get("0") or cur
# always show canonical base form in parens
base_s = "%s:%s" % (cur, fmt_num(val))
if val == 0:
    print("0 %s (%s)" % (base_name, base_s))
    raise SystemExit(0)

# scales: power of 10 relative to unit "0"
scales = []
for k, name in alt.items():
    try:
        scales.append((int(k), str(name)))
    except Exception:
        pass
scales.sort(key=lambda x: -x[0])  # largest unit first

# pick largest scale where |value| >= 10^scale (for scale>=0),
# or for fractions the finest unit that makes the coefficient >= 1
chosen = None  # (scale, name, coeff)
absval = abs(val)
for sc, name in scales:
    unit = Decimal(10) ** sc
    if unit <= 0:
        continue
    coeff = absval / unit
    if coeff >= 1:
        chosen = (sc, name, coeff if val >= 0 else -coeff)
        break
if chosen is None:
    # smaller than smallest unit — use base
    print("%s %s (%s)" % (fmt_num(val), base_name, base_s))
    raise SystemExit(0)

sc, name, coeff = chosen
# if base unit, prefer "GOA:12" style still with parens only when alt differs
if sc == 0:
    print("%s %s" % (fmt_num(val), name))
    raise SystemExit(0)
print("%s %s (%s)" % (fmt_num(coeff), name, base_s))
PY
}

# Format list of "CUR:n" amounts for plans / logs (space-separated → multiline or compact)
format_amount_list_alt() {
  local a out=""
  for a in "$@"; do
    [ -n "$out" ] && out="$out  "
    out="${out}$(format_amount_alt "$a")"
  done
  printf '%s\n' "$out"
}

# --- coin inventory from wallet-cli dump-coins ---
# Writes JSON to $1; prints one-line summary to stdout.
# Sets COINS_TOTAL, COINS_FRESH, COINS_SPENT, COINS_AMOUNT_CIRC, COINS_SUMMARY.
metrics_wallet_coins() {
  local out="${1:-$METRICS_DIR/coins.json}"
  local dump="${METRICS_DIR}/dump-coins.raw"
  mkdir -p "$(dirname "$out")" 2>/dev/null || true
  COINS_TOTAL=0
  COINS_FRESH=0
  COINS_SPENT=0
  COINS_AMOUNT_CIRC="0"
  COINS_SUMMARY="(no coins)"
  # Prefer wcli() from caller (e2e/ladder). Do not pass a leading timeout number —
  # ladder wcli() has no optional secs arg (would become a wallet subcommand).
  if type wcli >/dev/null 2>&1; then
    wcli advanced dump-coins >"$dump" 2>/dev/null || true
  elif [ -n "${CLI_JS:-}" ] && [ -f "${CLI_JS}" ]; then
    node "$CLI_JS" --wallet-db="${WDB:-}" --no-throttle advanced dump-coins >"$dump" 2>/dev/null || true
  elif [ -n "${WALLET_CLI:-}" ] && [ -n "${WDB:-}" ]; then
    node "$WALLET_CLI" --wallet-db="${WDB}" --no-throttle --skip-defaults advanced dump-coins >"$dump" 2>/dev/null || true
  else
    echo "$COINS_SUMMARY"
    printf '%s\n' '{"ok":false,"reason":"no-wcli"}' >"$out"
    return 1
  fi
  python3 - "$dump" "$out" "${CUR:-GOA}" "${ALT_UNITS_FILE:-}" <<'PY'
import json, re, sys
from collections import Counter, defaultdict
from decimal import Decimal, InvalidOperation, ROUND_HALF_UP

raw_path, out_path, cur = sys.argv[1:4]
alt_path = sys.argv[4] if len(sys.argv) > 4 else ""
raw = open(raw_path).read() if raw_path else ""
d = None
for m in re.finditer(r"\{", raw):
    try:
        d = json.loads(raw[m.start():])
        if isinstance(d, dict) and ("coins" in d or "coin" in d):
            break
    except Exception:
        d = None
coins = []
if isinstance(d, dict):
    coins = d.get("coins") or d.get("coin") or []

alt = {}
if alt_path:
    try:
        alt = json.load(open(alt_path))
    except Exception:
        alt = {}
if not alt:
    alt = {"0": cur}

def parse_amt(s):
    """'GOA:10' / '10' → (currency, Decimal) or (cur, 0)."""
    s = str(s or "").strip()
    if not s or s == "?":
        return cur, Decimal(0)
    if ":" in s:
        c, v = s.split(":", 1)
        try:
            return c, Decimal(v)
        except InvalidOperation:
            return c, Decimal(0)
    try:
        return cur, Decimal(s)
    except InvalidOperation:
        return cur, Decimal(0)

def fmt_num(v: Decimal) -> str:
    if v == v.to_integral():
        return format(int(v), "d")
    s = format(v.quantize(Decimal("0.00000001"), rounding=ROUND_HALF_UP), "f")
    return s.rstrip("0").rstrip(".")

def fmt_amt(c, v: Decimal) -> str:
    return "%s:%s" % (c, fmt_num(v))

def fmt_amt_alt(c, v: Decimal) -> str:
    """5 Kilo-GOA (GOA:5000) using alt_unit_names."""
    base_s = fmt_amt(c, v)
    base_name = alt.get("0") or c
    if v == 0:
        return "0 %s (%s)" % (base_name, base_s)
    scales = []
    for k, name in alt.items():
        try:
            scales.append((int(k), str(name)))
        except Exception:
            pass
    scales.sort(key=lambda x: -x[0])
    absval = abs(v)
    chosen = None
    for sc, name in scales:
        unit = Decimal(10) ** sc
        coeff = absval / unit
        if coeff >= 1:
            chosen = (sc, name, coeff if v >= 0 else -coeff)
            break
    if chosen is None:
        return "%s %s (%s)" % (fmt_num(v), base_name, base_s)
    sc, name, coeff = chosen
    if sc == 0:
        return "%s %s" % (fmt_num(v), name)
    return "%s %s (%s)" % (fmt_num(coeff), name, base_s)

by_denom_all = Counter()          # all coins
by_denom_circ = Counter()         # non-spent
by_denom_spent = Counter()
by_status = Counter()
amt_circ = defaultdict(lambda: Decimal(0))   # currency → amount
amt_spent = defaultdict(lambda: Decimal(0))
amt_all = defaultdict(lambda: Decimal(0))
circ_n = 0
spent_n = 0
for c in coins:
    if not isinstance(c, dict):
        continue
    dv = c.get("denomValue") or c.get("value") or c.get("denom_value") or "?"
    st = str(c.get("coinStatus") or c.get("status") or "?")
    by_denom_all[dv] += 1
    by_status[st] += 1
    ac, av = parse_amt(dv)
    amt_all[ac] += av
    st_l = st.lower()
    if "spent" in st_l or "delete" in st_l or "dirty" in st_l:
        spent_n += 1
        by_denom_spent[dv] += 1
        amt_spent[ac] += av
    else:
        circ_n += 1
        by_denom_circ[dv] += 1
        amt_circ[ac] += av

total = len(coins)

def denom_key(k):
    try:
        return float(str(k).split(":", 1)[-1])
    except Exception:
        return 0.0

def denom_list(counter, spent_counter=None):
    out = []
    for k in sorted(counter.keys(), key=denom_key):
        item = {"denom": k, "count": counter[k]}
        ac, av = parse_amt(k)
        item["unit_value"] = str(av)
        item["amount"] = fmt_amt(ac, av * counter[k])
        if spent_counter is not None:
            item["spent_count"] = spent_counter.get(k, 0)
        out.append(item)
    return out

# human: "10 GOA×3 (=30 GOA)  …" and alt: "3 Kilo-GOA (GOA:3000)×1"
parts = []
parts_alt = []
for item in denom_list(by_denom_circ):
    ac, av = parse_amt(item["denom"])
    total_v = av * item["count"]
    parts.append("%s×%d (=%s)" % (item["denom"], item["count"], item["amount"]))
    parts_alt.append(
        "%s×%d (=%s)"
        % (fmt_amt_alt(ac, av), item["count"], fmt_amt_alt(ac, total_v))
    )
circ_amt_s = " ".join(fmt_amt(c, amt_circ[c]) for c in sorted(amt_circ))
circ_amt_alt = " ".join(fmt_amt_alt(c, amt_circ[c]) for c in sorted(amt_circ))
spent_amt_s = " ".join(fmt_amt(c, amt_spent[c]) for c in sorted(amt_spent)) if amt_spent else "0"
spent_amt_alt = " ".join(fmt_amt_alt(c, amt_spent[c]) for c in sorted(amt_spent)) if amt_spent else "0"
if not circ_amt_s:
    circ_amt_s = "%s:0" % cur
    circ_amt_alt = "0 %s (%s:0)" % (alt.get("0") or cur, cur)
summary = (
    "coins=%d  in_circ=%d  spent=%d  amount_circ=%s  |  %s"
    % (total, circ_n, spent_n, circ_amt_alt, "  ".join(parts_alt) if parts_alt else "(empty)")
)
# enrich denom lists with human labels
def enrich(lst):
    out = []
    for item in lst:
        ac, av = parse_amt(item["denom"])
        total_v = av * int(item["count"])
        item = dict(item)
        item["denom_alt"] = fmt_amt_alt(ac, av)
        item["amount_alt"] = fmt_amt_alt(ac, total_v)
        out.append(item)
    return out

report = {
    "ok": True,
    "currency": cur,
    "total_coins": total,
    "in_circulation": circ_n,
    "spent": spent_n,
    "amount_in_circulation": {c: str(amt_circ[c]) for c in amt_circ},
    "amount_in_circulation_s": circ_amt_s,
    "amount_in_circulation_alt": circ_amt_alt,
    "amount_spent": {c: str(amt_spent[c]) for c in amt_spent},
    "amount_spent_s": spent_amt_s,
    "amount_spent_alt": spent_amt_alt,
    "by_status": dict(by_status),
    "by_denom": enrich(denom_list(by_denom_all, by_denom_spent)),
    "by_denom_circulation": enrich(denom_list(by_denom_circ)),
    "by_denom_spent": enrich(denom_list(by_denom_spent)),
    "summary": summary,
    "alt_unit_names": alt,
}
json.dump(report, open(out_path, "w"), indent=2)
print(summary)
open(out_path + ".total", "w").write(str(total))
open(out_path + ".circ", "w").write(str(circ_n))
open(out_path + ".spent", "w").write(str(spent_n))
open(out_path + ".amt", "w").write(circ_amt_s)
PY
  COINS_TOTAL=$(cat "${out}.total" 2>/dev/null || echo 0)
  COINS_FRESH=$(cat "${out}.circ" 2>/dev/null || echo 0)
  COINS_SPENT=$(cat "${out}.spent" 2>/dev/null || echo 0)
  COINS_AMOUNT_CIRC=$(cat "${out}.amt" 2>/dev/null || echo "0")
  COINS_SUMMARY=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("summary","?"))' "$out" 2>/dev/null || echo "$COINS_SUMMARY")
  # history TSV for end-of-run table
  if [ -n "${METRICS_DIR:-}" ]; then
    hist="${METRICS_DIR}/coins-history.tsv"
    if [ ! -f "$hist" ]; then
      printf 'ts\tlabel\ttotal\tin_circ\tspent\tamount_circ\tsummary\n' >"$hist"
    fi
    printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
      "$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date)" \
      "${METRICS_COINS_LABEL:-snap}" \
      "$COINS_TOTAL" "$COINS_FRESH" "$COINS_SPENT" "$COINS_AMOUNT_CIRC" \
      "$(printf '%s' "$COINS_SUMMARY" | tr '\t' ' ')" >>"$hist"
  fi
  echo "$COINS_SUMMARY"
}

# Emit coin inventory as monitoring info lines (label = step marker).
# $1=label  $2=optional json path (default METRICS_DIR/coins-<safe-label>.json)
metrics_report_coins() {
  local label="$1"
  local safe
  safe=$(printf '%s' "$label" | tr -c 'A-Za-z0-9._-' '_' | head -c 80)
  local out="${2:-${METRICS_DIR}/coins-${safe}.json}"
  local line prev="${METRICS_DIR}/coins-prev.json"
  if [ -z "${WDB:-}" ] && [ -z "${WALLET_CLI:-}" ] && ! type wcli >/dev/null 2>&1; then
    info "coins ${label}" "skipped (no wallet yet)"
    return 0
  fi
  METRICS_COINS_LABEL="$label"
  export METRICS_COINS_LABEL
  if ! metrics_wallet_coins "$out" >/dev/null; then
    warn "coins ${label}" "dump-coins failed / empty"
    return 1
  fi
  # multi-line detail from JSON
  while IFS= read -r line; do
    [ -n "$line" ] || continue
    info "coins ${label}" "$line"
  done < <(python3 - "$out" <<'PY'
import json, sys
d = json.load(open(sys.argv[1]))
if not d.get("ok"):
    print("unavailable: %s" % d.get("reason", "?"))
    raise SystemExit(0)
amt = d.get("amount_in_circulation_alt") or d.get("amount_in_circulation_s") or "0"
print(
    "n=%s  in_circulation=%s  spent=%s  amount_circ=%s"
    % (d.get("total_coins"), d.get("in_circulation"), d.get("spent"), amt)
)
st = d.get("by_status") or {}
if st:
    print("status " + " ".join("%s=%s" % (k, st[k]) for k in sorted(st)))
circ = d.get("by_denom_circulation") or []
if circ:
    bits = []
    for x in circ:
        dlab = x.get("denom_alt") or x.get("denom")
        alab = x.get("amount_alt") or x.get("amount")
        bits.append("%s×%d (=%s)" % (dlab, x["count"], alab))
    print("denoms_in_circ " + "  ".join(bits))
else:
    print("denoms_in_circ (none)")
spent = d.get("by_denom_spent") or []
if spent:
    bits = []
    for x in spent:
        dlab = x.get("denom_alt") or x.get("denom")
        alab = x.get("amount_alt") or x.get("amount")
        bits.append("%s×%d (=%s)" % (dlab, x["count"], alab))
    print("denoms_spent " + "  ".join(bits))
sp = d.get("amount_spent_alt") or d.get("amount_spent_s")
if sp and sp not in ("0", "0 GOA (GOA:0)"):
    print("amount_spent %s" % sp)
PY
  )
  # delta vs previous snapshot at this run
  if [ -f "$prev" ]; then
    local dsum
    dsum=$(metrics_coins_delta "$prev" "$out" "${METRICS_DIR}/coins-delta-${safe}.json" 2>/dev/null || true)
    if [ -n "$dsum" ]; then
      info "coins ${label} Δ" "$dsum"
    fi
  fi
  cp -f "$out" "$prev" 2>/dev/null || true
  cp -f "$out" "${METRICS_DIR}/coins-final.json" 2>/dev/null || true
  return 0
}

# Diff two coin JSON snapshots → new coins this step
metrics_coins_delta() {
  local before="${1:-}" after="${2:-}" out="${3:-$METRICS_DIR/coins-delta.json}"
  python3 - "$before" "$after" "$out" <<'PY'
import json, sys
from collections import Counter
def load(p):
    try:
        d=json.load(open(p))
        return d
    except Exception:
        return {}
b, a = load(sys.argv[1]), load(sys.argv[2])
outp = sys.argv[3]
def bag(d):
    c=Counter()
    for item in d.get("by_denom") or []:
        c[item.get("denom") or "?"] += int(item.get("count") or 0)
    return c
bb, aa = bag(b), bag(a)
delta = aa - bb
parts = ["%s×%+d" % (k, delta[k]) for k in sorted(delta.keys(), key=lambda x: float(str(x).split(":")[-1]) if ":" in str(x) or str(x).replace(".","").isdigit() else 0) if delta[k]]
new_total = int(a.get("total_coins") or 0) - int(b.get("total_coins") or 0)
rep = {
    "new_coins": new_total,
    "total_after": int(a.get("total_coins") or 0),
    "circulation_after": int(a.get("in_circulation") or 0),
    "delta_by_denom": {k: delta[k] for k in delta},
    "summary": ("new=%+d  total_now=%s  |  %s" % (
        new_total, a.get("total_coins"), "  ".join(parts) if parts else "(no denom change)")),
}
json.dump(rep, open(outp, "w"), indent=2)
print(rep["summary"])
PY
}

# --- Taler stack load on koopa (host + bank/exchange/merchant) ---
# Writes JSON to $1. RAM, process counts, DB sizes, disk I/O counters.
# Uses KOOPA_SSH with fallback to KOOPA_SSH_FALLBACKS (koopa-external).
metrics_taler_load() {
  local out="${1:-$METRICS_DIR/load.json}"
  local label="${2:-snap}"
  mkdir -p "$(dirname "$out")" 2>/dev/null || true
  if [ "${METRICS_LOAD}" = "0" ]; then
    printf '%s\n' "{\"ok\":false,\"reason\":\"disabled\",\"label\":\"$label\"}" >"$out"
    return 0
  fi
  local raw=""
  local remote_py
  remote_py=$(cat <<'PY'
import base64, json, os, re, subprocess, time
from collections import defaultdict

def sh(cmd, t=15):
    try:
        return subprocess.check_output(cmd, shell=True, text=True, stderr=subprocess.DEVNULL, timeout=t)
    except Exception:
        return ""

def podman_sh(name, script, t=25):
    """Run a shell script inside container without nested-quote hell (base64)."""
    b64 = base64.b64encode(script.encode()).decode()
    return sh(
        f"podman exec {name} sh -c 'echo {b64} | base64 -d | sh'",
        t=t,
    )

def loadavg():
    try:
        a,b,c = open("/proc/loadavg").read().split()[:3]
        return [float(a), float(b), float(c)]
    except Exception:
        return []

def meminfo():
    d = {}
    try:
        for line in open("/proc/meminfo"):
            k,v = line.split(":",1)
            d[k.strip()] = int(v.strip().split()[0]) * 1024
    except Exception:
        pass
    return {
        "mem_total_b": d.get("MemTotal"),
        "mem_available_b": d.get("MemAvailable"),
        "mem_free_b": d.get("MemFree"),
        "buffers_b": d.get("Buffers"),
        "cached_b": d.get("Cached"),
    }

def disk_io():
    r = w = 0
    try:
        for line in open("/proc/diskstats"):
            p = line.split()
            if len(p) < 14: continue
            name = p[2]
            if name.startswith(("loop","ram","dm-")): continue
            r += int(p[5]); w += int(p[9])
    except Exception:
        pass
    return {"sectors_read": r, "sectors_written": w,
            "approx_write_bytes": w*512, "approx_read_bytes": r*512}

CTRS = [("bank","taler-hacktivism-bank"),("merchant","taler-hacktivism"),
        ("exchange","taler-hacktivism-exchange-ansible")]

def running(name):
    return sh(f"podman inspect -f '{{{{.State.Running}}}}' {name}").strip() == "true"

def stats(name):
    # Prefer Go template (stable across podman versions); JSON field names vary.
    o = sh(
        f"podman stats --no-stream --format "
        f"'{{{{.CPUPerc}}}}|{{{{.MemUsage}}}}|{{{{.MemPerc}}}}|{{{{.BlockIO}}}}|{{{{.NetIO}}}}|{{{{.PIDs}}}}' "
        f"{name}",
        t=25,
    ).strip()
    if o and "|" in o:
        p = o.split("|")
        while len(p) < 6:
            p.append("")
        return {
            "cpu_pct": p[0] or None,
            "mem_usage": p[1] or None,
            "mem_pct": p[2] or None,
            "block_io": p[3] or None,
            "net_io": p[4] or None,
            "pids": p[5] or None,
        }
    o = sh(f"podman stats --no-stream --format json {name}", t=25)
    try:
        data = json.loads(o)
        if isinstance(data, list) and data:
            data = data[0]
        if not isinstance(data, dict):
            return {}
        return {
            "cpu_pct": data.get("CPU") or data.get("CPUPerc"),
            "mem_usage": data.get("MemUsage"),
            "mem_pct": data.get("MemPerc"),
            "block_io": data.get("BlockIO"),
            "net_io": data.get("NetIO"),
            "pids": data.get("PIDs"),
        }
    except Exception:
        return {}

def procs(name):
    # Pure shell /proc scan via base64 (containers often lack python3).
    script = r"""
n=0; rss=0
n_postgres=0; rss_postgres=0
n_libeufin=0; rss_libeufin=0
n_taler_merchant=0; rss_taler_merchant=0
n_taler_exchange=0; rss_taler_exchange=0
n_nginx=0; rss_nginx=0
n_java=0; rss_java=0
n_other=0; rss_other=0
for d in /proc/[0-9]*; do
  [ -r "$d/status" ] || continue
  r=$(sed -n 's/^VmRSS:[[:space:]]*\([0-9][0-9]*\).*/\1/p' "$d/status" | head -1)
  [ -n "$r" ] || continue
  n=$((n+1)); rss=$((rss+r))
  nm=$(sed -n 's/^Name:[[:space:]]*//p' "$d/status" | head -1)
  cmd=$(tr '\0' ' ' <"$d/cmdline" 2>/dev/null | head -c 240)
  blob=$(printf '%s %s' "$nm" "$cmd" | tr 'A-Z' 'a-z')
  case "$blob" in
    *postgres*|*postmaster*) n_postgres=$((n_postgres+1)); rss_postgres=$((rss_postgres+r)) ;;
    *libeufin*|*mainkt*) n_libeufin=$((n_libeufin+1)); rss_libeufin=$((rss_libeufin+r)) ;;
    *taler-merchant*) n_taler_merchant=$((n_taler_merchant+1)); rss_taler_merchant=$((rss_taler_merchant+r)) ;;
    *taler-exchange*) n_taler_exchange=$((n_taler_exchange+1)); rss_taler_exchange=$((rss_taler_exchange+r)) ;;
    *nginx*) n_nginx=$((n_nginx+1)); rss_nginx=$((rss_nginx+r)) ;;
    *java*) n_java=$((n_java+1)); rss_java=$((rss_java+r)) ;;
    *) n_other=$((n_other+1)); rss_other=$((rss_other+r)) ;;
  esac
done
printf 'TOTAL %s %s\n' "$n" "$rss"
[ "$n_postgres" -gt 0 ] && printf 'ROLE postgres %s %s\n' "$n_postgres" "$rss_postgres"
[ "$n_libeufin" -gt 0 ] && printf 'ROLE libeufin %s %s\n' "$n_libeufin" "$rss_libeufin"
[ "$n_taler_merchant" -gt 0 ] && printf 'ROLE taler-merchant %s %s\n' "$n_taler_merchant" "$rss_taler_merchant"
[ "$n_taler_exchange" -gt 0 ] && printf 'ROLE taler-exchange %s %s\n' "$n_taler_exchange" "$rss_taler_exchange"
[ "$n_nginx" -gt 0 ] && printf 'ROLE nginx %s %s\n' "$n_nginx" "$rss_nginx"
[ "$n_java" -gt 0 ] && printf 'ROLE java %s %s\n' "$n_java" "$rss_java"
[ "$n_other" -gt 0 ] && printf 'ROLE other %s %s\n' "$n_other" "$rss_other"
"""
    o = podman_sh(name, script, t=25)
    by = {}
    n = None
    rss_kb = None
    for line in o.splitlines():
        p = line.split()
        if not p:
            continue
        if p[0] == "TOTAL" and len(p) >= 3:
            try:
                n = int(p[1])
                rss_kb = int(p[2])
            except Exception:
                pass
        elif p[0] == "ROLE" and len(p) >= 4:
            try:
                by[p[1]] = {"n": int(p[2]), "rss_b": int(p[3]) * 1024}
            except Exception:
                pass
    if n is None:
        nn = sh(f"podman exec {name} sh -c 'ps -e --no-headers 2>/dev/null | wc -l'").strip()
        n = int(nn) if nn.isdigit() else None
    return {
        "proc_total": n,
        "rss_total_b": (rss_kb * 1024) if rss_kb is not None else None,
        "by_role": by,
    }

def dbs(name):
    o = sh(
        f"podman exec {name} su -s /bin/bash postgres -c "
        + json.dumps(
            "psql -Atc \"SELECT datname||'|'||pg_database_size(datname)||'|'||pg_size_pretty(pg_database_size(datname)) "
            "FROM pg_database WHERE datistemplate=false ORDER BY 1\""
        )
    )
    out=[]
    for line in o.splitlines():
        p=line.strip().split("|")
        if len(p)>=3:
            try: out.append({"name":p[0],"size_b":int(p[1]),"size_pretty":p[2]})
            except Exception: pass
    du=sh(f"podman exec {name} sh -c 'du -sb /var/lib/postgresql 2>/dev/null | cut -f1'").strip()
    pretty=sh(f"podman exec {name} sh -c 'du -sh /var/lib/postgresql 2>/dev/null | cut -f1'").strip()
    return {"databases": out, "pgdata_bytes": int(du) if du.isdigit() else None, "pgdata_pretty": pretty or None}

components={}
for role,cname in CTRS:
    if not running(cname):
        components[role]={"container":cname,"running":False}
        continue
    components[role]={
        "container":cname,"running":True,
        "podman_stats":stats(cname),
        "processes":procs(cname),
        "databases":dbs(cname),
    }

host_ps=sh("ps -eo comm=")
host_counts={k:sum(1 for line in host_ps.splitlines() if k in line.lower())
             for k in ("postgres","nginx","caddy","podman","conmon","pasta")}
print(json.dumps({
    "ok": True,
    "ts": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
    "host": {
        "hostname": sh("hostname").strip(),
        "loadavg": loadavg(),
        "nproc": os.cpu_count(),
        "memory": meminfo(),
        "disk_io": disk_io(),
        "process_counts": host_counts,
    },
    "taler": components,
}))
PY
)
  # Prefer SSH to koopa (LAN or koopa-external). Local podman only if we are on the host.
  if [ "${SKIP_SSH:-0}" != "1" ] && type koopa_ssh_ok >/dev/null 2>&1 && koopa_ssh_ok; then
    if type koopa_ssh_python >/dev/null 2>&1; then
      raw=$(printf '%s' "$remote_py" | koopa_ssh_python "${METRICS_LOAD_SSH_TIMEOUT:-90}" 2>/dev/null || true)
    else
      raw=$(printf '%s' "$remote_py" | with_timeout "${METRICS_LOAD_SSH_TIMEOUT:-90}" \
        ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" python3 - 2>/dev/null || true)
    fi
  elif command -v podman >/dev/null 2>&1; then
    raw=$(python3 -c "$remote_py" 2>/dev/null || true)
  fi
  if [ -z "$raw" ]; then
    printf '%s\n' "{\"ok\":false,\"reason\":\"probe-failed\",\"label\":\"$label\",\"ssh\":\"${KOOPA_SSH:-?}\"}" >"$out"
    return 1
  fi
  printf '%s\n' "$raw" | python3 -c '
import sys,json,re
t=sys.stdin.read(); obj=None
for m in re.finditer(r"\{", t):
  try:
    obj=json.loads(t[m.start():])
    if isinstance(obj, dict) and (obj.get("host") or obj.get("taler") or obj.get("ok") is False):
      break
  except Exception:
    obj=None
if not obj: obj={"ok":False,"reason":"parse"}
obj["label"]=sys.argv[1]
json.dump(obj, open(sys.argv[2],"w"), indent=2)
' "$label" "$out"
}

# One-line human summaries (no leading indent) for info()/ok() detail fields.
metrics_load_lines() {
  local f="$1"
  python3 - "$f" <<'PY'
import json, sys
try:
  d = json.load(open(sys.argv[1]))
except Exception as e:
  print(f"unreadable: {e}")
  raise SystemExit(0)
if not d.get("ok"):
  print(f"unavailable: {d.get('reason', 'n/a')} ssh={d.get('ssh', '')}".strip())
  raise SystemExit(0)
h = d.get("host") or {}
mem = h.get("memory") or {}
la = h.get("loadavg") or []

def gi(b):
  if b is None:
    return "?"
  return f"{b/1024/1024/1024:.2f}GiB"

def mi(b):
  if b is None:
    return "?"
  return f"{b/1024/1024:.0f}MiB"

la_s = " ".join(f"{x:.2f}" for x in la) if la else "?"
tot = mem.get("mem_total_b")
avail = mem.get("mem_available_b")
used = (tot - avail) if (tot is not None and avail is not None) else None
print(
  f"host loadavg=[{la_s}] nproc={h.get('nproc')} "
  f"mem_used={gi(used)} avail={gi(avail)} total={gi(tot)}"
)
pc = h.get("process_counts") or {}
if pc:
  bits = [f"{k}={v}" for k, v in sorted(pc.items()) if v]
  if bits:
    print("host procs " + " ".join(bits))
for role in ("bank", "exchange", "merchant"):
  c = (d.get("taler") or {}).get(role) or {}
  if not c:
    continue
  if not c.get("running"):
    print(f"{role} DOWN ({c.get('container')})")
    continue
  pr = c.get("processes") or {}
  st = c.get("podman_stats") or {}
  roles = pr.get("by_role") or {}
  role_bits = []
  for k in ("libeufin", "taler-exchange", "taler-merchant", "postgres", "nginx", "java"):
    if k in roles:
      role_bits.append(f"{k}={mi(roles[k].get('rss_b'))}")
  dbs = c.get("databases") or {}
  db_s = ",".join(
    f"{x.get('name')}={x.get('size_pretty')}"
    for x in (dbs.get("databases") or [])[:5]
  )
  # podman MemUsage often reports 0B under pasta/cgroup — prefer /proc RSS
  mem_u = st.get("mem_usage") or ""
  mem_bit = ""
  if mem_u and not str(mem_u).startswith("0B"):
    mem_bit = f" podman_mem={mem_u}"
  cpu = st.get("cpu_pct") or "?"
  blk = st.get("block_io") or ""
  blk_bit = ""
  if blk and blk not in ("0B / 0B", "0B/0B", "-- / --"):
    blk_bit = f" block={blk}"
  line = (
    f"{role} rss={gi(pr.get('rss_total_b'))} procs={pr.get('proc_total')} "
    f"cpu={cpu}{mem_bit}{blk_bit}"
  )
  if role_bits:
    line += " | " + " ".join(role_bits)
  if db_s:
    line += f" | db:{db_s}"
  if dbs.get("pgdata_pretty"):
    line += f" pgdata={dbs.get('pgdata_pretty')}"
  print(line)
PY
}

# Snapshot load and emit as monitoring info lines (withdraw/pay phase markers).
# $1=json path  $2=label (e.g. after-withdraw)
metrics_report_load() {
  local out="$1" label="$2"
  local line rc=0
  if [ "${METRICS_LOAD}" = "0" ]; then
    info "load ${label}" "skipped (METRICS_LOAD=0)"
    return 0
  fi
  if [ "${SKIP_SSH:-0}" = "1" ] && ! command -v podman >/dev/null 2>&1; then
    info "load ${label}" "skipped (no SSH / no local podman)"
    return 0
  fi
  if ! metrics_taler_load "$out" "$label"; then
    warn "load ${label}" "probe failed via ${KOOPA_SSH:-?} — try KOOPA_SSH=koopa-external"
    return 1
  fi
  while IFS= read -r line; do
    [ -n "$line" ] || continue
    case "$line" in
      unavailable:*|unreadable:*)
        warn "load ${label}" "$line"
        rc=1
        ;;
      *)
        info "load ${label}" "$line"
        ;;
    esac
  done < <(metrics_load_lines "$out")
  return "$rc"
}

# Human one-screen summary of a load JSON
metrics_print_load() {
  local f="$1" title="${2:-load}"
  python3 - "$f" "$title" <<'PY'
import json,sys
try:
  d=json.load(open(sys.argv[1]))
except Exception as e:
  print(f"  ({sys.argv[2]}: unreadable {e})")
  raise SystemExit
if not d.get("ok"):
  print(f"  ({sys.argv[2]}: {d.get('reason','n/a')})")
  raise SystemExit
h=d.get("host") or {}
mem=h.get("memory") or {}
la=h.get("loadavg") or []
def gi(b):
  if b is None: return "?"
  return f"{b/1024/1024/1024:.2f} GiB"
print(f"  host loadavg {la}  nproc={h.get('nproc')}  mem_avail={gi(mem.get('mem_available_b'))}/{gi(mem.get('mem_total_b'))}")
dio=h.get("disk_io") or {}
if dio:
  print(f"  host disk Δsectors read={dio.get('sectors_read')} written={dio.get('sectors_written')} (~write {gi(dio.get('approx_write_bytes'))})")
for role in ("bank","exchange","merchant"):
  c=(d.get("taler") or {}).get(role) or {}
  if not c:
    continue
  if not c.get("running"):
    print(f"  {role:8} DOWN ({c.get('container')})")
    continue
  pr=c.get("processes") or {}
  rss=pr.get("rss_total_b")
  n=pr.get("proc_total")
  roles=pr.get("by_role") or {}
  bits=[]
  for k in ("libeufin","taler-exchange","taler-merchant","postgres","nginx"):
    if k in roles:
      bits.append(f"{k}:n={roles[k].get('n')} rss={gi(roles[k].get('rss_b'))}")
  dbs=c.get("databases") or {}
  db_s=", ".join(f"{x.get('name')}={x.get('size_pretty')}" for x in (dbs.get("databases") or [])[:6])
  pg=dbs.get("pgdata_pretty") or ""
  st=c.get("podman_stats") or {}
  print(f"  {role:8} procs={n} rss={gi(rss)} cpu={st.get('cpu_pct','?')} block={st.get('block_io','?')}")
  if bits:
    print(f"           " + "  ".join(bits))
  if db_s or pg:
    print(f"           db: {db_s}" + (f"  pgdata={pg}" if pg else ""))
PY
}

# Diff two load snaps: highlight RAM/proc/DB growth for taler roles
metrics_print_load_delta() {
  local before="$1" after="$2"
  python3 - "$before" "$after" <<'PY'
import json,sys
def load(p):
  try: return json.load(open(p))
  except Exception: return {}
b,a=load(sys.argv[1]),load(sys.argv[2])
if not b.get("ok") or not a.get("ok"):
  print("  (load delta unavailable)")
  raise SystemExit
def gi(x):
  if x is None: return None
  return x/1024/1024/1024
print("  --- delta (after − before) ---")
# host load
bla=(b.get("host") or {}).get("loadavg") or [0,0,0]
ala=(a.get("host") or {}).get("loadavg") or [0,0,0]
if bla and ala:
  print(f"  loadavg1  {bla[0]:.2f} → {ala[0]:.2f}  (Δ {ala[0]-bla[0]:+.2f})")
bm=(b.get("host") or {}).get("memory") or {}
am=(a.get("host") or {}).get("memory") or {}
if bm.get("mem_available_b") is not None and am.get("mem_available_b") is not None:
  print(f"  mem_avail {gi(bm['mem_available_b']):.2f} → {gi(am['mem_available_b']):.2f} GiB  (Δ {gi(am['mem_available_b'])-gi(bm['mem_available_b']):+.3f} GiB)")
bd=(b.get("host") or {}).get("disk_io") or {}
ad=(a.get("host") or {}).get("disk_io") or {}
if bd.get("sectors_written") is not None and ad.get("sectors_written") is not None:
  dw=(ad["sectors_written"]-bd["sectors_written"])*512
  dr=(ad["sectors_read"]-bd["sectors_read"])*512
  print(f"  disk I/O  write≈{dw/1024/1024:.1f} MiB  read≈{dr/1024/1024:.1f} MiB  (during phase)")
for role in ("bank","exchange","merchant"):
  bc=(b.get("taler") or {}).get(role) or {}
  ac=(a.get("taler") or {}).get(role) or {}
  if not ac.get("running"):
    continue
  br=(bc.get("processes") or {}).get("rss_total_b")
  ar=(ac.get("processes") or {}).get("rss_total_b")
  bn=(bc.get("processes") or {}).get("proc_total")
  an=(ac.get("processes") or {}).get("proc_total")
  line=f"  {role:8}"
  if br is not None and ar is not None:
    line+=f" rss {gi(br):.3f}→{gi(ar):.3f} GiB (Δ{gi(ar)-gi(br):+.3f})"
  if bn is not None and an is not None:
    line+=f"  procs {bn}→{an} (Δ{an-bn:+d})"
  # DB sizes
  def dbmap(c):
    m={}
    for x in ((c.get("databases") or {}).get("databases") or []):
      m[x.get("name")]=x.get("size_b")
    return m
  bdb,adb=dbmap(bc),dbmap(ac)
  dbits=[]
  for name in sorted(set(bdb)|set(adb)):
    bb,aa=bdb.get(name),adb.get(name)
    if bb is not None and aa is not None and aa!=bb:
      dbits.append(f"{name} {aa-bb:+d}B")
    elif aa is not None and bb is None:
      dbits.append(f"{name}={aa}B")
  if dbits:
    line+="  dbΔ["+", ".join(dbits)+"]"
  print(line)
PY
}

# Overall end-of-run statistics block
# Args via env / files:
#   METRICS_DIR, optional: WITHDRAW_REPORT, PAY_REPORT, phase timings JSON files
metrics_print_overall() {
  local title="${1:-overall statistics}"
  section "metrics · $title"
  if [ -f "${METRICS_DIR}/load-before.json" ]; then
    info "load BEFORE" ""
    metrics_print_load "${METRICS_DIR}/load-before.json" "before"
  fi
  if [ -f "${METRICS_DIR}/load-after.json" ]; then
    info "load AFTER" ""
    metrics_print_load "${METRICS_DIR}/load-after.json" "after"
    metrics_print_load_delta "${METRICS_DIR}/load-before.json" "${METRICS_DIR}/load-after.json"
  fi
  if [ -f "${METRICS_DIR}/coins-final.json" ]; then
    info "coins final" "$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("summary","?"))' "${METRICS_DIR}/coins-final.json" 2>/dev/null || echo n/a)"
  fi
  if [ -f "${METRICS_DIR}/coins-history.tsv" ]; then
    echo "  --- coins after each withdraw ---"
    # header + rows
    if [ -s "${METRICS_DIR}/coins-history.tsv" ]; then
      column -t -s $'\t' "${METRICS_DIR}/coins-history.tsv" 2>/dev/null \
        || cat "${METRICS_DIR}/coins-history.tsv"
    fi
  fi
  if [ -f "${METRICS_DIR}/perf-summary.json" ]; then
    info "performance" ""
    python3 - "${METRICS_DIR}/perf-summary.json" <<'PY'
import json,sys
d=json.load(open(sys.argv[1]))
for k,v in d.items():
  if isinstance(v, dict) and "n" in v:
    print(f"  {k:14} n={v.get('n')}  min={v.get('min_ms')}ms  p50={v.get('p50_ms')}ms  avg={v.get('avg_ms')}ms  max={v.get('max_ms')}ms")
  else:
    print(f"  {k}: {v}")
PY
  fi
  # free-form extras from caller
  [ -n "${METRICS_EXTRA_LINES:-}" ] && printf '%s\n' "$METRICS_EXTRA_LINES"
}