From 1e6a050ee7cac7ab3306c35d54cc7966972087d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Fri, 17 Jul 2026 01:19:31 +0200 Subject: [PATCH] feat(monitoring): load/memory probes, coin inventory, alt_unit_names --- scripts/taler-monitoring/metrics.sh | 695 ++++++++++++++++++++++++---- 1 file changed, 615 insertions(+), 80 deletions(-) diff --git a/scripts/taler-monitoring/metrics.sh b/scripts/taler-monitoring/metrics.sh index e862644..50ef343 100644 --- a/scripts/taler-monitoring/metrics.sh +++ b/scripts/taler-monitoring/metrics.sh @@ -14,31 +14,170 @@ : "${METRICS_DIR:=${SCRATCH:-/tmp}}" mkdir -p "$METRICS_DIR" 2>/dev/null || true +# --------------------------------------------------------------------------- +# alt_unit_names — human amounts (Kilo-GOA / Mega-GOA / …) with base in parens +# From exchange/bank currency_specification.alt_unit_names: +# "0"→GOA, "3"→Kilo-GOA, "6"→Mega-GOA, … "-3"→Milli-GOA, … +# Example: GOA:5000 → 5 Kilo-GOA (GOA:5000) +# --------------------------------------------------------------------------- +: "${ALT_UNITS_FILE:=${METRICS_DIR}/alt_unit_names.json}" + +# Load map from public /config into ALT_UNITS_FILE. $1=optional config URL. +metrics_load_alt_units() { + local url="${1:-${EXCHANGE_PUBLIC:-https://exchange.hacktivism.ch}/config}" + local code + mkdir -p "$(dirname "$ALT_UNITS_FILE")" 2>/dev/null || true + code=$(curl -skS -m 12 -o "${ALT_UNITS_FILE}.raw" -w '%{http_code}' "$url" 2>/dev/null || echo 000) + if [ "$code" != "200" ]; then + # soft fallback: SI-style names if live config unreachable + printf '%s\n' '{"0":"GOA","3":"Kilo-GOA","6":"Mega-GOA","9":"Giga-GOA","12":"Tera-GOA","15":"Peta-GOA","18":"Exa-GOA","21":"Zetta-GOA","24":"Yotta-GOA","-1":"Deci-GOA","-2":"Centi-GOA","-3":"Milli-GOA","-6":"Micro-GOA","-8":"Atomic-GOA"}' \ + >"$ALT_UNITS_FILE" + return 1 + fi + python3 - "${ALT_UNITS_FILE}.raw" "$ALT_UNITS_FILE" <<'PY' +import json, sys +src, dst = sys.argv[1:3] +d = json.load(open(src)) +au = None +cs = d.get("currency_specification") +if isinstance(cs, dict): + au = cs.get("alt_unit_names") +if not au and isinstance(d.get("currencies"), dict): + # merchant-style: currencies.GOA.alt_unit_names + for _code, spec in d["currencies"].items(): + if isinstance(spec, dict) and spec.get("alt_unit_names"): + au = spec["alt_unit_names"] + break +if not isinstance(au, dict) or "0" not in au: + au = {"0": d.get("currency") or "GOA"} +json.dump(au, open(dst, "w"), indent=2, sort_keys=True) +PY + export ALT_UNITS_FILE + return 0 +} + +# Format one amount: "GOA:5000" → "5 Kilo-GOA (GOA:5000)" +# Uses ALT_UNITS_FILE if present. Pure stdout. +format_amount_alt() { + local amt="${1:-}" + [ -n "$amt" ] || return 0 + python3 - "$amt" "${ALT_UNITS_FILE:-}" <<'PY' +import json, sys +from decimal import Decimal, InvalidOperation, ROUND_HALF_UP + +amt = sys.argv[1].strip() +path = sys.argv[2] if len(sys.argv) > 2 else "" +alt = {} +if path: + try: + alt = json.load(open(path)) + except Exception: + alt = {} +if not alt: + alt = {"0": "GOA"} + +def parse(s): + if ":" in s: + c, v = s.split(":", 1) + return c, Decimal(v) + return "GOA", Decimal(s) + +def fmt_num(v: Decimal) -> str: + if v == v.to_integral(): + return format(int(v), "d") + s = format(v.quantize(Decimal("0.00000001"), rounding=ROUND_HALF_UP), "f") + return s.rstrip("0").rstrip(".") + +try: + cur, val = parse(amt) +except (InvalidOperation, ValueError): + print(amt) + raise SystemExit(0) + +base_name = alt.get("0") or cur +# always show canonical base form in parens +base_s = "%s:%s" % (cur, fmt_num(val)) +if val == 0: + print("0 %s (%s)" % (base_name, base_s)) + raise SystemExit(0) + +# scales: power of 10 relative to unit "0" +scales = [] +for k, name in alt.items(): + try: + scales.append((int(k), str(name))) + except Exception: + pass +scales.sort(key=lambda x: -x[0]) # largest unit first + +# pick largest scale where |value| >= 10^scale (for scale>=0), +# or for fractions the finest unit that makes the coefficient >= 1 +chosen = None # (scale, name, coeff) +absval = abs(val) +for sc, name in scales: + unit = Decimal(10) ** sc + if unit <= 0: + continue + coeff = absval / unit + if coeff >= 1: + chosen = (sc, name, coeff if val >= 0 else -coeff) + break +if chosen is None: + # smaller than smallest unit — use base + print("%s %s (%s)" % (fmt_num(val), base_name, base_s)) + raise SystemExit(0) + +sc, name, coeff = chosen +# if base unit, prefer "GOA:12" style still with parens only when alt differs +if sc == 0: + print("%s %s" % (fmt_num(val), name)) + raise SystemExit(0) +print("%s %s (%s)" % (fmt_num(coeff), name, base_s)) +PY +} + +# Format list of "CUR:n" amounts for plans / logs (space-separated → multiline or compact) +format_amount_list_alt() { + local a out="" + for a in "$@"; do + [ -n "$out" ] && out="$out " + out="${out}$(format_amount_alt "$a")" + done + printf '%s\n' "$out" +} + # --- coin inventory from wallet-cli dump-coins --- # Writes JSON to $1; prints one-line summary to stdout. -# Sets COINS_TOTAL, COINS_FRESH, COINS_SUMMARY (human one-liner). +# Sets COINS_TOTAL, COINS_FRESH, COINS_SPENT, COINS_AMOUNT_CIRC, COINS_SUMMARY. metrics_wallet_coins() { local out="${1:-$METRICS_DIR/coins.json}" - local dump="$METRICS_DIR/dump-coins.raw" + local dump="${METRICS_DIR}/dump-coins.raw" + mkdir -p "$(dirname "$out")" 2>/dev/null || true COINS_TOTAL=0 COINS_FRESH=0 + COINS_SPENT=0 + COINS_AMOUNT_CIRC="0" COINS_SUMMARY="(no coins)" - # Prefer wcli() from caller (bash function in e2e/ladder) + # Prefer wcli() from caller (e2e/ladder). Do not pass a leading timeout number — + # ladder wcli() has no optional secs arg (would become a wallet subcommand). if type wcli >/dev/null 2>&1; then wcli advanced dump-coins >"$dump" 2>/dev/null || true elif [ -n "${CLI_JS:-}" ] && [ -f "${CLI_JS}" ]; then node "$CLI_JS" --wallet-db="${WDB:-}" --no-throttle advanced dump-coins >"$dump" 2>/dev/null || true - elif [ -n "${WALLET_CLI:-}" ]; then - node "$WALLET_CLI" --wallet-db="${WDB:-}" --no-throttle advanced dump-coins >"$dump" 2>/dev/null || true + elif [ -n "${WALLET_CLI:-}" ] && [ -n "${WDB:-}" ]; then + node "$WALLET_CLI" --wallet-db="${WDB}" --no-throttle --skip-defaults advanced dump-coins >"$dump" 2>/dev/null || true else echo "$COINS_SUMMARY" printf '%s\n' '{"ok":false,"reason":"no-wcli"}' >"$out" return 1 fi - python3 - "$dump" "$out" "${CUR:-GOA}" <<'PY' + python3 - "$dump" "$out" "${CUR:-GOA}" "${ALT_UNITS_FILE:-}" <<'PY' import json, re, sys -from collections import Counter +from collections import Counter, defaultdict +from decimal import Decimal, InvalidOperation, ROUND_HALF_UP + raw_path, out_path, cur = sys.argv[1:4] +alt_path = sys.argv[4] if len(sys.argv) > 4 else "" raw = open(raw_path).read() if raw_path else "" d = None for m in re.finditer(r"\{", raw): @@ -51,60 +190,269 @@ for m in re.finditer(r"\{", raw): coins = [] if isinstance(d, dict): coins = d.get("coins") or d.get("coin") or [] -by_denom = Counter() + +alt = {} +if alt_path: + try: + alt = json.load(open(alt_path)) + except Exception: + alt = {} +if not alt: + alt = {"0": cur} + +def parse_amt(s): + """'GOA:10' / '10' → (currency, Decimal) or (cur, 0).""" + s = str(s or "").strip() + if not s or s == "?": + return cur, Decimal(0) + if ":" in s: + c, v = s.split(":", 1) + try: + return c, Decimal(v) + except InvalidOperation: + return c, Decimal(0) + try: + return cur, Decimal(s) + except InvalidOperation: + return cur, Decimal(0) + +def fmt_num(v: Decimal) -> str: + if v == v.to_integral(): + return format(int(v), "d") + s = format(v.quantize(Decimal("0.00000001"), rounding=ROUND_HALF_UP), "f") + return s.rstrip("0").rstrip(".") + +def fmt_amt(c, v: Decimal) -> str: + return "%s:%s" % (c, fmt_num(v)) + +def fmt_amt_alt(c, v: Decimal) -> str: + """5 Kilo-GOA (GOA:5000) using alt_unit_names.""" + base_s = fmt_amt(c, v) + base_name = alt.get("0") or c + if v == 0: + return "0 %s (%s)" % (base_name, base_s) + scales = [] + for k, name in alt.items(): + try: + scales.append((int(k), str(name))) + except Exception: + pass + scales.sort(key=lambda x: -x[0]) + absval = abs(v) + chosen = None + for sc, name in scales: + unit = Decimal(10) ** sc + coeff = absval / unit + if coeff >= 1: + chosen = (sc, name, coeff if v >= 0 else -coeff) + break + if chosen is None: + return "%s %s (%s)" % (fmt_num(v), base_name, base_s) + sc, name, coeff = chosen + if sc == 0: + return "%s %s" % (fmt_num(v), name) + return "%s %s (%s)" % (fmt_num(coeff), name, base_s) + +by_denom_all = Counter() # all coins +by_denom_circ = Counter() # non-spent +by_denom_spent = Counter() by_status = Counter() -fresh = 0 +amt_circ = defaultdict(lambda: Decimal(0)) # currency → amount +amt_spent = defaultdict(lambda: Decimal(0)) +amt_all = defaultdict(lambda: Decimal(0)) +circ_n = 0 +spent_n = 0 for c in coins: if not isinstance(c, dict): continue - dv = c.get("denomValue") or c.get("value") or "?" + dv = c.get("denomValue") or c.get("value") or c.get("denom_value") or "?" st = str(c.get("coinStatus") or c.get("status") or "?") - by_denom[dv] += 1 + by_denom_all[dv] += 1 by_status[st] += 1 - if st.lower() in ("fresh", "pending", "dormant", "usable", ""): - # count non-spent-looking as "in circulation" for summary - pass - if "spent" not in st.lower() and "delete" not in st.lower(): - fresh += 1 -# prefer explicit status -fresh = sum(n for s, n in by_status.items() if "spent" not in s.lower() and "delete" not in s.lower()) + ac, av = parse_amt(dv) + amt_all[ac] += av + st_l = st.lower() + if "spent" in st_l or "delete" in st_l or "dirty" in st_l: + spent_n += 1 + by_denom_spent[dv] += 1 + amt_spent[ac] += av + else: + circ_n += 1 + by_denom_circ[dv] += 1 + amt_circ[ac] += av + total = len(coins) -# stable sort denoms by numeric value if CUR:num + def denom_key(k): try: return float(str(k).split(":", 1)[-1]) except Exception: return 0.0 -denom_list = [ - {"denom": k, "count": by_denom[k]} - for k in sorted(by_denom.keys(), key=denom_key) -] -# human bar: "GOA:10×3 GOA:1×5" + +def denom_list(counter, spent_counter=None): + out = [] + for k in sorted(counter.keys(), key=denom_key): + item = {"denom": k, "count": counter[k]} + ac, av = parse_amt(k) + item["unit_value"] = str(av) + item["amount"] = fmt_amt(ac, av * counter[k]) + if spent_counter is not None: + item["spent_count"] = spent_counter.get(k, 0) + out.append(item) + return out + +# human: "10 GOA×3 (=30 GOA) …" and alt: "3 Kilo-GOA (GOA:3000)×1" parts = [] -for item in denom_list: - parts.append("%s×%d" % (item["denom"], item["count"])) -summary = ("total=%d in_wallet=%d | %s" % (total, fresh, " ".join(parts))) if parts else ("total=%d" % total) +parts_alt = [] +for item in denom_list(by_denom_circ): + ac, av = parse_amt(item["denom"]) + total_v = av * item["count"] + parts.append("%s×%d (=%s)" % (item["denom"], item["count"], item["amount"])) + parts_alt.append( + "%s×%d (=%s)" + % (fmt_amt_alt(ac, av), item["count"], fmt_amt_alt(ac, total_v)) + ) +circ_amt_s = " ".join(fmt_amt(c, amt_circ[c]) for c in sorted(amt_circ)) +circ_amt_alt = " ".join(fmt_amt_alt(c, amt_circ[c]) for c in sorted(amt_circ)) +spent_amt_s = " ".join(fmt_amt(c, amt_spent[c]) for c in sorted(amt_spent)) if amt_spent else "0" +spent_amt_alt = " ".join(fmt_amt_alt(c, amt_spent[c]) for c in sorted(amt_spent)) if amt_spent else "0" +if not circ_amt_s: + circ_amt_s = "%s:0" % cur + circ_amt_alt = "0 %s (%s:0)" % (alt.get("0") or cur, cur) +summary = ( + "coins=%d in_circ=%d spent=%d amount_circ=%s | %s" + % (total, circ_n, spent_n, circ_amt_alt, " ".join(parts_alt) if parts_alt else "(empty)") +) +# enrich denom lists with human labels +def enrich(lst): + out = [] + for item in lst: + ac, av = parse_amt(item["denom"]) + total_v = av * int(item["count"]) + item = dict(item) + item["denom_alt"] = fmt_amt_alt(ac, av) + item["amount_alt"] = fmt_amt_alt(ac, total_v) + out.append(item) + return out + report = { "ok": True, "currency": cur, "total_coins": total, - "in_circulation": fresh, + "in_circulation": circ_n, + "spent": spent_n, + "amount_in_circulation": {c: str(amt_circ[c]) for c in amt_circ}, + "amount_in_circulation_s": circ_amt_s, + "amount_in_circulation_alt": circ_amt_alt, + "amount_spent": {c: str(amt_spent[c]) for c in amt_spent}, + "amount_spent_s": spent_amt_s, + "amount_spent_alt": spent_amt_alt, "by_status": dict(by_status), - "by_denom": denom_list, + "by_denom": enrich(denom_list(by_denom_all, by_denom_spent)), + "by_denom_circulation": enrich(denom_list(by_denom_circ)), + "by_denom_spent": enrich(denom_list(by_denom_spent)), "summary": summary, + "alt_unit_names": alt, } json.dump(report, open(out_path, "w"), indent=2) print(summary) -# side channel for bash via file open(out_path + ".total", "w").write(str(total)) -open(out_path + ".circ", "w").write(str(fresh)) +open(out_path + ".circ", "w").write(str(circ_n)) +open(out_path + ".spent", "w").write(str(spent_n)) +open(out_path + ".amt", "w").write(circ_amt_s) PY COINS_TOTAL=$(cat "${out}.total" 2>/dev/null || echo 0) COINS_FRESH=$(cat "${out}.circ" 2>/dev/null || echo 0) + COINS_SPENT=$(cat "${out}.spent" 2>/dev/null || echo 0) + COINS_AMOUNT_CIRC=$(cat "${out}.amt" 2>/dev/null || echo "0") COINS_SUMMARY=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("summary","?"))' "$out" 2>/dev/null || echo "$COINS_SUMMARY") + # history TSV for end-of-run table + if [ -n "${METRICS_DIR:-}" ]; then + hist="${METRICS_DIR}/coins-history.tsv" + if [ ! -f "$hist" ]; then + printf 'ts\tlabel\ttotal\tin_circ\tspent\tamount_circ\tsummary\n' >"$hist" + fi + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date)" \ + "${METRICS_COINS_LABEL:-snap}" \ + "$COINS_TOTAL" "$COINS_FRESH" "$COINS_SPENT" "$COINS_AMOUNT_CIRC" \ + "$(printf '%s' "$COINS_SUMMARY" | tr '\t' ' ')" >>"$hist" + fi echo "$COINS_SUMMARY" } +# Emit coin inventory as monitoring info lines (label = step marker). +# $1=label $2=optional json path (default METRICS_DIR/coins-.json) +metrics_report_coins() { + local label="$1" + local safe + safe=$(printf '%s' "$label" | tr -c 'A-Za-z0-9._-' '_' | head -c 80) + local out="${2:-${METRICS_DIR}/coins-${safe}.json}" + local line prev="${METRICS_DIR}/coins-prev.json" + if [ -z "${WDB:-}" ] && [ -z "${WALLET_CLI:-}" ] && ! type wcli >/dev/null 2>&1; then + info "coins ${label}" "skipped (no wallet yet)" + return 0 + fi + METRICS_COINS_LABEL="$label" + export METRICS_COINS_LABEL + if ! metrics_wallet_coins "$out" >/dev/null; then + warn "coins ${label}" "dump-coins failed / empty" + return 1 + fi + # multi-line detail from JSON + while IFS= read -r line; do + [ -n "$line" ] || continue + info "coins ${label}" "$line" + done < <(python3 - "$out" <<'PY' +import json, sys +d = json.load(open(sys.argv[1])) +if not d.get("ok"): + print("unavailable: %s" % d.get("reason", "?")) + raise SystemExit(0) +amt = d.get("amount_in_circulation_alt") or d.get("amount_in_circulation_s") or "0" +print( + "n=%s in_circulation=%s spent=%s amount_circ=%s" + % (d.get("total_coins"), d.get("in_circulation"), d.get("spent"), amt) +) +st = d.get("by_status") or {} +if st: + print("status " + " ".join("%s=%s" % (k, st[k]) for k in sorted(st))) +circ = d.get("by_denom_circulation") or [] +if circ: + bits = [] + for x in circ: + dlab = x.get("denom_alt") or x.get("denom") + alab = x.get("amount_alt") or x.get("amount") + bits.append("%s×%d (=%s)" % (dlab, x["count"], alab)) + print("denoms_in_circ " + " ".join(bits)) +else: + print("denoms_in_circ (none)") +spent = d.get("by_denom_spent") or [] +if spent: + bits = [] + for x in spent: + dlab = x.get("denom_alt") or x.get("denom") + alab = x.get("amount_alt") or x.get("amount") + bits.append("%s×%d (=%s)" % (dlab, x["count"], alab)) + print("denoms_spent " + " ".join(bits)) +sp = d.get("amount_spent_alt") or d.get("amount_spent_s") +if sp and sp not in ("0", "0 GOA (GOA:0)"): + print("amount_spent %s" % sp) +PY + ) + # delta vs previous snapshot at this run + if [ -f "$prev" ]; then + local dsum + dsum=$(metrics_coins_delta "$prev" "$out" "${METRICS_DIR}/coins-delta-${safe}.json" 2>/dev/null || true) + if [ -n "$dsum" ]; then + info "coins ${label} Δ" "$dsum" + fi + fi + cp -f "$out" "$prev" 2>/dev/null || true + cp -f "$out" "${METRICS_DIR}/coins-final.json" 2>/dev/null || true + return 0 +} + # Diff two coin JSON snapshots → new coins this step metrics_coins_delta() { local before="${1:-}" after="${2:-}" out="${3:-$METRICS_DIR/coins-delta.json}" @@ -143,17 +491,19 @@ PY # --- Taler stack load on koopa (host + bank/exchange/merchant) --- # Writes JSON to $1. RAM, process counts, DB sizes, disk I/O counters. +# Uses KOOPA_SSH with fallback to KOOPA_SSH_FALLBACKS (koopa-external). metrics_taler_load() { local out="${1:-$METRICS_DIR/load.json}" local label="${2:-snap}" - if [ "${METRICS_LOAD}" = "0" ] || [ "${LADDER_LOAD:-1}" = "0" ]; then + mkdir -p "$(dirname "$out")" 2>/dev/null || true + if [ "${METRICS_LOAD}" = "0" ]; then printf '%s\n' "{\"ok\":false,\"reason\":\"disabled\",\"label\":\"$label\"}" >"$out" return 0 fi local raw="" local remote_py remote_py=$(cat <<'PY' -import json, os, re, subprocess, time +import base64, json, os, re, subprocess, time from collections import defaultdict def sh(cmd, t=15): @@ -162,6 +512,14 @@ def sh(cmd, t=15): except Exception: return "" +def podman_sh(name, script, t=25): + """Run a shell script inside container without nested-quote hell (base64).""" + b64 = base64.b64encode(script.encode()).decode() + return sh( + f"podman exec {name} sh -c 'echo {b64} | base64 -d | sh'", + t=t, + ) + def loadavg(): try: a,b,c = open("/proc/loadavg").read().split()[:3] @@ -206,52 +564,108 @@ def running(name): return sh(f"podman inspect -f '{{{{.State.Running}}}}' {name}").strip() == "true" def stats(name): - o = sh(f"podman stats --no-stream --format json {name}") + # Prefer Go template (stable across podman versions); JSON field names vary. + o = sh( + f"podman stats --no-stream --format " + f"'{{{{.CPUPerc}}}}|{{{{.MemUsage}}}}|{{{{.MemPerc}}}}|{{{{.BlockIO}}}}|{{{{.NetIO}}}}|{{{{.PIDs}}}}' " + f"{name}", + t=25, + ).strip() + if o and "|" in o: + p = o.split("|") + while len(p) < 6: + p.append("") + return { + "cpu_pct": p[0] or None, + "mem_usage": p[1] or None, + "mem_pct": p[2] or None, + "block_io": p[3] or None, + "net_io": p[4] or None, + "pids": p[5] or None, + } + o = sh(f"podman stats --no-stream --format json {name}", t=25) try: data = json.loads(o) - if isinstance(data, list) and data: data = data[0] - if not isinstance(data, dict): return {} - return {"cpu_pct": data.get("CPU") or data.get("CPUPerc"), - "mem_usage": data.get("MemUsage"), - "mem_pct": data.get("MemPerc"), - "block_io": data.get("BlockIO"), - "net_io": data.get("NetIO"), - "pids": data.get("PIDs")} + if isinstance(data, list) and data: + data = data[0] + if not isinstance(data, dict): + return {} + return { + "cpu_pct": data.get("CPU") or data.get("CPUPerc"), + "mem_usage": data.get("MemUsage"), + "mem_pct": data.get("MemPerc"), + "block_io": data.get("BlockIO"), + "net_io": data.get("NetIO"), + "pids": data.get("PIDs"), + } except Exception: return {} def procs(name): - # count + RSS by role inside container - code = r''' -import os,re,json -from collections import defaultdict -by=defaultdict(lambda:{"n":0,"rss_b":0}); n=0; rss=0 -for ent in os.listdir("/proc"): - if not ent.isdigit(): continue - try: st=open(f"/proc/{ent}/status").read() - except Exception: continue - m=re.search(r"^VmRSS:\s+(\d+)",st,re.M) - if not m: continue - rb=int(m.group(1))*1024 - nm=(re.search(r"^Name:\s+(\S+)",st,re.M) or type("x",(object,),{"group":lambda s,i: "?"})()).group(1) - try: cmd=open(f"/proc/{ent}/cmdline","rb").read().decode("utf-8","replace").replace("\0"," ") - except Exception: cmd="" - blob=(nm+" "+cmd).lower(); key="other" - if "postgres" in blob or "postmaster" in blob: key="postgres" - elif "libeufin" in blob or "mainkt" in blob: key="libeufin" - elif "java" in blob: key="java" - elif "taler-merchant" in blob: key="taler-merchant" - elif "taler-exchange" in blob: key="taler-exchange" - elif "nginx" in blob: key="nginx" - elif "apache" in blob: key="apache" - by[key]["n"]+=1; by[key]["rss_b"]+=rb; n+=1; rss+=rb -print(json.dumps({"proc_total":n,"rss_total_b":rss,"by_role":dict(by)})) -''' - o = sh(f"podman exec {name} python3 -c {json.dumps(code)}") - try: return json.loads(o) - except Exception: - n = sh(f"podman exec {name} sh -c 'ps -e --no-headers 2>/dev/null | wc -l'").strip() - return {"proc_total": int(n) if n.isdigit() else None, "rss_total_b": None, "by_role": {}} + # Pure shell /proc scan via base64 (containers often lack python3). + script = r""" +n=0; rss=0 +n_postgres=0; rss_postgres=0 +n_libeufin=0; rss_libeufin=0 +n_taler_merchant=0; rss_taler_merchant=0 +n_taler_exchange=0; rss_taler_exchange=0 +n_nginx=0; rss_nginx=0 +n_java=0; rss_java=0 +n_other=0; rss_other=0 +for d in /proc/[0-9]*; do + [ -r "$d/status" ] || continue + r=$(sed -n 's/^VmRSS:[[:space:]]*\([0-9][0-9]*\).*/\1/p' "$d/status" | head -1) + [ -n "$r" ] || continue + n=$((n+1)); rss=$((rss+r)) + nm=$(sed -n 's/^Name:[[:space:]]*//p' "$d/status" | head -1) + cmd=$(tr '\0' ' ' <"$d/cmdline" 2>/dev/null | head -c 240) + blob=$(printf '%s %s' "$nm" "$cmd" | tr 'A-Z' 'a-z') + case "$blob" in + *postgres*|*postmaster*) n_postgres=$((n_postgres+1)); rss_postgres=$((rss_postgres+r)) ;; + *libeufin*|*mainkt*) n_libeufin=$((n_libeufin+1)); rss_libeufin=$((rss_libeufin+r)) ;; + *taler-merchant*) n_taler_merchant=$((n_taler_merchant+1)); rss_taler_merchant=$((rss_taler_merchant+r)) ;; + *taler-exchange*) n_taler_exchange=$((n_taler_exchange+1)); rss_taler_exchange=$((rss_taler_exchange+r)) ;; + *nginx*) n_nginx=$((n_nginx+1)); rss_nginx=$((rss_nginx+r)) ;; + *java*) n_java=$((n_java+1)); rss_java=$((rss_java+r)) ;; + *) n_other=$((n_other+1)); rss_other=$((rss_other+r)) ;; + esac +done +printf 'TOTAL %s %s\n' "$n" "$rss" +[ "$n_postgres" -gt 0 ] && printf 'ROLE postgres %s %s\n' "$n_postgres" "$rss_postgres" +[ "$n_libeufin" -gt 0 ] && printf 'ROLE libeufin %s %s\n' "$n_libeufin" "$rss_libeufin" +[ "$n_taler_merchant" -gt 0 ] && printf 'ROLE taler-merchant %s %s\n' "$n_taler_merchant" "$rss_taler_merchant" +[ "$n_taler_exchange" -gt 0 ] && printf 'ROLE taler-exchange %s %s\n' "$n_taler_exchange" "$rss_taler_exchange" +[ "$n_nginx" -gt 0 ] && printf 'ROLE nginx %s %s\n' "$n_nginx" "$rss_nginx" +[ "$n_java" -gt 0 ] && printf 'ROLE java %s %s\n' "$n_java" "$rss_java" +[ "$n_other" -gt 0 ] && printf 'ROLE other %s %s\n' "$n_other" "$rss_other" +""" + o = podman_sh(name, script, t=25) + by = {} + n = None + rss_kb = None + for line in o.splitlines(): + p = line.split() + if not p: + continue + if p[0] == "TOTAL" and len(p) >= 3: + try: + n = int(p[1]) + rss_kb = int(p[2]) + except Exception: + pass + elif p[0] == "ROLE" and len(p) >= 4: + try: + by[p[1]] = {"n": int(p[2]), "rss_b": int(p[3]) * 1024} + except Exception: + pass + if n is None: + nn = sh(f"podman exec {name} sh -c 'ps -e --no-headers 2>/dev/null | wc -l'").strip() + n = int(nn) if nn.isdigit() else None + return { + "proc_total": n, + "rss_total_b": (rss_kb * 1024) if rss_kb is not None else None, + "by_role": by, + } def dbs(name): o = sh( @@ -301,31 +715,152 @@ print(json.dumps({ })) PY ) + # Prefer SSH to koopa (LAN or koopa-external). Local podman only if we are on the host. if [ "${SKIP_SSH:-0}" != "1" ] && type koopa_ssh_ok >/dev/null 2>&1 && koopa_ssh_ok; then - raw=$(printf '%s\n' "$remote_py" | koopa_ssh_bash 50 'python3 -' 2>/dev/null || true) - # koopa_ssh_bash only runs bash -s; pipe python via bash - if [ -z "$raw" ]; then - raw=$(printf 'python3 - <<'"'"'PY'"'"'\n%s\nPY\n' "$remote_py" | koopa_ssh_bash 50 2>/dev/null || true) + if type koopa_ssh_python >/dev/null 2>&1; then + raw=$(printf '%s' "$remote_py" | koopa_ssh_python "${METRICS_LOAD_SSH_TIMEOUT:-90}" 2>/dev/null || true) + else + raw=$(printf '%s' "$remote_py" | with_timeout "${METRICS_LOAD_SSH_TIMEOUT:-90}" \ + ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" python3 - 2>/dev/null || true) fi elif command -v podman >/dev/null 2>&1; then raw=$(python3 -c "$remote_py" 2>/dev/null || true) fi if [ -z "$raw" ]; then - printf '%s\n' "{\"ok\":false,\"reason\":\"probe-failed\",\"label\":\"$label\"}" >"$out" + printf '%s\n' "{\"ok\":false,\"reason\":\"probe-failed\",\"label\":\"$label\",\"ssh\":\"${KOOPA_SSH:-?}\"}" >"$out" return 1 fi printf '%s\n' "$raw" | python3 -c ' import sys,json,re t=sys.stdin.read(); obj=None for m in re.finditer(r"\{", t): - try: obj=json.loads(t[m.start():]) - except Exception: pass + try: + obj=json.loads(t[m.start():]) + if isinstance(obj, dict) and (obj.get("host") or obj.get("taler") or obj.get("ok") is False): + break + except Exception: + obj=None if not obj: obj={"ok":False,"reason":"parse"} obj["label"]=sys.argv[1] json.dump(obj, open(sys.argv[2],"w"), indent=2) ' "$label" "$out" } +# One-line human summaries (no leading indent) for info()/ok() detail fields. +metrics_load_lines() { + local f="$1" + python3 - "$f" <<'PY' +import json, sys +try: + d = json.load(open(sys.argv[1])) +except Exception as e: + print(f"unreadable: {e}") + raise SystemExit(0) +if not d.get("ok"): + print(f"unavailable: {d.get('reason', 'n/a')} ssh={d.get('ssh', '')}".strip()) + raise SystemExit(0) +h = d.get("host") or {} +mem = h.get("memory") or {} +la = h.get("loadavg") or [] + +def gi(b): + if b is None: + return "?" + return f"{b/1024/1024/1024:.2f}GiB" + +def mi(b): + if b is None: + return "?" + return f"{b/1024/1024:.0f}MiB" + +la_s = " ".join(f"{x:.2f}" for x in la) if la else "?" +tot = mem.get("mem_total_b") +avail = mem.get("mem_available_b") +used = (tot - avail) if (tot is not None and avail is not None) else None +print( + f"host loadavg=[{la_s}] nproc={h.get('nproc')} " + f"mem_used={gi(used)} avail={gi(avail)} total={gi(tot)}" +) +pc = h.get("process_counts") or {} +if pc: + bits = [f"{k}={v}" for k, v in sorted(pc.items()) if v] + if bits: + print("host procs " + " ".join(bits)) +for role in ("bank", "exchange", "merchant"): + c = (d.get("taler") or {}).get(role) or {} + if not c: + continue + if not c.get("running"): + print(f"{role} DOWN ({c.get('container')})") + continue + pr = c.get("processes") or {} + st = c.get("podman_stats") or {} + roles = pr.get("by_role") or {} + role_bits = [] + for k in ("libeufin", "taler-exchange", "taler-merchant", "postgres", "nginx", "java"): + if k in roles: + role_bits.append(f"{k}={mi(roles[k].get('rss_b'))}") + dbs = c.get("databases") or {} + db_s = ",".join( + f"{x.get('name')}={x.get('size_pretty')}" + for x in (dbs.get("databases") or [])[:5] + ) + # podman MemUsage often reports 0B under pasta/cgroup — prefer /proc RSS + mem_u = st.get("mem_usage") or "" + mem_bit = "" + if mem_u and not str(mem_u).startswith("0B"): + mem_bit = f" podman_mem={mem_u}" + cpu = st.get("cpu_pct") or "?" + blk = st.get("block_io") or "" + blk_bit = "" + if blk and blk not in ("0B / 0B", "0B/0B", "-- / --"): + blk_bit = f" block={blk}" + line = ( + f"{role} rss={gi(pr.get('rss_total_b'))} procs={pr.get('proc_total')} " + f"cpu={cpu}{mem_bit}{blk_bit}" + ) + if role_bits: + line += " | " + " ".join(role_bits) + if db_s: + line += f" | db:{db_s}" + if dbs.get("pgdata_pretty"): + line += f" pgdata={dbs.get('pgdata_pretty')}" + print(line) +PY +} + +# Snapshot load and emit as monitoring info lines (withdraw/pay phase markers). +# $1=json path $2=label (e.g. after-withdraw) +metrics_report_load() { + local out="$1" label="$2" + local line rc=0 + if [ "${METRICS_LOAD}" = "0" ]; then + info "load ${label}" "skipped (METRICS_LOAD=0)" + return 0 + fi + if [ "${SKIP_SSH:-0}" = "1" ] && ! command -v podman >/dev/null 2>&1; then + info "load ${label}" "skipped (no SSH / no local podman)" + return 0 + fi + if ! metrics_taler_load "$out" "$label"; then + warn "load ${label}" "probe failed via ${KOOPA_SSH:-?} — try KOOPA_SSH=koopa-external" + return 1 + fi + while IFS= read -r line; do + [ -n "$line" ] || continue + case "$line" in + unavailable:*|unreadable:*) + warn "load ${label}" "$line" + rc=1 + ;; + *) + info "load ${label}" "$line" + ;; + esac + done < <(metrics_load_lines "$out") + return "$rc" +} + # Human one-screen summary of a load JSON metrics_print_load() { local f="$1" title="${2:-load}"