landing: central container memory/load stats for all three sites

Collect RSS groups, top processes, and loadavg from inside each podman
container (not host /proc), merge into performance.memory, show compact
labels with cgroup limits and byte tooltips, and cover this in the test suite.
This commit is contained in:
Hernâni Marques 2026-07-17 08:55:00 +02:00
parent 9e0f85024b
commit 78bbd80d1f
No known key found for this signature in database
11 changed files with 474 additions and 69 deletions

View file

@ -59,6 +59,12 @@ Installed paths:
3. **Merchant** — same for merchant container.
4. **Resources (all three)**`collect_container_resources.sh` runs
`mem-snapshot` **inside** each podman container (not host `/proc`):
container RSS (+ cgroup limit label), postgres/java/taler/nginx groups,
top-10 processes, loadavg. Merged into `performance.memory` /
`performance.loadavg` via `merge_resources.py`.
`exchange` is skipped in the bank **flow** scan so the same GOA is not counted once as customer withdraw and again as exchange credit. **Admin**, **explorer**, and every auto-account are included.
## Secrets

View file

@ -3,6 +3,7 @@
#
# - Bank: full account+ledger scan via collect_bank_stats.py → bank container
# - Exchange / merchant: existing in-container scripts, then amount_alt enrich
# - All three: container RSS / loadavg / top procs via mem-snapshot (podman exec)
# - Never wipes a good stats.json on failure (writes stats-run.json only)
#
# Install: scripts/taler-landing/install-landing-stats-host.sh
@ -50,6 +51,50 @@ ec_bank=0
ec_ex=0
ec_mer=0
# Resolve helper scripts (checkout or ~/.local/lib)
COLLECT_RES="$LIB/collect_container_resources.sh"
MERGE_RES="$LIB/merge_resources.py"
if [ ! -f "$COLLECT_RES" ] && [ -f "$ROOT/collect_container_resources.sh" ]; then
COLLECT_RES="$ROOT/collect_container_resources.sh"
fi
if [ ! -f "$MERGE_RES" ] && [ -f "$ROOT/merge_resources.py" ]; then
MERGE_RES="$ROOT/merge_resources.py"
fi
MEM_SRC="${MEM_SNAPSHOT_SRC:-$ADMIN_LOG/scripts/taler-shared/mem-snapshot.sh}"
merge_container_resources() {
local label="$1" ctr="$2" stats_file="$3"
[ -f "$stats_file" ] || return 0
if ! ctr_running "$ctr"; then
log "WARN: $label resources: $ctr not running"
return 1
fi
if [ ! -x "$COLLECT_RES" ] && [ -f "$COLLECT_RES" ]; then
chmod +x "$COLLECT_RES" 2>/dev/null || true
fi
if [ ! -f "$COLLECT_RES" ] || [ ! -f "$MERGE_RES" ]; then
log "WARN: $label resources: helpers missing ($COLLECT_RES / $MERGE_RES)"
return 1
fi
local resf="$WORKDIR/${label}-resources.json"
set +e
ADMIN_LOG="$ADMIN_LOG" MEM_SNAPSHOT_SRC="$MEM_SRC" \
bash "$COLLECT_RES" "$ctr" "$resf" >>"$LOG_DIR/${label}-resources.log" 2>&1
local ec=$?
set -e
if [ "$ec" -ne 0 ] || [ ! -s "$resf" ]; then
log "WARN: $label resources: collect failed (ec=$ec)"
return 1
fi
if ! "$PY" -c 'import json,sys; d=json.load(open(sys.argv[1])); sys.exit(0 if d.get("ok") and d.get("memory") else 1)' "$resf" 2>/dev/null; then
log "WARN: $label resources: bad payload"
return 1
fi
"$PY" "$MERGE_RES" "$stats_file" "$resf" >>"$LOG_DIR/${label}-resources.log" 2>&1
log "$label: resources merged (RSS + loadavg from $ctr)"
return 0
}
read_pass() {
local name="$1" f
for f in \
@ -136,28 +181,8 @@ collect_bank() {
set -e
if [ "$ec_bank" -eq 0 ] && [ -f "$WORKDIR/bank-stats.json" ]; then
# merge in-container memory snapshot when helper exists
if ctr_running "$BANK_CTR" && podman exec "$BANK_CTR" test -f /usr/local/lib/landing-mem-snapshot.sh 2>/dev/null; then
set +e
mem_json=$(podman exec "$BANK_CTR" bash -c '
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
MEM_JSON=""
# shellcheck disable=SC1091
. /usr/local/lib/landing-mem-snapshot.sh
mem_snapshot_json 2>/dev/null || true
# print MEM_JSON lines only if function set them via echo — fallback empty
if [ -n "${MEM_JSON:-}" ]; then printf "%s" "$MEM_JSON"; fi
' 2>/dev/null)
set -e
if [ -n "${mem_json:-}" ]; then
"$PY" - "$WORKDIR/bank-stats.json" <<'PY' || true
import json, sys
path = sys.argv[1]
# optional: leave memory as-is if merge fails
print("mem merge skipped (structured merge via podman helper optional)", file=sys.stderr)
PY
fi
fi
# Always attach in-container RSS/loadavg (not host /proc)
merge_container_resources bank "$BANK_CTR" "$WORKDIR/bank-stats.json" || true
publish_json "$BANK_CTR" "$BANK_LANDING_IN" \
"$WORKDIR/bank-stats.json" "$WORKDIR/bank-stats-run.json"
log "bank: OK → ${BANK_CTR}:${BANK_LANDING_IN}/stats.json"
@ -220,14 +245,16 @@ run_incontainer_stats() {
log "ERROR: $label stats failed (ec=$ec)"
return "$ec"
fi
# enrich with alt units on host
# enrich with alt units + ensure resources (RSS/loadavg) on host
podman cp "${ctr}:${landing}/stats.json" "$WORKDIR/${label}-stats.json" 2>/dev/null || return 0
set +e
"$PY" "$LIB/enrich_stats_alt.py" "$WORKDIR/${label}-stats.json" \
--exchange-config "$EXCHANGE_CONFIG_URL" >>"$LOG_DIR/${label}.log" 2>&1
set -e
# Prefer fresh container snapshot (also fills gaps if in-container mem helper missing)
merge_container_resources "$label" "$ctr" "$WORKDIR/${label}-stats.json" || true
podman cp "$WORKDIR/${label}-stats.json" "${ctr}:${landing}/stats.json"
log "$label: OK + alt enrich → ${ctr}:${landing}/stats.json"
log "$label: OK + alt + resources${ctr}:${landing}/stats.json"
return 0
}

View file

@ -629,13 +629,9 @@ def main() -> int:
if int_http != "200":
int_ms, int_http = measure_ms(f"{bank}/taler-integration/config")
# loadavg filled later by host merge from *inside* the bank container
# (host /proc would be wrong when this script runs on koopa outside podman)
loadavg = ""
try:
with open("/proc/loadavg", encoding="utf-8") as f:
parts = f.read().split()
loadavg = ",".join(parts[:3])
except Exception:
pass
total_pack = pack_amt(total_wd)
@ -755,7 +751,8 @@ def main() -> int:
"loadavg": loadavg,
"memory": {
"container_rss_human": "",
"note": "memory filled by host collector merge when available",
"container_rss_label": "",
"note": "filled by collect_container_resources.sh (in-container /proc + cgroup)",
},
},
"alt_unit_names": alt,

View file

@ -0,0 +1,70 @@
#!/usr/bin/env bash
# Snapshot loadavg + RSS groups inside a podman container (for landing stats).
# Usage: collect_container_resources.sh CONTAINER [OUT.json]
# Prints JSON to stdout (and writes OUT when given).
set -euo pipefail
CTR="${1:-}"
OUT="${2:-}"
if [ -z "$CTR" ]; then
echo "usage: $0 CONTAINER [OUT.json]" >&2
exit 2
fi
ADMIN_LOG="${ADMIN_LOG:-${HOME}/src/koopa/koopa-admin-log}"
MEM_SRC="${MEM_SNAPSHOT_SRC:-$ADMIN_LOG/scripts/taler-shared/mem-snapshot.sh}"
MEM_DST="${MEM_SNAPSHOT_DST:-/usr/local/lib/landing-mem-snapshot.sh}"
if ! podman inspect -f '{{.State.Running}}' "$CTR" 2>/dev/null | grep -qx true; then
echo "{\"ok\":false,\"error\":\"container not running: $CTR\"}"
exit 1
fi
if [ -f "$MEM_SRC" ]; then
podman exec "$CTR" mkdir -p "$(dirname "$MEM_DST")" 2>/dev/null || true
podman cp "$MEM_SRC" "${CTR}:${MEM_DST}"
fi
# Run emit inside container (needs /proc + cgroup of that container)
set +e
raw=$(podman exec "$CTR" bash -c '
set -e
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
if [ ! -f /usr/local/lib/landing-mem-snapshot.sh ]; then
echo "{\"ok\":false,\"error\":\"mem-snapshot helper missing\"}"
exit 1
fi
# shellcheck disable=SC1091
. /usr/local/lib/landing-mem-snapshot.sh
if ! declare -F mem_snapshot_emit >/dev/null 2>&1; then
# older helper without emit — synthesize
mem_snapshot_json
loadavg=""
[ -r /proc/loadavg ] && loadavg=$(awk "{print \$1\",\"\$2\",\"\$3}" /proc/loadavg)
printf "{\"ok\":true,\"source\":\"mem-snapshot-legacy\",\"loadavg\":%s,\"memory\":{%s}}\n" \
"\"$loadavg\"" "$MEM_JSON"
else
mem_snapshot_emit
fi
' 2>/tmp/landing-mem-err.$$)
ec=$?
set -e
if [ "$ec" -ne 0 ] || [ -z "$raw" ]; then
err=$(tr '\n' ' ' </tmp/landing-mem-err.$$ 2>/dev/null | head -c 200 || true)
rm -f /tmp/landing-mem-err.$$
printf '{"ok":false,"error":"podman exec failed: %s"}\n' "${err//\"/\'}"
exit 1
fi
rm -f /tmp/landing-mem-err.$$
# Validate JSON
if ! printf '%s' "$raw" | python3 -c 'import json,sys; json.load(sys.stdin)' 2>/dev/null; then
printf '{"ok":false,"error":"invalid json from container"}\n'
exit 1
fi
if [ -n "$OUT" ]; then
printf '%s\n' "$raw" >"$OUT"
fi
printf '%s\n' "$raw"

View file

@ -26,7 +26,16 @@ mkdir -p "$BIN_DST" "$LIB_DST" "$UNIT_DST" "$STATE_DST"
install -m 0755 "$SRC/collect-landing-stats.sh" "$BIN_DST/collect-landing-stats.sh"
install -m 0755 "$SRC/collect_bank_stats.py" "$LIB_DST/collect_bank_stats.py"
install -m 0755 "$SRC/enrich_stats_alt.py" "$LIB_DST/enrich_stats_alt.py"
install -m 0755 "$SRC/merge_resources.py" "$LIB_DST/merge_resources.py"
install -m 0755 "$SRC/collect_container_resources.sh" "$LIB_DST/collect_container_resources.sh"
install -m 0755 "$SRC/test-landing-stats.sh" "$BIN_DST/test-landing-stats.sh" 2>/dev/null || \
install -m 0755 "$SRC/test-landing-stats.sh" "$LIB_DST/test-landing-stats.sh"
install -m 0644 "$SRC/goa_amounts.py" "$LIB_DST/goa_amounts.py"
# mem-snapshot helper lives in taler-shared (copied into containers at collect time)
if [ -f "$ADMIN_LOG/scripts/taler-shared/mem-snapshot.sh" ]; then
install -m 0644 "$ADMIN_LOG/scripts/taler-shared/mem-snapshot.sh" \
"$LIB_DST/mem-snapshot.sh"
fi
# Wrapper always uses installed lib next to itself when LIB discovery works;
# also point ADMIN_LOG for in-container script refresh.

View file

@ -0,0 +1,106 @@
#!/usr/bin/env python3
"""Merge container resource snapshot into landing stats.json performance block.
Usage:
merge_resources.py STATS.json RESOURCES.json [-o OUT.json]
RESOURCES shape (from mem_snapshot_emit / collect_container_resources.sh):
{ "ok": true, "loadavg": "0.1,0.2,0.3", "memory": { ... } }
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from pathlib import Path
from typing import Any, Dict
def compact_memory(mem: Dict[str, Any]) -> Dict[str, Any]:
"""Ensure human labels + short top cmd for UI width."""
if not isinstance(mem, dict):
return mem
# Prefer explicit label (with limit); else human
if not mem.get("container_rss_label"):
h = mem.get("container_rss_human") or ""
lim = mem.get("cgroup_limit_human")
if lim:
mem["container_rss_label"] = f"{h} / {lim}"
else:
mem["container_rss_label"] = h
tops = mem.get("top")
if isinstance(tops, list):
for t in tops:
if not isinstance(t, dict):
continue
cmd = str(t.get("cmd") or "")
if len(cmd) > 72:
t["cmd_full"] = cmd
t["cmd"] = cmd[:69] + ""
# ensure human present
if not t.get("rss_human") and t.get("rss_bytes") is not None:
try:
b = int(t["rss_bytes"])
if b < 1024:
t["rss_human"] = f"{b} B"
elif b < 1048576:
t["rss_human"] = f"{b/1024:.1f} KiB"
elif b < 1073741824:
t["rss_human"] = f"{b/1048576:.1f} MiB"
else:
t["rss_human"] = f"{b/1073741824:.2f} GiB"
except Exception:
pass
return mem
def merge(stats: Dict[str, Any], resources: Dict[str, Any]) -> Dict[str, Any]:
perf = stats.setdefault("performance", {})
if not isinstance(perf, dict):
perf = {}
stats["performance"] = perf
if resources.get("loadavg"):
perf["loadavg"] = resources["loadavg"]
perf["loadavg_source"] = "container"
mem = resources.get("memory")
if isinstance(mem, dict) and mem:
perf["memory"] = compact_memory(dict(mem))
perf["memory"]["source"] = resources.get("source") or "mem-snapshot"
return stats
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("stats")
ap.add_argument("resources")
ap.add_argument("-o", "--out", default="")
args = ap.parse_args()
stats = json.loads(Path(args.stats).read_text(encoding="utf-8"))
resources = json.loads(Path(args.resources).read_text(encoding="utf-8"))
if not isinstance(stats, dict) or not stats.get("ok"):
print("skip: stats not ok", file=sys.stderr)
return 1
if not isinstance(resources, dict) or resources.get("ok") is False:
print("skip: resources not ok", file=sys.stderr)
return 1
merge(stats, resources)
out = Path(args.out) if args.out else Path(args.stats)
tmp = out.with_suffix(out.suffix + f".tmp.{os.getpid()}")
tmp.write_text(json.dumps(stats, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
tmp.replace(out)
mem = (stats.get("performance") or {}).get("memory") or {}
print(
f"merged memory container={mem.get('container_rss_human')} "
f"pg={mem.get('postgres_rss_human')} loadavg={stats.get('performance', {}).get('loadavg')}",
file=sys.stderr,
)
return 0
if __name__ == "__main__":
raise SystemExit(main())

View file

@ -102,6 +102,56 @@ then ok "enrich shapes bank/exchange/merchant"
else bad "enrich unit failed"
fi
echo
echo "=== unit: merge_resources + mem labels ==="
if "$PY" - <<PY
import json, sys, tempfile, os
sys.path.insert(0, "$ROOT/scripts/taler-landing")
# import as script module path
import importlib.util
spec = importlib.util.spec_from_file_location("merge_resources", "$ROOT/scripts/taler-landing/merge_resources.py")
mr = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mr)
stats = {"ok": True, "performance": {"config_ms": 12, "memory": {"container_rss_human": "—"}}}
res = {
"ok": True,
"loadavg": "0.5,0.4,0.3",
"source": "mem-snapshot",
"memory": {
"container_rss_bytes": 726663168,
"container_rss_human": "693.4 MiB",
"cgroup_limit_bytes": 2147483648,
"cgroup_limit_human": "2.00 GiB",
"postgres_rss_human": "281.1 MiB",
"postgres_n": 17,
"java_rss_human": "353.7 MiB",
"java_n": 2,
"top": [{"rss_bytes": 367312896, "rss_human": "350.3 MiB", "comm": "java",
"cmd": "java -classpath /usr/lib/libeufin-bank-all.jar " + ("x" * 120)}],
},
}
mr.merge(stats, res)
mem = stats["performance"]["memory"]
assert mem["container_rss_label"].startswith("693.4 MiB")
assert "/ 2.00 GiB" in mem["container_rss_label"]
assert stats["performance"]["loadavg"] == "0.5,0.4,0.3"
assert len(mem["top"][0]["cmd"]) <= 75
assert mem["top"][0].get("cmd_full")
print("merge ok", mem["container_rss_label"])
PY
then ok "merge_resources compact labels"
else bad "merge_resources unit failed"
fi
# mem-snapshot syntax
if bash -n "$ROOT/scripts/taler-shared/mem-snapshot.sh" \
&& bash -n "$ROOT/scripts/taler-landing/collect_container_resources.sh"; then
ok "mem-snapshot + collect_container_resources bash -n"
else
bad "bash -n resources helpers"
fi
echo
echo "=== public HTTPS (live stack) ==="
for name_base in "bank|$BANK" "exchange|$EX" "merchant|$MER"; do
@ -129,6 +179,51 @@ for name_base in "bank|$BANK" "exchange|$EX" "merchant|$MER"; do
else
bad "$name live stats enrich failed"
fi
# resources / memory block
if "$PY" - <<'PY'
import json, sys
d = json.load(open("/tmp/landing-test.body"))
p = d.get("performance") or {}
mem = p.get("memory") or {}
fail = []
if not p:
fail.append("no performance")
# latency probes differ per site but config_ms is common
if p.get("config_ms") is None and p.get("keys_ms") is None:
fail.append("no latency ms")
ctr = mem.get("container_rss_human") or ""
if not ctr or ctr in ("—", "-", "0"):
fail.append("container_rss_human missing")
if not isinstance(mem.get("top"), list) or len(mem.get("top") or []) < 1:
fail.append("top processes empty")
# role groups should exist as keys
for k in ("postgres_rss_human", "nginx_rss_human"):
if k not in mem:
fail.append("missing " + k)
if fail:
print("resources issues:", ", ".join(fail))
sys.exit(1)
print("resources ok container=", ctr, "top_n=", len(mem.get("top") or []),
"loadavg=", p.get("loadavg"))
sys.exit(0)
PY
then ok "$name performance.memory present (RSS + top)"
else bad "$name performance.memory incomplete"
fi
# merge dry-run on live stats (re-apply compact labels)
if "$PY" "$ROOT/scripts/taler-landing/merge_resources.py" /tmp/landing-test.body \
<("$PY" -c 'import json; d=json.load(open("/tmp/landing-test.body")); p=d.get("performance") or {}; print(json.dumps({"ok":True,"loadavg":p.get("loadavg") or "","memory":p.get("memory") or {}}))') \
-o /tmp/landing-test.resmerged.json 2>/dev/null; then
ok "$name resource merge on live payload"
else
# fallback without process substitution for macOS bash
"$PY" -c 'import json; d=json.load(open("/tmp/landing-test.body")); p=d.get("performance") or {}; json.dump({"ok":True,"loadavg":p.get("loadavg") or "","memory":p.get("memory") or {}}, open("/tmp/landing-test.res.json","w"))'
if "$PY" "$ROOT/scripts/taler-landing/merge_resources.py" /tmp/landing-test.body /tmp/landing-test.res.json -o /tmp/landing-test.resmerged.json 2>/dev/null; then
ok "$name resource merge on live payload"
else
bad "$name resource merge failed"
fi
fi
else
bad "$name stats.json HTTP $code / not ok"
fi