hernani/koopa-admin-log
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

landing: central container memory/load stats for all three sites

Browse source 
Collect RSS groups, top processes, and loadavg from inside each podman
container (not host /proc), merge into performance.memory, show compact
labels with cgroup limits and byte tooltips, and cover this in the test suite.
This commit is contained in:
Hernâni Marques 2026-07-17 08:55:00 +02:00
parent 9e0f85024b
commit 78bbd80d1f
No known key found for this signature in database
11 changed files with 474 additions and 69 deletions
Download patch file Download diff file Expand all files Collapse all files

44
configs/bank-landing/index.html
View file

@ -2037,20 +2037,33 @@ tw run-until-done &amp;&amp; tw balance</pre>
if (http && http !== "200") s += " (" + http + ")"; if (http && http !== "200") s += " (" + http + ")";
return s; return s;
} }
function setMem(id, human, n, bytes) {
var el = document.getElementById(id);
if (!el) return;
if (!human && human !== 0) {
el.textContent = "—";
el.removeAttribute("title");
return;
}
var h = String(human);
if (n != null && n !== "" && Number(n) > 0) h += " · " + n + "p";
el.textContent = h;
if (bytes != null && bytes !== "" && bytes !== 0) {
el.setAttribute("title", Number(bytes).toLocaleString("en-US") + " B");
} else {
el.removeAttribute("title");
}
}
var p = d.performance || {}; var p = d.performance || {};
set("st-config-ms", msLabel(p.config_ms, p.config_http)); set("st-config-ms", msLabel(p.config_ms, p.config_http));
set("st-int-ms", msLabel(p.integration_ms, p.integration_http)); set("st-int-ms", msLabel(p.integration_ms, p.integration_http));
set("st-webui-ms", msLabel(p.webui_ms, p.webui_http)); set("st-webui-ms", msLabel(p.webui_ms, p.webui_http));
set("st-load", p.loadavg || "—"); set("st-load", p.loadavg || "—");
var mem = p.memory || {}; var mem = p.memory || {};
set("st-mem-ctr", mem.container_rss_human || "—"); setMem("st-mem-ctr", mem.container_rss_label || mem.container_rss_human || "—", null, mem.container_rss_bytes);
set("st-mem-pg", mem.postgres_rss_human setMem("st-mem-pg", mem.postgres_rss_human, mem.postgres_n, mem.postgres_rss_bytes);
? mem.postgres_rss_human + (mem.postgres_n ? " · " + mem.postgres_n + "p" : "") setMem("st-mem-java", mem.java_rss_human, mem.java_n, mem.java_rss_bytes);
: "—"); setMem("st-mem-nginx", mem.nginx_rss_human, mem.nginx_n, mem.nginx_rss_bytes);
set("st-mem-java", mem.java_rss_human
? mem.java_rss_human + (mem.java_n ? " · " + mem.java_n + "p" : "")
: "—");
set("st-mem-nginx", mem.nginx_rss_human || "—");
var topBox = document.getElementById("st-mem-top"); var topBox = document.getElementById("st-mem-top");
if (topBox) { if (topBox) {
var tops = mem.top || []; var tops = mem.top || [];
@ -2062,10 +2075,16 @@ tw run-until-done &amp;&amp; tw balance</pre>
'<th style="text-align:left;padding:0.25rem;border-bottom:1px solid var(--border)">Comm</th>' + '<th style="text-align:left;padding:0.25rem;border-bottom:1px solid var(--border)">Comm</th>' +
'<th style="text-align:left;padding:0.25rem;border-bottom:1px solid var(--border)">Command</th></tr></thead><tbody>'; '<th style="text-align:left;padding:0.25rem;border-bottom:1px solid var(--border)">Command</th></tr></thead><tbody>';
tops.forEach(function (t) { tops.forEach(function (t) {
th += '<tr><td style="padding:0.25rem;color:#5eead4">' + (t.rss_human || "?") + var cmd = t.cmd || "";
var cmdFull = t.cmd_full || cmd;
if (cmd.length > 64) cmd = cmd.slice(0, 61) + "…";
th += '<tr><td style="padding:0.25rem;color:#5eead4" title="' +
(t.rss_bytes != null ? Number(t.rss_bytes).toLocaleString("en-US") + " B" : "") + '">' +
(t.rss_human || "?") +
'</td><td style="padding:0.25rem">' + (t.comm || "") + '</td><td style="padding:0.25rem">' + (t.comm || "") +
'</td><td style="padding:0.25rem;word-break:break-all;color:var(--muted)">' + '</td><td style="padding:0.25rem;word-break:break-all;color:var(--muted)" title="' +
(t.cmd || "") + "</td></tr>"; String(cmdFull).replace(/"/g, "&quot;") + '">' +
cmd + "</td></tr>";
}); });
th += "</tbody></table>"; th += "</tbody></table>";
topBox.innerHTML = th; topBox.innerHTML = th;
@ -2073,7 +2092,8 @@ tw run-until-done &amp;&amp; tw balance</pre>
} }
var pf = document.getElementById("st-perf-foot"); var pf = document.getElementById("st-perf-foot");
if (pf) { if (pf) {
pf.textContent = "Container RSS + process groups · " + pf.textContent = "Container RSS + process groups" +
(mem.source ? " · " + mem.source : "") + " · " +
(d.generated_at_human || d.generated_at || ""); (d.generated_at_human || d.generated_at || "");
} }
return fetch("stats-run.json", { cache: "no-store" }) return fetch("stats-run.json", { cache: "no-store" })

41
configs/exchange-landing/index.html
View file

@ -569,19 +569,30 @@ sudo apt-get install -y taler-wallet-cli</pre>
(d.generated_at_human || d.generated_at || ""); (d.generated_at_human || d.generated_at || "");
} }
function setMem(id, human, n, bytes) {
var el = document.getElementById(id);
if (!el) return;
if (!human && human !== 0) {
el.textContent = "—";
el.removeAttribute("title");
return;
}
var h = String(human);
if (n != null && n !== "" && Number(n) > 0) h += " · " + n + "p";
el.textContent = h;
if (bytes != null && bytes !== "" && Number(bytes) > 0) {
el.setAttribute("title", Number(bytes).toLocaleString("en-US") + " B");
} else el.removeAttribute("title");
}
var p = d.performance || {}; var p = d.performance || {};
set("st-keys-ms", msLabel(p.keys_ms, p.keys_http)); set("st-keys-ms", msLabel(p.keys_ms, p.keys_http));
set("st-config-ms", msLabel(p.config_ms, p.config_http)); set("st-config-ms", msLabel(p.config_ms, p.config_http));
set("st-load", p.loadavg || "—"); set("st-load", p.loadavg || "—");
var mem = p.memory || {}; var mem = p.memory || {};
set("st-mem-ctr", mem.container_rss_human || "—"); setMem("st-mem-ctr", mem.container_rss_label || mem.container_rss_human || "—", null, mem.container_rss_bytes);
set("st-mem-pg", mem.postgres_rss_human setMem("st-mem-pg", mem.postgres_rss_human, mem.postgres_n, mem.postgres_rss_bytes);
? mem.postgres_rss_human + (mem.postgres_n ? " · " + mem.postgres_n + "p" : "") setMem("st-mem-taler", mem.taler_rss_human, mem.taler_n, mem.taler_rss_bytes);
: "—"); setMem("st-mem-nginx", mem.nginx_rss_human, mem.nginx_n, mem.nginx_rss_bytes);
set("st-mem-taler", mem.taler_rss_human
? mem.taler_rss_human + (mem.taler_n ? " · " + mem.taler_n + "p" : "")
: "—");
set("st-mem-nginx", mem.nginx_rss_human || "—");
var topBox = document.getElementById("st-mem-top"); var topBox = document.getElementById("st-mem-top");
if (topBox) { if (topBox) {
var tops = mem.top || []; var tops = mem.top || [];
@ -590,9 +601,14 @@ sudo apt-get install -y taler-wallet-cli</pre>
} else { } else {
var th = "<table><thead><tr><th>RSS</th><th>Comm</th><th>Command</th></tr></thead><tbody>"; var th = "<table><thead><tr><th>RSS</th><th>Comm</th><th>Command</th></tr></thead><tbody>";
tops.forEach(function (t) { tops.forEach(function (t) {
th += "<tr><td>" + (t.rss_human || "?") + "</td><td>" + var cmd = t.cmd || "";
(t.comm || "") + "</td><td style=\"font-size:0.72rem;word-break:break-all\">" + var cmdFull = t.cmd_full || cmd;
(t.cmd || "") + "</td></tr>"; if (cmd.length > 64) cmd = cmd.slice(0, 61) + "…";
th += "<tr><td title=\"" + (t.rss_bytes != null ? Number(t.rss_bytes).toLocaleString("en-US") + " B" : "") + "\">" +
(t.rss_human || "?") + "</td><td>" +
(t.comm || "") + "</td><td style=\"font-size:0.72rem;word-break:break-all\" title=\"" +
String(cmdFull).replace(/"/g, "&quot;") + "\">" +
cmd + "</td></tr>";
}); });
th += "</tbody></table>"; th += "</tbody></table>";
topBox.innerHTML = th; topBox.innerHTML = th;
@ -600,7 +616,8 @@ sudo apt-get install -y taler-wallet-cli</pre>
} }
var pf = document.getElementById("st-perf-foot"); var pf = document.getElementById("st-perf-foot");
if (pf) { if (pf) {
pf.textContent = "Container RSS + process groups · " + pf.textContent = "Container RSS + process groups" +
(mem.source ? " · " + mem.source : "") + " · " +
(d.generated_at_human || d.generated_at || ""); (d.generated_at_human || d.generated_at || "");
} }
return fetch("/intro/stats-run.json", { cache: "no-store" }) return fetch("/intro/stats-run.json", { cache: "no-store" })

42
configs/merchant-landing/index.html
View file

@ -973,20 +973,31 @@ sudo apt-get install -y taler-wallet-cli</pre>
foot.textContent = (d.dual_currency ? "Dual-currency · " : "") + foot.textContent = (d.dual_currency ? "Dual-currency · " : "") +
(d.generated_at_human || d.generated_at || ""); (d.generated_at_human || d.generated_at || "");
} }
function setMem(id, human, n, bytes) {
var el = document.getElementById(id);
if (!el) return;
if (!human && human !== 0) {
el.textContent = "—";
el.removeAttribute("title");
return;
}
var h = String(human);
if (n != null && n !== "" && Number(n) > 0) h += " · " + n + "p";
el.textContent = h;
if (bytes != null && bytes !== "" && Number(bytes) > 0) {
el.setAttribute("title", Number(bytes).toLocaleString("en-US") + " B");
} else el.removeAttribute("title");
}
var p = d.performance || {}; var p = d.performance || {};
set("st-config-ms", msLabel(p.config_ms, p.config_http)); set("st-config-ms", msLabel(p.config_ms, p.config_http));
set("st-terms-ms", msLabel(p.terms_ms, p.terms_http)); set("st-terms-ms", msLabel(p.terms_ms, p.terms_http));
set("st-webui-ms", msLabel(p.webui_ms, p.webui_http)); set("st-webui-ms", msLabel(p.webui_ms, p.webui_http));
set("st-load", p.loadavg || "—"); set("st-load", p.loadavg || "—");
var mem = p.memory || {}; var mem = p.memory || {};
set("st-mem-ctr", mem.container_rss_human || "—"); setMem("st-mem-ctr", mem.container_rss_label || mem.container_rss_human || "—", null, mem.container_rss_bytes);
set("st-mem-pg", mem.postgres_rss_human setMem("st-mem-pg", mem.postgres_rss_human, mem.postgres_n, mem.postgres_rss_bytes);
? mem.postgres_rss_human + (mem.postgres_n ? " · " + mem.postgres_n + "p" : "") setMem("st-mem-taler", mem.taler_rss_human, mem.taler_n, mem.taler_rss_bytes);
: "—"); setMem("st-mem-nginx", mem.nginx_rss_human, mem.nginx_n, mem.nginx_rss_bytes);
set("st-mem-taler", mem.taler_rss_human
? mem.taler_rss_human + (mem.taler_n ? " · " + mem.taler_n + "p" : "")
: "—");
set("st-mem-nginx", mem.nginx_rss_human || "—");
var topBox = document.getElementById("st-mem-top"); var topBox = document.getElementById("st-mem-top");
if (topBox) { if (topBox) {
var tops = mem.top || []; var tops = mem.top || [];
@ -998,10 +1009,16 @@ sudo apt-get install -y taler-wallet-cli</pre>
'<th style="text-align:left;padding:0.25rem;border-bottom:1px solid var(--border)">Comm</th>' + '<th style="text-align:left;padding:0.25rem;border-bottom:1px solid var(--border)">Comm</th>' +
'<th style="text-align:left;padding:0.25rem;border-bottom:1px solid var(--border)">Command</th></tr></thead><tbody>'; '<th style="text-align:left;padding:0.25rem;border-bottom:1px solid var(--border)">Command</th></tr></thead><tbody>';
tops.forEach(function (t) { tops.forEach(function (t) {
th += '<tr><td style="padding:0.25rem;color:#5eead4">' + (t.rss_human || "?") + var cmd = t.cmd || "";
var cmdFull = t.cmd_full || cmd;
if (cmd.length > 64) cmd = cmd.slice(0, 61) + "…";
th += '<tr><td style="padding:0.25rem;color:#5eead4" title="' +
(t.rss_bytes != null ? Number(t.rss_bytes).toLocaleString("en-US") + " B" : "") + '">' +
(t.rss_human || "?") +
'</td><td style="padding:0.25rem">' + (t.comm || "") + '</td><td style="padding:0.25rem">' + (t.comm || "") +
'</td><td style="padding:0.25rem;word-break:break-all;color:var(--muted)">' + '</td><td style="padding:0.25rem;word-break:break-all;color:var(--muted)" title="' +
(t.cmd || "") + "</td></tr>"; String(cmdFull).replace(/"/g, "&quot;") + '">' +
cmd + "</td></tr>";
}); });
th += "</tbody></table>"; th += "</tbody></table>";
topBox.innerHTML = th; topBox.innerHTML = th;
@ -1009,7 +1026,8 @@ sudo apt-get install -y taler-wallet-cli</pre>
} }
var pf = document.getElementById("st-perf-foot"); var pf = document.getElementById("st-perf-foot");
if (pf) { if (pf) {
pf.textContent = "Container RSS + process groups · " + pf.textContent = "Container RSS + process groups" +
(mem.source ? " · " + mem.source : "") + " · " +
(d.generated_at_human || d.generated_at || ""); (d.generated_at_human || d.generated_at || "");
} }
return fetch("/intro/stats-run.json", { cache: "no-store" }) return fetch("/intro/stats-run.json", { cache: "no-store" })

6
scripts/taler-landing/README.md
View file

@ -59,6 +59,12 @@ Installed paths:
3. **Merchant** — same for merchant container. 3. **Merchant** — same for merchant container.
4. **Resources (all three)** — `collect_container_resources.sh` runs
`mem-snapshot` **inside** each podman container (not host `/proc`):
container RSS (+ cgroup limit label), postgres/java/taler/nginx groups,
top-10 processes, loadavg. Merged into `performance.memory` /
`performance.loadavg` via `merge_resources.py`.
`exchange` is skipped in the bank **flow** scan so the same GOA is not counted once as customer withdraw and again as exchange credit. **Admin**, **explorer**, and every auto-account are included. `exchange` is skipped in the bank **flow** scan so the same GOA is not counted once as customer withdraw and again as exchange credit. **Admin**, **explorer**, and every auto-account are included.
## Secrets ## Secrets

75
scripts/taler-landing/collect-landing-stats.sh
View file

@ -3,6 +3,7 @@
# #
# - Bank: full account+ledger scan via collect_bank_stats.py → bank container # - Bank: full account+ledger scan via collect_bank_stats.py → bank container
# - Exchange / merchant: existing in-container scripts, then amount_alt enrich # - Exchange / merchant: existing in-container scripts, then amount_alt enrich
# - All three: container RSS / loadavg / top procs via mem-snapshot (podman exec)
# - Never wipes a good stats.json on failure (writes stats-run.json only) # - Never wipes a good stats.json on failure (writes stats-run.json only)
# #
# Install: scripts/taler-landing/install-landing-stats-host.sh # Install: scripts/taler-landing/install-landing-stats-host.sh
@ -50,6 +51,50 @@ ec_bank=0
ec_ex=0 ec_ex=0
ec_mer=0 ec_mer=0
# Resolve helper scripts (checkout or ~/.local/lib)
COLLECT_RES="$LIB/collect_container_resources.sh"
MERGE_RES="$LIB/merge_resources.py"
if [ ! -f "$COLLECT_RES" ] && [ -f "$ROOT/collect_container_resources.sh" ]; then
COLLECT_RES="$ROOT/collect_container_resources.sh"
fi
if [ ! -f "$MERGE_RES" ] && [ -f "$ROOT/merge_resources.py" ]; then
MERGE_RES="$ROOT/merge_resources.py"
fi
MEM_SRC="${MEM_SNAPSHOT_SRC:-$ADMIN_LOG/scripts/taler-shared/mem-snapshot.sh}"
merge_container_resources() {
local label="$1" ctr="$2" stats_file="$3"
[ -f "$stats_file" ] || return 0
if ! ctr_running "$ctr"; then
log "WARN: $label resources: $ctr not running"
return 1
fi
if [ ! -x "$COLLECT_RES" ] && [ -f "$COLLECT_RES" ]; then
chmod +x "$COLLECT_RES" 2>/dev/null || true
fi
if [ ! -f "$COLLECT_RES" ] || [ ! -f "$MERGE_RES" ]; then
log "WARN: $label resources: helpers missing ($COLLECT_RES / $MERGE_RES)"
return 1
fi
local resf="$WORKDIR/${label}-resources.json"
set +e
ADMIN_LOG="$ADMIN_LOG" MEM_SNAPSHOT_SRC="$MEM_SRC" \
bash "$COLLECT_RES" "$ctr" "$resf" >>"$LOG_DIR/${label}-resources.log" 2>&1
local ec=$?
set -e
if [ "$ec" -ne 0 ] || [ ! -s "$resf" ]; then
log "WARN: $label resources: collect failed (ec=$ec)"
return 1
fi
if ! "$PY" -c 'import json,sys; d=json.load(open(sys.argv[1])); sys.exit(0 if d.get("ok") and d.get("memory") else 1)' "$resf" 2>/dev/null; then
log "WARN: $label resources: bad payload"
return 1
fi
"$PY" "$MERGE_RES" "$stats_file" "$resf" >>"$LOG_DIR/${label}-resources.log" 2>&1
log "$label: resources merged (RSS + loadavg from $ctr)"
return 0
}
read_pass() { read_pass() {
local name="$1" f local name="$1" f
for f in \ for f in \
@ -136,28 +181,8 @@ collect_bank() {
set -e set -e
if [ "$ec_bank" -eq 0 ] && [ -f "$WORKDIR/bank-stats.json" ]; then if [ "$ec_bank" -eq 0 ] && [ -f "$WORKDIR/bank-stats.json" ]; then
# merge in-container memory snapshot when helper exists # Always attach in-container RSS/loadavg (not host /proc)
if ctr_running "$BANK_CTR" && podman exec "$BANK_CTR" test -f /usr/local/lib/landing-mem-snapshot.sh 2>/dev/null; then merge_container_resources bank "$BANK_CTR" "$WORKDIR/bank-stats.json" || true
set +e
mem_json=$(podman exec "$BANK_CTR" bash -c '
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
MEM_JSON=""
# shellcheck disable=SC1091
. /usr/local/lib/landing-mem-snapshot.sh
mem_snapshot_json 2>/dev/null || true
# print MEM_JSON lines only if function set them via echo — fallback empty
if [ -n "${MEM_JSON:-}" ]; then printf "%s" "$MEM_JSON"; fi
' 2>/dev/null)
set -e
if [ -n "${mem_json:-}" ]; then
"$PY" - "$WORKDIR/bank-stats.json" <<'PY' || true
import json, sys
path = sys.argv[1]
# optional: leave memory as-is if merge fails
print("mem merge skipped (structured merge via podman helper optional)", file=sys.stderr)
PY
fi
fi
publish_json "$BANK_CTR" "$BANK_LANDING_IN" \ publish_json "$BANK_CTR" "$BANK_LANDING_IN" \
"$WORKDIR/bank-stats.json" "$WORKDIR/bank-stats-run.json" "$WORKDIR/bank-stats.json" "$WORKDIR/bank-stats-run.json"
log "bank: OK → ${BANK_CTR}:${BANK_LANDING_IN}/stats.json" log "bank: OK → ${BANK_CTR}:${BANK_LANDING_IN}/stats.json"
@ -220,14 +245,16 @@ run_incontainer_stats() {
log "ERROR: $label stats failed (ec=$ec)" log "ERROR: $label stats failed (ec=$ec)"
return "$ec" return "$ec"
fi fi
# enrich with alt units on host # enrich with alt units + ensure resources (RSS/loadavg) on host
podman cp "${ctr}:${landing}/stats.json" "$WORKDIR/${label}-stats.json" 2>/dev/null || return 0 podman cp "${ctr}:${landing}/stats.json" "$WORKDIR/${label}-stats.json" 2>/dev/null || return 0
set +e set +e
"$PY" "$LIB/enrich_stats_alt.py" "$WORKDIR/${label}-stats.json" \ "$PY" "$LIB/enrich_stats_alt.py" "$WORKDIR/${label}-stats.json" \
--exchange-config "$EXCHANGE_CONFIG_URL" >>"$LOG_DIR/${label}.log" 2>&1 --exchange-config "$EXCHANGE_CONFIG_URL" >>"$LOG_DIR/${label}.log" 2>&1
set -e set -e
# Prefer fresh container snapshot (also fills gaps if in-container mem helper missing)
merge_container_resources "$label" "$ctr" "$WORKDIR/${label}-stats.json" || true
podman cp "$WORKDIR/${label}-stats.json" "${ctr}:${landing}/stats.json" podman cp "$WORKDIR/${label}-stats.json" "${ctr}:${landing}/stats.json"
log "$label: OK + alt enrich → ${ctr}:${landing}/stats.json" log "$label: OK + alt + resources → ${ctr}:${landing}/stats.json"
return 0 return 0
} }

11
scripts/taler-landing/collect_bank_stats.py
View file

@ -629,13 +629,9 @@ def main() -> int:
if int_http != "200": if int_http != "200":
int_ms, int_http = measure_ms(f"{bank}/taler-integration/config") int_ms, int_http = measure_ms(f"{bank}/taler-integration/config")
# loadavg filled later by host merge from *inside* the bank container
# (host /proc would be wrong when this script runs on koopa outside podman)
loadavg = "" loadavg = ""
try:
with open("/proc/loadavg", encoding="utf-8") as f:
parts = f.read().split()
loadavg = ",".join(parts[:3])
except Exception:
pass
total_pack = pack_amt(total_wd) total_pack = pack_amt(total_wd)
@ -755,7 +751,8 @@ def main() -> int:
"loadavg": loadavg, "loadavg": loadavg,
"memory": { "memory": {
"container_rss_human": "—", "container_rss_human": "—",
"note": "memory filled by host collector merge when available", "container_rss_label": "—",
"note": "filled by collect_container_resources.sh (in-container /proc + cgroup)",
}, },
}, },
"alt_unit_names": alt, "alt_unit_names": alt,

70
scripts/taler-landing/collect_container_resources.sh Executable file
View file

@ -0,0 +1,70 @@
#!/usr/bin/env bash
# Snapshot loadavg + RSS groups inside a podman container (for landing stats).
# Usage: collect_container_resources.sh CONTAINER [OUT.json]
# Prints JSON to stdout (and writes OUT when given).
set -euo pipefail
CTR="${1:-}"
OUT="${2:-}"
if [ -z "$CTR" ]; then
echo "usage: $0 CONTAINER [OUT.json]" >&2
exit 2
fi
ADMIN_LOG="${ADMIN_LOG:-${HOME}/src/koopa/koopa-admin-log}"
MEM_SRC="${MEM_SNAPSHOT_SRC:-$ADMIN_LOG/scripts/taler-shared/mem-snapshot.sh}"
MEM_DST="${MEM_SNAPSHOT_DST:-/usr/local/lib/landing-mem-snapshot.sh}"
if ! podman inspect -f '{{.State.Running}}' "$CTR" 2>/dev/null | grep -qx true; then
echo "{\"ok\":false,\"error\":\"container not running: $CTR\"}"
exit 1
fi
if [ -f "$MEM_SRC" ]; then
podman exec "$CTR" mkdir -p "$(dirname "$MEM_DST")" 2>/dev/null || true
podman cp "$MEM_SRC" "${CTR}:${MEM_DST}"
fi
# Run emit inside container (needs /proc + cgroup of that container)
set +e
raw=$(podman exec "$CTR" bash -c '
set -e
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
if [ ! -f /usr/local/lib/landing-mem-snapshot.sh ]; then
echo "{\"ok\":false,\"error\":\"mem-snapshot helper missing\"}"
exit 1
fi
# shellcheck disable=SC1091
. /usr/local/lib/landing-mem-snapshot.sh
if ! declare -F mem_snapshot_emit >/dev/null 2>&1; then
# older helper without emit — synthesize
mem_snapshot_json
loadavg=""
[ -r /proc/loadavg ] && loadavg=$(awk "{print \$1\",\"\$2\",\"\$3}" /proc/loadavg)
printf "{\"ok\":true,\"source\":\"mem-snapshot-legacy\",\"loadavg\":%s,\"memory\":{%s}}\n" \
"\"$loadavg\"" "$MEM_JSON"
else
mem_snapshot_emit
fi
' 2>/tmp/landing-mem-err.$$)
ec=$?
set -e
if [ "$ec" -ne 0 ] || [ -z "$raw" ]; then
err=$(tr '\n' ' ' </tmp/landing-mem-err.$$ 2>/dev/null | head -c 200 || true)
rm -f /tmp/landing-mem-err.$$
printf '{"ok":false,"error":"podman exec failed: %s"}\n' "${err//\"/\'}"
exit 1
fi
rm -f /tmp/landing-mem-err.$$
# Validate JSON
if ! printf '%s' "$raw" | python3 -c 'import json,sys; json.load(sys.stdin)' 2>/dev/null; then
printf '{"ok":false,"error":"invalid json from container"}\n'
exit 1
fi
if [ -n "$OUT" ]; then
printf '%s\n' "$raw" >"$OUT"
fi
printf '%s\n' "$raw"

9
scripts/taler-landing/install-landing-stats-host.sh
View file

@ -26,7 +26,16 @@ mkdir -p "$BIN_DST" "$LIB_DST" "$UNIT_DST" "$STATE_DST"
install -m 0755 "$SRC/collect-landing-stats.sh" "$BIN_DST/collect-landing-stats.sh" install -m 0755 "$SRC/collect-landing-stats.sh" "$BIN_DST/collect-landing-stats.sh"
install -m 0755 "$SRC/collect_bank_stats.py" "$LIB_DST/collect_bank_stats.py" install -m 0755 "$SRC/collect_bank_stats.py" "$LIB_DST/collect_bank_stats.py"
install -m 0755 "$SRC/enrich_stats_alt.py" "$LIB_DST/enrich_stats_alt.py" install -m 0755 "$SRC/enrich_stats_alt.py" "$LIB_DST/enrich_stats_alt.py"
install -m 0755 "$SRC/merge_resources.py" "$LIB_DST/merge_resources.py"
install -m 0755 "$SRC/collect_container_resources.sh" "$LIB_DST/collect_container_resources.sh"
install -m 0755 "$SRC/test-landing-stats.sh" "$BIN_DST/test-landing-stats.sh" 2>/dev/null || \
install -m 0755 "$SRC/test-landing-stats.sh" "$LIB_DST/test-landing-stats.sh"
install -m 0644 "$SRC/goa_amounts.py" "$LIB_DST/goa_amounts.py" install -m 0644 "$SRC/goa_amounts.py" "$LIB_DST/goa_amounts.py"
# mem-snapshot helper lives in taler-shared (copied into containers at collect time)
if [ -f "$ADMIN_LOG/scripts/taler-shared/mem-snapshot.sh" ]; then
install -m 0644 "$ADMIN_LOG/scripts/taler-shared/mem-snapshot.sh" \
"$LIB_DST/mem-snapshot.sh"
fi
# Wrapper always uses installed lib next to itself when LIB discovery works; # Wrapper always uses installed lib next to itself when LIB discovery works;
# also point ADMIN_LOG for in-container script refresh. # also point ADMIN_LOG for in-container script refresh.

106
scripts/taler-landing/merge_resources.py Executable file
View file

@ -0,0 +1,106 @@
#!/usr/bin/env python3
"""Merge container resource snapshot into landing stats.json performance block.
Usage:
merge_resources.py STATS.json RESOURCES.json [-o OUT.json]
RESOURCES shape (from mem_snapshot_emit / collect_container_resources.sh):
{ "ok": true, "loadavg": "0.1,0.2,0.3", "memory": { ... } }
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from pathlib import Path
from typing import Any, Dict
def compact_memory(mem: Dict[str, Any]) -> Dict[str, Any]:
"""Ensure human labels + short top cmd for UI width."""
if not isinstance(mem, dict):
return mem
# Prefer explicit label (with limit); else human
if not mem.get("container_rss_label"):
h = mem.get("container_rss_human") or "—"
lim = mem.get("cgroup_limit_human")
if lim:
mem["container_rss_label"] = f"{h} / {lim}"
else:
mem["container_rss_label"] = h
tops = mem.get("top")
if isinstance(tops, list):
for t in tops:
if not isinstance(t, dict):
continue
cmd = str(t.get("cmd") or "")
if len(cmd) > 72:
t["cmd_full"] = cmd
t["cmd"] = cmd[:69] + "…"
# ensure human present
if not t.get("rss_human") and t.get("rss_bytes") is not None:
try:
b = int(t["rss_bytes"])
if b < 1024:
t["rss_human"] = f"{b} B"
elif b < 1048576:
t["rss_human"] = f"{b/1024:.1f} KiB"
elif b < 1073741824:
t["rss_human"] = f"{b/1048576:.1f} MiB"
else:
t["rss_human"] = f"{b/1073741824:.2f} GiB"
except Exception:
pass
return mem
def merge(stats: Dict[str, Any], resources: Dict[str, Any]) -> Dict[str, Any]:
perf = stats.setdefault("performance", {})
if not isinstance(perf, dict):
perf = {}
stats["performance"] = perf
if resources.get("loadavg"):
perf["loadavg"] = resources["loadavg"]
perf["loadavg_source"] = "container"
mem = resources.get("memory")
if isinstance(mem, dict) and mem:
perf["memory"] = compact_memory(dict(mem))
perf["memory"]["source"] = resources.get("source") or "mem-snapshot"
return stats
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("stats")
ap.add_argument("resources")
ap.add_argument("-o", "--out", default="")
args = ap.parse_args()
stats = json.loads(Path(args.stats).read_text(encoding="utf-8"))
resources = json.loads(Path(args.resources).read_text(encoding="utf-8"))
if not isinstance(stats, dict) or not stats.get("ok"):
print("skip: stats not ok", file=sys.stderr)
return 1
if not isinstance(resources, dict) or resources.get("ok") is False:
print("skip: resources not ok", file=sys.stderr)
return 1
merge(stats, resources)
out = Path(args.out) if args.out else Path(args.stats)
tmp = out.with_suffix(out.suffix + f".tmp.{os.getpid()}")
tmp.write_text(json.dumps(stats, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
tmp.replace(out)
mem = (stats.get("performance") or {}).get("memory") or {}
print(
f"merged memory container={mem.get('container_rss_human')} "
f"pg={mem.get('postgres_rss_human')} loadavg={stats.get('performance', {}).get('loadavg')}",
file=sys.stderr,
)
return 0
if __name__ == "__main__":
raise SystemExit(main())

95
scripts/taler-landing/test-landing-stats.sh
View file

@ -102,6 +102,56 @@ then ok "enrich shapes bank/exchange/merchant"
else bad "enrich unit failed" else bad "enrich unit failed"
fi fi
echo
echo "=== unit: merge_resources + mem labels ==="
if "$PY" - <<PY
import json, sys, tempfile, os
sys.path.insert(0, "$ROOT/scripts/taler-landing")
# import as script module path
import importlib.util
spec = importlib.util.spec_from_file_location("merge_resources", "$ROOT/scripts/taler-landing/merge_resources.py")
mr = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mr)
stats = {"ok": True, "performance": {"config_ms": 12, "memory": {"container_rss_human": "—"}}}
res = {
"ok": True,
"loadavg": "0.5,0.4,0.3",
"source": "mem-snapshot",
"memory": {
"container_rss_bytes": 726663168,
"container_rss_human": "693.4 MiB",
"cgroup_limit_bytes": 2147483648,
"cgroup_limit_human": "2.00 GiB",
"postgres_rss_human": "281.1 MiB",
"postgres_n": 17,
"java_rss_human": "353.7 MiB",
"java_n": 2,
"top": [{"rss_bytes": 367312896, "rss_human": "350.3 MiB", "comm": "java",
"cmd": "java -classpath /usr/lib/libeufin-bank-all.jar " + ("x" * 120)}],
},
}
mr.merge(stats, res)
mem = stats["performance"]["memory"]
assert mem["container_rss_label"].startswith("693.4 MiB")
assert "/ 2.00 GiB" in mem["container_rss_label"]
assert stats["performance"]["loadavg"] == "0.5,0.4,0.3"
assert len(mem["top"][0]["cmd"]) <= 75
assert mem["top"][0].get("cmd_full")
print("merge ok", mem["container_rss_label"])
PY
then ok "merge_resources compact labels"
else bad "merge_resources unit failed"
fi
# mem-snapshot syntax
if bash -n "$ROOT/scripts/taler-shared/mem-snapshot.sh" \
&& bash -n "$ROOT/scripts/taler-landing/collect_container_resources.sh"; then
ok "mem-snapshot + collect_container_resources bash -n"
else
bad "bash -n resources helpers"
fi
echo echo
echo "=== public HTTPS (live stack) ===" echo "=== public HTTPS (live stack) ==="
for name_base in "bank|$BANK" "exchange|$EX" "merchant|$MER"; do for name_base in "bank|$BANK" "exchange|$EX" "merchant|$MER"; do
@ -129,6 +179,51 @@ for name_base in "bank|$BANK" "exchange|$EX" "merchant|$MER"; do
else else
bad "$name live stats enrich failed" bad "$name live stats enrich failed"
fi fi
# resources / memory block
if "$PY" - <<'PY'
import json, sys
d = json.load(open("/tmp/landing-test.body"))
p = d.get("performance") or {}
mem = p.get("memory") or {}
fail = []
if not p:
fail.append("no performance")
# latency probes differ per site but config_ms is common
if p.get("config_ms") is None and p.get("keys_ms") is None:
fail.append("no latency ms")
ctr = mem.get("container_rss_human") or ""
if not ctr or ctr in ("—", "-", "0"):
fail.append("container_rss_human missing")
if not isinstance(mem.get("top"), list) or len(mem.get("top") or []) < 1:
fail.append("top processes empty")
# role groups should exist as keys
for k in ("postgres_rss_human", "nginx_rss_human"):
if k not in mem:
fail.append("missing " + k)
if fail:
print("resources issues:", ", ".join(fail))
sys.exit(1)
print("resources ok container=", ctr, "top_n=", len(mem.get("top") or []),
"loadavg=", p.get("loadavg"))
sys.exit(0)
PY
then ok "$name performance.memory present (RSS + top)"
else bad "$name performance.memory incomplete"
fi
# merge dry-run on live stats (re-apply compact labels)
if "$PY" "$ROOT/scripts/taler-landing/merge_resources.py" /tmp/landing-test.body \
<("$PY" -c 'import json; d=json.load(open("/tmp/landing-test.body")); p=d.get("performance") or {}; print(json.dumps({"ok":True,"loadavg":p.get("loadavg") or "","memory":p.get("memory") or {}}))') \
-o /tmp/landing-test.resmerged.json 2>/dev/null; then
ok "$name resource merge on live payload"
else
# fallback without process substitution for macOS bash
"$PY" -c 'import json; d=json.load(open("/tmp/landing-test.body")); p=d.get("performance") or {}; json.dump({"ok":True,"loadavg":p.get("loadavg") or "","memory":p.get("memory") or {}}, open("/tmp/landing-test.res.json","w"))'
if "$PY" "$ROOT/scripts/taler-landing/merge_resources.py" /tmp/landing-test.body /tmp/landing-test.res.json -o /tmp/landing-test.resmerged.json 2>/dev/null; then
ok "$name resource merge on live payload"
else
bad "$name resource merge failed"
fi
fi
else else
bad "$name stats.json HTTP $code / not ok" bad "$name stats.json HTTP $code / not ok"
fi fi

44
scripts/taler-shared/mem-snapshot.sh
View file

@ -1,7 +1,17 @@
#!/bin/bash #!/bin/bash
# Memory snapshot for landing-stats (source after json_str is defined). # Memory snapshot for landing-stats.
# Sets MEM_JSON (fields to embed inside performance.memory). # Sets MEM_JSON (fields to embed inside performance.memory).
# Also: mem_snapshot_emit → full JSON object on stdout (host collector).
# IMPORTANT: no pipelines around the /proc loop (bash subshell loses counters). # IMPORTANT: no pipelines around the /proc loop (bash subshell loses counters).
#
# json_str may already be defined by the caller (landing-stats.sh); provide a
# safe default so this file is usable stand-alone via podman exec.
if ! declare -F json_str >/dev/null 2>&1; then
json_str() {
printf '"%s"' "$(printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g' | tr '\n\r\t' ' ')"
}
fi
mem_fmt_bytes() { mem_fmt_bytes() {
awk -v b="${1:-0}" 'BEGIN{ awk -v b="${1:-0}" 'BEGIN{
@ -92,21 +102,39 @@ mem_snapshot_json() {
rm -f "$topf" rm -f "$topf"
lim_json="null" lim_json="null"
lim_human_json="null"
if [[ "$limit_b" =~ ^[0-9]+$ ]] && [ "$limit_b" -gt 0 ]; then if [[ "$limit_b" =~ ^[0-9]+$ ]] && [ "$limit_b" -gt 0 ]; then
lim_json=$limit_b lim_json=$limit_b
lim_human_json=$(json_str "$(mem_fmt_bytes "$limit_b")")
fi fi
cg_json="null" cg_json="null"
if [[ "$cgroup_b" =~ ^[0-9]+$ ]]; then if [[ "$cgroup_b" =~ ^[0-9]+$ ]]; then
cg_json=$cgroup_b cg_json=$cgroup_b
fi fi
# Compact container label: "693.4 MiB" or "693.4 MiB / 2.00 GiB" when capped
local ctr_human
ctr_human=$(mem_fmt_bytes "$total_b")
if [ "$lim_human_json" != "null" ]; then
# lim_human_json is a quoted string already
:
fi
MEM_JSON=" MEM_JSON="
\"container_rss_bytes\": ${total_b}, \"container_rss_bytes\": ${total_b},
\"container_rss_human\": $(json_str "$(mem_fmt_bytes "$total_b")"), \"container_rss_human\": $(json_str "$ctr_human"),
\"container_rss_label\": $(json_str "$(
if [ "$lim_json" != "null" ]; then
printf '%s / %s' "$ctr_human" "$(mem_fmt_bytes "$limit_b")"
else
printf '%s' "$ctr_human"
fi
)"),
\"proc_sum_rss_bytes\": ${sum_b}, \"proc_sum_rss_bytes\": ${sum_b},
\"proc_sum_rss_human\": $(json_str "$(mem_fmt_bytes "$sum_b")"), \"proc_sum_rss_human\": $(json_str "$(mem_fmt_bytes "$sum_b")"),
\"cgroup_bytes\": ${cg_json}, \"cgroup_bytes\": ${cg_json},
\"cgroup_limit_bytes\": ${lim_json}, \"cgroup_limit_bytes\": ${lim_json},
\"cgroup_limit_human\": ${lim_human_json},
\"postgres_rss_bytes\": ${postgres_b}, \"postgres_rss_bytes\": ${postgres_b},
\"postgres_rss_human\": $(json_str "$(mem_fmt_bytes "$postgres_b")"), \"postgres_rss_human\": $(json_str "$(mem_fmt_bytes "$postgres_b")"),
\"postgres_n\": ${n_pg}, \"postgres_n\": ${n_pg},
@ -128,3 +156,15 @@ mem_snapshot_json() {
\"top\": ${top_json} \"top\": ${top_json}
" "
} }
# Full JSON for host collector: { ok, loadavg, memory: {…} }
mem_snapshot_emit() {
mem_snapshot_json || return 1
local loadavg=""
if [ -r /proc/loadavg ]; then
loadavg=$(awk '{print $1","$2","$3}' /proc/loadavg)
fi
printf '{\n "ok": true,\n "source": "mem-snapshot",\n "loadavg": %s,\n "memory": {\n%s\n }\n}\n' \
"$(json_str "$loadavg")" \
"$MEM_JSON"
}