landing: central container memory/load stats for all three sites

Collect RSS groups, top processes, and loadavg from inside each podman
container (not host /proc), merge into performance.memory, show compact
labels with cgroup limits and byte tooltips, and cover this in the test suite.

scripts/taler-landing/test-landing-stats.sh

@@ -102,6 +102,56 @@ then ok "enrich shapes bank/exchange/merchant"
 else bad "enrich unit failed"
 fi
 
+echo
+echo "=== unit: merge_resources + mem labels ==="
+if "$PY" - <<PY
+import json, sys, tempfile, os
+sys.path.insert(0, "$ROOT/scripts/taler-landing")
+# import as script module path
+import importlib.util
+spec = importlib.util.spec_from_file_location("merge_resources", "$ROOT/scripts/taler-landing/merge_resources.py")
+mr = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(mr)
+
+stats = {"ok": True, "performance": {"config_ms": 12, "memory": {"container_rss_human": "—"}}}
+res = {
+  "ok": True,
+  "loadavg": "0.5,0.4,0.3",
+  "source": "mem-snapshot",
+  "memory": {
+    "container_rss_bytes": 726663168,
+    "container_rss_human": "693.4 MiB",
+    "cgroup_limit_bytes": 2147483648,
+    "cgroup_limit_human": "2.00 GiB",
+    "postgres_rss_human": "281.1 MiB",
+    "postgres_n": 17,
+    "java_rss_human": "353.7 MiB",
+    "java_n": 2,
+    "top": [{"rss_bytes": 367312896, "rss_human": "350.3 MiB", "comm": "java",
+             "cmd": "java -classpath /usr/lib/libeufin-bank-all.jar " + ("x" * 120)}],
+  },
+}
+mr.merge(stats, res)
+mem = stats["performance"]["memory"]
+assert mem["container_rss_label"].startswith("693.4 MiB")
+assert "/ 2.00 GiB" in mem["container_rss_label"]
+assert stats["performance"]["loadavg"] == "0.5,0.4,0.3"
+assert len(mem["top"][0]["cmd"]) <= 75
+assert mem["top"][0].get("cmd_full")
+print("merge ok", mem["container_rss_label"])
+PY
+then ok "merge_resources compact labels"
+else bad "merge_resources unit failed"
+fi
+
+# mem-snapshot syntax
+if bash -n "$ROOT/scripts/taler-shared/mem-snapshot.sh" \
+  && bash -n "$ROOT/scripts/taler-landing/collect_container_resources.sh"; then
+  ok "mem-snapshot + collect_container_resources bash -n"
+else
+  bad "bash -n resources helpers"
+fi
+
 echo
 echo "=== public HTTPS (live stack) ==="
 for name_base in "bank|$BANK" "exchange|$EX" "merchant|$MER"; do
@@ -129,6 +179,51 @@ for name_base in "bank|$BANK" "exchange|$EX" "merchant|$MER"; do
     else
       bad "$name live stats enrich failed"
     fi
+    # resources / memory block
+    if "$PY" - <<'PY'
+import json, sys
+d = json.load(open("/tmp/landing-test.body"))
+p = d.get("performance") or {}
+mem = p.get("memory") or {}
+fail = []
+if not p:
+    fail.append("no performance")
+# latency probes differ per site but config_ms is common
+if p.get("config_ms") is None and p.get("keys_ms") is None:
+    fail.append("no latency ms")
+ctr = mem.get("container_rss_human") or ""
+if not ctr or ctr in ("—", "-", "0"):
+    fail.append("container_rss_human missing")
+if not isinstance(mem.get("top"), list) or len(mem.get("top") or []) < 1:
+    fail.append("top processes empty")
+# role groups should exist as keys
+for k in ("postgres_rss_human", "nginx_rss_human"):
+    if k not in mem:
+        fail.append("missing " + k)
+if fail:
+    print("resources issues:", ", ".join(fail))
+    sys.exit(1)
+print("resources ok container=", ctr, "top_n=", len(mem.get("top") or []),
+      "loadavg=", p.get("loadavg"))
+sys.exit(0)
+PY
+    then ok "$name performance.memory present (RSS + top)"
+    else bad "$name performance.memory incomplete"
+    fi
+    # merge dry-run on live stats (re-apply compact labels)
+    if "$PY" "$ROOT/scripts/taler-landing/merge_resources.py" /tmp/landing-test.body \
+         <("$PY" -c 'import json; d=json.load(open("/tmp/landing-test.body")); p=d.get("performance") or {}; print(json.dumps({"ok":True,"loadavg":p.get("loadavg") or "","memory":p.get("memory") or {}}))') \
+         -o /tmp/landing-test.resmerged.json 2>/dev/null; then
+      ok "$name resource merge on live payload"
+    else
+      # fallback without process substitution for macOS bash
+      "$PY" -c 'import json; d=json.load(open("/tmp/landing-test.body")); p=d.get("performance") or {}; json.dump({"ok":True,"loadavg":p.get("loadavg") or "","memory":p.get("memory") or {}}, open("/tmp/landing-test.res.json","w"))'
+      if "$PY" "$ROOT/scripts/taler-landing/merge_resources.py" /tmp/landing-test.body /tmp/landing-test.res.json -o /tmp/landing-test.resmerged.json 2>/dev/null; then
+        ok "$name resource merge on live payload"
+      else
+        bad "$name resource merge failed"
+      fi
+    fi
   else
     bad "$name stats.json HTTP $code / not ok"
   fi