# shellcheck shell=bash # Shared Taler stack metrics for e2e + ladder (source after lib.sh). # # - Host/container load (RAM, processes, DB sizes) before/after a phase # - Wallet coin inventory (total + by denomination) # - Final overall statistics block # # Env: # METRICS_DIR where JSON snapshots go (default $SCRATCH or /tmp) # METRICS_LOAD=0 skip remote/host load probes # KOOPA_SSH / SKIP_SSH as in lib.sh : "${METRICS_LOAD:=1}" : "${METRICS_DIR:=${SCRATCH:-/tmp}}" mkdir -p "$METRICS_DIR" 2>/dev/null || true # --------------------------------------------------------------------------- # alt_unit_names — human amounts (Kilo-GOA / Mega-GOA / …) with base in parens # From exchange/bank currency_specification.alt_unit_names: # "0"→GOA, "3"→Kilo-GOA, "6"→Mega-GOA, … "-3"→Milli-GOA, … # Example: GOA:5000 → 5 Kilo-GOA (GOA:5000) # --------------------------------------------------------------------------- : "${ALT_UNITS_FILE:=${METRICS_DIR}/alt_unit_names.json}" # Load map from public /config into ALT_UNITS_FILE. $1=optional config URL. metrics_load_alt_units() { local url="${1:-${EXCHANGE_PUBLIC:-https://exchange.hacktivism.ch}/config}" local code mkdir -p "$(dirname "$ALT_UNITS_FILE")" 2>/dev/null || true code=$(curl -skS -m 12 -o "${ALT_UNITS_FILE}.raw" -w '%{http_code}' "$url" 2>/dev/null || echo 000) if [ "$code" != "200" ]; then # soft fallback: SI-style names if live config unreachable printf '%s\n' '{"0":"GOA","3":"Kilo-GOA","6":"Mega-GOA","9":"Giga-GOA","12":"Tera-GOA","15":"Peta-GOA","18":"Exa-GOA","21":"Zetta-GOA","24":"Yotta-GOA","-1":"Deci-GOA","-2":"Centi-GOA","-3":"Milli-GOA","-6":"Micro-GOA","-8":"Atomic-GOA"}' \ >"$ALT_UNITS_FILE" return 1 fi python3 - "${ALT_UNITS_FILE}.raw" "$ALT_UNITS_FILE" <<'PY' import json, sys src, dst = sys.argv[1:3] d = json.load(open(src)) au = None cs = d.get("currency_specification") if isinstance(cs, dict): au = cs.get("alt_unit_names") if not au and isinstance(d.get("currencies"), dict): # merchant-style: currencies.GOA.alt_unit_names for _code, spec in d["currencies"].items(): if isinstance(spec, dict) and spec.get("alt_unit_names"): au = spec["alt_unit_names"] break if not isinstance(au, dict) or "0" not in au: au = {"0": d.get("currency") or "GOA"} json.dump(au, open(dst, "w"), indent=2, sort_keys=True) PY export ALT_UNITS_FILE return 0 } # Format one amount: "GOA:5000" → "5 Kilo-GOA (GOA:5000)" # Uses ALT_UNITS_FILE if present. Pure stdout. format_amount_alt() { local amt="${1:-}" [ -n "$amt" ] || return 0 python3 - "$amt" "${ALT_UNITS_FILE:-}" <<'PY' import json, sys from decimal import Decimal, InvalidOperation, ROUND_HALF_UP amt = sys.argv[1].strip() path = sys.argv[2] if len(sys.argv) > 2 else "" alt = {} if path: try: alt = json.load(open(path)) except Exception: alt = {} if not alt: alt = {"0": "GOA"} def parse(s): if ":" in s: c, v = s.split(":", 1) return c, Decimal(v) return "GOA", Decimal(s) def fmt_num(v: Decimal) -> str: if v == v.to_integral(): return format(int(v), "d") s = format(v.quantize(Decimal("0.00000001"), rounding=ROUND_HALF_UP), "f") return s.rstrip("0").rstrip(".") try: cur, val = parse(amt) except (InvalidOperation, ValueError): print(amt) raise SystemExit(0) base_name = alt.get("0") or cur # always show canonical base form in parens base_s = "%s:%s" % (cur, fmt_num(val)) if val == 0: print("0 %s (%s)" % (base_name, base_s)) raise SystemExit(0) # scales: power of 10 relative to unit "0" scales = [] for k, name in alt.items(): try: scales.append((int(k), str(name))) except Exception: pass scales.sort(key=lambda x: -x[0]) # largest unit first # pick largest scale where |value| >= 10^scale (for scale>=0), # or for fractions the finest unit that makes the coefficient >= 1 chosen = None # (scale, name, coeff) absval = abs(val) for sc, name in scales: unit = Decimal(10) ** sc if unit <= 0: continue coeff = absval / unit if coeff >= 1: chosen = (sc, name, coeff if val >= 0 else -coeff) break if chosen is None: # smaller than smallest unit — use base print("%s %s (%s)" % (fmt_num(val), base_name, base_s)) raise SystemExit(0) sc, name, coeff = chosen # if base unit, prefer "GOA:12" style still with parens only when alt differs if sc == 0: print("%s %s" % (fmt_num(val), name)) raise SystemExit(0) print("%s %s (%s)" % (fmt_num(coeff), name, base_s)) PY } # Format list of "CUR:n" amounts for plans / logs (space-separated → multiline or compact) format_amount_list_alt() { local a out="" for a in "$@"; do [ -n "$out" ] && out="$out " out="${out}$(format_amount_alt "$a")" done printf '%s\n' "$out" } # --- coin inventory from wallet-cli dump-coins --- # Writes JSON to $1; prints one-line summary to stdout. # Sets COINS_TOTAL, COINS_FRESH, COINS_SPENT, COINS_AMOUNT_CIRC, COINS_SUMMARY. metrics_wallet_coins() { local out="${1:-$METRICS_DIR/coins.json}" local dump="${METRICS_DIR}/dump-coins.raw" mkdir -p "$(dirname "$out")" 2>/dev/null || true COINS_TOTAL=0 COINS_FRESH=0 COINS_SPENT=0 COINS_AMOUNT_CIRC="0" COINS_SUMMARY="(no coins)" # Prefer wcli() from caller (e2e/ladder). Do not pass a leading timeout number — # ladder wcli() has no optional secs arg (would become a wallet subcommand). if type wcli >/dev/null 2>&1; then wcli advanced dump-coins >"$dump" 2>/dev/null || true elif [ -n "${CLI_JS:-}" ] && [ -f "${CLI_JS}" ]; then node "$CLI_JS" --wallet-db="${WDB:-}" --no-throttle advanced dump-coins >"$dump" 2>/dev/null || true elif [ -n "${WALLET_CLI:-}" ] && [ -n "${WDB:-}" ]; then node "$WALLET_CLI" --wallet-db="${WDB}" --no-throttle --skip-defaults advanced dump-coins >"$dump" 2>/dev/null || true else echo "$COINS_SUMMARY" printf '%s\n' '{"ok":false,"reason":"no-wcli"}' >"$out" return 1 fi python3 - "$dump" "$out" "${CUR:-GOA}" "${ALT_UNITS_FILE:-}" <<'PY' import json, re, sys from collections import Counter, defaultdict from decimal import Decimal, InvalidOperation, ROUND_HALF_UP raw_path, out_path, cur = sys.argv[1:4] alt_path = sys.argv[4] if len(sys.argv) > 4 else "" raw = open(raw_path).read() if raw_path else "" d = None for m in re.finditer(r"\{", raw): try: d = json.loads(raw[m.start():]) if isinstance(d, dict) and ("coins" in d or "coin" in d): break except Exception: d = None coins = [] if isinstance(d, dict): coins = d.get("coins") or d.get("coin") or [] alt = {} if alt_path: try: alt = json.load(open(alt_path)) except Exception: alt = {} if not alt: alt = {"0": cur} def parse_amt(s): """'GOA:10' / '10' → (currency, Decimal) or (cur, 0).""" s = str(s or "").strip() if not s or s == "?": return cur, Decimal(0) if ":" in s: c, v = s.split(":", 1) try: return c, Decimal(v) except InvalidOperation: return c, Decimal(0) try: return cur, Decimal(s) except InvalidOperation: return cur, Decimal(0) def fmt_num(v: Decimal) -> str: if v == v.to_integral(): return format(int(v), "d") s = format(v.quantize(Decimal("0.00000001"), rounding=ROUND_HALF_UP), "f") return s.rstrip("0").rstrip(".") def fmt_amt(c, v: Decimal) -> str: return "%s:%s" % (c, fmt_num(v)) def fmt_amt_alt(c, v: Decimal) -> str: """5 Kilo-GOA (GOA:5000) using alt_unit_names.""" base_s = fmt_amt(c, v) base_name = alt.get("0") or c if v == 0: return "0 %s (%s)" % (base_name, base_s) scales = [] for k, name in alt.items(): try: scales.append((int(k), str(name))) except Exception: pass scales.sort(key=lambda x: -x[0]) absval = abs(v) chosen = None for sc, name in scales: unit = Decimal(10) ** sc coeff = absval / unit if coeff >= 1: chosen = (sc, name, coeff if v >= 0 else -coeff) break if chosen is None: return "%s %s (%s)" % (fmt_num(v), base_name, base_s) sc, name, coeff = chosen if sc == 0: return "%s %s" % (fmt_num(v), name) return "%s %s (%s)" % (fmt_num(coeff), name, base_s) by_denom_all = Counter() # all coins by_denom_circ = Counter() # non-spent by_denom_spent = Counter() by_status = Counter() amt_circ = defaultdict(lambda: Decimal(0)) # currency → amount amt_spent = defaultdict(lambda: Decimal(0)) amt_all = defaultdict(lambda: Decimal(0)) circ_n = 0 spent_n = 0 for c in coins: if not isinstance(c, dict): continue dv = c.get("denomValue") or c.get("value") or c.get("denom_value") or "?" st = str(c.get("coinStatus") or c.get("status") or "?") by_denom_all[dv] += 1 by_status[st] += 1 ac, av = parse_amt(dv) amt_all[ac] += av st_l = st.lower() if "spent" in st_l or "delete" in st_l or "dirty" in st_l: spent_n += 1 by_denom_spent[dv] += 1 amt_spent[ac] += av else: circ_n += 1 by_denom_circ[dv] += 1 amt_circ[ac] += av total = len(coins) def denom_key(k): try: return float(str(k).split(":", 1)[-1]) except Exception: return 0.0 def denom_list(counter, spent_counter=None): out = [] for k in sorted(counter.keys(), key=denom_key): item = {"denom": k, "count": counter[k]} ac, av = parse_amt(k) item["unit_value"] = str(av) item["amount"] = fmt_amt(ac, av * counter[k]) if spent_counter is not None: item["spent_count"] = spent_counter.get(k, 0) out.append(item) return out # human: "10 GOA×3 (=30 GOA) …" and alt: "3 Kilo-GOA (GOA:3000)×1" parts = [] parts_alt = [] for item in denom_list(by_denom_circ): ac, av = parse_amt(item["denom"]) total_v = av * item["count"] parts.append("%s×%d (=%s)" % (item["denom"], item["count"], item["amount"])) parts_alt.append( "%s×%d (=%s)" % (fmt_amt_alt(ac, av), item["count"], fmt_amt_alt(ac, total_v)) ) circ_amt_s = " ".join(fmt_amt(c, amt_circ[c]) for c in sorted(amt_circ)) circ_amt_alt = " ".join(fmt_amt_alt(c, amt_circ[c]) for c in sorted(amt_circ)) spent_amt_s = " ".join(fmt_amt(c, amt_spent[c]) for c in sorted(amt_spent)) if amt_spent else "0" spent_amt_alt = " ".join(fmt_amt_alt(c, amt_spent[c]) for c in sorted(amt_spent)) if amt_spent else "0" if not circ_amt_s: circ_amt_s = "%s:0" % cur circ_amt_alt = "0 %s (%s:0)" % (alt.get("0") or cur, cur) summary = ( "coins=%d in_circ=%d spent=%d amount_circ=%s | %s" % (total, circ_n, spent_n, circ_amt_alt, " ".join(parts_alt) if parts_alt else "(empty)") ) # enrich denom lists with human labels def enrich(lst): out = [] for item in lst: ac, av = parse_amt(item["denom"]) total_v = av * int(item["count"]) item = dict(item) item["denom_alt"] = fmt_amt_alt(ac, av) item["amount_alt"] = fmt_amt_alt(ac, total_v) out.append(item) return out report = { "ok": True, "currency": cur, "total_coins": total, "in_circulation": circ_n, "spent": spent_n, "amount_in_circulation": {c: str(amt_circ[c]) for c in amt_circ}, "amount_in_circulation_s": circ_amt_s, "amount_in_circulation_alt": circ_amt_alt, "amount_spent": {c: str(amt_spent[c]) for c in amt_spent}, "amount_spent_s": spent_amt_s, "amount_spent_alt": spent_amt_alt, "by_status": dict(by_status), "by_denom": enrich(denom_list(by_denom_all, by_denom_spent)), "by_denom_circulation": enrich(denom_list(by_denom_circ)), "by_denom_spent": enrich(denom_list(by_denom_spent)), "summary": summary, "alt_unit_names": alt, } json.dump(report, open(out_path, "w"), indent=2) print(summary) open(out_path + ".total", "w").write(str(total)) open(out_path + ".circ", "w").write(str(circ_n)) open(out_path + ".spent", "w").write(str(spent_n)) open(out_path + ".amt", "w").write(circ_amt_s) PY COINS_TOTAL=$(cat "${out}.total" 2>/dev/null || echo 0) COINS_FRESH=$(cat "${out}.circ" 2>/dev/null || echo 0) COINS_SPENT=$(cat "${out}.spent" 2>/dev/null || echo 0) COINS_AMOUNT_CIRC=$(cat "${out}.amt" 2>/dev/null || echo "0") COINS_SUMMARY=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("summary","?"))' "$out" 2>/dev/null || echo "$COINS_SUMMARY") # history TSV for end-of-run table if [ -n "${METRICS_DIR:-}" ]; then hist="${METRICS_DIR}/coins-history.tsv" if [ ! -f "$hist" ]; then printf 'ts\tlabel\ttotal\tin_circ\tspent\tamount_circ\tsummary\n' >"$hist" fi printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ "$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date)" \ "${METRICS_COINS_LABEL:-snap}" \ "$COINS_TOTAL" "$COINS_FRESH" "$COINS_SPENT" "$COINS_AMOUNT_CIRC" \ "$(printf '%s' "$COINS_SUMMARY" | tr '\t' ' ')" >>"$hist" fi echo "$COINS_SUMMARY" } # Emit coin inventory as monitoring info lines (label = step marker). # $1=label $2=optional json path (default METRICS_DIR/coins-.json) metrics_report_coins() { local label="$1" local safe safe=$(printf '%s' "$label" | tr -c 'A-Za-z0-9._-' '_' | head -c 80) local out="${2:-${METRICS_DIR}/coins-${safe}.json}" local line prev="${METRICS_DIR}/coins-prev.json" if [ -z "${WDB:-}" ] && [ -z "${WALLET_CLI:-}" ] && ! type wcli >/dev/null 2>&1; then info "coins ${label}" "skipped (no wallet yet)" return 0 fi METRICS_COINS_LABEL="$label" export METRICS_COINS_LABEL if ! metrics_wallet_coins "$out" >/dev/null; then warn "coins ${label}" "dump-coins failed / empty" return 1 fi # multi-line detail from JSON while IFS= read -r line; do [ -n "$line" ] || continue info "coins ${label}" "$line" done < <(python3 - "$out" <<'PY' import json, sys d = json.load(open(sys.argv[1])) if not d.get("ok"): print("unavailable: %s" % d.get("reason", "?")) raise SystemExit(0) amt = d.get("amount_in_circulation_alt") or d.get("amount_in_circulation_s") or "0" print( "n=%s in_circulation=%s spent=%s amount_circ=%s" % (d.get("total_coins"), d.get("in_circulation"), d.get("spent"), amt) ) st = d.get("by_status") or {} if st: print("status " + " ".join("%s=%s" % (k, st[k]) for k in sorted(st))) circ = d.get("by_denom_circulation") or [] if circ: bits = [] for x in circ: dlab = x.get("denom_alt") or x.get("denom") alab = x.get("amount_alt") or x.get("amount") bits.append("%s×%d (=%s)" % (dlab, x["count"], alab)) print("denoms_in_circ " + " ".join(bits)) else: print("denoms_in_circ (none)") spent = d.get("by_denom_spent") or [] if spent: bits = [] for x in spent: dlab = x.get("denom_alt") or x.get("denom") alab = x.get("amount_alt") or x.get("amount") bits.append("%s×%d (=%s)" % (dlab, x["count"], alab)) print("denoms_spent " + " ".join(bits)) sp = d.get("amount_spent_alt") or d.get("amount_spent_s") if sp and sp not in ("0", "0 GOA (GOA:0)"): print("amount_spent %s" % sp) PY ) # delta vs previous snapshot at this run if [ -f "$prev" ]; then local dsum dsum=$(metrics_coins_delta "$prev" "$out" "${METRICS_DIR}/coins-delta-${safe}.json" 2>/dev/null || true) if [ -n "$dsum" ]; then info "coins ${label} Δ" "$dsum" fi fi cp -f "$out" "$prev" 2>/dev/null || true cp -f "$out" "${METRICS_DIR}/coins-final.json" 2>/dev/null || true return 0 } # Diff two coin JSON snapshots → new coins this step metrics_coins_delta() { local before="${1:-}" after="${2:-}" out="${3:-$METRICS_DIR/coins-delta.json}" python3 - "$before" "$after" "$out" <<'PY' import json, sys from collections import Counter def load(p): try: d=json.load(open(p)) return d except Exception: return {} b, a = load(sys.argv[1]), load(sys.argv[2]) outp = sys.argv[3] def bag(d): c=Counter() for item in d.get("by_denom") or []: c[item.get("denom") or "?"] += int(item.get("count") or 0) return c bb, aa = bag(b), bag(a) delta = aa - bb parts = ["%s×%+d" % (k, delta[k]) for k in sorted(delta.keys(), key=lambda x: float(str(x).split(":")[-1]) if ":" in str(x) or str(x).replace(".","").isdigit() else 0) if delta[k]] new_total = int(a.get("total_coins") or 0) - int(b.get("total_coins") or 0) rep = { "new_coins": new_total, "total_after": int(a.get("total_coins") or 0), "circulation_after": int(a.get("in_circulation") or 0), "delta_by_denom": {k: delta[k] for k in delta}, "summary": ("new=%+d total_now=%s | %s" % ( new_total, a.get("total_coins"), " ".join(parts) if parts else "(no denom change)")), } json.dump(rep, open(outp, "w"), indent=2) print(rep["summary"]) PY } # --- Taler stack load (host + bank/exchange/merchant) --- # Writes JSON to $1. RAM, process counts, DB sizes, disk I/O counters. # koopa: KOOPA_SSH (+ fallbacks). stage-lfp: INSIDE_SSH (stagepaysan) + stage-lfp-* names. metrics_taler_load() { local out="${1:-$METRICS_DIR/load.json}" local label="${2:-snap}" mkdir -p "$(dirname "$out")" 2>/dev/null || true if [ "${METRICS_LOAD}" = "0" ]; then printf '%s\n' "{\"ok\":false,\"reason\":\"disabled\",\"label\":\"$label\"}" >"$out" return 0 fi local raw="" local remote_py local _m_bank _m_mer _m_ex _m_ssh _m_bank="taler-hacktivism-bank" _m_mer="taler-hacktivism" _m_ex="taler-hacktivism-exchange-ansible" _m_ssh="" if [ "${INSIDE_PROFILE:-}" = "stage-lfp" ] \ || [ "${EXPECT_CURRENCY:-}" = "TESTPAYSAN" ] \ || { [ -n "${INSIDE_SSH:-}" ] && [ "${LOCAL_STACK:-0}" != "1" ]; }; then _m_bank="${INSIDE_BANK_CTR:-stage-lfp-bank}" _m_mer="${INSIDE_MERCHANT_CTR:-stage-lfp-merchant}" _m_ex="${INSIDE_EXCHANGE_CTR:-stage-lfp-exchange-ansible}" _m_ssh="${INSIDE_SSH:-}" fi remote_py=$(cat </dev/null | head -c 240) blob=$(printf '%s %s' "$nm" "$cmd" | tr 'A-Z' 'a-z') case "$blob" in *postgres*|*postmaster*) n_postgres=$((n_postgres+1)); rss_postgres=$((rss_postgres+r)) ;; *libeufin*|*mainkt*) n_libeufin=$((n_libeufin+1)); rss_libeufin=$((rss_libeufin+r)) ;; *taler-merchant*) n_taler_merchant=$((n_taler_merchant+1)); rss_taler_merchant=$((rss_taler_merchant+r)) ;; *taler-exchange*) n_taler_exchange=$((n_taler_exchange+1)); rss_taler_exchange=$((rss_taler_exchange+r)) ;; *nginx*) n_nginx=$((n_nginx+1)); rss_nginx=$((rss_nginx+r)) ;; *java*) n_java=$((n_java+1)); rss_java=$((rss_java+r)) ;; *) n_other=$((n_other+1)); rss_other=$((rss_other+r)) ;; esac done printf 'TOTAL %s %s\n' "$n" "$rss" [ "$n_postgres" -gt 0 ] && printf 'ROLE postgres %s %s\n' "$n_postgres" "$rss_postgres" [ "$n_libeufin" -gt 0 ] && printf 'ROLE libeufin %s %s\n' "$n_libeufin" "$rss_libeufin" [ "$n_taler_merchant" -gt 0 ] && printf 'ROLE taler-merchant %s %s\n' "$n_taler_merchant" "$rss_taler_merchant" [ "$n_taler_exchange" -gt 0 ] && printf 'ROLE taler-exchange %s %s\n' "$n_taler_exchange" "$rss_taler_exchange" [ "$n_nginx" -gt 0 ] && printf 'ROLE nginx %s %s\n' "$n_nginx" "$rss_nginx" [ "$n_java" -gt 0 ] && printf 'ROLE java %s %s\n' "$n_java" "$rss_java" [ "$n_other" -gt 0 ] && printf 'ROLE other %s %s\n' "$n_other" "$rss_other" """ o = podman_sh(name, script, t=25) by = {} n = None rss_kb = None for line in o.splitlines(): p = line.split() if not p: continue if p[0] == "TOTAL" and len(p) >= 3: try: n = int(p[1]) rss_kb = int(p[2]) except Exception: pass elif p[0] == "ROLE" and len(p) >= 4: try: by[p[1]] = {"n": int(p[2]), "rss_b": int(p[3]) * 1024} except Exception: pass if n is None: nn = sh(f"podman exec {name} sh -c 'ps -e --no-headers 2>/dev/null | wc -l'").strip() n = int(nn) if nn.isdigit() else None return { "proc_total": n, "rss_total_b": (rss_kb * 1024) if rss_kb is not None else None, "by_role": by, } def dbs(name): o = sh( f"podman exec {name} su -s /bin/bash postgres -c " + json.dumps( "psql -Atc \"SELECT datname||'|'||pg_database_size(datname)||'|'||pg_size_pretty(pg_database_size(datname)) " "FROM pg_database WHERE datistemplate=false ORDER BY 1\"" ) ) out=[] for line in o.splitlines(): p=line.strip().split("|") if len(p)>=3: try: out.append({"name":p[0],"size_b":int(p[1]),"size_pretty":p[2]}) except Exception: pass du=sh(f"podman exec {name} sh -c 'du -sb /var/lib/postgresql 2>/dev/null | cut -f1'").strip() pretty=sh(f"podman exec {name} sh -c 'du -sh /var/lib/postgresql 2>/dev/null | cut -f1'").strip() return {"databases": out, "pgdata_bytes": int(du) if du.isdigit() else None, "pgdata_pretty": pretty or None} components={} for role,cname in CTRS: if not running(cname): components[role]={"container":cname,"running":False} continue components[role]={ "container":cname,"running":True, "podman_stats":stats(cname), "processes":procs(cname), "databases":dbs(cname), } host_ps=sh("ps -eo comm=") host_counts={k:sum(1 for line in host_ps.splitlines() if k in line.lower()) for k in ("postgres","nginx","caddy","podman","conmon","pasta")} print(json.dumps({ "ok": True, "ts": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "host": { "hostname": sh("hostname").strip(), "loadavg": loadavg(), "nproc": os.cpu_count(), "memory": meminfo(), "disk_io": disk_io(), "process_counts": host_counts, }, "taler": components, })) PY ) # Prefer stagepaysan (stage-lfp), else koopa SSH, else local podman only on LOCAL_STACK. if [ -n "${_m_ssh}" ] && type mon_ssh_ok >/dev/null 2>&1 && mon_ssh_ok "${_m_ssh}"; then raw=$(printf '%s' "$remote_py" | with_timeout "${METRICS_LOAD_SSH_TIMEOUT:-90}" \ ssh "${SSH_BASE_OPTS[@]}" "${_m_ssh}" python3 - 2>/dev/null || true) elif [ "${SKIP_SSH:-0}" != "1" ] && type koopa_ssh_ok >/dev/null 2>&1 && koopa_ssh_ok; then if type koopa_ssh_python >/dev/null 2>&1; then raw=$(printf '%s' "$remote_py" | koopa_ssh_python "${METRICS_LOAD_SSH_TIMEOUT:-90}" 2>/dev/null || true) else raw=$(printf '%s' "$remote_py" | with_timeout "${METRICS_LOAD_SSH_TIMEOUT:-90}" \ ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" python3 - 2>/dev/null || true) fi elif [ "${LOCAL_STACK:-0}" = "1" ] && command -v podman >/dev/null 2>&1; then raw=$(printf '%s' "$remote_py" | python3 - 2>/dev/null || true) else printf '%s\n' "{\"ok\":false,\"reason\":\"no-ssh-for-load\",\"label\":\"$label\"}" >"$out" return 0 fi if [ -z "$raw" ]; then printf '%s\n' "{\"ok\":false,\"reason\":\"probe-failed\",\"label\":\"$label\",\"ssh\":\"${_m_ssh:-${KOOPA_SSH:-?}}\"}" >"$out" return 1 fi printf '%s\n' "$raw" | python3 -c ' import sys,json,re t=sys.stdin.read(); obj=None for m in re.finditer(r"\{", t): try: obj=json.loads(t[m.start():]) if isinstance(obj, dict) and (obj.get("host") or obj.get("taler") or obj.get("ok") is False): break except Exception: obj=None if not obj: obj={"ok":False,"reason":"parse"} obj["label"]=sys.argv[1] json.dump(obj, open(sys.argv[2],"w"), indent=2) ' "$label" "$out" } # One-line human summaries (no leading indent) for info()/ok() detail fields. metrics_load_lines() { local f="$1" python3 - "$f" <<'PY' import json, sys try: d = json.load(open(sys.argv[1])) except Exception as e: print(f"unreadable: {e}") raise SystemExit(0) if not d.get("ok"): print(f"unavailable: {d.get('reason', 'n/a')} ssh={d.get('ssh', '')}".strip()) raise SystemExit(0) h = d.get("host") or {} mem = h.get("memory") or {} la = h.get("loadavg") or [] def gi(b): if b is None: return "?" return f"{b/1024/1024/1024:.2f}GiB" def mi(b): if b is None: return "?" return f"{b/1024/1024:.0f}MiB" la_s = " ".join(f"{x:.2f}" for x in la) if la else "?" tot = mem.get("mem_total_b") avail = mem.get("mem_available_b") used = (tot - avail) if (tot is not None and avail is not None) else None print( f"host loadavg=[{la_s}] nproc={h.get('nproc')} " f"mem_used={gi(used)} avail={gi(avail)} total={gi(tot)}" ) pc = h.get("process_counts") or {} if pc: bits = [f"{k}={v}" for k, v in sorted(pc.items()) if v] if bits: print("host procs " + " ".join(bits)) for role in ("bank", "exchange", "merchant"): c = (d.get("taler") or {}).get(role) or {} if not c: continue if not c.get("running"): print(f"{role} DOWN ({c.get('container')})") continue pr = c.get("processes") or {} st = c.get("podman_stats") or {} roles = pr.get("by_role") or {} role_bits = [] for k in ("libeufin", "taler-exchange", "taler-merchant", "postgres", "nginx", "java"): if k in roles: role_bits.append(f"{k}={mi(roles[k].get('rss_b'))}") dbs = c.get("databases") or {} db_s = ",".join( f"{x.get('name')}={x.get('size_pretty')}" for x in (dbs.get("databases") or [])[:5] ) # podman MemUsage often reports 0B under pasta/cgroup — prefer /proc RSS mem_u = st.get("mem_usage") or "" mem_bit = "" if mem_u and not str(mem_u).startswith("0B"): mem_bit = f" podman_mem={mem_u}" cpu = st.get("cpu_pct") or "?" blk = st.get("block_io") or "" blk_bit = "" if blk and blk not in ("0B / 0B", "0B/0B", "-- / --"): blk_bit = f" block={blk}" line = ( f"{role} rss={gi(pr.get('rss_total_b'))} procs={pr.get('proc_total')} " f"cpu={cpu}{mem_bit}{blk_bit}" ) if role_bits: line += " | " + " ".join(role_bits) if db_s: line += f" | db:{db_s}" if dbs.get("pgdata_pretty"): line += f" pgdata={dbs.get('pgdata_pretty')}" print(line) PY } # Snapshot load and emit as monitoring info lines (withdraw/pay phase markers). # $1=json path $2=label (e.g. after-withdraw) metrics_report_load() { local out="$1" label="$2" local line rc=0 if [ "${METRICS_LOAD}" = "0" ]; then info "load ${label}" "skipped (METRICS_LOAD=0)" return 0 fi if [ "${SKIP_SSH:-0}" = "1" ] && ! command -v podman >/dev/null 2>&1; then info "load ${label}" "skipped (no SSH / no local podman)" return 0 fi if ! metrics_taler_load "$out" "$label"; then warn "load ${label}" "probe failed via ${KOOPA_SSH:-?} — set KOOPA_SSH / KOOPA_SSH_FALLBACKS in env" return 1 fi while IFS= read -r line; do [ -n "$line" ] || continue case "$line" in unavailable:*|unreadable:*) warn "load ${label}" "$line" rc=1 ;; *) info "load ${label}" "$line" ;; esac done < <(metrics_load_lines "$out") return "$rc" } # Human one-screen summary of a load JSON metrics_print_load() { local f="$1" title="${2:-load}" python3 - "$f" "$title" <<'PY' import json,sys try: d=json.load(open(sys.argv[1])) except Exception as e: print(f" ({sys.argv[2]}: unreadable {e})") raise SystemExit if not d.get("ok"): print(f" ({sys.argv[2]}: {d.get('reason','n/a')})") raise SystemExit h=d.get("host") or {} mem=h.get("memory") or {} la=h.get("loadavg") or [] def gi(b): if b is None: return "?" return f"{b/1024/1024/1024:.2f} GiB" print(f" host loadavg {la} nproc={h.get('nproc')} mem_avail={gi(mem.get('mem_available_b'))}/{gi(mem.get('mem_total_b'))}") dio=h.get("disk_io") or {} if dio: print(f" host disk Δsectors read={dio.get('sectors_read')} written={dio.get('sectors_written')} (~write {gi(dio.get('approx_write_bytes'))})") for role in ("bank","exchange","merchant"): c=(d.get("taler") or {}).get(role) or {} if not c: continue if not c.get("running"): print(f" {role:8} DOWN ({c.get('container')})") continue pr=c.get("processes") or {} rss=pr.get("rss_total_b") n=pr.get("proc_total") roles=pr.get("by_role") or {} bits=[] for k in ("libeufin","taler-exchange","taler-merchant","postgres","nginx"): if k in roles: bits.append(f"{k}:n={roles[k].get('n')} rss={gi(roles[k].get('rss_b'))}") dbs=c.get("databases") or {} db_s=", ".join(f"{x.get('name')}={x.get('size_pretty')}" for x in (dbs.get("databases") or [])[:6]) pg=dbs.get("pgdata_pretty") or "" st=c.get("podman_stats") or {} print(f" {role:8} procs={n} rss={gi(rss)} cpu={st.get('cpu_pct','?')} block={st.get('block_io','?')}") if bits: print(f" " + " ".join(bits)) if db_s or pg: print(f" db: {db_s}" + (f" pgdata={pg}" if pg else "")) PY } # Diff two load snaps: highlight RAM/proc/DB growth for taler roles metrics_print_load_delta() { local before="$1" after="$2" python3 - "$before" "$after" <<'PY' import json,sys def load(p): try: return json.load(open(p)) except Exception: return {} b,a=load(sys.argv[1]),load(sys.argv[2]) if not b.get("ok") or not a.get("ok"): print(" (load delta unavailable)") raise SystemExit def gi(x): if x is None: return None return x/1024/1024/1024 print(" --- delta (after − before) ---") # host load bla=(b.get("host") or {}).get("loadavg") or [0,0,0] ala=(a.get("host") or {}).get("loadavg") or [0,0,0] if bla and ala: print(f" loadavg1 {bla[0]:.2f} → {ala[0]:.2f} (Δ {ala[0]-bla[0]:+.2f})") bm=(b.get("host") or {}).get("memory") or {} am=(a.get("host") or {}).get("memory") or {} if bm.get("mem_available_b") is not None and am.get("mem_available_b") is not None: print(f" mem_avail {gi(bm['mem_available_b']):.2f} → {gi(am['mem_available_b']):.2f} GiB (Δ {gi(am['mem_available_b'])-gi(bm['mem_available_b']):+.3f} GiB)") bd=(b.get("host") or {}).get("disk_io") or {} ad=(a.get("host") or {}).get("disk_io") or {} if bd.get("sectors_written") is not None and ad.get("sectors_written") is not None: dw=(ad["sectors_written"]-bd["sectors_written"])*512 dr=(ad["sectors_read"]-bd["sectors_read"])*512 print(f" disk I/O write≈{dw/1024/1024:.1f} MiB read≈{dr/1024/1024:.1f} MiB (during phase)") for role in ("bank","exchange","merchant"): bc=(b.get("taler") or {}).get(role) or {} ac=(a.get("taler") or {}).get(role) or {} if not ac.get("running"): continue br=(bc.get("processes") or {}).get("rss_total_b") ar=(ac.get("processes") or {}).get("rss_total_b") bn=(bc.get("processes") or {}).get("proc_total") an=(ac.get("processes") or {}).get("proc_total") line=f" {role:8}" if br is not None and ar is not None: line+=f" rss {gi(br):.3f}→{gi(ar):.3f} GiB (Δ{gi(ar)-gi(br):+.3f})" if bn is not None and an is not None: line+=f" procs {bn}→{an} (Δ{an-bn:+d})" # DB sizes def dbmap(c): m={} for x in ((c.get("databases") or {}).get("databases") or []): m[x.get("name")]=x.get("size_b") return m bdb,adb=dbmap(bc),dbmap(ac) dbits=[] for name in sorted(set(bdb)|set(adb)): bb,aa=bdb.get(name),adb.get(name) if bb is not None and aa is not None and aa!=bb: dbits.append(f"{name} {aa-bb:+d}B") elif aa is not None and bb is None: dbits.append(f"{name}={aa}B") if dbits: line+=" dbΔ["+", ".join(dbits)+"]" print(line) PY } # Record a successful withdraw/pay amount for end statistics (one amount per line). # $1=withdrawn|spent $2=amount (CUR:n) metrics_record_flow() { local kind="$1" amt="$2" [ -n "$amt" ] || return 0 mkdir -p "${METRICS_DIR}" 2>/dev/null || true case "$kind" in withdrawn|withdraw) printf '%s\n' "$amt" >>"${METRICS_DIR}/flow-withdrawn.txt" ;; spent|pay) printf '%s\n' "$amt" >>"${METRICS_DIR}/flow-spent.txt" ;; *) return 1 ;; esac } # Overall end-of-run statistics block # Files under METRICS_DIR: # coins-final.json, coins-history.tsv, perf-summary.json, # load-before/after.json, flow-withdrawn.txt, flow-spent.txt, # optional ladder TSV via METRICS_WITHDRAW_TSV / METRICS_PAY_TSV metrics_print_overall() { local title="${1:-overall statistics}" section "statistics · $title" metrics_print_final_stats || true [ -n "${METRICS_EXTRA_LINES:-}" ] && printf '%s\n' "$METRICS_EXTRA_LINES" } # Rich final dashboard: coins, withdrawn/spent, performance, tendencies metrics_print_final_stats() { python3 - \ "${METRICS_DIR:-/tmp}" \ "${ALT_UNITS_FILE:-}" \ "${METRICS_WITHDRAW_TSV:-}" \ "${METRICS_PAY_TSV:-}" \ "${CUR:-GOA}" \ <<'PY' import csv, json, os, sys from decimal import Decimal, InvalidOperation, ROUND_HALF_UP from pathlib import Path mdir = Path(sys.argv[1]) alt_path = sys.argv[2] or "" wd_tsv = sys.argv[3] or "" pay_tsv = sys.argv[4] or "" cur = sys.argv[5] or "GOA" alt = {} if alt_path and Path(alt_path).is_file(): try: alt = json.load(open(alt_path)) except Exception: alt = {} if not alt: alt = {"0": cur} def D(x, default=Decimal(0)): try: return Decimal(str(x)) except Exception: return default def parse_amt(s): s = str(s or "").strip() if not s: return cur, Decimal(0) if ":" in s: c, v = s.split(":", 1) try: return c, Decimal(v) except InvalidOperation: return c, Decimal(0) try: return cur, Decimal(s) except InvalidOperation: return cur, Decimal(0) def fmt_num(v: Decimal) -> str: if v == v.to_integral(): return format(int(v), "d") s = format(v.quantize(Decimal("0.00000001"), rounding=ROUND_HALF_UP), "f") return s.rstrip("0").rstrip(".") def fmt_amt(c, v: Decimal) -> str: return "%s:%s" % (c, fmt_num(v)) def fmt_alt(c, v: Decimal) -> str: base_s = fmt_amt(c, v) base_name = alt.get("0") or c if v == 0: return "0 %s (%s)" % (base_name, base_s) scales = [] for k, name in alt.items(): try: scales.append((int(k), str(name))) except Exception: pass scales.sort(key=lambda x: -x[0]) absval = abs(v) for sc, name in scales: unit = Decimal(10) ** sc coeff = absval / unit if coeff >= 1: if sc == 0: return "%s %s" % (fmt_num(v), name) return "%s %s (%s)" % (fmt_num(coeff if v >= 0 else -coeff), name, base_s) return "%s %s (%s)" % (fmt_num(v), base_name, base_s) def sum_amount_file(path: Path): total = Decimal(0) ccy = cur n = 0 if not path.is_file(): return ccy, total, n for line in path.read_text().splitlines(): line = line.strip() if not line: continue c, v = parse_amt(line) ccy = c total += v n += 1 return ccy, total, n def sum_tsv_amounts(path, amount_col="amount", status_ok=None): total = Decimal(0) ccy = cur n = 0 if not path or not Path(path).is_file(): return ccy, total, n with open(path, newline="") as f: for row in csv.DictReader(f, delimiter="\t"): st = (row.get("status") or "").upper() if status_ok is not None and st not in status_ok: continue c, v = parse_amt(row.get(amount_col) or "") if v <= 0: continue ccy = c total += v n += 1 return ccy, total, n def trend_label(delta, eps=Decimal("0.01")): if delta > eps: return "↑ rising" if delta < -eps: return "↓ falling" return "→ stable" def half_trend(values): """Compare avg of second half vs first half of a list of numbers.""" xs = [float(x) for x in values if x is not None] if len(xs) < 4: if len(xs) < 2: return "n/a (few samples)", None, None a, b = xs[0], xs[-1] return trend_label(Decimal(str(b - a))), a, b mid = len(xs) // 2 first = sum(xs[:mid]) / mid second = sum(xs[mid:]) / (len(xs) - mid) d = second - first lab = "↑ slowing (worse)" if d > 0.05 * max(abs(first), 1) else ( "↓ faster (better)" if d < -0.05 * max(abs(first), 1) else "→ steady" ) return lab, first, second print("") print("╔══════════════════════════════════════════════════════════════╗") print("║ FINAL STATISTICS ║") print("╚══════════════════════════════════════════════════════════════╝") # --- coins snapshot --- coins = {} cf = mdir / "coins-final.json" if cf.is_file(): try: coins = json.load(open(cf)) except Exception: coins = {} print("") print("── Coins (wallet) ─────────────────────────────────────────────") if coins.get("ok"): circ_n = coins.get("in_circulation") or 0 spent_n = coins.get("spent") or 0 total_n = coins.get("total_coins") or 0 amt_alt = coins.get("amount_in_circulation_alt") or coins.get("amount_in_circulation_s") or "?" spent_alt = coins.get("amount_spent_alt") or coins.get("amount_spent_s") or "0" print(" total coins %s" % total_n) print(" in circulation %s coins · %s" % (circ_n, amt_alt)) print(" spent (in dump) %s coins · %s" % (spent_n, spent_alt)) circ = coins.get("by_denom_circulation") or [] if circ: print(" denoms in circ:") for x in circ: print( " · %s × %s = %s" % ( x.get("denom_alt") or x.get("denom"), x.get("count"), x.get("amount_alt") or x.get("amount"), ) ) spent = coins.get("by_denom_spent") or [] if spent: print(" denoms spent:") for x in spent: print( " · %s × %s = %s" % ( x.get("denom_alt") or x.get("denom"), x.get("count"), x.get("amount_alt") or x.get("amount"), ) ) else: print(" (no coins-final.json — run with wallet snaps)") # --- withdrawn / spent flows --- print("") print("── Money flow (this run) ──────────────────────────────────────") # Prefer ladder TSV if provided; else flow-*.txt from e2e wd_ok = {"OK", "OK_BANK", "ZERO_REJECT", "ZERO_SKIP", "SKIP_DENOM", "CEILING_REJECT"} pay_ok = {"OK", "ZERO_SKIP", "CEILING_SKIP", "CEILING_REJECT"} c1, wd_sum, wd_n = sum_tsv_amounts(wd_tsv, "amount", None) # count successful withdraws by status prefix OK or soft if wd_tsv and Path(wd_tsv).is_file(): c1, wd_sum, wd_n = Decimal(0), Decimal(0), 0 ccy = cur with open(wd_tsv, newline="") as f: for row in csv.DictReader(f, delimiter="\t"): st = (row.get("status") or "").upper() # count value when mint/settle produced money in wallet if st in ("OK", "OK_BANK"): c, v = parse_amt(row.get("amount")) ccy, wd_sum, wd_n = c, wd_sum + v, wd_n + 1 c1 = ccy else: c1, wd_sum, wd_n = sum_amount_file(mdir / "flow-withdrawn.txt") if pay_tsv and Path(pay_tsv).is_file(): c2, pay_sum, pay_n = Decimal(0), Decimal(0), 0 ccy = cur with open(pay_tsv, newline="") as f: for row in csv.DictReader(f, delimiter="\t"): st = (row.get("status") or "").upper() if st == "OK": c, v = parse_amt(row.get("amount")) ccy, pay_sum, pay_n = c, pay_sum + v, pay_n + 1 c2 = ccy else: c2, pay_sum, pay_n = sum_amount_file(mdir / "flow-spent.txt") print(" withdrawn %s events · %s" % (wd_n, fmt_alt(c1, wd_sum))) print(" spent (paid) %s events · %s" % (pay_n, fmt_alt(c2, pay_sum))) net = wd_sum - pay_sum print(" net (wd − spent) %s %s" % (fmt_alt(c1, net), trend_label(net))) if coins.get("ok"): # circulation amount vs net flow circ_map = coins.get("amount_in_circulation") or {} circ_v = Decimal(0) for _c, vs in circ_map.items(): circ_v += D(vs) print(" wallet circ now %s" % fmt_alt(cur, circ_v)) print(" residual check wallet_circ vs net: Δ %s" % fmt_alt(cur, circ_v - net)) # --- coin history tendency --- print("") print("── Tendency · coins over run ─────────────────────────────────") hist = mdir / "coins-history.tsv" if hist.is_file() and hist.stat().st_size > 0: rows = list(csv.DictReader(open(hist), delimiter="\t")) if rows: def circ_amt(row): # amount_circ column may be "GOA:20" or alt form — try numeric from total/in_circ a = row.get("amount_circ") or "0" if ":" in a and " " not in a: return parse_amt(a)[1] # try extract GOA: from parentheses import re m = re.search(r"\(([^)]+:[0-9.]+)\)", a) if m: return parse_amt(m.group(1))[1] return D(row.get("in_circ") or 0) first, last = rows[0], rows[-1] c0, c1_ = D(first.get("in_circ") or 0), D(last.get("in_circ") or 0) a0, a1 = circ_amt(first), circ_amt(last) print(" samples %d snaps (%s → %s)" % (len(rows), first.get("label"), last.get("label"))) print(" coins in circ %s → %s (Δ %+d) %s" % (fmt_num(c0), fmt_num(c1_), int(c1_ - c0), trend_label(c1_ - c0))) print(" amount in circ %s → %s %s" % (fmt_alt(cur, a0), fmt_alt(cur, a1), trend_label(a1 - a0))) # simple linear slope on coin count if len(rows) >= 3: ys = [float(D(r.get("in_circ") or 0)) for r in rows] n = len(ys) xs = list(range(n)) xm, ym = sum(xs) / n, sum(ys) / n num = sum((x - xm) * (y - ym) for x, y in zip(xs, ys)) den = sum((x - xm) ** 2 for x in xs) or 1 slope = num / den print(" trend slope %+.3f coins/snap %s" % (slope, "↑ accumulating" if slope > 0.05 else ("↓ draining" if slope < -0.05 else "→ flat"))) print(" history (label · in_circ · amount):") for r in rows[-12:]: # last 12 print(" · %-28s circ=%-4s %s" % (r.get("label", "?")[:28], r.get("in_circ"), r.get("amount_circ"))) if len(rows) > 12: print(" · … (%d earlier snaps)" % (len(rows) - 12)) else: print(" (empty history)") else: print(" (no coins-history.tsv)") # --- performance --- print("") print("── Performance indicators ───────────────────────────────────") perf = {} pf = mdir / "perf-summary.json" if pf.is_file(): try: perf = json.load(open(pf)) except Exception: perf = {} def print_bucket(name, v): if not isinstance(v, dict) or not v.get("n"): return print( " %-14s n=%s min=%sms p50=%sms avg=%sms max=%sms" % (name, v.get("n"), v.get("min_ms"), v.get("p50_ms"), v.get("avg_ms"), v.get("max_ms")) ) if perf: for k, v in perf.items(): print_bucket(k, v) else: print(" (no perf-summary.json)") # per-rung timing tendency from ladder TSV for label, path, cols in ( ("withdraw rungs", wd_tsv, ("ms_total", "ms_mint", "ms_settle")), ("pay rungs", pay_tsv, ("ms_total", "ms_order", "ms_handle", "ms_settle")), ): if not path or not Path(path).is_file(): continue with open(path, newline="") as f: rrows = list(csv.DictReader(f, delimiter="\t")) if len(rrows) < 2: continue print(" tendency · %s:" % label) for col in cols: vals = [] for r in rrows: try: vals.append(int(r.get(col) or 0)) except Exception: pass vals = [v for v in vals if v > 0] if len(vals) < 2: continue lab, a, b = half_trend(vals) if a is None: print(" %-12s %s" % (col, lab)) else: print(" %-12s first-half avg=%.0fms → second-half avg=%.0fms %s" % (col, a, b, lab)) # --- load --- print("") print("── Host / container load ────────────────────────────────────") lb, la = mdir / "load-before.json", mdir / "load-after.json" def load_brief(path, tag): if not path.is_file(): print(" %s: (missing)" % tag) return None try: d = json.load(open(path)) except Exception: print(" %s: (unreadable)" % tag) return None if not d.get("ok"): print(" %s: %s" % (tag, d.get("reason", "?"))) return d h = d.get("host") or {} mem = h.get("memory") or {} la_ = h.get("loadavg") or [] avail = mem.get("mem_available_b") tot = mem.get("mem_total_b") def gi(b): return "?" if b is None else "%.2f GiB" % (b / 1024 / 1024 / 1024) used = (tot - avail) if (tot is not None and avail is not None) else None print(" %s loadavg=%s mem_used=%s / %s" % (tag, la_, gi(used), gi(tot))) for role in ("bank", "exchange", "merchant"): c = (d.get("taler") or {}).get(role) or {} if not c.get("running"): continue pr = c.get("processes") or {} st = c.get("podman_stats") or {} rss = pr.get("rss_total_b") rss_s = "?" if rss is None else "%.2f GiB" % (rss / 1024 / 1024 / 1024) print(" %-8s rss=%s cpu=%s procs=%s" % (role, rss_s, st.get("cpu_pct") or "?", pr.get("proc_total"))) return d db = load_brief(lb, "before") da = load_brief(la, "after") if db and da and db.get("ok") and da.get("ok"): bla = (db.get("host") or {}).get("loadavg") or [0] ala = (da.get("host") or {}).get("loadavg") or [0] if bla and ala: dload = float(ala[0]) - float(bla[0]) print(" loadavg1 tendency %.2f → %.2f (Δ %+.2f) %s" % (float(bla[0]), float(ala[0]), dload, "↑ higher load" if dload > 0.1 else ("↓ lower load" if dload < -0.1 else "→ steady"))) bm = (db.get("host") or {}).get("memory") or {} am = (da.get("host") or {}).get("memory") or {} if bm.get("mem_available_b") is not None and am.get("mem_available_b") is not None: dmem = (am["mem_available_b"] - bm["mem_available_b"]) / 1024 / 1024 / 1024 print(" mem_avail tendency %.2f → %.2f GiB (Δ %+.3f) %s" % (bm["mem_available_b"] / 1024**3, am["mem_available_b"] / 1024**3, dmem, "↑ more free" if dmem > 0.05 else ("↓ less free" if dmem < -0.05 else "→ steady"))) print("") print("──────────────────────────────────────────────────────────────") PY }