feat(monitoring): load/memory probes, coin inventory, alt_unit_names
This commit is contained in:
Hernâni Marques
parent
36556e3450
commit
1e6a050ee7
1 changed files with 615 additions and 80 deletions
|
|
@ -14,31 +14,170 @@
|
|||
: "${METRICS_DIR:=${SCRATCH:-/tmp}}"
|
||||
mkdir -p "$METRICS_DIR" 2>/dev/null || true
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# alt_unit_names — human amounts (Kilo-GOA / Mega-GOA / …) with base in parens
|
||||
# From exchange/bank currency_specification.alt_unit_names:
|
||||
# "0"→GOA, "3"→Kilo-GOA, "6"→Mega-GOA, … "-3"→Milli-GOA, …
|
||||
# Example: GOA:5000 → 5 Kilo-GOA (GOA:5000)
|
||||
# ---------------------------------------------------------------------------
|
||||
: "${ALT_UNITS_FILE:=${METRICS_DIR}/alt_unit_names.json}"
|
||||
|
||||
# Load map from public /config into ALT_UNITS_FILE. $1=optional config URL.
|
||||
metrics_load_alt_units() {
|
||||
local url="${1:-${EXCHANGE_PUBLIC:-https://exchange.hacktivism.ch}/config}"
|
||||
local code
|
||||
mkdir -p "$(dirname "$ALT_UNITS_FILE")" 2>/dev/null || true
|
||||
code=$(curl -skS -m 12 -o "${ALT_UNITS_FILE}.raw" -w '%{http_code}' "$url" 2>/dev/null || echo 000)
|
||||
if [ "$code" != "200" ]; then
|
||||
# soft fallback: SI-style names if live config unreachable
|
||||
printf '%s\n' '{"0":"GOA","3":"Kilo-GOA","6":"Mega-GOA","9":"Giga-GOA","12":"Tera-GOA","15":"Peta-GOA","18":"Exa-GOA","21":"Zetta-GOA","24":"Yotta-GOA","-1":"Deci-GOA","-2":"Centi-GOA","-3":"Milli-GOA","-6":"Micro-GOA","-8":"Atomic-GOA"}' \
|
||||
>"$ALT_UNITS_FILE"
|
||||
return 1
|
||||
fi
|
||||
python3 - "${ALT_UNITS_FILE}.raw" "$ALT_UNITS_FILE" <<'PY'
|
||||
import json, sys
|
||||
src, dst = sys.argv[1:3]
|
||||
d = json.load(open(src))
|
||||
au = None
|
||||
cs = d.get("currency_specification")
|
||||
if isinstance(cs, dict):
|
||||
au = cs.get("alt_unit_names")
|
||||
if not au and isinstance(d.get("currencies"), dict):
|
||||
# merchant-style: currencies.GOA.alt_unit_names
|
||||
for _code, spec in d["currencies"].items():
|
||||
if isinstance(spec, dict) and spec.get("alt_unit_names"):
|
||||
au = spec["alt_unit_names"]
|
||||
break
|
||||
if not isinstance(au, dict) or "0" not in au:
|
||||
au = {"0": d.get("currency") or "GOA"}
|
||||
json.dump(au, open(dst, "w"), indent=2, sort_keys=True)
|
||||
PY
|
||||
export ALT_UNITS_FILE
|
||||
return 0
|
||||
}
|
||||
|
||||
# Format one amount: "GOA:5000" → "5 Kilo-GOA (GOA:5000)"
|
||||
# Uses ALT_UNITS_FILE if present. Pure stdout.
|
||||
format_amount_alt() {
|
||||
local amt="${1:-}"
|
||||
[ -n "$amt" ] || return 0
|
||||
python3 - "$amt" "${ALT_UNITS_FILE:-}" <<'PY'
|
||||
import json, sys
|
||||
from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
|
||||
|
||||
amt = sys.argv[1].strip()
|
||||
path = sys.argv[2] if len(sys.argv) > 2 else ""
|
||||
alt = {}
|
||||
if path:
|
||||
try:
|
||||
alt = json.load(open(path))
|
||||
except Exception:
|
||||
alt = {}
|
||||
if not alt:
|
||||
alt = {"0": "GOA"}
|
||||
|
||||
def parse(s):
|
||||
if ":" in s:
|
||||
c, v = s.split(":", 1)
|
||||
return c, Decimal(v)
|
||||
return "GOA", Decimal(s)
|
||||
|
||||
def fmt_num(v: Decimal) -> str:
|
||||
if v == v.to_integral():
|
||||
return format(int(v), "d")
|
||||
s = format(v.quantize(Decimal("0.00000001"), rounding=ROUND_HALF_UP), "f")
|
||||
return s.rstrip("0").rstrip(".")
|
||||
|
||||
try:
|
||||
cur, val = parse(amt)
|
||||
except (InvalidOperation, ValueError):
|
||||
print(amt)
|
||||
raise SystemExit(0)
|
||||
|
||||
base_name = alt.get("0") or cur
|
||||
# always show canonical base form in parens
|
||||
base_s = "%s:%s" % (cur, fmt_num(val))
|
||||
if val == 0:
|
||||
print("0 %s (%s)" % (base_name, base_s))
|
||||
raise SystemExit(0)
|
||||
|
||||
# scales: power of 10 relative to unit "0"
|
||||
scales = []
|
||||
for k, name in alt.items():
|
||||
try:
|
||||
scales.append((int(k), str(name)))
|
||||
except Exception:
|
||||
pass
|
||||
scales.sort(key=lambda x: -x[0]) # largest unit first
|
||||
|
||||
# pick largest scale where |value| >= 10^scale (for scale>=0),
|
||||
# or for fractions the finest unit that makes the coefficient >= 1
|
||||
chosen = None # (scale, name, coeff)
|
||||
absval = abs(val)
|
||||
for sc, name in scales:
|
||||
unit = Decimal(10) ** sc
|
||||
if unit <= 0:
|
||||
continue
|
||||
coeff = absval / unit
|
||||
if coeff >= 1:
|
||||
chosen = (sc, name, coeff if val >= 0 else -coeff)
|
||||
break
|
||||
if chosen is None:
|
||||
# smaller than smallest unit — use base
|
||||
print("%s %s (%s)" % (fmt_num(val), base_name, base_s))
|
||||
raise SystemExit(0)
|
||||
|
||||
sc, name, coeff = chosen
|
||||
# if base unit, prefer "GOA:12" style still with parens only when alt differs
|
||||
if sc == 0:
|
||||
print("%s %s" % (fmt_num(val), name))
|
||||
raise SystemExit(0)
|
||||
print("%s %s (%s)" % (fmt_num(coeff), name, base_s))
|
||||
PY
|
||||
}
|
||||
|
||||
# Format list of "CUR:n" amounts for plans / logs (space-separated → multiline or compact)
|
||||
format_amount_list_alt() {
|
||||
local a out=""
|
||||
for a in "$@"; do
|
||||
[ -n "$out" ] && out="$out "
|
||||
out="${out}$(format_amount_alt "$a")"
|
||||
done
|
||||
printf '%s\n' "$out"
|
||||
}
|
||||
|
||||
# --- coin inventory from wallet-cli dump-coins ---
|
||||
# Writes JSON to $1; prints one-line summary to stdout.
|
||||
# Sets COINS_TOTAL, COINS_FRESH, COINS_SUMMARY (human one-liner).
|
||||
# Sets COINS_TOTAL, COINS_FRESH, COINS_SPENT, COINS_AMOUNT_CIRC, COINS_SUMMARY.
|
||||
metrics_wallet_coins() {
|
||||
local out="${1:-$METRICS_DIR/coins.json}"
|
||||
local dump="$METRICS_DIR/dump-coins.raw"
|
||||
local dump="${METRICS_DIR}/dump-coins.raw"
|
||||
mkdir -p "$(dirname "$out")" 2>/dev/null || true
|
||||
COINS_TOTAL=0
|
||||
COINS_FRESH=0
|
||||
COINS_SPENT=0
|
||||
COINS_AMOUNT_CIRC="0"
|
||||
COINS_SUMMARY="(no coins)"
|
||||
# Prefer wcli() from caller (bash function in e2e/ladder)
|
||||
# Prefer wcli() from caller (e2e/ladder). Do not pass a leading timeout number —
|
||||
# ladder wcli() has no optional secs arg (would become a wallet subcommand).
|
||||
if type wcli >/dev/null 2>&1; then
|
||||
wcli advanced dump-coins >"$dump" 2>/dev/null || true
|
||||
elif [ -n "${CLI_JS:-}" ] && [ -f "${CLI_JS}" ]; then
|
||||
node "$CLI_JS" --wallet-db="${WDB:-}" --no-throttle advanced dump-coins >"$dump" 2>/dev/null || true
|
||||
elif [ -n "${WALLET_CLI:-}" ]; then
|
||||
node "$WALLET_CLI" --wallet-db="${WDB:-}" --no-throttle advanced dump-coins >"$dump" 2>/dev/null || true
|
||||
elif [ -n "${WALLET_CLI:-}" ] && [ -n "${WDB:-}" ]; then
|
||||
node "$WALLET_CLI" --wallet-db="${WDB}" --no-throttle --skip-defaults advanced dump-coins >"$dump" 2>/dev/null || true
|
||||
else
|
||||
echo "$COINS_SUMMARY"
|
||||
printf '%s\n' '{"ok":false,"reason":"no-wcli"}' >"$out"
|
||||
return 1
|
||||
fi
|
||||
python3 - "$dump" "$out" "${CUR:-GOA}" <<'PY'
|
||||
python3 - "$dump" "$out" "${CUR:-GOA}" "${ALT_UNITS_FILE:-}" <<'PY'
|
||||
import json, re, sys
|
||||
from collections import Counter
|
||||
from collections import Counter, defaultdict
|
||||
from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
|
||||
|
||||
raw_path, out_path, cur = sys.argv[1:4]
|
||||
alt_path = sys.argv[4] if len(sys.argv) > 4 else ""
|
||||
raw = open(raw_path).read() if raw_path else ""
|
||||
d = None
|
||||
for m in re.finditer(r"\{", raw):
|
||||
|
|
@ -51,60 +190,269 @@ for m in re.finditer(r"\{", raw):
|
|||
coins = []
|
||||
if isinstance(d, dict):
|
||||
coins = d.get("coins") or d.get("coin") or []
|
||||
by_denom = Counter()
|
||||
|
||||
alt = {}
|
||||
if alt_path:
|
||||
try:
|
||||
alt = json.load(open(alt_path))
|
||||
except Exception:
|
||||
alt = {}
|
||||
if not alt:
|
||||
alt = {"0": cur}
|
||||
|
||||
def parse_amt(s):
|
||||
"""'GOA:10' / '10' → (currency, Decimal) or (cur, 0)."""
|
||||
s = str(s or "").strip()
|
||||
if not s or s == "?":
|
||||
return cur, Decimal(0)
|
||||
if ":" in s:
|
||||
c, v = s.split(":", 1)
|
||||
try:
|
||||
return c, Decimal(v)
|
||||
except InvalidOperation:
|
||||
return c, Decimal(0)
|
||||
try:
|
||||
return cur, Decimal(s)
|
||||
except InvalidOperation:
|
||||
return cur, Decimal(0)
|
||||
|
||||
def fmt_num(v: Decimal) -> str:
|
||||
if v == v.to_integral():
|
||||
return format(int(v), "d")
|
||||
s = format(v.quantize(Decimal("0.00000001"), rounding=ROUND_HALF_UP), "f")
|
||||
return s.rstrip("0").rstrip(".")
|
||||
|
||||
def fmt_amt(c, v: Decimal) -> str:
|
||||
return "%s:%s" % (c, fmt_num(v))
|
||||
|
||||
def fmt_amt_alt(c, v: Decimal) -> str:
|
||||
"""5 Kilo-GOA (GOA:5000) using alt_unit_names."""
|
||||
base_s = fmt_amt(c, v)
|
||||
base_name = alt.get("0") or c
|
||||
if v == 0:
|
||||
return "0 %s (%s)" % (base_name, base_s)
|
||||
scales = []
|
||||
for k, name in alt.items():
|
||||
try:
|
||||
scales.append((int(k), str(name)))
|
||||
except Exception:
|
||||
pass
|
||||
scales.sort(key=lambda x: -x[0])
|
||||
absval = abs(v)
|
||||
chosen = None
|
||||
for sc, name in scales:
|
||||
unit = Decimal(10) ** sc
|
||||
coeff = absval / unit
|
||||
if coeff >= 1:
|
||||
chosen = (sc, name, coeff if v >= 0 else -coeff)
|
||||
break
|
||||
if chosen is None:
|
||||
return "%s %s (%s)" % (fmt_num(v), base_name, base_s)
|
||||
sc, name, coeff = chosen
|
||||
if sc == 0:
|
||||
return "%s %s" % (fmt_num(v), name)
|
||||
return "%s %s (%s)" % (fmt_num(coeff), name, base_s)
|
||||
|
||||
by_denom_all = Counter() # all coins
|
||||
by_denom_circ = Counter() # non-spent
|
||||
by_denom_spent = Counter()
|
||||
by_status = Counter()
|
||||
fresh = 0
|
||||
amt_circ = defaultdict(lambda: Decimal(0)) # currency → amount
|
||||
amt_spent = defaultdict(lambda: Decimal(0))
|
||||
amt_all = defaultdict(lambda: Decimal(0))
|
||||
circ_n = 0
|
||||
spent_n = 0
|
||||
for c in coins:
|
||||
if not isinstance(c, dict):
|
||||
continue
|
||||
dv = c.get("denomValue") or c.get("value") or "?"
|
||||
dv = c.get("denomValue") or c.get("value") or c.get("denom_value") or "?"
|
||||
st = str(c.get("coinStatus") or c.get("status") or "?")
|
||||
by_denom[dv] += 1
|
||||
by_denom_all[dv] += 1
|
||||
by_status[st] += 1
|
||||
if st.lower() in ("fresh", "pending", "dormant", "usable", ""):
|
||||
# count non-spent-looking as "in circulation" for summary
|
||||
pass
|
||||
if "spent" not in st.lower() and "delete" not in st.lower():
|
||||
fresh += 1
|
||||
# prefer explicit status
|
||||
fresh = sum(n for s, n in by_status.items() if "spent" not in s.lower() and "delete" not in s.lower())
|
||||
ac, av = parse_amt(dv)
|
||||
amt_all[ac] += av
|
||||
st_l = st.lower()
|
||||
if "spent" in st_l or "delete" in st_l or "dirty" in st_l:
|
||||
spent_n += 1
|
||||
by_denom_spent[dv] += 1
|
||||
amt_spent[ac] += av
|
||||
else:
|
||||
circ_n += 1
|
||||
by_denom_circ[dv] += 1
|
||||
amt_circ[ac] += av
|
||||
|
||||
total = len(coins)
|
||||
# stable sort denoms by numeric value if CUR:num
|
||||
|
||||
def denom_key(k):
|
||||
try:
|
||||
return float(str(k).split(":", 1)[-1])
|
||||
except Exception:
|
||||
return 0.0
|
||||
denom_list = [
|
||||
{"denom": k, "count": by_denom[k]}
|
||||
for k in sorted(by_denom.keys(), key=denom_key)
|
||||
]
|
||||
# human bar: "GOA:10×3 GOA:1×5"
|
||||
|
||||
def denom_list(counter, spent_counter=None):
|
||||
out = []
|
||||
for k in sorted(counter.keys(), key=denom_key):
|
||||
item = {"denom": k, "count": counter[k]}
|
||||
ac, av = parse_amt(k)
|
||||
item["unit_value"] = str(av)
|
||||
item["amount"] = fmt_amt(ac, av * counter[k])
|
||||
if spent_counter is not None:
|
||||
item["spent_count"] = spent_counter.get(k, 0)
|
||||
out.append(item)
|
||||
return out
|
||||
|
||||
# human: "10 GOA×3 (=30 GOA) …" and alt: "3 Kilo-GOA (GOA:3000)×1"
|
||||
parts = []
|
||||
for item in denom_list:
|
||||
parts.append("%s×%d" % (item["denom"], item["count"]))
|
||||
summary = ("total=%d in_wallet=%d | %s" % (total, fresh, " ".join(parts))) if parts else ("total=%d" % total)
|
||||
parts_alt = []
|
||||
for item in denom_list(by_denom_circ):
|
||||
ac, av = parse_amt(item["denom"])
|
||||
total_v = av * item["count"]
|
||||
parts.append("%s×%d (=%s)" % (item["denom"], item["count"], item["amount"]))
|
||||
parts_alt.append(
|
||||
"%s×%d (=%s)"
|
||||
% (fmt_amt_alt(ac, av), item["count"], fmt_amt_alt(ac, total_v))
|
||||
)
|
||||
circ_amt_s = " ".join(fmt_amt(c, amt_circ[c]) for c in sorted(amt_circ))
|
||||
circ_amt_alt = " ".join(fmt_amt_alt(c, amt_circ[c]) for c in sorted(amt_circ))
|
||||
spent_amt_s = " ".join(fmt_amt(c, amt_spent[c]) for c in sorted(amt_spent)) if amt_spent else "0"
|
||||
spent_amt_alt = " ".join(fmt_amt_alt(c, amt_spent[c]) for c in sorted(amt_spent)) if amt_spent else "0"
|
||||
if not circ_amt_s:
|
||||
circ_amt_s = "%s:0" % cur
|
||||
circ_amt_alt = "0 %s (%s:0)" % (alt.get("0") or cur, cur)
|
||||
summary = (
|
||||
"coins=%d in_circ=%d spent=%d amount_circ=%s | %s"
|
||||
% (total, circ_n, spent_n, circ_amt_alt, " ".join(parts_alt) if parts_alt else "(empty)")
|
||||
)
|
||||
# enrich denom lists with human labels
|
||||
def enrich(lst):
|
||||
out = []
|
||||
for item in lst:
|
||||
ac, av = parse_amt(item["denom"])
|
||||
total_v = av * int(item["count"])
|
||||
item = dict(item)
|
||||
item["denom_alt"] = fmt_amt_alt(ac, av)
|
||||
item["amount_alt"] = fmt_amt_alt(ac, total_v)
|
||||
out.append(item)
|
||||
return out
|
||||
|
||||
report = {
|
||||
"ok": True,
|
||||
"currency": cur,
|
||||
"total_coins": total,
|
||||
"in_circulation": fresh,
|
||||
"in_circulation": circ_n,
|
||||
"spent": spent_n,
|
||||
"amount_in_circulation": {c: str(amt_circ[c]) for c in amt_circ},
|
||||
"amount_in_circulation_s": circ_amt_s,
|
||||
"amount_in_circulation_alt": circ_amt_alt,
|
||||
"amount_spent": {c: str(amt_spent[c]) for c in amt_spent},
|
||||
"amount_spent_s": spent_amt_s,
|
||||
"amount_spent_alt": spent_amt_alt,
|
||||
"by_status": dict(by_status),
|
||||
"by_denom": denom_list,
|
||||
"by_denom": enrich(denom_list(by_denom_all, by_denom_spent)),
|
||||
"by_denom_circulation": enrich(denom_list(by_denom_circ)),
|
||||
"by_denom_spent": enrich(denom_list(by_denom_spent)),
|
||||
"summary": summary,
|
||||
"alt_unit_names": alt,
|
||||
}
|
||||
json.dump(report, open(out_path, "w"), indent=2)
|
||||
print(summary)
|
||||
# side channel for bash via file
|
||||
open(out_path + ".total", "w").write(str(total))
|
||||
open(out_path + ".circ", "w").write(str(fresh))
|
||||
open(out_path + ".circ", "w").write(str(circ_n))
|
||||
open(out_path + ".spent", "w").write(str(spent_n))
|
||||
open(out_path + ".amt", "w").write(circ_amt_s)
|
||||
PY
|
||||
COINS_TOTAL=$(cat "${out}.total" 2>/dev/null || echo 0)
|
||||
COINS_FRESH=$(cat "${out}.circ" 2>/dev/null || echo 0)
|
||||
COINS_SPENT=$(cat "${out}.spent" 2>/dev/null || echo 0)
|
||||
COINS_AMOUNT_CIRC=$(cat "${out}.amt" 2>/dev/null || echo "0")
|
||||
COINS_SUMMARY=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("summary","?"))' "$out" 2>/dev/null || echo "$COINS_SUMMARY")
|
||||
# history TSV for end-of-run table
|
||||
if [ -n "${METRICS_DIR:-}" ]; then
|
||||
hist="${METRICS_DIR}/coins-history.tsv"
|
||||
if [ ! -f "$hist" ]; then
|
||||
printf 'ts\tlabel\ttotal\tin_circ\tspent\tamount_circ\tsummary\n' >"$hist"
|
||||
fi
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date)" \
|
||||
"${METRICS_COINS_LABEL:-snap}" \
|
||||
"$COINS_TOTAL" "$COINS_FRESH" "$COINS_SPENT" "$COINS_AMOUNT_CIRC" \
|
||||
"$(printf '%s' "$COINS_SUMMARY" | tr '\t' ' ')" >>"$hist"
|
||||
fi
|
||||
echo "$COINS_SUMMARY"
|
||||
}
|
||||
|
||||
# Emit coin inventory as monitoring info lines (label = step marker).
|
||||
# $1=label $2=optional json path (default METRICS_DIR/coins-<safe-label>.json)
|
||||
metrics_report_coins() {
|
||||
local label="$1"
|
||||
local safe
|
||||
safe=$(printf '%s' "$label" | tr -c 'A-Za-z0-9._-' '_' | head -c 80)
|
||||
local out="${2:-${METRICS_DIR}/coins-${safe}.json}"
|
||||
local line prev="${METRICS_DIR}/coins-prev.json"
|
||||
if [ -z "${WDB:-}" ] && [ -z "${WALLET_CLI:-}" ] && ! type wcli >/dev/null 2>&1; then
|
||||
info "coins ${label}" "skipped (no wallet yet)"
|
||||
return 0
|
||||
fi
|
||||
METRICS_COINS_LABEL="$label"
|
||||
export METRICS_COINS_LABEL
|
||||
if ! metrics_wallet_coins "$out" >/dev/null; then
|
||||
warn "coins ${label}" "dump-coins failed / empty"
|
||||
return 1
|
||||
fi
|
||||
# multi-line detail from JSON
|
||||
while IFS= read -r line; do
|
||||
[ -n "$line" ] || continue
|
||||
info "coins ${label}" "$line"
|
||||
done < <(python3 - "$out" <<'PY'
|
||||
import json, sys
|
||||
d = json.load(open(sys.argv[1]))
|
||||
if not d.get("ok"):
|
||||
print("unavailable: %s" % d.get("reason", "?"))
|
||||
raise SystemExit(0)
|
||||
amt = d.get("amount_in_circulation_alt") or d.get("amount_in_circulation_s") or "0"
|
||||
print(
|
||||
"n=%s in_circulation=%s spent=%s amount_circ=%s"
|
||||
% (d.get("total_coins"), d.get("in_circulation"), d.get("spent"), amt)
|
||||
)
|
||||
st = d.get("by_status") or {}
|
||||
if st:
|
||||
print("status " + " ".join("%s=%s" % (k, st[k]) for k in sorted(st)))
|
||||
circ = d.get("by_denom_circulation") or []
|
||||
if circ:
|
||||
bits = []
|
||||
for x in circ:
|
||||
dlab = x.get("denom_alt") or x.get("denom")
|
||||
alab = x.get("amount_alt") or x.get("amount")
|
||||
bits.append("%s×%d (=%s)" % (dlab, x["count"], alab))
|
||||
print("denoms_in_circ " + " ".join(bits))
|
||||
else:
|
||||
print("denoms_in_circ (none)")
|
||||
spent = d.get("by_denom_spent") or []
|
||||
if spent:
|
||||
bits = []
|
||||
for x in spent:
|
||||
dlab = x.get("denom_alt") or x.get("denom")
|
||||
alab = x.get("amount_alt") or x.get("amount")
|
||||
bits.append("%s×%d (=%s)" % (dlab, x["count"], alab))
|
||||
print("denoms_spent " + " ".join(bits))
|
||||
sp = d.get("amount_spent_alt") or d.get("amount_spent_s")
|
||||
if sp and sp not in ("0", "0 GOA (GOA:0)"):
|
||||
print("amount_spent %s" % sp)
|
||||
PY
|
||||
)
|
||||
# delta vs previous snapshot at this run
|
||||
if [ -f "$prev" ]; then
|
||||
local dsum
|
||||
dsum=$(metrics_coins_delta "$prev" "$out" "${METRICS_DIR}/coins-delta-${safe}.json" 2>/dev/null || true)
|
||||
if [ -n "$dsum" ]; then
|
||||
info "coins ${label} Δ" "$dsum"
|
||||
fi
|
||||
fi
|
||||
cp -f "$out" "$prev" 2>/dev/null || true
|
||||
cp -f "$out" "${METRICS_DIR}/coins-final.json" 2>/dev/null || true
|
||||
return 0
|
||||
}
|
||||
|
||||
# Diff two coin JSON snapshots → new coins this step
|
||||
metrics_coins_delta() {
|
||||
local before="${1:-}" after="${2:-}" out="${3:-$METRICS_DIR/coins-delta.json}"
|
||||
|
|
@ -143,17 +491,19 @@ PY
|
|||
|
||||
# --- Taler stack load on koopa (host + bank/exchange/merchant) ---
|
||||
# Writes JSON to $1. RAM, process counts, DB sizes, disk I/O counters.
|
||||
# Uses KOOPA_SSH with fallback to KOOPA_SSH_FALLBACKS (koopa-external).
|
||||
metrics_taler_load() {
|
||||
local out="${1:-$METRICS_DIR/load.json}"
|
||||
local label="${2:-snap}"
|
||||
if [ "${METRICS_LOAD}" = "0" ] || [ "${LADDER_LOAD:-1}" = "0" ]; then
|
||||
mkdir -p "$(dirname "$out")" 2>/dev/null || true
|
||||
if [ "${METRICS_LOAD}" = "0" ]; then
|
||||
printf '%s\n' "{\"ok\":false,\"reason\":\"disabled\",\"label\":\"$label\"}" >"$out"
|
||||
return 0
|
||||
fi
|
||||
local raw=""
|
||||
local remote_py
|
||||
remote_py=$(cat <<'PY'
|
||||
import json, os, re, subprocess, time
|
||||
import base64, json, os, re, subprocess, time
|
||||
from collections import defaultdict
|
||||
|
||||
def sh(cmd, t=15):
|
||||
|
|
@ -162,6 +512,14 @@ def sh(cmd, t=15):
|
|||
except Exception:
|
||||
return ""
|
||||
|
||||
def podman_sh(name, script, t=25):
|
||||
"""Run a shell script inside container without nested-quote hell (base64)."""
|
||||
b64 = base64.b64encode(script.encode()).decode()
|
||||
return sh(
|
||||
f"podman exec {name} sh -c 'echo {b64} | base64 -d | sh'",
|
||||
t=t,
|
||||
)
|
||||
|
||||
def loadavg():
|
||||
try:
|
||||
a,b,c = open("/proc/loadavg").read().split()[:3]
|
||||
|
|
@ -206,52 +564,108 @@ def running(name):
|
|||
return sh(f"podman inspect -f '{{{{.State.Running}}}}' {name}").strip() == "true"
|
||||
|
||||
def stats(name):
|
||||
o = sh(f"podman stats --no-stream --format json {name}")
|
||||
# Prefer Go template (stable across podman versions); JSON field names vary.
|
||||
o = sh(
|
||||
f"podman stats --no-stream --format "
|
||||
f"'{{{{.CPUPerc}}}}|{{{{.MemUsage}}}}|{{{{.MemPerc}}}}|{{{{.BlockIO}}}}|{{{{.NetIO}}}}|{{{{.PIDs}}}}' "
|
||||
f"{name}",
|
||||
t=25,
|
||||
).strip()
|
||||
if o and "|" in o:
|
||||
p = o.split("|")
|
||||
while len(p) < 6:
|
||||
p.append("")
|
||||
return {
|
||||
"cpu_pct": p[0] or None,
|
||||
"mem_usage": p[1] or None,
|
||||
"mem_pct": p[2] or None,
|
||||
"block_io": p[3] or None,
|
||||
"net_io": p[4] or None,
|
||||
"pids": p[5] or None,
|
||||
}
|
||||
o = sh(f"podman stats --no-stream --format json {name}", t=25)
|
||||
try:
|
||||
data = json.loads(o)
|
||||
if isinstance(data, list) and data: data = data[0]
|
||||
if not isinstance(data, dict): return {}
|
||||
return {"cpu_pct": data.get("CPU") or data.get("CPUPerc"),
|
||||
"mem_usage": data.get("MemUsage"),
|
||||
"mem_pct": data.get("MemPerc"),
|
||||
"block_io": data.get("BlockIO"),
|
||||
"net_io": data.get("NetIO"),
|
||||
"pids": data.get("PIDs")}
|
||||
if isinstance(data, list) and data:
|
||||
data = data[0]
|
||||
if not isinstance(data, dict):
|
||||
return {}
|
||||
return {
|
||||
"cpu_pct": data.get("CPU") or data.get("CPUPerc"),
|
||||
"mem_usage": data.get("MemUsage"),
|
||||
"mem_pct": data.get("MemPerc"),
|
||||
"block_io": data.get("BlockIO"),
|
||||
"net_io": data.get("NetIO"),
|
||||
"pids": data.get("PIDs"),
|
||||
}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
def procs(name):
|
||||
# count + RSS by role inside container
|
||||
code = r'''
|
||||
import os,re,json
|
||||
from collections import defaultdict
|
||||
by=defaultdict(lambda:{"n":0,"rss_b":0}); n=0; rss=0
|
||||
for ent in os.listdir("/proc"):
|
||||
if not ent.isdigit(): continue
|
||||
try: st=open(f"/proc/{ent}/status").read()
|
||||
except Exception: continue
|
||||
m=re.search(r"^VmRSS:\s+(\d+)",st,re.M)
|
||||
if not m: continue
|
||||
rb=int(m.group(1))*1024
|
||||
nm=(re.search(r"^Name:\s+(\S+)",st,re.M) or type("x",(object,),{"group":lambda s,i: "?"})()).group(1)
|
||||
try: cmd=open(f"/proc/{ent}/cmdline","rb").read().decode("utf-8","replace").replace("\0"," ")
|
||||
except Exception: cmd=""
|
||||
blob=(nm+" "+cmd).lower(); key="other"
|
||||
if "postgres" in blob or "postmaster" in blob: key="postgres"
|
||||
elif "libeufin" in blob or "mainkt" in blob: key="libeufin"
|
||||
elif "java" in blob: key="java"
|
||||
elif "taler-merchant" in blob: key="taler-merchant"
|
||||
elif "taler-exchange" in blob: key="taler-exchange"
|
||||
elif "nginx" in blob: key="nginx"
|
||||
elif "apache" in blob: key="apache"
|
||||
by[key]["n"]+=1; by[key]["rss_b"]+=rb; n+=1; rss+=rb
|
||||
print(json.dumps({"proc_total":n,"rss_total_b":rss,"by_role":dict(by)}))
|
||||
'''
|
||||
o = sh(f"podman exec {name} python3 -c {json.dumps(code)}")
|
||||
try: return json.loads(o)
|
||||
except Exception:
|
||||
n = sh(f"podman exec {name} sh -c 'ps -e --no-headers 2>/dev/null | wc -l'").strip()
|
||||
return {"proc_total": int(n) if n.isdigit() else None, "rss_total_b": None, "by_role": {}}
|
||||
# Pure shell /proc scan via base64 (containers often lack python3).
|
||||
script = r"""
|
||||
n=0; rss=0
|
||||
n_postgres=0; rss_postgres=0
|
||||
n_libeufin=0; rss_libeufin=0
|
||||
n_taler_merchant=0; rss_taler_merchant=0
|
||||
n_taler_exchange=0; rss_taler_exchange=0
|
||||
n_nginx=0; rss_nginx=0
|
||||
n_java=0; rss_java=0
|
||||
n_other=0; rss_other=0
|
||||
for d in /proc/[0-9]*; do
|
||||
[ -r "$d/status" ] || continue
|
||||
r=$(sed -n 's/^VmRSS:[[:space:]]*\([0-9][0-9]*\).*/\1/p' "$d/status" | head -1)
|
||||
[ -n "$r" ] || continue
|
||||
n=$((n+1)); rss=$((rss+r))
|
||||
nm=$(sed -n 's/^Name:[[:space:]]*//p' "$d/status" | head -1)
|
||||
cmd=$(tr '\0' ' ' <"$d/cmdline" 2>/dev/null | head -c 240)
|
||||
blob=$(printf '%s %s' "$nm" "$cmd" | tr 'A-Z' 'a-z')
|
||||
case "$blob" in
|
||||
*postgres*|*postmaster*) n_postgres=$((n_postgres+1)); rss_postgres=$((rss_postgres+r)) ;;
|
||||
*libeufin*|*mainkt*) n_libeufin=$((n_libeufin+1)); rss_libeufin=$((rss_libeufin+r)) ;;
|
||||
*taler-merchant*) n_taler_merchant=$((n_taler_merchant+1)); rss_taler_merchant=$((rss_taler_merchant+r)) ;;
|
||||
*taler-exchange*) n_taler_exchange=$((n_taler_exchange+1)); rss_taler_exchange=$((rss_taler_exchange+r)) ;;
|
||||
*nginx*) n_nginx=$((n_nginx+1)); rss_nginx=$((rss_nginx+r)) ;;
|
||||
*java*) n_java=$((n_java+1)); rss_java=$((rss_java+r)) ;;
|
||||
*) n_other=$((n_other+1)); rss_other=$((rss_other+r)) ;;
|
||||
esac
|
||||
done
|
||||
printf 'TOTAL %s %s\n' "$n" "$rss"
|
||||
[ "$n_postgres" -gt 0 ] && printf 'ROLE postgres %s %s\n' "$n_postgres" "$rss_postgres"
|
||||
[ "$n_libeufin" -gt 0 ] && printf 'ROLE libeufin %s %s\n' "$n_libeufin" "$rss_libeufin"
|
||||
[ "$n_taler_merchant" -gt 0 ] && printf 'ROLE taler-merchant %s %s\n' "$n_taler_merchant" "$rss_taler_merchant"
|
||||
[ "$n_taler_exchange" -gt 0 ] && printf 'ROLE taler-exchange %s %s\n' "$n_taler_exchange" "$rss_taler_exchange"
|
||||
[ "$n_nginx" -gt 0 ] && printf 'ROLE nginx %s %s\n' "$n_nginx" "$rss_nginx"
|
||||
[ "$n_java" -gt 0 ] && printf 'ROLE java %s %s\n' "$n_java" "$rss_java"
|
||||
[ "$n_other" -gt 0 ] && printf 'ROLE other %s %s\n' "$n_other" "$rss_other"
|
||||
"""
|
||||
o = podman_sh(name, script, t=25)
|
||||
by = {}
|
||||
n = None
|
||||
rss_kb = None
|
||||
for line in o.splitlines():
|
||||
p = line.split()
|
||||
if not p:
|
||||
continue
|
||||
if p[0] == "TOTAL" and len(p) >= 3:
|
||||
try:
|
||||
n = int(p[1])
|
||||
rss_kb = int(p[2])
|
||||
except Exception:
|
||||
pass
|
||||
elif p[0] == "ROLE" and len(p) >= 4:
|
||||
try:
|
||||
by[p[1]] = {"n": int(p[2]), "rss_b": int(p[3]) * 1024}
|
||||
except Exception:
|
||||
pass
|
||||
if n is None:
|
||||
nn = sh(f"podman exec {name} sh -c 'ps -e --no-headers 2>/dev/null | wc -l'").strip()
|
||||
n = int(nn) if nn.isdigit() else None
|
||||
return {
|
||||
"proc_total": n,
|
||||
"rss_total_b": (rss_kb * 1024) if rss_kb is not None else None,
|
||||
"by_role": by,
|
||||
}
|
||||
|
||||
def dbs(name):
|
||||
o = sh(
|
||||
|
|
@ -301,31 +715,152 @@ print(json.dumps({
|
|||
}))
|
||||
PY
|
||||
)
|
||||
# Prefer SSH to koopa (LAN or koopa-external). Local podman only if we are on the host.
|
||||
if [ "${SKIP_SSH:-0}" != "1" ] && type koopa_ssh_ok >/dev/null 2>&1 && koopa_ssh_ok; then
|
||||
raw=$(printf '%s\n' "$remote_py" | koopa_ssh_bash 50 'python3 -' 2>/dev/null || true)
|
||||
# koopa_ssh_bash only runs bash -s; pipe python via bash
|
||||
if [ -z "$raw" ]; then
|
||||
raw=$(printf 'python3 - <<'"'"'PY'"'"'\n%s\nPY\n' "$remote_py" | koopa_ssh_bash 50 2>/dev/null || true)
|
||||
if type koopa_ssh_python >/dev/null 2>&1; then
|
||||
raw=$(printf '%s' "$remote_py" | koopa_ssh_python "${METRICS_LOAD_SSH_TIMEOUT:-90}" 2>/dev/null || true)
|
||||
else
|
||||
raw=$(printf '%s' "$remote_py" | with_timeout "${METRICS_LOAD_SSH_TIMEOUT:-90}" \
|
||||
ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" python3 - 2>/dev/null || true)
|
||||
fi
|
||||
elif command -v podman >/dev/null 2>&1; then
|
||||
raw=$(python3 -c "$remote_py" 2>/dev/null || true)
|
||||
fi
|
||||
if [ -z "$raw" ]; then
|
||||
printf '%s\n' "{\"ok\":false,\"reason\":\"probe-failed\",\"label\":\"$label\"}" >"$out"
|
||||
printf '%s\n' "{\"ok\":false,\"reason\":\"probe-failed\",\"label\":\"$label\",\"ssh\":\"${KOOPA_SSH:-?}\"}" >"$out"
|
||||
return 1
|
||||
fi
|
||||
printf '%s\n' "$raw" | python3 -c '
|
||||
import sys,json,re
|
||||
t=sys.stdin.read(); obj=None
|
||||
for m in re.finditer(r"\{", t):
|
||||
try: obj=json.loads(t[m.start():])
|
||||
except Exception: pass
|
||||
try:
|
||||
obj=json.loads(t[m.start():])
|
||||
if isinstance(obj, dict) and (obj.get("host") or obj.get("taler") or obj.get("ok") is False):
|
||||
break
|
||||
except Exception:
|
||||
obj=None
|
||||
if not obj: obj={"ok":False,"reason":"parse"}
|
||||
obj["label"]=sys.argv[1]
|
||||
json.dump(obj, open(sys.argv[2],"w"), indent=2)
|
||||
' "$label" "$out"
|
||||
}
|
||||
|
||||
# One-line human summaries (no leading indent) for info()/ok() detail fields.
|
||||
metrics_load_lines() {
|
||||
local f="$1"
|
||||
python3 - "$f" <<'PY'
|
||||
import json, sys
|
||||
try:
|
||||
d = json.load(open(sys.argv[1]))
|
||||
except Exception as e:
|
||||
print(f"unreadable: {e}")
|
||||
raise SystemExit(0)
|
||||
if not d.get("ok"):
|
||||
print(f"unavailable: {d.get('reason', 'n/a')} ssh={d.get('ssh', '')}".strip())
|
||||
raise SystemExit(0)
|
||||
h = d.get("host") or {}
|
||||
mem = h.get("memory") or {}
|
||||
la = h.get("loadavg") or []
|
||||
|
||||
def gi(b):
|
||||
if b is None:
|
||||
return "?"
|
||||
return f"{b/1024/1024/1024:.2f}GiB"
|
||||
|
||||
def mi(b):
|
||||
if b is None:
|
||||
return "?"
|
||||
return f"{b/1024/1024:.0f}MiB"
|
||||
|
||||
la_s = " ".join(f"{x:.2f}" for x in la) if la else "?"
|
||||
tot = mem.get("mem_total_b")
|
||||
avail = mem.get("mem_available_b")
|
||||
used = (tot - avail) if (tot is not None and avail is not None) else None
|
||||
print(
|
||||
f"host loadavg=[{la_s}] nproc={h.get('nproc')} "
|
||||
f"mem_used={gi(used)} avail={gi(avail)} total={gi(tot)}"
|
||||
)
|
||||
pc = h.get("process_counts") or {}
|
||||
if pc:
|
||||
bits = [f"{k}={v}" for k, v in sorted(pc.items()) if v]
|
||||
if bits:
|
||||
print("host procs " + " ".join(bits))
|
||||
for role in ("bank", "exchange", "merchant"):
|
||||
c = (d.get("taler") or {}).get(role) or {}
|
||||
if not c:
|
||||
continue
|
||||
if not c.get("running"):
|
||||
print(f"{role} DOWN ({c.get('container')})")
|
||||
continue
|
||||
pr = c.get("processes") or {}
|
||||
st = c.get("podman_stats") or {}
|
||||
roles = pr.get("by_role") or {}
|
||||
role_bits = []
|
||||
for k in ("libeufin", "taler-exchange", "taler-merchant", "postgres", "nginx", "java"):
|
||||
if k in roles:
|
||||
role_bits.append(f"{k}={mi(roles[k].get('rss_b'))}")
|
||||
dbs = c.get("databases") or {}
|
||||
db_s = ",".join(
|
||||
f"{x.get('name')}={x.get('size_pretty')}"
|
||||
for x in (dbs.get("databases") or [])[:5]
|
||||
)
|
||||
# podman MemUsage often reports 0B under pasta/cgroup — prefer /proc RSS
|
||||
mem_u = st.get("mem_usage") or ""
|
||||
mem_bit = ""
|
||||
if mem_u and not str(mem_u).startswith("0B"):
|
||||
mem_bit = f" podman_mem={mem_u}"
|
||||
cpu = st.get("cpu_pct") or "?"
|
||||
blk = st.get("block_io") or ""
|
||||
blk_bit = ""
|
||||
if blk and blk not in ("0B / 0B", "0B/0B", "-- / --"):
|
||||
blk_bit = f" block={blk}"
|
||||
line = (
|
||||
f"{role} rss={gi(pr.get('rss_total_b'))} procs={pr.get('proc_total')} "
|
||||
f"cpu={cpu}{mem_bit}{blk_bit}"
|
||||
)
|
||||
if role_bits:
|
||||
line += " | " + " ".join(role_bits)
|
||||
if db_s:
|
||||
line += f" | db:{db_s}"
|
||||
if dbs.get("pgdata_pretty"):
|
||||
line += f" pgdata={dbs.get('pgdata_pretty')}"
|
||||
print(line)
|
||||
PY
|
||||
}
|
||||
|
||||
# Snapshot load and emit as monitoring info lines (withdraw/pay phase markers).
|
||||
# $1=json path $2=label (e.g. after-withdraw)
|
||||
metrics_report_load() {
|
||||
local out="$1" label="$2"
|
||||
local line rc=0
|
||||
if [ "${METRICS_LOAD}" = "0" ]; then
|
||||
info "load ${label}" "skipped (METRICS_LOAD=0)"
|
||||
return 0
|
||||
fi
|
||||
if [ "${SKIP_SSH:-0}" = "1" ] && ! command -v podman >/dev/null 2>&1; then
|
||||
info "load ${label}" "skipped (no SSH / no local podman)"
|
||||
return 0
|
||||
fi
|
||||
if ! metrics_taler_load "$out" "$label"; then
|
||||
warn "load ${label}" "probe failed via ${KOOPA_SSH:-?} — try KOOPA_SSH=koopa-external"
|
||||
return 1
|
||||
fi
|
||||
while IFS= read -r line; do
|
||||
[ -n "$line" ] || continue
|
||||
case "$line" in
|
||||
unavailable:*|unreadable:*)
|
||||
warn "load ${label}" "$line"
|
||||
rc=1
|
||||
;;
|
||||
*)
|
||||
info "load ${label}" "$line"
|
||||
;;
|
||||
esac
|
||||
done < <(metrics_load_lines "$out")
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
# Human one-screen summary of a load JSON
|
||||
metrics_print_load() {
|
||||
local f="$1" title="${2:-load}"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue