monitoring: stats.json freshness + display fields outside-in
This commit is contained in:
Hernâni Marques
parent
62d0e426f4
commit
3d5f412af3
5 changed files with 273 additions and 60 deletions
|
|
@ -294,11 +294,18 @@ fi
|
|||
info "perf note" "RTT measured from this host (outside-in)"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Landing load stats (stats.json written *inside* containers, usually public).
|
||||
# Prefer HTTPS /intro/stats.json (no SSH). If missing and SSH works, cat from
|
||||
# the container. Report loadavg + RSS + in-container probe ms — one line each.
|
||||
# Landing stats.json (public outside-in) — freshness + display fields.
|
||||
# Timestamps (required for external measurement):
|
||||
# generated_at_unix — epoch seconds (preferred)
|
||||
# generated_at — ISO-8601
|
||||
# generated_at_human — footer / “updated” UI string
|
||||
# Bank collector always sets all three; exchange/merchant scripts set unix too.
|
||||
# Env:
|
||||
# STATS_STALE_SECS warn if age > this (default 900 = 15m; timer often 5–15m)
|
||||
# STATS_FAIL_SECS hard fail if age > this (default 3600); 0 = never hard-fail on age
|
||||
# ---------------------------------------------------------------------------
|
||||
: "${STATS_STALE_SECS:=900}" # warn if generated_at older than 15m
|
||||
: "${STATS_STALE_SECS:=900}"
|
||||
: "${STATS_FAIL_SECS:=3600}"
|
||||
|
||||
_fetch_landing_stats_json() {
|
||||
# $1=name bank|exchange|merchant $2=base URL → writes $tmp/stats-$1.json, exit 0 if ok
|
||||
|
|
@ -326,82 +333,265 @@ _fetch_landing_stats_json() {
|
|||
return 1
|
||||
}
|
||||
|
||||
report_landing_load_stats() {
|
||||
# Validate one stats.json for monitoring display + outside age measurement.
|
||||
# Prints lines: OK|WARN|ERR|STALE|FAIL <detail>
|
||||
# Sets $tmp/stats-meta-$name.txt with gen_unix|source for shared-feed compare.
|
||||
check_landing_stats_json() {
|
||||
local name="$1" base="$2"
|
||||
local f="$tmp/stats-${name}.json"
|
||||
local line age_s
|
||||
local line hard_missing=0
|
||||
|
||||
if ! _fetch_landing_stats_json "$name" "$base"; then
|
||||
if [ "${CHECK_LANDING:-1}" = "1" ] || [ "${LOCAL_STACK:-0}" = "1" ]; then
|
||||
warn "perf ${name} landing-stats" "stats.json not available (public or SSH)"
|
||||
fail "stats ${name}" "HTTP/body missing · ${base}/intro/stats.json"
|
||||
else
|
||||
warn "stats ${name}" "stats.json not available"
|
||||
fi
|
||||
return
|
||||
return 1
|
||||
fi
|
||||
ok "stats ${name} reachable" "${base}/intro/stats.json"
|
||||
|
||||
# Multi-line verdict from python (STATUS detail)
|
||||
while IFS= read -r line; do
|
||||
case "$line" in
|
||||
META\ *)
|
||||
printf '%s\n' "${line#META }" >"$tmp/stats-meta-${name}.txt"
|
||||
;;
|
||||
OK\ *)
|
||||
ok "stats ${name}" "${line#OK }"
|
||||
;;
|
||||
WARN\ *)
|
||||
warn "stats ${name}" "${line#WARN }"
|
||||
;;
|
||||
STALE\ *)
|
||||
warn "stats ${name} freshness" "stale · ${line#STALE }"
|
||||
;;
|
||||
FAIL\ *)
|
||||
fail "stats ${name}" "${line#FAIL }"
|
||||
hard_missing=1
|
||||
;;
|
||||
ERR\ *)
|
||||
if [ "${LOCAL_STACK:-0}" = "1" ] || [ "${CHECK_LANDING:-1}" = "1" ]; then
|
||||
fail "stats ${name}" "${line#ERR }"
|
||||
else
|
||||
warn "stats ${name}" "${line#ERR }"
|
||||
fi
|
||||
hard_missing=1
|
||||
;;
|
||||
*)
|
||||
[ -n "$line" ] && warn "stats ${name}" "unparsed · $line"
|
||||
;;
|
||||
esac
|
||||
done < <(python3 - "$f" "$name" "${STATS_STALE_SECS}" "${STATS_FAIL_SECS}" <<'PY' 2>/dev/null || echo "ERR python check failed"
|
||||
import json, sys, time, re
|
||||
from datetime import datetime, timezone
|
||||
|
||||
path, role, stale_s, fail_s = sys.argv[1], sys.argv[2], int(sys.argv[3]), int(sys.argv[4])
|
||||
now = int(time.time())
|
||||
|
||||
def out(kind, msg):
|
||||
print("%s %s" % (kind, msg))
|
||||
|
||||
line=$(python3 - "$f" "${STATS_STALE_SECS}" <<'PY' 2>/dev/null || true
|
||||
import json, sys, time
|
||||
path, stale = sys.argv[1], int(sys.argv[2])
|
||||
try:
|
||||
d = json.load(open(path))
|
||||
except Exception as e:
|
||||
print("ERR parse:" + str(e)[:80])
|
||||
out("ERR", "JSON parse: %s" % str(e)[:100])
|
||||
sys.exit(0)
|
||||
if not d.get("ok", True):
|
||||
print("ERR ok=false " + str(d.get("error") or "")[:80])
|
||||
|
||||
if not isinstance(d, dict):
|
||||
out("ERR", "body is not a JSON object")
|
||||
sys.exit(0)
|
||||
p = d.get("performance") or {}
|
||||
if not isinstance(p, dict) or not p:
|
||||
print("ERR no performance block")
|
||||
|
||||
if d.get("ok") is False:
|
||||
out("ERR", "ok=false · %s" % str(d.get("error") or d.get("hint") or "")[:80])
|
||||
sys.exit(0)
|
||||
mem = p.get("memory") if isinstance(p.get("memory"), dict) else {}
|
||||
rss = mem.get("container_rss_human") or mem.get("proc_sum_rss_human") or "?"
|
||||
load = p.get("loadavg") or "?"
|
||||
# latency fields differ by site
|
||||
bits = []
|
||||
for k in ("config_ms", "integration_ms", "webui_ms", "keys_ms", "terms_ms"):
|
||||
if k in p and p[k] is not None:
|
||||
bits.append("%s=%sms" % (k.replace("_ms", ""), p[k]))
|
||||
lat = " ".join(bits) if bits else "latency=?"
|
||||
gen = d.get("generated_at_human") or d.get("generated_at") or "?"
|
||||
# staleness
|
||||
age = ""
|
||||
try:
|
||||
gu = d.get("generated_at_unix")
|
||||
if gu is not None:
|
||||
age_s = int(time.time()) - int(gu)
|
||||
age = " age=%ss" % age_s
|
||||
if age_s > stale:
|
||||
print("STALE loadavg=%s RSS=%s %s gen=%s%s" % (load, rss, lat, gen, age))
|
||||
sys.exit(0)
|
||||
except Exception:
|
||||
pass
|
||||
print("OK loadavg=%s RSS=%s %s gen=%s%s" % (load, rss, lat, gen, age))
|
||||
|
||||
# --- timestamps (outside-in age measurement) ---
|
||||
gu = d.get("generated_at_unix")
|
||||
giso = d.get("generated_at") or d.get("generated_at_iso")
|
||||
ghum = d.get("generated_at_human")
|
||||
age = None
|
||||
src_u = None
|
||||
|
||||
if gu is not None and str(gu).strip() != "":
|
||||
try:
|
||||
src_u = int(float(gu))
|
||||
# tolerate ms accidental timestamps
|
||||
if src_u > 10_000_000_000:
|
||||
src_u //= 1000
|
||||
age = now - src_u
|
||||
except Exception:
|
||||
out("ERR", "generated_at_unix not numeric: %r" % gu)
|
||||
src_u = None
|
||||
elif giso:
|
||||
# parse ISO when unix missing (legacy exchange/merchant before unix field)
|
||||
try:
|
||||
s = str(giso).strip()
|
||||
if s.endswith("Z"):
|
||||
s = s[:-1] + "+00:00"
|
||||
# allow +0200 without colon
|
||||
s = re.sub(r"([+-]\d{2})(\d{2})$", r"\1:\2", s)
|
||||
dt = datetime.fromisoformat(s)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
src_u = int(dt.timestamp())
|
||||
age = now - src_u
|
||||
out("WARN", "no generated_at_unix — derived age from generated_at ISO (deploy collector with unix)")
|
||||
except Exception as e:
|
||||
out("ERR", "cannot parse generated_at=%r (%s)" % (giso, e))
|
||||
else:
|
||||
out("ERR", "missing generated_at_unix and generated_at — cannot measure freshness outside")
|
||||
|
||||
if not ghum and not giso:
|
||||
out("ERR", "missing generated_at_human/generated_at — UI cannot show “updated” time")
|
||||
elif ghum:
|
||||
out("OK", "timestamp human=%s" % ghum)
|
||||
else:
|
||||
out("OK", "timestamp iso=%s" % giso)
|
||||
|
||||
if src_u is not None:
|
||||
out("META", "%s|%s|%s" % (src_u, d.get("source") or "", d.get("currency") or ""))
|
||||
if age is not None and age < -120:
|
||||
out("WARN", "clock skew · generated_at_unix in the future by %ss" % (-age))
|
||||
if age is not None:
|
||||
out("OK", "generated_at_unix=%s age=%ss (now=%s)" % (src_u, age, now))
|
||||
if fail_s > 0 and age > fail_s:
|
||||
out("FAIL", "age=%ss > STATS_FAIL_SECS=%s · collector not updating" % (age, fail_s))
|
||||
elif age > stale_s:
|
||||
out("STALE", "age=%ss > STATS_STALE_SECS=%s · timer lag or stuck publish" % (age, stale_s))
|
||||
|
||||
# --- display fields for landings / monitoring ---
|
||||
src = str(d.get("source") or "")
|
||||
# Bank-shaped (incl. stage shared feed on all three landings)
|
||||
bankish = (
|
||||
role == "bank"
|
||||
or "collect_bank_stats" in src
|
||||
or "shared" in src
|
||||
or isinstance(d.get("withdraws"), dict)
|
||||
or isinstance(d.get("bank_accounts"), dict)
|
||||
)
|
||||
if bankish:
|
||||
missing = []
|
||||
if not isinstance(d.get("withdraws"), dict):
|
||||
missing.append("withdraws")
|
||||
ba = d.get("bank_accounts")
|
||||
wl = d.get("wallets")
|
||||
if not isinstance(ba, dict) and not isinstance(wl, dict):
|
||||
missing.append("bank_accounts|wallets")
|
||||
if d.get("currency") in (None, ""):
|
||||
missing.append("currency")
|
||||
if missing:
|
||||
out("ERR", "bank-shaped display missing: %s" % ",".join(missing))
|
||||
else:
|
||||
w = d.get("withdraws") or {}
|
||||
ba = ba if isinstance(ba, dict) else {}
|
||||
n_acc = ba.get("total") if ba.get("total") is not None else ba.get("users")
|
||||
out(
|
||||
"OK",
|
||||
"display bank-shape currency=%s accounts=%s withdraws.count=%s source=%s"
|
||||
% (d.get("currency"), n_acc, w.get("count"), (src or "?")[:48]),
|
||||
)
|
||||
elif src == "exchange-db" or (role == "exchange" and "reserves" in d):
|
||||
need = []
|
||||
for k in ("reserves", "wire_in_count", "known_coins"):
|
||||
if k not in d and k.replace("known_coins", "coins_live") not in d:
|
||||
if k == "known_coins" and ("coins_live" in d or "coins_remaining_amount" in d):
|
||||
continue
|
||||
need.append(k)
|
||||
if need:
|
||||
out("WARN", "exchange display keys missing: %s" % ",".join(need))
|
||||
else:
|
||||
out(
|
||||
"OK",
|
||||
"display exchange-db reserves=%s wire_in=%s coins=%s"
|
||||
% (d.get("reserves"), d.get("wire_in_count"), d.get("known_coins") or d.get("coins_live")),
|
||||
)
|
||||
elif src == "merchant-db" or (role == "merchant" and "instances" in d):
|
||||
if "instances" not in d and "orders" not in d:
|
||||
out("ERR", "merchant display missing instances/orders")
|
||||
else:
|
||||
out(
|
||||
"OK",
|
||||
"display merchant-db instances=%s orders=%s paid=%s"
|
||||
% (d.get("instances"), d.get("orders"), d.get("paid")),
|
||||
)
|
||||
else:
|
||||
out("WARN", "unknown stats schema role=%s source=%s keys=%s" % (role, src[:40], ",".join(list(d.keys())[:8])))
|
||||
|
||||
# performance block optional (stage shared bank feed may omit heavy probes)
|
||||
p = d.get("performance")
|
||||
if isinstance(p, dict) and p:
|
||||
mem = p.get("memory") if isinstance(p.get("memory"), dict) else {}
|
||||
rss = mem.get("container_rss_human") or mem.get("proc_sum_rss_human") or "—"
|
||||
load = p.get("loadavg") or "—"
|
||||
bits = []
|
||||
for k in ("config_ms", "integration_ms", "webui_ms", "keys_ms", "terms_ms"):
|
||||
if k in p and p[k] is not None:
|
||||
bits.append("%s=%s" % (k.replace("_ms", ""), p[k]))
|
||||
lat = " ".join(bits) if bits else "latency=—"
|
||||
out("OK", "performance loadavg=%s RSS=%s %s" % (load, rss, lat))
|
||||
else:
|
||||
out("OK", "performance block optional/absent")
|
||||
PY
|
||||
)
|
||||
|
||||
case "$line" in
|
||||
OK\ *)
|
||||
ok "perf ${name} landing-stats" "${line#OK }"
|
||||
;;
|
||||
STALE\ *)
|
||||
warn "perf ${name} landing-stats" "stale · ${line#STALE }"
|
||||
;;
|
||||
ERR\ *)
|
||||
warn "perf ${name} landing-stats" "${line#ERR }"
|
||||
;;
|
||||
*)
|
||||
warn "perf ${name} landing-stats" "unreadable stats.json"
|
||||
;;
|
||||
esac
|
||||
return 0
|
||||
}
|
||||
|
||||
if [ "${CHECK_LANDING:-1}" = "1" ] || [ "${LOCAL_STACK:-0}" = "1" ]; then
|
||||
set_group stats
|
||||
section "www · performance · landing load stats (stats.json · in-container probes)"
|
||||
report_landing_load_stats "bank" "$BANK_PUBLIC"
|
||||
report_landing_load_stats "exchange" "$EXCHANGE_PUBLIC"
|
||||
report_landing_load_stats "merchant" "$MERCHANT_PUBLIC"
|
||||
info "perf landing-stats note" "from /intro/stats.json (public); SSH container fallback if needed"
|
||||
section "www · landing stats.json (freshness + display fields · outside-in)"
|
||||
info "stats policy" "STALE≥${STATS_STALE_SECS}s WARN · FAIL≥${STATS_FAIL_SECS}s ERROR (0=disable fail) · need generated_at_unix"
|
||||
check_landing_stats_json "bank" "$BANK_PUBLIC" || true
|
||||
check_landing_stats_json "exchange" "$EXCHANGE_PUBLIC" || true
|
||||
check_landing_stats_json "merchant" "$MERCHANT_PUBLIC" || true
|
||||
|
||||
# Shared feed: stage-style publish copies same bank stats to all three URLs
|
||||
if [ -f "$tmp/stats-meta-bank.txt" ] && [ -f "$tmp/stats-meta-exchange.txt" ] && [ -f "$tmp/stats-meta-merchant.txt" ]; then
|
||||
shared_line=$(python3 - "$tmp/stats-meta-bank.txt" "$tmp/stats-meta-exchange.txt" "$tmp/stats-meta-merchant.txt" <<'PY' 2>/dev/null || true
|
||||
import sys
|
||||
def parse(p):
|
||||
try:
|
||||
t = open(p).read().strip().split("|", 2)
|
||||
return t[0], (t[1] if len(t) > 1 else ""), (t[2] if len(t) > 2 else "")
|
||||
except Exception:
|
||||
return "", "", ""
|
||||
b, bs, bc = parse(sys.argv[1])
|
||||
e, es, ec = parse(sys.argv[2])
|
||||
m, ms, mc = parse(sys.argv[3])
|
||||
if not b or not e or not m:
|
||||
print("SKIP incomplete meta")
|
||||
elif b == e == m:
|
||||
print("SHARED gen_unix=%s bank_src=%s" % (b, (bs or "?")[:50]))
|
||||
else:
|
||||
# Shared feed only when all three claim the same bank-collector source
|
||||
# (e.g. stage TESTPAYSAN "… shared"). GOA uses independent exchange-db /
|
||||
# merchant-db collectors — different gen_unix is expected.
|
||||
def bankish(s):
|
||||
s = (s or "").lower()
|
||||
return "shared" in s or "collect_bank_stats" in s
|
||||
if bankish(bs) and bankish(es) and bankish(ms):
|
||||
print("DRIFT bank=%s exchange=%s merchant=%s (expected equal for shared feed)" % (b, e, m))
|
||||
else:
|
||||
print("INDEPENDENT bank=%s exchange=%s merchant=%s" % (b, e, m))
|
||||
PY
|
||||
)
|
||||
case "$shared_line" in
|
||||
SHARED\ *)
|
||||
ok "stats shared feed" "${shared_line#SHARED }"
|
||||
;;
|
||||
DRIFT\ *)
|
||||
warn "stats shared feed" "timestamps differ · ${shared_line#DRIFT }"
|
||||
;;
|
||||
INDEPENDENT\ *)
|
||||
info "stats feeds" "independent collectors · ${shared_line#INDEPENDENT }"
|
||||
;;
|
||||
*)
|
||||
info "stats feeds" "${shared_line:-n/a}"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
info "stats note" "public GET /intro/stats.json; measure age via generated_at_unix vs wall clock"
|
||||
fi
|
||||
# Rollup + optional JSON for metrics_print_overall
|
||||
if [ -s "$PERF_TSV" ]; then
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue