diff --git a/scripts/taler-exchange/landing-stats-exchange.sh b/scripts/taler-exchange/landing-stats-exchange.sh index 35452d4..987e631 100644 --- a/scripts/taler-exchange/landing-stats-exchange.sh +++ b/scripts/taler-exchange/landing-stats-exchange.sh @@ -238,6 +238,7 @@ fi GEN=$(now_iso) HUMAN=$(now_human) +GEN_UNIX=$(date +%s) num_or_null() { case "${1:-}" in ''|null) echo null ;; *) echo "$1" ;; esac; } MEM_JSON='"container_rss_human": "—"' @@ -257,6 +258,7 @@ cat >"$TMP" < this (default 900 = 15m; timer often 5–15m) +# STATS_FAIL_SECS hard fail if age > this (default 3600); 0 = never hard-fail on age # --------------------------------------------------------------------------- -: "${STATS_STALE_SECS:=900}" # warn if generated_at older than 15m +: "${STATS_STALE_SECS:=900}" +: "${STATS_FAIL_SECS:=3600}" _fetch_landing_stats_json() { # $1=name bank|exchange|merchant $2=base URL → writes $tmp/stats-$1.json, exit 0 if ok @@ -326,82 +333,265 @@ _fetch_landing_stats_json() { return 1 } -report_landing_load_stats() { +# Validate one stats.json for monitoring display + outside age measurement. +# Prints lines: OK|WARN|ERR|STALE|FAIL +# Sets $tmp/stats-meta-$name.txt with gen_unix|source for shared-feed compare. +check_landing_stats_json() { local name="$1" base="$2" local f="$tmp/stats-${name}.json" - local line age_s + local line hard_missing=0 if ! _fetch_landing_stats_json "$name" "$base"; then if [ "${CHECK_LANDING:-1}" = "1" ] || [ "${LOCAL_STACK:-0}" = "1" ]; then - warn "perf ${name} landing-stats" "stats.json not available (public or SSH)" + fail "stats ${name}" "HTTP/body missing · ${base}/intro/stats.json" + else + warn "stats ${name}" "stats.json not available" fi - return + return 1 fi + ok "stats ${name} reachable" "${base}/intro/stats.json" + + # Multi-line verdict from python (STATUS detail) + while IFS= read -r line; do + case "$line" in + META\ *) + printf '%s\n' "${line#META }" >"$tmp/stats-meta-${name}.txt" + ;; + OK\ *) + ok "stats ${name}" "${line#OK }" + ;; + WARN\ *) + warn "stats ${name}" "${line#WARN }" + ;; + STALE\ *) + warn "stats ${name} freshness" "stale · ${line#STALE }" + ;; + FAIL\ *) + fail "stats ${name}" "${line#FAIL }" + hard_missing=1 + ;; + ERR\ *) + if [ "${LOCAL_STACK:-0}" = "1" ] || [ "${CHECK_LANDING:-1}" = "1" ]; then + fail "stats ${name}" "${line#ERR }" + else + warn "stats ${name}" "${line#ERR }" + fi + hard_missing=1 + ;; + *) + [ -n "$line" ] && warn "stats ${name}" "unparsed · $line" + ;; + esac + done < <(python3 - "$f" "$name" "${STATS_STALE_SECS}" "${STATS_FAIL_SECS}" <<'PY' 2>/dev/null || echo "ERR python check failed" +import json, sys, time, re +from datetime import datetime, timezone + +path, role, stale_s, fail_s = sys.argv[1], sys.argv[2], int(sys.argv[3]), int(sys.argv[4]) +now = int(time.time()) + +def out(kind, msg): + print("%s %s" % (kind, msg)) - line=$(python3 - "$f" "${STATS_STALE_SECS}" <<'PY' 2>/dev/null || true -import json, sys, time -path, stale = sys.argv[1], int(sys.argv[2]) try: d = json.load(open(path)) except Exception as e: - print("ERR parse:" + str(e)[:80]) + out("ERR", "JSON parse: %s" % str(e)[:100]) sys.exit(0) -if not d.get("ok", True): - print("ERR ok=false " + str(d.get("error") or "")[:80]) + +if not isinstance(d, dict): + out("ERR", "body is not a JSON object") sys.exit(0) -p = d.get("performance") or {} -if not isinstance(p, dict) or not p: - print("ERR no performance block") + +if d.get("ok") is False: + out("ERR", "ok=false · %s" % str(d.get("error") or d.get("hint") or "")[:80]) sys.exit(0) -mem = p.get("memory") if isinstance(p.get("memory"), dict) else {} -rss = mem.get("container_rss_human") or mem.get("proc_sum_rss_human") or "?" -load = p.get("loadavg") or "?" -# latency fields differ by site -bits = [] -for k in ("config_ms", "integration_ms", "webui_ms", "keys_ms", "terms_ms"): - if k in p and p[k] is not None: - bits.append("%s=%sms" % (k.replace("_ms", ""), p[k])) -lat = " ".join(bits) if bits else "latency=?" -gen = d.get("generated_at_human") or d.get("generated_at") or "?" -# staleness -age = "" -try: - gu = d.get("generated_at_unix") - if gu is not None: - age_s = int(time.time()) - int(gu) - age = " age=%ss" % age_s - if age_s > stale: - print("STALE loadavg=%s RSS=%s %s gen=%s%s" % (load, rss, lat, gen, age)) - sys.exit(0) -except Exception: - pass -print("OK loadavg=%s RSS=%s %s gen=%s%s" % (load, rss, lat, gen, age)) + +# --- timestamps (outside-in age measurement) --- +gu = d.get("generated_at_unix") +giso = d.get("generated_at") or d.get("generated_at_iso") +ghum = d.get("generated_at_human") +age = None +src_u = None + +if gu is not None and str(gu).strip() != "": + try: + src_u = int(float(gu)) + # tolerate ms accidental timestamps + if src_u > 10_000_000_000: + src_u //= 1000 + age = now - src_u + except Exception: + out("ERR", "generated_at_unix not numeric: %r" % gu) + src_u = None +elif giso: + # parse ISO when unix missing (legacy exchange/merchant before unix field) + try: + s = str(giso).strip() + if s.endswith("Z"): + s = s[:-1] + "+00:00" + # allow +0200 without colon + s = re.sub(r"([+-]\d{2})(\d{2})$", r"\1:\2", s) + dt = datetime.fromisoformat(s) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + src_u = int(dt.timestamp()) + age = now - src_u + out("WARN", "no generated_at_unix — derived age from generated_at ISO (deploy collector with unix)") + except Exception as e: + out("ERR", "cannot parse generated_at=%r (%s)" % (giso, e)) +else: + out("ERR", "missing generated_at_unix and generated_at — cannot measure freshness outside") + +if not ghum and not giso: + out("ERR", "missing generated_at_human/generated_at — UI cannot show “updated” time") +elif ghum: + out("OK", "timestamp human=%s" % ghum) +else: + out("OK", "timestamp iso=%s" % giso) + +if src_u is not None: + out("META", "%s|%s|%s" % (src_u, d.get("source") or "", d.get("currency") or "")) + if age is not None and age < -120: + out("WARN", "clock skew · generated_at_unix in the future by %ss" % (-age)) + if age is not None: + out("OK", "generated_at_unix=%s age=%ss (now=%s)" % (src_u, age, now)) + if fail_s > 0 and age > fail_s: + out("FAIL", "age=%ss > STATS_FAIL_SECS=%s · collector not updating" % (age, fail_s)) + elif age > stale_s: + out("STALE", "age=%ss > STATS_STALE_SECS=%s · timer lag or stuck publish" % (age, stale_s)) + +# --- display fields for landings / monitoring --- +src = str(d.get("source") or "") +# Bank-shaped (incl. stage shared feed on all three landings) +bankish = ( + role == "bank" + or "collect_bank_stats" in src + or "shared" in src + or isinstance(d.get("withdraws"), dict) + or isinstance(d.get("bank_accounts"), dict) +) +if bankish: + missing = [] + if not isinstance(d.get("withdraws"), dict): + missing.append("withdraws") + ba = d.get("bank_accounts") + wl = d.get("wallets") + if not isinstance(ba, dict) and not isinstance(wl, dict): + missing.append("bank_accounts|wallets") + if d.get("currency") in (None, ""): + missing.append("currency") + if missing: + out("ERR", "bank-shaped display missing: %s" % ",".join(missing)) + else: + w = d.get("withdraws") or {} + ba = ba if isinstance(ba, dict) else {} + n_acc = ba.get("total") if ba.get("total") is not None else ba.get("users") + out( + "OK", + "display bank-shape currency=%s accounts=%s withdraws.count=%s source=%s" + % (d.get("currency"), n_acc, w.get("count"), (src or "?")[:48]), + ) +elif src == "exchange-db" or (role == "exchange" and "reserves" in d): + need = [] + for k in ("reserves", "wire_in_count", "known_coins"): + if k not in d and k.replace("known_coins", "coins_live") not in d: + if k == "known_coins" and ("coins_live" in d or "coins_remaining_amount" in d): + continue + need.append(k) + if need: + out("WARN", "exchange display keys missing: %s" % ",".join(need)) + else: + out( + "OK", + "display exchange-db reserves=%s wire_in=%s coins=%s" + % (d.get("reserves"), d.get("wire_in_count"), d.get("known_coins") or d.get("coins_live")), + ) +elif src == "merchant-db" or (role == "merchant" and "instances" in d): + if "instances" not in d and "orders" not in d: + out("ERR", "merchant display missing instances/orders") + else: + out( + "OK", + "display merchant-db instances=%s orders=%s paid=%s" + % (d.get("instances"), d.get("orders"), d.get("paid")), + ) +else: + out("WARN", "unknown stats schema role=%s source=%s keys=%s" % (role, src[:40], ",".join(list(d.keys())[:8]))) + +# performance block optional (stage shared bank feed may omit heavy probes) +p = d.get("performance") +if isinstance(p, dict) and p: + mem = p.get("memory") if isinstance(p.get("memory"), dict) else {} + rss = mem.get("container_rss_human") or mem.get("proc_sum_rss_human") or "—" + load = p.get("loadavg") or "—" + bits = [] + for k in ("config_ms", "integration_ms", "webui_ms", "keys_ms", "terms_ms"): + if k in p and p[k] is not None: + bits.append("%s=%s" % (k.replace("_ms", ""), p[k])) + lat = " ".join(bits) if bits else "latency=—" + out("OK", "performance loadavg=%s RSS=%s %s" % (load, rss, lat)) +else: + out("OK", "performance block optional/absent") PY ) - case "$line" in - OK\ *) - ok "perf ${name} landing-stats" "${line#OK }" - ;; - STALE\ *) - warn "perf ${name} landing-stats" "stale · ${line#STALE }" - ;; - ERR\ *) - warn "perf ${name} landing-stats" "${line#ERR }" - ;; - *) - warn "perf ${name} landing-stats" "unreadable stats.json" - ;; - esac + return 0 } if [ "${CHECK_LANDING:-1}" = "1" ] || [ "${LOCAL_STACK:-0}" = "1" ]; then set_group stats - section "www · performance · landing load stats (stats.json · in-container probes)" - report_landing_load_stats "bank" "$BANK_PUBLIC" - report_landing_load_stats "exchange" "$EXCHANGE_PUBLIC" - report_landing_load_stats "merchant" "$MERCHANT_PUBLIC" - info "perf landing-stats note" "from /intro/stats.json (public); SSH container fallback if needed" + section "www · landing stats.json (freshness + display fields · outside-in)" + info "stats policy" "STALE≥${STATS_STALE_SECS}s WARN · FAIL≥${STATS_FAIL_SECS}s ERROR (0=disable fail) · need generated_at_unix" + check_landing_stats_json "bank" "$BANK_PUBLIC" || true + check_landing_stats_json "exchange" "$EXCHANGE_PUBLIC" || true + check_landing_stats_json "merchant" "$MERCHANT_PUBLIC" || true + + # Shared feed: stage-style publish copies same bank stats to all three URLs + if [ -f "$tmp/stats-meta-bank.txt" ] && [ -f "$tmp/stats-meta-exchange.txt" ] && [ -f "$tmp/stats-meta-merchant.txt" ]; then + shared_line=$(python3 - "$tmp/stats-meta-bank.txt" "$tmp/stats-meta-exchange.txt" "$tmp/stats-meta-merchant.txt" <<'PY' 2>/dev/null || true +import sys +def parse(p): + try: + t = open(p).read().strip().split("|", 2) + return t[0], (t[1] if len(t) > 1 else ""), (t[2] if len(t) > 2 else "") + except Exception: + return "", "", "" +b, bs, bc = parse(sys.argv[1]) +e, es, ec = parse(sys.argv[2]) +m, ms, mc = parse(sys.argv[3]) +if not b or not e or not m: + print("SKIP incomplete meta") +elif b == e == m: + print("SHARED gen_unix=%s bank_src=%s" % (b, (bs or "?")[:50])) +else: + # Shared feed only when all three claim the same bank-collector source + # (e.g. stage TESTPAYSAN "… shared"). GOA uses independent exchange-db / + # merchant-db collectors — different gen_unix is expected. + def bankish(s): + s = (s or "").lower() + return "shared" in s or "collect_bank_stats" in s + if bankish(bs) and bankish(es) and bankish(ms): + print("DRIFT bank=%s exchange=%s merchant=%s (expected equal for shared feed)" % (b, e, m)) + else: + print("INDEPENDENT bank=%s exchange=%s merchant=%s" % (b, e, m)) +PY +) + case "$shared_line" in + SHARED\ *) + ok "stats shared feed" "${shared_line#SHARED }" + ;; + DRIFT\ *) + warn "stats shared feed" "timestamps differ · ${shared_line#DRIFT }" + ;; + INDEPENDENT\ *) + info "stats feeds" "independent collectors · ${shared_line#INDEPENDENT }" + ;; + *) + info "stats feeds" "${shared_line:-n/a}" + ;; + esac + fi + info "stats note" "public GET /intro/stats.json; measure age via generated_at_unix vs wall clock" fi # Rollup + optional JSON for metrics_print_overall if [ -s "$PERF_TSV" ]; then