monitoring: compact landings + stats load

This commit is contained in:
Hernâni Marques 2026-07-17 10:18:23 +02:00
parent d38999d5ad
commit 9c2d3077a5
No known key found for this signature in database
GPG key ID: CB5738652768F7E9

View file

@ -136,9 +136,13 @@ PY
fi
fi
fi
if [ "${CHECK_LANDING:-1}" = "1" ]; then
check_url_soft "exchange /intro/" 200 "$EXCHANGE_PUBLIC/intro/"
# Root should land on intro (302/301 then 200 on follow is checked separately)
check_url_soft "exchange /" 302,301,200 "$EXCHANGE_PUBLIC/"
else
info "exchange /intro" "skipped (CHECK_LANDING=0 · no public landings for this stack)"
fi
# ---------------------------------------------------------------------------
# Performance — outside-in public HTTPS latency (this runner, not loopback)
@ -188,21 +192,137 @@ check_perf() {
# Bank first (wallet-critical paths before UI chrome)
check_perf "perf bank /taler-integration/config" "$BANK_PUBLIC/taler-integration/config"
check_perf "perf bank /config" "$BANK_PUBLIC/config"
if [ "${CHECK_LANDING:-1}" = "1" ]; then
check_perf "perf bank /intro/" "$BANK_PUBLIC/intro/"
check_perf "perf bank /intro/stats.json" "$BANK_PUBLIC/intro/stats.json" 200
fi
check_perf "perf bank /webui/" "$BANK_PUBLIC/webui/" 200,301,302
# Exchange
check_perf "perf exchange /config" "$EXCHANGE_PUBLIC/config"
check_perf "perf exchange /keys" "$EXCHANGE_PUBLIC/keys"
if [ "${CHECK_LANDING:-1}" = "1" ]; then
check_perf "perf exchange /intro/" "$EXCHANGE_PUBLIC/intro/"
fi
# Merchant
check_perf "perf merchant /config" "$MERCHANT_PUBLIC/config"
check_perf "perf merchant /webui/" "$MERCHANT_PUBLIC/webui/" 200,301,302
if [ "${CHECK_LANDING:-1}" = "1" ]; then
check_perf "perf merchant /intro/" "$MERCHANT_PUBLIC/intro/"
fi
info "perf note" "measured from this host (outside-in); not container loopback"
info "perf note" "RTT measured from this host (outside-in)"
# ---------------------------------------------------------------------------
# Landing load stats (stats.json written *inside* containers, usually public).
# Prefer HTTPS /intro/stats.json (no SSH). If missing and SSH works, cat from
# the container. Report loadavg + RSS + in-container probe ms — one line each.
# ---------------------------------------------------------------------------
: "${STATS_STALE_SECS:=900}" # warn if generated_at older than 15m
_fetch_landing_stats_json() {
# $1=name bank|exchange|merchant $2=base URL → writes $tmp/stats-$1.json, exit 0 if ok
local name="$1" base="$2"
local f="$tmp/stats-${name}.json"
local code ctr path
rm -f "$f"
code=$(http_body "${base}/intro/stats.json" "$f" 2>/dev/null || echo 000)
if [ "$code" = "200" ] && [ -s "$f" ]; then
return 0
fi
# Optional inside-container fallback (no public path or empty)
if [ "${LOCAL_STACK:-0}" = "1" ] && [ "${SKIP_SSH:-0}" != "1" ] && koopa_ssh_ok 2>/dev/null; then
case "$name" in
bank) ctr=taler-hacktivism-bank; path=/var/www/bank-landing/stats.json ;;
exchange) ctr=taler-hacktivism-exchange-ansible; path=/var/www/exchange-landing/stats.json ;;
merchant) ctr=taler-hacktivism; path=/var/www/merchant-landing/stats.json ;;
*) return 1 ;;
esac
if koopa_ssh_run 12 "podman exec ${ctr} cat ${path} 2>/dev/null" >"$f" 2>/dev/null \
&& [ -s "$f" ]; then
return 0
fi
fi
return 1
}
report_landing_load_stats() {
local name="$1" base="$2"
local f="$tmp/stats-${name}.json"
local line age_s
if ! _fetch_landing_stats_json "$name" "$base"; then
if [ "${CHECK_LANDING:-1}" = "1" ] || [ "${LOCAL_STACK:-0}" = "1" ]; then
warn "perf ${name} landing-stats" "stats.json not available (public or SSH)"
fi
return
fi
line=$(python3 - "$f" "${STATS_STALE_SECS}" <<'PY' 2>/dev/null || true
import json, sys, time
path, stale = sys.argv[1], int(sys.argv[2])
try:
d = json.load(open(path))
except Exception as e:
print("ERR parse:" + str(e)[:80])
sys.exit(0)
if not d.get("ok", True):
print("ERR ok=false " + str(d.get("error") or "")[:80])
sys.exit(0)
p = d.get("performance") or {}
if not isinstance(p, dict) or not p:
print("ERR no performance block")
sys.exit(0)
mem = p.get("memory") if isinstance(p.get("memory"), dict) else {}
rss = mem.get("container_rss_human") or mem.get("proc_sum_rss_human") or "?"
load = p.get("loadavg") or "?"
# latency fields differ by site
bits = []
for k in ("config_ms", "integration_ms", "webui_ms", "keys_ms", "terms_ms"):
if k in p and p[k] is not None:
bits.append("%s=%sms" % (k.replace("_ms", ""), p[k]))
lat = " ".join(bits) if bits else "latency=?"
gen = d.get("generated_at_human") or d.get("generated_at") or "?"
# staleness
age = ""
try:
gu = d.get("generated_at_unix")
if gu is not None:
age_s = int(time.time()) - int(gu)
age = " age=%ss" % age_s
if age_s > stale:
print("STALE loadavg=%s RSS=%s %s gen=%s%s" % (load, rss, lat, gen, age))
sys.exit(0)
except Exception:
pass
print("OK loadavg=%s RSS=%s %s gen=%s%s" % (load, rss, lat, gen, age))
PY
)
case "$line" in
OK\ *)
ok "perf ${name} landing-stats" "${line#OK }"
;;
STALE\ *)
warn "perf ${name} landing-stats" "stale · ${line#STALE }"
;;
ERR\ *)
warn "perf ${name} landing-stats" "${line#ERR }"
;;
*)
warn "perf ${name} landing-stats" "unreadable stats.json"
;;
esac
}
if [ "${CHECK_LANDING:-1}" = "1" ] || [ "${LOCAL_STACK:-0}" = "1" ]; then
section "www · performance · landing load stats (stats.json · in-container probes)"
report_landing_load_stats "bank" "$BANK_PUBLIC"
report_landing_load_stats "exchange" "$EXCHANGE_PUBLIC"
report_landing_load_stats "merchant" "$MERCHANT_PUBLIC"
info "perf landing-stats note" "from /intro/stats.json (public); SSH container fallback if needed"
fi
# Terms + privacy (legal docs)
@ -241,6 +361,7 @@ if [ "$code" = "200" ]; then
check_url_soft "bank /taler-integration/config" 200 "$BANK_PUBLIC/taler-integration/config"
fi
check_url_soft "bank /webui/" 200 "$BANK_PUBLIC/webui/"
if [ "${CHECK_LANDING:-1}" = "1" ]; then
check_url_soft "bank /intro/" 200 "$BANK_PUBLIC/intro/"
check_url_soft "bank /" 302,301,200 "$BANK_PUBLIC/"
# Auto-account: credentials + shared-pool taler://withdraw (like step 2)
@ -282,6 +403,7 @@ PY
;;
esac
fi
fi
# Bank legal docs (landing nginx via Caddy /terms* /privacy* or /intro/*)
check_legal_doc "bank /terms" "$BANK_PUBLIC/terms" "terms|GOA|exploration|bank|FADP|revDSG"
@ -346,7 +468,9 @@ PY
rm -f /tmp/alt-mer.$$
# Follow each exchange listed in merchant /config and require its /config alt_unit_names
check_merchant_listed_exchanges_alt_units "$tmp/mc.json"
if [ "${CHECK_LANDING:-1}" = "1" ]; then
check_url_soft "merchant /intro/" 200 "$MERCHANT_PUBLIC/intro/"
fi
check_url_soft "merchant /webui/" 200 "$MERCHANT_PUBLIC/webui/"
check_url_soft "merchant /" 302,301,200 "$MERCHANT_PUBLIC/"
fi
@ -360,49 +484,92 @@ case "$code" in
*) warn "merchant /terms/" "HTTP $code (expect 302 → /terms)" ;;
esac
# ---------------------------------------------------------------------------
# Paivana paywall (local GOA stack) — public front only; pay path is e2e
# ---------------------------------------------------------------------------
if [ "${LOCAL_STACK:-1}" = "1" ] && [ "${E2E_PAIVANA:-1}" != "0" ]; then
section "www · paivana paywall"
: "${PAIVANA_PUBLIC:=https://paivana.hacktivism.ch}"
PAIVANA_PUBLIC="${PAIVANA_PUBLIC%/}"
hdr=$(curl -skS -m "${TIMEOUT}" -D - -o /dev/null "${PAIVANA_PUBLIC}/" 2>/dev/null || true)
pcode=$(printf '%s' "$hdr" | awk 'BEGIN{c="000"} /^HTTP/{c=$2} END{print c}')
loc=$(printf '%s' "$hdr" | awk 'BEGIN{IGNORECASE=1} /^location:/{sub(/\r$/,""); sub(/^location:[[:space:]]*/,""); print; exit}')
case "$pcode" in
301|302|303|307|308)
if printf '%s' "$loc" | grep -qiE 'paivana|templates|well-known'; then
ok "paivana /" "HTTP $pcode → template flow"
else
ok "paivana /" "HTTP $pcode redirect"
fi
info "paivana Location" "${loc:0:140}"
;;
200)
warn "paivana /" "HTTP 200 (expected paywall redirect to template)"
;;
*)
warn "paivana /" "HTTP ${pcode:-000}${PAIVANA_PUBLIC}/ (e2e pay may still work via template)"
;;
esac
fi
# ---------------------------------------------------------------------------
# Landing pages: every HTTPS link exposed on bank / merchant / exchange intros
# + required static assets + bank withdraw mint (taler://withdraw only)
# Skipped when CHECK_LANDING=0 (e.g. taler-ops.ch — no GOA-style landings).
# ---------------------------------------------------------------------------
if [ "${CHECK_LANDING:-1}" != "1" ]; then
section "www · landing pages"
info "landing checks" "skipped (CHECK_LANDING=0 · stack has no public /intro landings)"
summary
exit 0
fi
section "www · landing exposed links · bank / merchant / exchange"
# Known-good landing static paths (relative to each host base)
# Checked even if HTML parse misses them.
check_landing_asset() {
local label="$1" url="$2" soft="${3:-0}"
# Probe one URL: print code to stdout (200 after following redirects counts as 200).
# Sets _landing_code. Exit 0 if OK (200 or redirect→200), 1 otherwise.
_landing_probe() {
local url="$1"
local code
code=$(http_code "$url")
case "$code" in
200) ok "$label" "HTTP 200 · $url" ;;
200) _landing_code=200; return 0 ;;
301|302|303|307|308)
# follow once for assets that redirect
code=$(curl -skS --max-redirs 3 -L -m "${TIMEOUT}" -o /dev/null -w '%{http_code}' "$url" 2>/dev/null || echo 000)
if [ "$code" = "200" ]; then
ok "$label" "HTTP redirect→200 · $url"
elif [ "$soft" = "1" ]; then
warn "$label" "HTTP $code$url"
else
fail "$label" "HTTP $code after redirect — $url"
fi
code=$(curl -skS --max-redirs 5 -L -m "${TIMEOUT}" -o /dev/null -w '%{http_code}' "$url" 2>/dev/null || echo 000)
_landing_code="$code"
[ "$code" = "200" ] && return 0
return 1
;;
*)
if [ "$soft" = "1" ]; then
warn "$label" "HTTP $code$url"
else
fail "$label" "HTTP $code$url"
fi
_landing_code="$code"
return 1
;;
esac
}
# Soft external (app stores / upstream docs): WARN if down, never ERROR
# Known-good landing static paths — one report line via caller aggregate, or
# soft=1 single warn. Returns 0 if ok.
check_landing_asset() {
local label="$1" url="$2" soft="${3:-0}"
if _landing_probe "$url"; then
return 0
fi
if [ "$soft" = "1" ]; then
warn "$label" "HTTP ${_landing_code:-?}$url"
else
fail "$label" "HTTP ${_landing_code:-?}$url"
fi
return 1
}
# Soft external: never ERROR; used only for failures in aggregated external probe.
check_external_soft() {
local label="$1" url="$2"
local code
code=$(curl -skS --max-redirs 5 -L -m "${TIMEOUT}" -o /dev/null -w '%{http_code}' "$url" 2>/dev/null || echo 000)
case "$code" in
200|204|301|302|303|307|308) ok "$label" "HTTP $code · $url" ;;
*) warn "$label" "HTTP $code (external soft) · $url" ;;
200|204|301|302|303|307|308) return 0 ;;
*) warn "$label" "HTTP $code · $url"; return 1 ;;
esac
}
@ -499,95 +666,139 @@ check_one_landing() {
local html="$tmp/landing-${name}.html"
local pref="$tmp/urls-${name}"
local code n own_n ext_n
local a_ok=0 a_fail=0 a_soft=0
local own_ok=0 own_fail=0
local ext_ok=0 ext_fail=0
local fail_sample="" soft_sample=""
code=$(http_body "${base}/intro/" "$html")
if [ "$code" != "200" ]; then
if [ "${LOCAL_STACK:-1}" = "1" ]; then
fail "landing ${name} /intro/" "HTTP $code"
fail "landing ${name}" "/intro/ HTTP $code — skip assets/links"
else
warn "landing ${name} /intro/" "HTTP $code"
warn "landing ${name}" "/intro/ HTTP $code — skip assets/links"
fi
return
fi
ok "landing ${name} /intro/" "HTTP 200 · $(wc -c <"$html" | tr -d ' ') bytes"
# Required static assets (hard on local)
check_landing_asset "landing ${name} qrcode.min.js" "${base}/intro/qrcode.min.js"
check_landing_asset "landing ${name} og-goa-shop.png" "${base}/intro/og-goa-shop.png"
check_landing_asset "landing ${name} qr-logo.png" "${base}/intro/qr-logo.png" 1
# Required static assets (hard: qrcode + og; soft: qr-logo)
for url in \
"${base}/intro/qrcode.min.js" \
"${base}/intro/og-goa-shop.png"
do
if _landing_probe "$url"; then
a_ok=$((a_ok + 1))
else
a_fail=$((a_fail + 1))
fail_sample="${fail_sample}${fail_sample:+; }HTTP ${_landing_code} $url"
fi
done
if _landing_probe "${base}/intro/qr-logo.png"; then
a_ok=$((a_ok + 1))
else
a_soft=$((a_soft + 1))
soft_sample="${soft_sample}${soft_sample:+; }HTTP ${_landing_code} qr-logo.png"
fi
n=$(extract_landing_urls "$base" "$html" "$pref" 2>/dev/null || echo "own=0 ext=0")
info "landing ${name} link extract" "$n"
own_n=0
ext_n=0
[ -f "${pref}.own" ] && own_n=$(grep -c . "${pref}.own" 2>/dev/null || echo 0)
[ -f "${pref}.ext" ] && ext_n=$(grep -c . "${pref}.ext" 2>/dev/null || echo 0)
if [ "${own_n:-0}" -lt 1 ]; then
fail "landing ${name} own-stack links" "none extracted from HTML"
else
ok "landing ${name} own-stack links" "${own_n} URLs to probe"
fi
# strip newlines from grep -c edge cases
own_n=${own_n//[^0-9]/}
ext_n=${ext_n//[^0-9]/}
own_n=${own_n:-0}
ext_n=${ext_n:-0}
# Probe every own-stack URL from the page
# Probe own-stack URLs — count only; list failures
if [ -f "${pref}.own" ]; then
while IFS= read -r u; do
[ -n "$u" ] || continue
# skip mint endpoints that create resources on GET if any (auto-account creates accounts)
case "$u" in
*/intro/auto-account.json)
# shape checked separately; still require 200 GET
;;
esac
code=$(http_code "$u")
case "$code" in
200) ok "landing ${name} link" "HTTP 200 · $u" ;;
301|302|303|307|308)
code=$(curl -skS --max-redirs 5 -L -m "${TIMEOUT}" -o /dev/null -w '%{http_code}' "$u" 2>/dev/null || echo 000)
if [ "$code" = "200" ]; then
ok "landing ${name} link" "redirect→200 · $u"
if _landing_probe "$u"; then
own_ok=$((own_ok + 1))
else
fail "landing ${name} link" "HTTP $code after redirect · $u"
own_fail=$((own_fail + 1))
# keep a few samples (max ~3)
if [ "$own_fail" -le 3 ]; then
fail_sample="${fail_sample}${fail_sample:+; }own HTTP ${_landing_code} $u"
fi
;;
405|501)
# some APIs reject wrong method — try GET already failed; soft note
fail "landing ${name} link" "HTTP $code · $u"
;;
*)
if [ "${LOCAL_STACK:-1}" = "1" ]; then
fail "landing ${name} link" "HTTP $code · $u"
else
warn "landing ${name} link" "HTTP $code · $u"
fi
;;
esac
done < "${pref}.own"
fi
# External store / docs: soft
# External: soft counts
if [ -f "${pref}.ext" ]; then
while IFS= read -r u; do
[ -n "$u" ] || continue
check_external_soft "landing ${name} external" "$u"
code=$(curl -skS --max-redirs 5 -L -m "${TIMEOUT}" -o /dev/null -w '%{http_code}' "$u" 2>/dev/null || echo 000)
case "$code" in
200|204|301|302|303|307|308) ext_ok=$((ext_ok + 1)) ;;
*)
ext_fail=$((ext_fail + 1))
if [ "$ext_fail" -le 3 ]; then
soft_sample="${soft_sample}${soft_sample:+; }ext HTTP $code $u"
fi
;;
esac
done < "${pref}.ext"
fi
# One primary line per landing
local detail
detail="/intro $(wc -c <"$html" | tr -d ' ')B · assets ${a_ok}/$((a_ok + a_fail + a_soft)) · own-links ${own_ok}/${own_n} · external ${ext_ok}/${ext_n}"
if [ "$a_fail" -gt 0 ] || [ "$own_fail" -gt 0 ] || [ "${own_n:-0}" -lt 1 ]; then
if [ "${LOCAL_STACK:-1}" = "1" ]; then
fail "landing ${name}" "$detail${fail_sample:+ · $fail_sample}"
else
warn "landing ${name}" "$detail${fail_sample:+ · $fail_sample}"
fi
else
ok "landing ${name}" "$detail"
fi
if [ "$a_soft" -gt 0 ] || [ "$ext_fail" -gt 0 ]; then
warn "landing ${name} soft" "${soft_sample:-soft issues}"
fi
}
check_one_landing "bank" "$BANK_PUBLIC"
check_one_landing "merchant" "$MERCHANT_PUBLIC"
check_one_landing "exchange" "$EXCHANGE_PUBLIC"
# Cross-links between the three landings (always on local stack)
# Cross-links: one line
if [ "${LOCAL_STACK:-1}" = "1" ]; then
check_landing_asset "cross bank→merchant intro" "$MERCHANT_PUBLIC/intro/"
check_landing_asset "cross bank→exchange intro" "$EXCHANGE_PUBLIC/intro/"
check_landing_asset "cross merchant→bank intro" "$BANK_PUBLIC/intro/"
check_landing_asset "cross exchange→bank intro" "$BANK_PUBLIC/intro/"
_cx_ok=0
_cx_fail=0
_cx_detail=""
for pair in \
"bank→merchant|$MERCHANT_PUBLIC/intro/" \
"bank→exchange|$EXCHANGE_PUBLIC/intro/" \
"merchant→bank|$BANK_PUBLIC/intro/" \
"exchange→bank|$BANK_PUBLIC/intro/"
do
_cx_name="${pair%%|*}"
_cx_url="${pair#*|}"
if _landing_probe "$_cx_url"; then
_cx_ok=$((_cx_ok + 1))
else
_cx_fail=$((_cx_fail + 1))
_cx_detail="${_cx_detail}${_cx_detail:+; }${_cx_name} HTTP ${_landing_code}"
fi
done
if [ "$_cx_fail" -eq 0 ]; then
ok "landing cross-links" "${_cx_ok}/4 intros reachable"
else
fail "landing cross-links" "${_cx_ok}/4 ok · ${_cx_detail}"
fi
fi
# Bank-only: shared-pool withdraw mint + static withdraw files + shop assets
# Bank withdraw mint + shop assets — compact
if [ "${LOCAL_STACK:-1}" = "1" ] || [ -n "${BANK_PUBLIC:-}" ]; then
check_landing_asset "bank shop-pay.js" "$BANK_PUBLIC/intro/shop-pay.js" 1
check_landing_asset "bank shop-pay.css" "$BANK_PUBLIC/intro/shop-pay.css" 1
_ba_ok=0
_ba_soft=0
_ba_msg=""
if _landing_probe "$BANK_PUBLIC/intro/shop-pay.js"; then _ba_ok=$((_ba_ok + 1)); else _ba_soft=$((_ba_soft + 1)); _ba_msg="${_ba_msg}shop-pay.js; "; fi
if _landing_probe "$BANK_PUBLIC/intro/shop-pay.css"; then _ba_ok=$((_ba_ok + 1)); else _ba_soft=$((_ba_soft + 1)); _ba_msg="${_ba_msg}shop-pay.css; "; fi
dw_code=$(http_body "$BANK_PUBLIC/intro/demo-withdraw.json" "$tmp/dw.json")
case "$dw_code" in
200)
@ -606,31 +817,44 @@ print(u[:88])
sys.exit(0)
PY
then
ok "bank /intro/demo-withdraw.json" "$(python3 -c 'import json;print(json.load(open("'"$tmp/dw.json"'")).get("taler_withdraw_uri","")[:80])' 2>/dev/null || true)"
_ba_ok=$((_ba_ok + 1))
wid=$(python3 -c 'import json;print(json.load(open("'"$tmp/dw.json"'")).get("withdrawal_id",""))' 2>/dev/null || true)
if [ -n "$wid" ]; then
check_landing_asset "bank taler-integration withdraw op" \
"$BANK_PUBLIC/taler-integration/withdrawal-operation/${wid}"
fi
if _landing_probe "$BANK_PUBLIC/taler-integration/withdrawal-operation/${wid}"; then
_ba_ok=$((_ba_ok + 1))
else
fail "bank /intro/demo-withdraw.json" "invalid taler://withdraw shape"
fail "landing bank withdraw-op" "HTTP ${_landing_code} · id=$wid"
fi
fi
ok "landing bank withdraw/shop" "demo-withdraw + shop assets ok (${_ba_ok} checks)"
else
fail "landing bank demo-withdraw" "invalid taler://withdraw shape"
fi
;;
405|501|404|502|503|000)
fail "bank /intro/demo-withdraw.json" "HTTP $dw_code (want 200)"
fail "landing bank demo-withdraw" "HTTP $dw_code (want 200)"
;;
*)
if [ "${LOCAL_STACK:-1}" = "1" ]; then
fail "bank /intro/demo-withdraw.json" "HTTP $dw_code want 200"
fail "landing bank demo-withdraw" "HTTP $dw_code want 200"
else
warn "bank /intro/demo-withdraw.json" "HTTP $dw_code"
warn "landing bank demo-withdraw" "HTTP $dw_code"
fi
;;
esac
if [ "$_ba_soft" -gt 0 ]; then
warn "landing bank shop assets" "${_ba_msg}soft-missing"
fi
fi
# Merchant landing shop assets
check_landing_asset "merchant shop-pay.js" "$MERCHANT_PUBLIC/intro/shop-pay.js" 1
check_landing_asset "merchant shop-pay.css" "$MERCHANT_PUBLIC/intro/shop-pay.css" 1
# Merchant shop assets — one soft line
_ma=0
_landing_probe "$MERCHANT_PUBLIC/intro/shop-pay.js" && _ma=$((_ma + 1))
_landing_probe "$MERCHANT_PUBLIC/intro/shop-pay.css" && _ma=$((_ma + 1))
if [ "$_ma" -eq 2 ]; then
ok "landing merchant shop assets" "shop-pay.js + .css"
else
warn "landing merchant shop assets" "${_ma}/2 present (soft)"
fi
summary