monitoring: taler-monitoring www/inside/sanity/server/e2e scaffold
This commit is contained in:
Hernâni Marques
parent
fed083d0d5
commit
95499a7f59
11 changed files with 2828 additions and 0 deletions
281
scripts/taler-monitoring/check_sanity.sh
Executable file
281
scripts/taler-monitoring/check_sanity.sh
Executable file
|
|
@ -0,0 +1,281 @@
|
|||
#!/usr/bin/env bash
|
||||
# Sanity checks for bank · exchange · merchant (public + server-side).
|
||||
# Sections are independent; continues after failures.
|
||||
set -euo pipefail
|
||||
ROOT=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=lib.sh
|
||||
source "$ROOT/lib.sh"
|
||||
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT
|
||||
|
||||
expect_code() {
|
||||
local label="$1" want="$2" url="$3"
|
||||
local code
|
||||
code=$(http_code "$url")
|
||||
case ",$want," in
|
||||
*",$code,"*) ok "$label" ;;
|
||||
*) fail "$label" "HTTP $code (want $want) $url" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
json_currency() {
|
||||
python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); print(d.get("currency") or "")' "$1" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# Area sanity-### — public + optional server-side per component
|
||||
set_area sanity
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
section "sanity · bank"
|
||||
# ---------------------------------------------------------------------------
|
||||
expect_code "bank public /config" 200 "$BANK_PUBLIC/config"
|
||||
expect_code "bank public /taler-integration/config" 200 "$BANK_PUBLIC/taler-integration/config"
|
||||
expect_code "bank public /webui/" 200 "$BANK_PUBLIC/webui/"
|
||||
|
||||
code=$(http_body "$BANK_PUBLIC/config" "$tmp/bank-config.json")
|
||||
if [ "$code" = "200" ]; then
|
||||
cur=$(json_currency "$tmp/bank-config.json")
|
||||
[ "$cur" = "GOA" ] && ok "bank currency GOA" || fail "bank currency" "got ${cur:-?}"
|
||||
# wire type / name if present
|
||||
python3 - "$tmp/bank-config.json" <<'PY' 2>/dev/null && ok "bank config JSON object" || fail "bank config JSON"
|
||||
import json,sys
|
||||
d=json.load(open(sys.argv[1]))
|
||||
sys.exit(0 if isinstance(d, dict) and d.get("currency") else 1)
|
||||
PY
|
||||
if json_has_alt_unit_names "$tmp/bank-config.json" >/tmp/alt-bank-s.$$ 2>&1; then
|
||||
ok "bank /config alt_unit_names" "$(tr '\n' '; ' </tmp/alt-bank-s.$$ | sed 's/; $//')"
|
||||
else
|
||||
fail "bank /config alt_unit_names" "$(tr '\n' '; ' </tmp/alt-bank-s.$$ | sed 's/; $//')"
|
||||
fi
|
||||
rm -f /tmp/alt-bank-s.$$
|
||||
fi
|
||||
|
||||
# server-side bank
|
||||
if koopa_ssh_ok; then
|
||||
BOUT=$(koopa_ssh_bash 40 <<'REMOTE' || true
|
||||
set +e
|
||||
BANK=$(podman ps --format '{{.Names}}' | grep -iE 'hacktivism-bank|taler-bank' | head -1)
|
||||
[ -z "$BANK" ] && BANK=$(podman ps --format '{{.Names}}' | grep -i bank | head -1)
|
||||
echo "CTR=$BANK"
|
||||
if [ -z "$BANK" ]; then echo "NOCTR"; exit 0; fi
|
||||
code=$(curl -sS -m 5 -o /dev/null -w '%{http_code}' http://127.0.0.1:9012/config 2>/dev/null || echo 000)
|
||||
echo "LOCAL_CONFIG=$code"
|
||||
code2=$(curl -sS -m 5 -o /dev/null -w '%{http_code}' http://127.0.0.1:9012/taler-integration/config 2>/dev/null || echo 000)
|
||||
echo "LOCAL_INT=$code2"
|
||||
if podman exec "$BANK" bash -c 'pgrep -f "MainKt serve|libeufin-bank serve" >/dev/null' 2>/dev/null; then
|
||||
echo "LIBEUFIN=1"
|
||||
else
|
||||
echo "LIBEUFIN=0"
|
||||
fi
|
||||
if podman exec "$BANK" bash -c 'pg_isready -q' 2>/dev/null; then
|
||||
echo "PG=1"
|
||||
else
|
||||
echo "PG=0"
|
||||
fi
|
||||
# in-container health if present
|
||||
if podman exec "$BANK" test -x /usr/local/bin/check_bank-health.sh 2>/dev/null; then
|
||||
podman exec "$BANK" /usr/local/bin/check_bank-health.sh 2>&1 | sed 's/^/HEALTH /' | tail -20
|
||||
echo "HEALTH_EC=${PIPESTATUS[0]}"
|
||||
fi
|
||||
REMOTE
|
||||
)
|
||||
echo "$BOUT" | grep -q '^CTR=.\+' && ok "bank container $(echo "$BOUT" | sed -n 's/^CTR=//p' | head -1)" || fail "bank container" "not found"
|
||||
echo "$BOUT" | grep -q 'LOCAL_CONFIG=200' && ok "bank local :9012/config" || fail "bank local :9012/config"
|
||||
echo "$BOUT" | grep -q 'LOCAL_INT=200' && ok "bank local :9012/taler-integration/config" || fail "bank local integration"
|
||||
echo "$BOUT" | grep -q 'LIBEUFIN=1' && ok "bank libeufin-bank process" || fail "bank libeufin-bank process"
|
||||
echo "$BOUT" | grep -q 'PG=1' && ok "bank postgres ready" || warn "bank postgres" "pg_isready failed"
|
||||
if echo "$BOUT" | grep -q 'HEALTH '; then
|
||||
if echo "$BOUT" | grep -qE 'HEALTH_EC=0|ALL CRITICAL CHECKS PASSED'; then
|
||||
ok "bank check_bank-health.sh"
|
||||
else
|
||||
# health script may false-fail process grep; warn not fail if local config ok
|
||||
warn "bank check_bank-health.sh" "non-zero or incomplete"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
warn "bank server-side" "ssh ${KOOPA_SSH} unavailable"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
section "sanity · exchange"
|
||||
# ---------------------------------------------------------------------------
|
||||
expect_code "exchange public /config" 200 "$EXCHANGE_PUBLIC/config"
|
||||
expect_code "exchange public /keys" 200 "$EXCHANGE_PUBLIC/keys"
|
||||
expect_code "exchange public /terms" 200 "$EXCHANGE_PUBLIC/terms"
|
||||
|
||||
code=$(http_body "$EXCHANGE_PUBLIC/config" "$tmp/ex-config.json")
|
||||
if [ "$code" = "200" ]; then
|
||||
cur=$(json_currency "$tmp/ex-config.json")
|
||||
[ "$cur" = "GOA" ] && ok "exchange currency GOA" || fail "exchange currency" "got ${cur:-?}"
|
||||
if json_has_alt_unit_names "$tmp/ex-config.json" "GOA" >/tmp/alt-ex-s.$$ 2>&1; then
|
||||
ok "exchange /config alt_unit_names" "$(tr '\n' '; ' </tmp/alt-ex-s.$$ | sed 's/; $//')"
|
||||
else
|
||||
fail "exchange /config alt_unit_names" "$(tr '\n' '; ' </tmp/alt-ex-s.$$ | sed 's/; $//')"
|
||||
fi
|
||||
rm -f /tmp/alt-ex-s.$$
|
||||
fi
|
||||
|
||||
code=$(http_body "$EXCHANGE_PUBLIC/keys" "$tmp/ex-keys.json")
|
||||
if [ "$code" = "200" ]; then
|
||||
python3 - "$tmp/ex-keys.json" <<'PY'
|
||||
import json,sys,time
|
||||
d=json.load(open(sys.argv[1]))
|
||||
sk=d.get("signkeys") or []
|
||||
acc=d.get("accounts") or []
|
||||
den=d.get("denominations") or []
|
||||
# count denoms roughly
|
||||
n=0
|
||||
for g in den:
|
||||
if isinstance(g, dict):
|
||||
n += len(g.get("denoms") or [])
|
||||
print(f"signkeys={len(sk)} accounts={len(acc)} denom_groups={len(den)} denoms~={n}")
|
||||
sys.exit(0 if sk and (acc or n) else 1)
|
||||
PY
|
||||
ec=$?
|
||||
detail=$(python3 - "$tmp/ex-keys.json" <<'PY'
|
||||
import json,sys
|
||||
d=json.load(open(sys.argv[1]))
|
||||
sk=d.get("signkeys") or []
|
||||
acc=d.get("accounts") or []
|
||||
den=d.get("denominations") or []
|
||||
n=sum(len(g.get("denoms") or []) for g in den if isinstance(g, dict))
|
||||
print(f"signkeys={len(sk)} accounts={len(acc)} denoms~={n}")
|
||||
PY
|
||||
)
|
||||
[ "$ec" -eq 0 ] && ok "exchange /keys usable ($detail)" || fail "exchange /keys usable" "$detail"
|
||||
fi
|
||||
|
||||
# /wire if exposed
|
||||
wcode=$(http_code "$EXCHANGE_PUBLIC/wire")
|
||||
if [ "$wcode" = "200" ]; then
|
||||
ok "exchange public /wire"
|
||||
else
|
||||
warn "exchange public /wire" "HTTP $wcode (accounts may only be in /keys)"
|
||||
fi
|
||||
|
||||
if koopa_ssh_ok; then
|
||||
EOUT=$(koopa_ssh_bash 40 <<'REMOTE' || true
|
||||
set +e
|
||||
EX=$(podman ps --format '{{.Names}}' | grep -i exchange | head -1)
|
||||
echo "CTR=$EX"
|
||||
code=$(curl -sS -m 5 -o /dev/null -w '%{http_code}' http://127.0.0.1:9011/config 2>/dev/null || echo 000)
|
||||
echo "LOCAL_CONFIG=$code"
|
||||
codek=$(curl -sS -m 8 -o /dev/null -w '%{http_code}' http://127.0.0.1:9011/keys 2>/dev/null || echo 000)
|
||||
echo "LOCAL_KEYS=$codek"
|
||||
if [ -n "$EX" ]; then
|
||||
if podman exec "$EX" bash -c 'pgrep -f taler-exchange-httpd >/dev/null' 2>/dev/null; then
|
||||
echo "HTTPD=1"
|
||||
else
|
||||
echo "HTTPD=0"
|
||||
fi
|
||||
for p in taler-exchange-secmod-rsa taler-exchange-secmod-eddsa taler-exchange-wirewatch taler-exchange-aggregator; do
|
||||
if podman exec "$EX" bash -c "pgrep -f $p >/dev/null" 2>/dev/null; then
|
||||
echo "PROC_$p=1"
|
||||
else
|
||||
echo "PROC_$p=0"
|
||||
fi
|
||||
done
|
||||
if podman exec "$EX" test -x /usr/local/bin/check_exchange-health.sh 2>/dev/null; then
|
||||
SKIP_ENSURE=1 podman exec -e SKIP_ENSURE=1 "$EX" /usr/local/bin/check_exchange-health.sh 2>&1 | sed 's/^/HEALTH /' | tail -25
|
||||
fi
|
||||
fi
|
||||
REMOTE
|
||||
)
|
||||
echo "$EOUT" | grep -q '^CTR=.\+' && ok "exchange container $(echo "$EOUT" | sed -n 's/^CTR=//p' | head -1)" || fail "exchange container"
|
||||
echo "$EOUT" | grep -q 'LOCAL_CONFIG=200' && ok "exchange local :9011/config" || fail "exchange local :9011/config"
|
||||
echo "$EOUT" | grep -q 'LOCAL_KEYS=200' && ok "exchange local :9011/keys" || fail "exchange local :9011/keys"
|
||||
echo "$EOUT" | grep -q 'HTTPD=1' && ok "exchange-httpd process" || warn "exchange-httpd process" "not detected"
|
||||
echo "$EOUT" | grep -q 'PROC_taler-exchange-wirewatch=1' && ok "exchange wirewatch" || warn "exchange wirewatch" "not running"
|
||||
echo "$EOUT" | grep -q 'PROC_taler-exchange-aggregator=1' && ok "exchange aggregator" || warn "exchange aggregator" "not running"
|
||||
else
|
||||
warn "exchange server-side" "ssh unavailable"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
section "sanity · merchant"
|
||||
# ---------------------------------------------------------------------------
|
||||
expect_code "merchant public /config" 200 "$MERCHANT_PUBLIC/config"
|
||||
expect_code "merchant public /webui/" 200 "$MERCHANT_PUBLIC/webui/"
|
||||
|
||||
code=$(http_body "$MERCHANT_PUBLIC/config" "$tmp/mer-config.json")
|
||||
if [ "$code" = "200" ]; then
|
||||
if python3 - "$tmp/mer-config.json" "$EXCHANGE_PUBLIC" <<'PY'
|
||||
import json,sys
|
||||
d=json.load(open(sys.argv[1]))
|
||||
want=sys.argv[2].rstrip("/")
|
||||
curs=list((d.get("currencies") or {}).keys())
|
||||
ex=d.get("exchanges") or []
|
||||
urls=[]
|
||||
for e in ex:
|
||||
if isinstance(e, dict):
|
||||
u=e.get("base_url") or e.get("url") or e.get("exchange_base_url") or ""
|
||||
if u: urls.append(u.rstrip("/"))
|
||||
elif isinstance(e, str):
|
||||
urls.append(e.rstrip("/"))
|
||||
ok_goa = "GOA" in curs or any((e.get("currency") if isinstance(e, dict) else None)=="GOA" for e in ex)
|
||||
ok_ex = any(want in u or "exchange.hacktivism.ch" in u for u in urls)
|
||||
print("currencies", curs)
|
||||
print("exchanges", urls[:5])
|
||||
sys.exit(0 if ok_goa and ok_ex else 1)
|
||||
PY
|
||||
then
|
||||
ok "merchant config GOA + exchange.hacktivism.ch"
|
||||
else
|
||||
fail "merchant config GOA + exchange" "see currencies/exchanges"
|
||||
fi
|
||||
if json_has_alt_unit_names "$tmp/mer-config.json" >/tmp/alt-mer-s.$$ 2>&1; then
|
||||
ok "merchant currencies alt_unit_names" "$(tr '\n' '; ' </tmp/alt-mer-s.$$ | sed 's/; $//')"
|
||||
else
|
||||
fail "merchant currencies alt_unit_names" "$(tr '\n' '; ' </tmp/alt-mer-s.$$ | sed 's/; $//')"
|
||||
fi
|
||||
rm -f /tmp/alt-mer-s.$$
|
||||
# Each exchange listed on merchant /config must publish alt_unit_names on its own /config
|
||||
check_merchant_listed_exchanges_alt_units "$tmp/mer-config.json"
|
||||
fi
|
||||
|
||||
# demo instance reachable?
|
||||
INST="${MERCHANT_INSTANCE}"
|
||||
icode=$(http_code "$MERCHANT_PUBLIC/instances/${INST}/config")
|
||||
if [ "$icode" = "200" ]; then
|
||||
ok "merchant instance ${INST} /config"
|
||||
else
|
||||
# some deployments use private only
|
||||
warn "merchant instance ${INST} /config" "HTTP $icode"
|
||||
fi
|
||||
|
||||
if koopa_ssh_ok; then
|
||||
MOUT=$(koopa_ssh_bash 40 <<'REMOTE' || true
|
||||
set +e
|
||||
MER=$(podman ps --format '{{.Names}}' | grep -E '^taler-hacktivism$' | head -1)
|
||||
[ -z "$MER" ] && MER=$(podman ps --format '{{.Names}}' | grep -iE 'merchant|hacktivism' | grep -viE 'bank|exchange' | head -1)
|
||||
echo "CTR=$MER"
|
||||
code=$(curl -skS -m 5 -o /dev/null -w '%{http_code}' https://127.0.0.1:9010/config 2>/dev/null || echo 000)
|
||||
echo "LOCAL_CONFIG=$code"
|
||||
if [ -n "$MER" ]; then
|
||||
if podman exec "$MER" bash -c 'pgrep -f taler-merchant-httpd >/dev/null' 2>/dev/null; then
|
||||
echo "HTTPD=1"
|
||||
else
|
||||
echo "HTTPD=0"
|
||||
fi
|
||||
if podman exec "$MER" test -x /usr/local/bin/check_merchant-health.sh 2>/dev/null; then
|
||||
SKIP_ENSURE=1 podman exec -e SKIP_ENSURE=1 "$MER" /usr/local/bin/check_merchant-health.sh 2>&1 | sed 's/^/HEALTH /' | tail -30
|
||||
fi
|
||||
fi
|
||||
REMOTE
|
||||
)
|
||||
echo "$MOUT" | grep -q '^CTR=.\+' && ok "merchant container $(echo "$MOUT" | sed -n 's/^CTR=//p' | head -1)" || fail "merchant container"
|
||||
echo "$MOUT" | grep -q 'LOCAL_CONFIG=200' && ok "merchant local :9010/config" || fail "merchant local :9010/config"
|
||||
echo "$MOUT" | grep -q 'HTTPD=1' && ok "merchant-httpd process" || warn "merchant-httpd process" "not detected"
|
||||
if echo "$MOUT" | grep -q 'HEALTH '; then
|
||||
if echo "$MOUT" | grep -qiE 'ALL CRITICAL|HEALTH_EC=0|\[OK\]'; then
|
||||
ok "merchant check_merchant-health.sh (sample OK)"
|
||||
else
|
||||
warn "merchant check_merchant-health.sh" "see remote output"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
warn "merchant server-side" "ssh unavailable"
|
||||
fi
|
||||
|
||||
summary
|
||||
Loading…
Add table
Add a link
Reference in a new issue