monitoring: taler-monitoring www/inside/sanity/server/e2e scaffold
This commit is contained in:
Hernâni Marques
parent
fed083d0d5
commit
95499a7f59
11 changed files with 2828 additions and 0 deletions
141
scripts/taler-monitoring/check_inside.sh
Executable file
141
scripts/taler-monitoring/check_inside.sh
Executable file
|
|
@ -0,0 +1,141 @@
|
|||
#!/usr/bin/env bash
|
||||
# Inside status for bank / exchange / merchant. Hard-capped SSH — never hang forever.
|
||||
set -euo pipefail
|
||||
ROOT=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=lib.sh
|
||||
source "$ROOT/lib.sh"
|
||||
|
||||
# Area inside-### — container / process state on koopa (SSH)
|
||||
set_area inside
|
||||
section "inside · collect from koopa"
|
||||
|
||||
if [ "${SKIP_SSH}" = "1" ]; then
|
||||
warn "ssh" "SKIP_SSH=1 — skipped"
|
||||
summary
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if ! koopa_ssh_ok; then
|
||||
err "ssh" "cannot reach ${KOOPA_SSH} in ${SSH_CONNECT_TIMEOUT}s — set SKIP_SSH=1 to skip inside"
|
||||
summary
|
||||
exit 1
|
||||
fi
|
||||
ok "ssh ${KOOPA_SSH}"
|
||||
|
||||
# One short remote script (≤ SSH_CMD_TIMEOUT). Every slow step is local curl -m 3 or quick pgrep.
|
||||
RAW=$(
|
||||
koopa_ssh_bash "${SSH_CMD_TIMEOUT}" <<'REMOTE' || true
|
||||
set +e
|
||||
emit() { printf 'E|%s|%s|%s|%s\n' "$1" "$2" "$3" "$(printf '%s' "${4:-}" | tr '\n\r' ' ' | head -c 200)"; }
|
||||
# quick curl
|
||||
hc() { curl -skS -m 3 -o /tmp/mb -w '%{http_code}' "$1" 2>/dev/null || echo 000; }
|
||||
# quick process check inside container (pgrep only)
|
||||
hasp() { podman exec "$1" pgrep -f "$2" >/dev/null 2>&1; }
|
||||
|
||||
BANK=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -iE 'hacktivism-bank|taler-bank' | head -1)
|
||||
[ -z "$BANK" ] && BANK=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -i bank | head -1)
|
||||
EX=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -i exchange | head -1)
|
||||
MER=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -E '^taler-hacktivism$' | head -1)
|
||||
[ -z "$MER" ] && MER=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -iE 'merchant|hacktivism' | grep -viE 'bank|exchange' | head -1)
|
||||
|
||||
# Domain resolve inside container (wirewatch needs bank.hacktivism.ch → real IP)
|
||||
# emit: comp LEVEL dns "host → ip" or fail
|
||||
check_dns() {
|
||||
local comp="$1" ctr="$2" host="$3"
|
||||
local line ip code
|
||||
# Prefer IPv4 (wirewatch/libcurl often happier; avoid dead AAAA)
|
||||
line=$(podman exec "$ctr" getent ahostsv4 "$host" 2>/dev/null | head -1)
|
||||
[ -z "$line" ] && line=$(podman exec "$ctr" getent hosts "$host" 2>/dev/null | head -1)
|
||||
ip=$(echo "$line" | awk '{print $1}')
|
||||
if [ -z "$ip" ]; then
|
||||
emit "$comp" ERROR "dns $host" "no resolve — run pin-container-hosts.sh"
|
||||
return 1
|
||||
fi
|
||||
# 127.0.0.1 is almost always wrong for public bank/exchange from inside pasta
|
||||
if [ "$ip" = "127.0.0.1" ] || [ "$ip" = "::1" ]; then
|
||||
emit "$comp" ERROR "dns $host" "$ip (loopback — wirewatch will fail)"
|
||||
return 1
|
||||
fi
|
||||
code=$(podman exec "$ctr" curl -skS -m 3 -o /dev/null -w '%{http_code}' "https://${host}/config" 2>/dev/null || echo 000)
|
||||
if [ "$code" = "200" ]; then
|
||||
emit "$comp" OK "dns $host" "→ $ip /config=$code"
|
||||
else
|
||||
emit "$comp" WARN "dns $host" "→ $ip /config=$code"
|
||||
fi
|
||||
}
|
||||
|
||||
if [ -z "$BANK" ]; then emit bank ERROR container "not running"
|
||||
else
|
||||
emit bank INFO container "$(podman ps --filter name=$BANK --format '{{.Names}} {{.Status}}' | head -1)"
|
||||
emit bank INFO ports "$(podman ps --filter name=$BANK --format '{{.Ports}}' | head -1)"
|
||||
hasp "$BANK" 'MainKt serve|libeufin-bank serve' && emit bank OK libeufin "running" || emit bank ERROR libeufin "not running — API/withdraw dead"
|
||||
podman exec "$BANK" pg_isready -q 2>/dev/null && emit bank OK postgres "ready" || emit bank ERROR postgres "not ready"
|
||||
c=$(hc http://127.0.0.1:9012/config)
|
||||
[ "$c" = "200" ] && emit bank OK "local /config" "HTTP $c" || emit bank ERROR "local /config" "HTTP $c"
|
||||
c=$(hc http://127.0.0.1:9012/taler-integration/config)
|
||||
[ "$c" = "200" ] && emit bank OK "local integration" "HTTP $c" || emit bank ERROR "local integration" "HTTP $c"
|
||||
hasp "$BANK" 'nginx' && emit bank OK nginx ":9013" || emit bank WARN nginx "not running"
|
||||
check_dns bank "$BANK" bank.hacktivism.ch || true
|
||||
check_dns bank "$BANK" exchange.hacktivism.ch || true
|
||||
fi
|
||||
|
||||
if [ -z "$EX" ]; then emit exchange ERROR container "not running"
|
||||
else
|
||||
emit exchange INFO container "$(podman ps --filter name=$EX --format '{{.Names}} {{.Status}}' | head -1)"
|
||||
c=$(hc http://127.0.0.1:9011/config)
|
||||
[ "$c" = "200" ] && emit exchange OK "local /config" "HTTP $c" || emit exchange ERROR "local /config" "HTTP $c"
|
||||
c=$(curl -sS -m 5 -o /dev/null -w '%{http_code}' http://127.0.0.1:9011/keys 2>/dev/null || echo 000)
|
||||
[ "$c" = "200" ] && emit exchange OK "local /keys" "HTTP $c" || emit exchange ERROR "local /keys" "HTTP $c"
|
||||
hasp "$EX" 'taler-exchange-httpd' && emit exchange OK httpd "running" || emit exchange ERROR httpd "not running"
|
||||
hasp "$EX" 'taler-exchange-wirewatch' && emit exchange OK wirewatch "running" || emit exchange ERROR wirewatch "not running — withdraw stuck after bank confirm"
|
||||
hasp "$EX" 'taler-exchange-aggregator' && emit exchange OK aggregator "running" || emit exchange WARN aggregator "not running"
|
||||
hasp "$EX" 'taler-exchange-transfer' && emit exchange OK transfer "running" || emit exchange WARN transfer "not running"
|
||||
# critical for wire gateway
|
||||
check_dns exchange "$EX" bank.hacktivism.ch || true
|
||||
check_dns exchange "$EX" exchange.hacktivism.ch || true
|
||||
check_dns exchange "$EX" taler.hacktivism.ch || true
|
||||
fi
|
||||
|
||||
if [ -z "$MER" ]; then emit merchant ERROR container "not running"
|
||||
else
|
||||
emit merchant INFO container "$(podman ps --filter name=$MER --format '{{.Names}} {{.Status}}' | head -1)"
|
||||
c=$(hc https://127.0.0.1:9010/config)
|
||||
[ "$c" = "200" ] && emit merchant OK "local /config" "HTTP $c" || emit merchant ERROR "local /config" "HTTP $c"
|
||||
hasp "$MER" 'taler-merchant-httpd' && emit merchant OK httpd "running" || emit merchant ERROR httpd "not running"
|
||||
hasp "$MER" 'taler-merchant-wirewatch' && emit merchant OK wirewatch "running" || emit merchant WARN wirewatch "not running"
|
||||
hasp "$MER" 'taler-merchant-depositcheck' && emit merchant OK depositcheck "running" || emit merchant WARN depositcheck "not running"
|
||||
check_dns merchant "$MER" bank.hacktivism.ch || true
|
||||
check_dns merchant "$MER" exchange.hacktivism.ch || true
|
||||
check_dns merchant "$MER" taler.hacktivism.ch || true
|
||||
fi
|
||||
|
||||
if systemctl is-active caddy >/dev/null 2>&1 || pgrep -x caddy >/dev/null 2>&1; then
|
||||
emit caddy OK process "active"
|
||||
else
|
||||
emit caddy ERROR process "not active"
|
||||
fi
|
||||
echo DONE
|
||||
REMOTE
|
||||
)
|
||||
|
||||
if [ -z "$RAW" ] || ! echo "$RAW" | grep -q '^E|'; then
|
||||
err "ssh" "remote timed out or empty (cap ${SSH_CMD_TIMEOUT}s)"
|
||||
summary
|
||||
exit 1
|
||||
fi
|
||||
|
||||
while IFS= read -r line; do
|
||||
case "$line" in
|
||||
E\|*)
|
||||
IFS='|' read -r _ comp level key detail <<<"$line"
|
||||
case "$level" in
|
||||
OK) ok "[$comp] $key${detail:+ ($detail)}" ;;
|
||||
ERROR) err "$comp" "$key" "$detail" ;;
|
||||
WARN) warn "[$comp] $key" "$detail" ;;
|
||||
INFO) info "[$comp] $key" "$detail" ;;
|
||||
esac
|
||||
;;
|
||||
esac
|
||||
done <<<"$RAW"
|
||||
|
||||
summary
|
||||
Loading…
Add table
Add a link
Reference in a new issue