141 lines
6.5 KiB
Bash
Executable file
141 lines
6.5 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Inside status for bank / exchange / merchant. Hard-capped SSH — never hang forever.
|
|
set -euo pipefail
|
|
ROOT=$(cd "$(dirname "$0")" && pwd)
|
|
# shellcheck source=lib.sh
|
|
source "$ROOT/lib.sh"
|
|
|
|
# Area inside-### — container / process state on koopa (SSH)
|
|
set_area inside
|
|
section "inside · collect from koopa"
|
|
|
|
if [ "${SKIP_SSH}" = "1" ]; then
|
|
warn "ssh" "SKIP_SSH=1 — skipped"
|
|
summary
|
|
exit 0
|
|
fi
|
|
|
|
if ! koopa_ssh_ok; then
|
|
err "ssh" "cannot reach ${KOOPA_SSH} in ${SSH_CONNECT_TIMEOUT}s — set SKIP_SSH=1 to skip inside"
|
|
summary
|
|
exit 1
|
|
fi
|
|
ok "ssh ${KOOPA_SSH}"
|
|
|
|
# One short remote script (≤ SSH_CMD_TIMEOUT). Every slow step is local curl -m 3 or quick pgrep.
|
|
RAW=$(
|
|
koopa_ssh_bash "${SSH_CMD_TIMEOUT}" <<'REMOTE' || true
|
|
set +e
|
|
emit() { printf 'E|%s|%s|%s|%s\n' "$1" "$2" "$3" "$(printf '%s' "${4:-}" | tr '\n\r' ' ' | head -c 200)"; }
|
|
# quick curl
|
|
hc() { curl -skS -m 3 -o /tmp/mb -w '%{http_code}' "$1" 2>/dev/null || echo 000; }
|
|
# quick process check inside container (pgrep only)
|
|
hasp() { podman exec "$1" pgrep -f "$2" >/dev/null 2>&1; }
|
|
|
|
BANK=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -iE 'hacktivism-bank|taler-bank' | head -1)
|
|
[ -z "$BANK" ] && BANK=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -i bank | head -1)
|
|
EX=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -i exchange | head -1)
|
|
MER=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -E '^taler-hacktivism$' | head -1)
|
|
[ -z "$MER" ] && MER=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -iE 'merchant|hacktivism' | grep -viE 'bank|exchange' | head -1)
|
|
|
|
# Domain resolve inside container (wirewatch needs bank.hacktivism.ch → real IP)
|
|
# emit: comp LEVEL dns "host → ip" or fail
|
|
check_dns() {
|
|
local comp="$1" ctr="$2" host="$3"
|
|
local line ip code
|
|
# Prefer IPv4 (wirewatch/libcurl often happier; avoid dead AAAA)
|
|
line=$(podman exec "$ctr" getent ahostsv4 "$host" 2>/dev/null | head -1)
|
|
[ -z "$line" ] && line=$(podman exec "$ctr" getent hosts "$host" 2>/dev/null | head -1)
|
|
ip=$(echo "$line" | awk '{print $1}')
|
|
if [ -z "$ip" ]; then
|
|
emit "$comp" ERROR "dns $host" "no resolve — run pin-container-hosts.sh"
|
|
return 1
|
|
fi
|
|
# 127.0.0.1 is almost always wrong for public bank/exchange from inside pasta
|
|
if [ "$ip" = "127.0.0.1" ] || [ "$ip" = "::1" ]; then
|
|
emit "$comp" ERROR "dns $host" "$ip (loopback — wirewatch will fail)"
|
|
return 1
|
|
fi
|
|
code=$(podman exec "$ctr" curl -skS -m 3 -o /dev/null -w '%{http_code}' "https://${host}/config" 2>/dev/null || echo 000)
|
|
if [ "$code" = "200" ]; then
|
|
emit "$comp" OK "dns $host" "→ $ip /config=$code"
|
|
else
|
|
emit "$comp" WARN "dns $host" "→ $ip /config=$code"
|
|
fi
|
|
}
|
|
|
|
if [ -z "$BANK" ]; then emit bank ERROR container "not running"
|
|
else
|
|
emit bank INFO container "$(podman ps --filter name=$BANK --format '{{.Names}} {{.Status}}' | head -1)"
|
|
emit bank INFO ports "$(podman ps --filter name=$BANK --format '{{.Ports}}' | head -1)"
|
|
hasp "$BANK" 'MainKt serve|libeufin-bank serve' && emit bank OK libeufin "running" || emit bank ERROR libeufin "not running — API/withdraw dead"
|
|
podman exec "$BANK" pg_isready -q 2>/dev/null && emit bank OK postgres "ready" || emit bank ERROR postgres "not ready"
|
|
c=$(hc http://127.0.0.1:9012/config)
|
|
[ "$c" = "200" ] && emit bank OK "local /config" "HTTP $c" || emit bank ERROR "local /config" "HTTP $c"
|
|
c=$(hc http://127.0.0.1:9012/taler-integration/config)
|
|
[ "$c" = "200" ] && emit bank OK "local integration" "HTTP $c" || emit bank ERROR "local integration" "HTTP $c"
|
|
hasp "$BANK" 'nginx' && emit bank OK nginx ":9013" || emit bank WARN nginx "not running"
|
|
check_dns bank "$BANK" bank.hacktivism.ch || true
|
|
check_dns bank "$BANK" exchange.hacktivism.ch || true
|
|
fi
|
|
|
|
if [ -z "$EX" ]; then emit exchange ERROR container "not running"
|
|
else
|
|
emit exchange INFO container "$(podman ps --filter name=$EX --format '{{.Names}} {{.Status}}' | head -1)"
|
|
c=$(hc http://127.0.0.1:9011/config)
|
|
[ "$c" = "200" ] && emit exchange OK "local /config" "HTTP $c" || emit exchange ERROR "local /config" "HTTP $c"
|
|
c=$(curl -sS -m 5 -o /dev/null -w '%{http_code}' http://127.0.0.1:9011/keys 2>/dev/null || echo 000)
|
|
[ "$c" = "200" ] && emit exchange OK "local /keys" "HTTP $c" || emit exchange ERROR "local /keys" "HTTP $c"
|
|
hasp "$EX" 'taler-exchange-httpd' && emit exchange OK httpd "running" || emit exchange ERROR httpd "not running"
|
|
hasp "$EX" 'taler-exchange-wirewatch' && emit exchange OK wirewatch "running" || emit exchange ERROR wirewatch "not running — withdraw stuck after bank confirm"
|
|
hasp "$EX" 'taler-exchange-aggregator' && emit exchange OK aggregator "running" || emit exchange WARN aggregator "not running"
|
|
hasp "$EX" 'taler-exchange-transfer' && emit exchange OK transfer "running" || emit exchange WARN transfer "not running"
|
|
# critical for wire gateway
|
|
check_dns exchange "$EX" bank.hacktivism.ch || true
|
|
check_dns exchange "$EX" exchange.hacktivism.ch || true
|
|
check_dns exchange "$EX" taler.hacktivism.ch || true
|
|
fi
|
|
|
|
if [ -z "$MER" ]; then emit merchant ERROR container "not running"
|
|
else
|
|
emit merchant INFO container "$(podman ps --filter name=$MER --format '{{.Names}} {{.Status}}' | head -1)"
|
|
c=$(hc https://127.0.0.1:9010/config)
|
|
[ "$c" = "200" ] && emit merchant OK "local /config" "HTTP $c" || emit merchant ERROR "local /config" "HTTP $c"
|
|
hasp "$MER" 'taler-merchant-httpd' && emit merchant OK httpd "running" || emit merchant ERROR httpd "not running"
|
|
hasp "$MER" 'taler-merchant-wirewatch' && emit merchant OK wirewatch "running" || emit merchant WARN wirewatch "not running"
|
|
hasp "$MER" 'taler-merchant-depositcheck' && emit merchant OK depositcheck "running" || emit merchant WARN depositcheck "not running"
|
|
check_dns merchant "$MER" bank.hacktivism.ch || true
|
|
check_dns merchant "$MER" exchange.hacktivism.ch || true
|
|
check_dns merchant "$MER" taler.hacktivism.ch || true
|
|
fi
|
|
|
|
if systemctl is-active caddy >/dev/null 2>&1 || pgrep -x caddy >/dev/null 2>&1; then
|
|
emit caddy OK process "active"
|
|
else
|
|
emit caddy ERROR process "not active"
|
|
fi
|
|
echo DONE
|
|
REMOTE
|
|
)
|
|
|
|
if [ -z "$RAW" ] || ! echo "$RAW" | grep -q '^E|'; then
|
|
err "ssh" "remote timed out or empty (cap ${SSH_CMD_TIMEOUT}s)"
|
|
summary
|
|
exit 1
|
|
fi
|
|
|
|
while IFS= read -r line; do
|
|
case "$line" in
|
|
E\|*)
|
|
IFS='|' read -r _ comp level key detail <<<"$line"
|
|
case "$level" in
|
|
OK) ok "[$comp] $key${detail:+ ($detail)}" ;;
|
|
ERROR) err "$comp" "$key" "$detail" ;;
|
|
WARN) warn "[$comp] $key" "$detail" ;;
|
|
INFO) info "[$comp] $key" "$detail" ;;
|
|
esac
|
|
;;
|
|
esac
|
|
done <<<"$RAW"
|
|
|
|
summary
|