596 lines
24 KiB
Bash
Executable file
596 lines
24 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Inside status for bank / exchange / merchant containers.
|
|
#
|
|
# Access modes (shown in the log as flags):
|
|
# host-podman — this host runs podman exec into containers (no SSH).
|
|
# INSIDE_PODMAN=1 or INSIDE_MODE=local-podman, or auto on GOA host.
|
|
# ssh — from a laptop/remote: SSH to KOOPA_SSH / INSIDE_SSH, then podman there.
|
|
#
|
|
# Profiles:
|
|
# koopa (default LOCAL_STACK=1) — taler-hacktivism* containers
|
|
# stage-lfp (TESTPAYSAN) — SSH INSIDE_SSH (stagepaysan), low-priv podman
|
|
set -euo pipefail
|
|
ROOT=$(cd "$(dirname "$0")" && pwd)
|
|
# shellcheck source=lib.sh
|
|
source "$ROOT/lib.sh"
|
|
# shellcheck source=metrics.sh
|
|
source "$ROOT/metrics.sh"
|
|
|
|
set_area inside
|
|
|
|
PROFILE="${INSIDE_PROFILE:-}"
|
|
if [ -z "$PROFILE" ]; then
|
|
if [ "${LOCAL_STACK:-0}" = "1" ]; then
|
|
PROFILE=koopa
|
|
elif [ "${EXPECT_CURRENCY:-}" = "TESTPAYSAN" ] || [ -n "${INSIDE_SSH:-}" ]; then
|
|
PROFILE=stage-lfp
|
|
else
|
|
PROFILE=koopa
|
|
fi
|
|
fi
|
|
|
|
# Resolve host-podman vs ssh before first check ID.
|
|
# stage-lfp on the FP host itself uses local podman (INSIDE_PODMAN=1 /
|
|
# INSIDE_MODE=local-podman); laptop → stage still uses INSIDE_SSH.
|
|
_use_local_podman=0
|
|
if [ "${INSIDE_PODMAN:-0}" = "1" ] || [ "${INSIDE_MODE:-}" = "local-podman" ]; then
|
|
_use_local_podman=1
|
|
elif [ "$PROFILE" != "stage-lfp" ]; then
|
|
if command -v podman >/dev/null 2>&1 \
|
|
&& podman ps --format '{{.Names}}' 2>/dev/null | grep -qE 'taler-hacktivism'; then
|
|
_use_local_podman=1
|
|
fi
|
|
fi
|
|
if [ "$_use_local_podman" = "1" ]; then
|
|
INSIDE_ACCESS=host-podman
|
|
set_group host
|
|
else
|
|
INSIDE_ACCESS=ssh
|
|
set_group ssh
|
|
fi
|
|
export INSIDE_ACCESS
|
|
|
|
section "inside · collect (${PROFILE} · access=${INSIDE_ACCESS})"
|
|
info "flags" "INSIDE_ACCESS=${INSIDE_ACCESS} INSIDE_PODMAN=${INSIDE_PODMAN:-0} INSIDE_MODE=${INSIDE_MODE:-} LOCAL_STACK=${LOCAL_STACK:-0} SKIP_SSH=${SKIP_SSH:-0} KOOPA_SSH=${KOOPA_SSH:-} INSIDE_SSH=${INSIDE_SSH:-}"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# stage-lfp: stagepaysan podman (host-local or SSH INSIDE_SSH from laptop)
|
|
# ---------------------------------------------------------------------------
|
|
if [ "$PROFILE" = "stage-lfp" ]; then
|
|
SSH_HOST="${INSIDE_SSH:-}"
|
|
BANK_CTR="${INSIDE_BANK_CTR:-stage-lfp-bank}"
|
|
EX_CTR="${INSIDE_EXCHANGE_CTR:-stage-lfp-exchange-ansible}"
|
|
MER_CTR="${INSIDE_MERCHANT_CTR:-stage-lfp-merchant}"
|
|
BANK_PORT="${INSIDE_BANK_PORT:-9032}"
|
|
EX_PORT="${INSIDE_EXCHANGE_PORT:-9031}"
|
|
MER_PORT="${INSIDE_MERCHANT_PORT:-9030}"
|
|
DNS_BANK="${INSIDE_DNS_BANK:-stage.bank.lefrancpaysan.ch}"
|
|
DNS_EX="${INSIDE_DNS_EXCHANGE:-stage.exchange.lefrancpaysan.ch}"
|
|
DNS_MER="${INSIDE_DNS_MERCHANT:-stage.monnaie.lefrancpaysan.ch}"
|
|
# Stage remote has more podman execs than koopa; allow a bit more wall time
|
|
STAGE_SSH_T="${INSIDE_SSH_TIMEOUT:-${SSH_CMD_TIMEOUT:-24}}"
|
|
if [ "${STAGE_SSH_T}" -lt 24 ] 2>/dev/null; then STAGE_SSH_T=24; fi
|
|
|
|
if [ "$_use_local_podman" = "1" ]; then
|
|
if ! command -v podman >/dev/null 2>&1; then
|
|
err "host" "INSIDE_PODMAN/host-podman but podman missing"
|
|
summary
|
|
exit 1
|
|
fi
|
|
ok "host→container" "podman exec on this host (INSIDE_ACCESS=host-podman · no SSH)"
|
|
elif ! mon_ssh_ok "$SSH_HOST"; then
|
|
err "ssh" "cannot reach ${SSH_HOST:-?} (stagepaysan low-priv) — set INSIDE_SSH= or INSIDE_PODMAN=1"
|
|
summary
|
|
exit 1
|
|
else
|
|
ok "ssh ${SSH_HOST}" "stagepaysan (podman, no sudo)"
|
|
fi
|
|
|
|
# Inject names/ports (ssh bash -s does not inherit local env; local bash does).
|
|
_stage_lfp_script() {
|
|
{
|
|
printf 'BANK_CTR=%q; EX_CTR=%q; MER_CTR=%q\n' "$BANK_CTR" "$EX_CTR" "$MER_CTR"
|
|
printf 'BANK_PORT=%q; EX_PORT=%q; MER_PORT=%q\n' "$BANK_PORT" "$EX_PORT" "$MER_PORT"
|
|
printf 'DNS_BANK=%q; DNS_EX=%q; DNS_MER=%q\n' "$DNS_BANK" "$DNS_EX" "$DNS_MER"
|
|
cat <<'REMOTE'
|
|
set +e
|
|
emit() { printf 'E|%s|%s|%s|%s\n' "$1" "$2" "$3" "$(printf '%s' "${4:-}" | tr '\n\r' ' ' | head -c 200)"; }
|
|
hc() { curl -skS -m 3 -o /tmp/mb -w '%{http_code}' "$1" 2>/dev/null || echo 000; }
|
|
hasp() { podman exec "$1" pgrep -f "$2" >/dev/null 2>&1; }
|
|
|
|
BANK="${BANK_CTR}"
|
|
EX="${EX_CTR}"
|
|
MER="${MER_CTR}"
|
|
podman ps --format '{{.Names}}' 2>/dev/null | grep -qx "$BANK" || BANK=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -iE 'stage.*bank|lfp-bank' | head -1)
|
|
podman ps --format '{{.Names}}' 2>/dev/null | grep -qx "$EX" || EX=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -iE 'stage.*exchange|lfp-exchange' | head -1)
|
|
podman ps --format '{{.Names}}' 2>/dev/null | grep -qx "$MER" || MER=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -iE 'stage.*merchant|lfp-merchant' | head -1)
|
|
|
|
check_dns() {
|
|
local comp="$1" ctr="$2" host="$3"
|
|
local line ip code
|
|
[ -n "$host" ] || { emit "$comp" WARN "dns" "empty host"; return 1; }
|
|
line=$(podman exec "$ctr" getent ahostsv4 "$host" 2>/dev/null | head -1)
|
|
[ -z "$line" ] && line=$(podman exec "$ctr" getent hosts "$host" 2>/dev/null | head -1)
|
|
ip=$(echo "$line" | awk '{print $1}')
|
|
if [ -z "$ip" ]; then
|
|
emit "$comp" WARN "dns $host" "no resolve inside container"
|
|
return 1
|
|
fi
|
|
if [ "$ip" = "127.0.0.1" ] || [ "$ip" = "::1" ]; then
|
|
emit "$comp" WARN "dns $host" "$ip (loopback)"
|
|
return 1
|
|
fi
|
|
# 169.254.x = pasta host-gateway pin is OK for stage
|
|
code=$(podman exec "$ctr" curl -skS -m 3 -o /dev/null -w '%{http_code}' "https://${host}/config" 2>/dev/null || echo 000)
|
|
if [ "$code" = "200" ]; then
|
|
emit "$comp" OK "dns $host" "→ $ip /config=$code"
|
|
else
|
|
emit "$comp" WARN "dns $host" "→ $ip /config=$code"
|
|
fi
|
|
}
|
|
|
|
if [ -z "$BANK" ]; then emit bank ERROR container "not running"
|
|
else
|
|
emit bank INFO container "$(podman ps --filter name=^${BANK}$ --format '{{.Names}} {{.Status}}' | head -1)"
|
|
emit bank INFO ports "$(podman ps --filter name=^${BANK}$ --format '{{.Ports}}' | head -1)"
|
|
hasp "$BANK" 'MainKt serve|libeufin-bank serve' && emit bank OK libeufin "running" || emit bank ERROR libeufin "not running"
|
|
podman exec "$BANK" pg_isready -q 2>/dev/null && emit bank OK postgres "ready" || emit bank WARN postgres "pg_isready failed"
|
|
c=$(hc "http://127.0.0.1:${BANK_PORT}/config")
|
|
[ "$c" = "200" ] && emit bank OK "host :${BANK_PORT}/config" "HTTP $c" || emit bank ERROR "host :${BANK_PORT}/config" "HTTP $c"
|
|
c=$(hc "http://127.0.0.1:${BANK_PORT}/taler-integration/config")
|
|
[ "$c" = "200" ] && emit bank OK "host integration" "HTTP $c" || emit bank ERROR "host integration" "HTTP $c"
|
|
check_dns bank "$BANK" "$DNS_BANK" || true
|
|
check_dns bank "$BANK" "$DNS_EX" || true
|
|
fi
|
|
|
|
if [ -z "$EX" ]; then emit exchange ERROR container "not running"
|
|
else
|
|
emit exchange INFO container "$(podman ps --filter name=^${EX}$ --format '{{.Names}} {{.Status}}' | head -1)"
|
|
c=$(hc "http://127.0.0.1:${EX_PORT}/config")
|
|
[ "$c" = "200" ] && emit exchange OK "host :${EX_PORT}/config" "HTTP $c" || emit exchange ERROR "host :${EX_PORT}/config" "HTTP $c"
|
|
c=$(curl -sS -m 5 -o /dev/null -w '%{http_code}' "http://127.0.0.1:${EX_PORT}/keys" 2>/dev/null || echo 000)
|
|
[ "$c" = "200" ] && emit exchange OK "host :${EX_PORT}/keys" "HTTP $c" || emit exchange ERROR "host :${EX_PORT}/keys" "HTTP $c"
|
|
hasp "$EX" 'taler-exchange-httpd' && emit exchange OK httpd "running" || emit exchange ERROR httpd "not running"
|
|
hasp "$EX" 'taler-exchange-wirewatch' && emit exchange OK wirewatch "running" || emit exchange ERROR wirewatch "not running"
|
|
hasp "$EX" 'taler-exchange-aggregator' && emit exchange OK aggregator "running" || emit exchange WARN aggregator "not running"
|
|
hasp "$EX" 'taler-exchange-transfer' && emit exchange OK transfer "running" || emit exchange WARN transfer "not running"
|
|
check_dns exchange "$EX" "$DNS_BANK" || true
|
|
check_dns exchange "$EX" "$DNS_EX" || true
|
|
check_dns exchange "$EX" "$DNS_MER" || true
|
|
fi
|
|
|
|
if [ -z "$MER" ]; then emit merchant ERROR container "not running"
|
|
else
|
|
emit merchant INFO container "$(podman ps --filter name=^${MER}$ --format '{{.Names}} {{.Status}}' | head -1)"
|
|
c=$(hc "http://127.0.0.1:${MER_PORT}/config")
|
|
[ "$c" = "200" ] && emit merchant OK "host :${MER_PORT}/config" "HTTP $c" || emit merchant ERROR "host :${MER_PORT}/config" "HTTP $c"
|
|
hasp "$MER" 'taler-merchant-httpd' && emit merchant OK httpd "running" || emit merchant ERROR httpd "not running"
|
|
# Stage merchant often has no wirewatch unit — INFO not WARN
|
|
hasp "$MER" 'taler-merchant-wirewatch' && emit merchant OK wirewatch "running" || emit merchant INFO wirewatch "not running (optional on stage)"
|
|
hasp "$MER" 'taler-merchant-depositcheck' && emit merchant OK depositcheck "running" || emit merchant INFO depositcheck "not running (optional on stage)"
|
|
hasp "$MER" 'taler-merchant-exchangekeyupdate' && emit merchant OK exchangekeyupdate "running" || emit merchant INFO exchangekeyupdate "not running"
|
|
check_dns merchant "$MER" "$DNS_BANK" || true
|
|
check_dns merchant "$MER" "$DNS_EX" || true
|
|
check_dns merchant "$MER" "$DNS_MER" || true
|
|
fi
|
|
|
|
# Caddy is host-wide (often root); stagepaysan can only see the process
|
|
if pgrep -x caddy >/dev/null 2>&1; then
|
|
emit caddy OK process "running (host)"
|
|
else
|
|
emit caddy WARN process "not seen as stagepaysan (may still run as root)"
|
|
fi
|
|
|
|
echo DONE
|
|
REMOTE
|
|
}
|
|
}
|
|
if [ "$_use_local_podman" = "1" ]; then
|
|
RAW=$(_stage_lfp_script | bash || true)
|
|
else
|
|
RAW=$(_stage_lfp_script | mon_ssh_bash "$SSH_HOST" "${STAGE_SSH_T}" || true)
|
|
fi
|
|
|
|
if [ -z "$RAW" ] || ! echo "$RAW" | grep -q '^E|'; then
|
|
if [ "$_use_local_podman" = "1" ]; then
|
|
err "host" "stage collect timed out or empty (local podman)"
|
|
else
|
|
err "ssh" "stage remote timed out or empty (cap ${STAGE_SSH_T}s · host=${SSH_HOST})"
|
|
fi
|
|
summary
|
|
exit 1
|
|
fi
|
|
|
|
_last_inside_grp=""
|
|
while IFS= read -r line; do
|
|
case "$line" in
|
|
E\|*)
|
|
IFS='|' read -r _ comp level key detail <<<"$line"
|
|
case "$comp" in
|
|
bank|exchange|merchant|caddy) _g="$comp" ;;
|
|
*) _g="$INSIDE_ACCESS" ;;
|
|
esac
|
|
if [ "$_g" != "$_last_inside_grp" ]; then
|
|
set_group "$_g"
|
|
_last_inside_grp="$_g"
|
|
fi
|
|
case "$level" in
|
|
OK) ok "[$comp] $key${detail:+ ($detail)}" ;;
|
|
ERROR) err "$comp" "$key" "$detail" ;;
|
|
WARN) warn "[$comp] $key" "$detail" ;;
|
|
INFO) info "[$comp] $key" "$detail" ;;
|
|
esac
|
|
;;
|
|
esac
|
|
done <<<"$RAW"
|
|
|
|
# Stats from outside (laptop) — public HTTPS, no stagepaysan write needed
|
|
set_group stats
|
|
section "inside · public stats (outside-in)"
|
|
_probe_public_stats() {
|
|
local comp="$1" url="$2"
|
|
local body code age unix now
|
|
body=$(mktemp)
|
|
code=$(curl -skS -m 8 -o "$body" -w '%{http_code}' "$url" 2>/dev/null || echo 000)
|
|
if [ "$code" != "200" ]; then
|
|
warn "[$comp] public stats.json" "HTTP $code · $url"
|
|
rm -f "$body"
|
|
return
|
|
fi
|
|
unix=$(python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); print(d.get("generated_at_unix") or 0)' "$body" 2>/dev/null || echo 0)
|
|
now=$(date +%s)
|
|
if [ "${unix:-0}" -gt 0 ] 2>/dev/null; then
|
|
age=$((now - unix))
|
|
if [ "$age" -lt 0 ]; then age=0; fi
|
|
if [ "$age" -le "${STATS_STALE_SECS:-900}" ]; then
|
|
ok "[$comp] public stats.json" "HTTP 200 age=${age}s · $url"
|
|
elif [ "$age" -le "${STATS_FAIL_SECS:-3600}" ]; then
|
|
warn "[$comp] public stats.json" "stale age=${age}s · $url"
|
|
else
|
|
err "$comp" "public stats.json" "too old age=${age}s · $url"
|
|
fi
|
|
else
|
|
ok "[$comp] public stats.json" "HTTP 200 · $url"
|
|
fi
|
|
rm -f "$body"
|
|
}
|
|
_probe_public_stats bank "https://${DNS_BANK}/intro/stats.json"
|
|
_probe_public_stats exchange "https://${DNS_EX}/intro/stats.json"
|
|
_probe_public_stats merchant "https://${DNS_MER}/intro/stats.json"
|
|
|
|
# Host load as stagepaysan (no container RSS from koopa metrics)
|
|
set_group load
|
|
section "inside · load (stagepaysan host)"
|
|
_load_py() {
|
|
python3 - <<'PY'
|
|
import os
|
|
la=os.getloadavg()
|
|
print("loadavg=%.2f,%.2f,%.2f" % la)
|
|
try:
|
|
with open("/proc/meminfo") as f:
|
|
d={}
|
|
for line in f:
|
|
k,v=line.split(":")[0], line.split(":")[1].strip().split()[0]
|
|
d[k]=int(v)
|
|
total=d.get("MemTotal",0)/1024/1024
|
|
avail=d.get("MemAvailable",0)/1024/1024
|
|
used=total-avail
|
|
print("mem_used=%.2fGiB avail=%.2fGiB total=%.2fGiB" % (used, avail, total))
|
|
except Exception:
|
|
print("mem=?")
|
|
PY
|
|
}
|
|
if [ "$_use_local_podman" = "1" ]; then
|
|
LOAD_LINE=$(_load_py || true)
|
|
else
|
|
LOAD_LINE=$(mon_ssh_bash "$SSH_HOST" 8 <<'EOF' || true
|
|
python3 - <<'PY'
|
|
import os
|
|
la=os.getloadavg()
|
|
print("loadavg=%.2f,%.2f,%.2f" % la)
|
|
try:
|
|
with open("/proc/meminfo") as f:
|
|
d={}
|
|
for line in f:
|
|
k,v=line.split(":")[0], line.split(":")[1].strip().split()[0]
|
|
d[k]=int(v)
|
|
total=d.get("MemTotal",0)/1024/1024
|
|
avail=d.get("MemAvailable",0)/1024/1024
|
|
used=total-avail
|
|
print("mem_used=%.2fGiB avail=%.2fGiB total=%.2fGiB" % (used, avail, total))
|
|
except Exception:
|
|
print("mem=?")
|
|
PY
|
|
EOF
|
|
)
|
|
fi
|
|
if [ -n "$LOAD_LINE" ]; then
|
|
info "stage host" "$(echo "$LOAD_LINE" | tr '\n' ' ')"
|
|
else
|
|
info "stage host" "load probe empty"
|
|
fi
|
|
|
|
set_group disk
|
|
section "inside · disk free space (stage host + containers)"
|
|
_disk_script() {
|
|
cat <<'DISK'
|
|
set +e
|
|
echo "###HOST###"
|
|
df -Pk / /var /home /tmp /mnt/data 2>/dev/null || df -Pk
|
|
echo "###CTRS###"
|
|
for c in $(podman ps --format '{{.Names}}' 2>/dev/null); do
|
|
echo "###CTR $c###"
|
|
podman exec "$c" df -Pk / /var /tmp 2>/dev/null || podman exec "$c" df -Pk 2>/dev/null
|
|
done
|
|
DISK
|
|
}
|
|
if [ "$_use_local_podman" = "1" ]; then
|
|
_disk_raw=$(_disk_script | bash || true)
|
|
_disk_label="host"
|
|
else
|
|
_disk_raw=$(_disk_script | mon_ssh_bash "$SSH_HOST" "${STAGE_SSH_T:-24}" || true)
|
|
_disk_label="ssh:${SSH_HOST}"
|
|
fi
|
|
_host_df=$(printf '%s\n' "$_disk_raw" | sed -n '/^###HOST###$/,/^###CTRS###$/p' | sed '1d;$d')
|
|
mon_disk_check_remote_text "$_disk_label" "$_host_df" || true
|
|
_ctr=""; _buf=""
|
|
while IFS= read -r _line || [ -n "$_line" ]; do
|
|
case "$_line" in
|
|
'###CTR '*)
|
|
if [ -n "$_ctr" ]; then mon_disk_check_remote_text "ctr:${_ctr}" "$_buf" || true; fi
|
|
_ctr=${_line####CTR }; _ctr=${_ctr%###}; _buf=""
|
|
;;
|
|
'###CTRS###'|'###HOST###') ;;
|
|
*) [ -n "$_ctr" ] && _buf="${_buf}${_line}"$'\n' ;;
|
|
esac
|
|
done <<<"$_disk_raw"
|
|
[ -n "$_ctr" ] && mon_disk_check_remote_text "ctr:${_ctr}" "$_buf" || true
|
|
|
|
summary
|
|
exit 0
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# koopa (default) — host-podman exec (on GOA host) or SSH to KOOPA_SSH (laptop)
|
|
# ---------------------------------------------------------------------------
|
|
section "inside · collect from koopa (access=${INSIDE_ACCESS})"
|
|
|
|
if [ "$_use_local_podman" = "1" ]; then
|
|
set_group host
|
|
if ! command -v podman >/dev/null 2>&1; then
|
|
err "host" "INSIDE_PODMAN/host-podman but podman missing"
|
|
summary
|
|
exit 1
|
|
fi
|
|
ok "host→container" "podman exec on this host (INSIDE_ACCESS=host-podman · no SSH)"
|
|
elif [ "${SKIP_SSH}" = "1" ] && [ "${LOCAL_STACK:-0}" != "1" ]; then
|
|
set_group ssh
|
|
warn "ssh" "SKIP_SSH=1 and not host-podman — skipped"
|
|
summary
|
|
exit 0
|
|
elif ! koopa_ssh_ok; then
|
|
set_group ssh
|
|
err "ssh" "cannot reach ${KOOPA_SSH} — from laptop use KOOPA_SSH=; on host set INSIDE_PODMAN=1"
|
|
summary
|
|
exit 1
|
|
else
|
|
set_group ssh
|
|
ok "ssh ${KOOPA_SSH}" "remote host then podman (INSIDE_ACCESS=ssh · laptop/remote)"
|
|
fi
|
|
|
|
# Collect script (file → bash local or ssh bash -s)
|
|
_INSIDE_SCRIPT=$(mktemp)
|
|
trap 'rm -f "${_INSIDE_SCRIPT:-}"' RETURN
|
|
cat >"$_INSIDE_SCRIPT" <<'REMOTE'
|
|
set +e
|
|
emit() { printf 'E|%s|%s|%s|%s\n' "$1" "$2" "$3" "$(printf '%s' "${4:-}" | tr '\n\r' ' ' | head -c 200)"; }
|
|
# quick curl
|
|
hc() { curl -skS -m 3 -o /tmp/mb -w '%{http_code}' "$1" 2>/dev/null || echo 000; }
|
|
# quick process check inside container (pgrep only)
|
|
hasp() { podman exec "$1" pgrep -f "$2" >/dev/null 2>&1; }
|
|
|
|
BANK=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -iE 'hacktivism-bank|taler-bank' | head -1)
|
|
[ -z "$BANK" ] && BANK=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -i bank | head -1)
|
|
EX=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -i exchange | head -1)
|
|
# Exact merchant container name first — never fall through to *-bank
|
|
MER=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -E '^taler-hacktivism$' | head -1)
|
|
[ -z "$MER" ] && MER=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -i merchant | grep -viE 'bank|exchange' | head -1)
|
|
[ -z "$MER" ] && MER=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -E 'hacktivism' | grep -viE 'bank|exchange' | head -1)
|
|
|
|
# Domain resolve inside container (wirewatch needs bank.hacktivism.ch → real IP)
|
|
# emit: comp LEVEL dns "host → ip" or fail
|
|
check_dns() {
|
|
local comp="$1" ctr="$2" host="$3"
|
|
local line ip code
|
|
# Prefer IPv4 (wirewatch/libcurl often happier; avoid dead AAAA)
|
|
line=$(podman exec "$ctr" getent ahostsv4 "$host" 2>/dev/null | head -1)
|
|
[ -z "$line" ] && line=$(podman exec "$ctr" getent hosts "$host" 2>/dev/null | head -1)
|
|
ip=$(echo "$line" | awk '{print $1}')
|
|
if [ -z "$ip" ]; then
|
|
emit "$comp" ERROR "dns $host" "no resolve — run pin-container-hosts.sh"
|
|
return 1
|
|
fi
|
|
# 127.0.0.1 is almost always wrong for public bank/exchange from inside pasta
|
|
if [ "$ip" = "127.0.0.1" ] || [ "$ip" = "::1" ]; then
|
|
emit "$comp" ERROR "dns $host" "$ip (loopback — wirewatch will fail)"
|
|
return 1
|
|
fi
|
|
code=$(podman exec "$ctr" curl -skS -m 3 -o /dev/null -w '%{http_code}' "https://${host}/config" 2>/dev/null || echo 000)
|
|
if [ "$code" = "200" ]; then
|
|
emit "$comp" OK "dns $host" "→ $ip /config=$code"
|
|
else
|
|
emit "$comp" WARN "dns $host" "→ $ip /config=$code"
|
|
fi
|
|
}
|
|
|
|
if [ -z "$BANK" ]; then emit bank ERROR container "not running"
|
|
else
|
|
emit bank INFO container "$(podman ps --filter name=$BANK --format '{{.Names}} {{.Status}}' | head -1)"
|
|
emit bank INFO ports "$(podman ps --filter name=$BANK --format '{{.Ports}}' | head -1)"
|
|
hasp "$BANK" 'MainKt serve|libeufin-bank serve' && emit bank OK libeufin "running" || emit bank ERROR libeufin "not running — API/withdraw dead"
|
|
podman exec "$BANK" pg_isready -q 2>/dev/null && emit bank OK postgres "ready" || emit bank ERROR postgres "not ready"
|
|
c=$(hc http://127.0.0.1:9012/config)
|
|
[ "$c" = "200" ] && emit bank OK "local /config" "HTTP $c" || emit bank ERROR "local /config" "HTTP $c"
|
|
c=$(hc http://127.0.0.1:9012/taler-integration/config)
|
|
[ "$c" = "200" ] && emit bank OK "local integration" "HTTP $c" || emit bank ERROR "local integration" "HTTP $c"
|
|
hasp "$BANK" 'nginx' && emit bank OK nginx ":9013" || emit bank WARN nginx "not running"
|
|
check_dns bank "$BANK" bank.hacktivism.ch || true
|
|
check_dns bank "$BANK" exchange.hacktivism.ch || true
|
|
fi
|
|
|
|
if [ -z "$EX" ]; then emit exchange ERROR container "not running"
|
|
else
|
|
emit exchange INFO container "$(podman ps --filter name=$EX --format '{{.Names}} {{.Status}}' | head -1)"
|
|
c=$(hc http://127.0.0.1:9011/config)
|
|
[ "$c" = "200" ] && emit exchange OK "local /config" "HTTP $c" || emit exchange ERROR "local /config" "HTTP $c"
|
|
c=$(curl -sS -m 5 -o /dev/null -w '%{http_code}' http://127.0.0.1:9011/keys 2>/dev/null || echo 000)
|
|
[ "$c" = "200" ] && emit exchange OK "local /keys" "HTTP $c" || emit exchange ERROR "local /keys" "HTTP $c"
|
|
hasp "$EX" 'taler-exchange-httpd' && emit exchange OK httpd "running" || emit exchange ERROR httpd "not running"
|
|
hasp "$EX" 'taler-exchange-wirewatch' && emit exchange OK wirewatch "running" || emit exchange ERROR wirewatch "not running — withdraw stuck after bank confirm"
|
|
hasp "$EX" 'taler-exchange-aggregator' && emit exchange OK aggregator "running" || emit exchange WARN aggregator "not running"
|
|
hasp "$EX" 'taler-exchange-transfer' && emit exchange OK transfer "running" || emit exchange WARN transfer "not running"
|
|
# critical for wire gateway
|
|
check_dns exchange "$EX" bank.hacktivism.ch || true
|
|
check_dns exchange "$EX" exchange.hacktivism.ch || true
|
|
check_dns exchange "$EX" taler.hacktivism.ch || true
|
|
fi
|
|
|
|
if [ -z "$MER" ]; then emit merchant ERROR container "not running"
|
|
else
|
|
emit merchant INFO container "$(podman ps --filter name=$MER --format '{{.Names}} {{.Status}}' | head -1)"
|
|
c=$(hc https://127.0.0.1:9010/config)
|
|
[ "$c" = "200" ] && emit merchant OK "local /config" "HTTP $c" || emit merchant ERROR "local /config" "HTTP $c"
|
|
hasp "$MER" 'taler-merchant-httpd' && emit merchant OK httpd "running" || emit merchant ERROR httpd "not running"
|
|
hasp "$MER" 'taler-merchant-wirewatch' && emit merchant OK wirewatch "running" || emit merchant WARN wirewatch "not running"
|
|
# depositcheck: may abort on exchange track 404 (merchant assertion) — try ensure once
|
|
if hasp "$MER" 'taler-merchant-depositcheck'; then
|
|
emit merchant OK depositcheck "running"
|
|
else
|
|
podman exec "$MER" /usr/local/bin/ensure_merchant_helpers.sh >/dev/null 2>&1 || true
|
|
sleep 1
|
|
if hasp "$MER" 'taler-merchant-depositcheck'; then
|
|
emit merchant OK depositcheck "running (after ensure_merchant_helpers)"
|
|
elif hasp "$MER" 'taler-merchant-wirewatch'; then
|
|
# Known: depositcheck can SIGABRT on exchange 404 for stale coins — not startable cleanly
|
|
emit merchant INFO depositcheck "not running (often aborts on track 404; wirewatch OK · settlement partial)"
|
|
else
|
|
emit merchant WARN depositcheck "not running"
|
|
fi
|
|
fi
|
|
check_dns merchant "$MER" bank.hacktivism.ch || true
|
|
check_dns merchant "$MER" exchange.hacktivism.ch || true
|
|
check_dns merchant "$MER" taler.hacktivism.ch || true
|
|
fi
|
|
|
|
if systemctl is-active caddy >/dev/null 2>&1 || pgrep -x caddy >/dev/null 2>&1; then
|
|
emit caddy OK process "active"
|
|
else
|
|
emit caddy ERROR process "not active"
|
|
fi
|
|
echo DONE
|
|
REMOTE
|
|
|
|
if [ "$_use_local_podman" = "1" ]; then
|
|
RAW=$(bash "$_INSIDE_SCRIPT" || true)
|
|
else
|
|
RAW=$(koopa_ssh_bash "${SSH_CMD_TIMEOUT}" <"$_INSIDE_SCRIPT" || true)
|
|
fi
|
|
rm -f "$_INSIDE_SCRIPT"
|
|
|
|
if [ -z "$RAW" ] || ! echo "$RAW" | grep -q '^E|'; then
|
|
if [ "$_use_local_podman" = "1" ]; then
|
|
err "host" "collect empty (INSIDE_ACCESS=host-podman · podman exec failed)"
|
|
else
|
|
err "ssh" "collect empty (INSIDE_ACCESS=ssh · cap ${SSH_CMD_TIMEOUT}s · host=${KOOPA_SSH})"
|
|
fi
|
|
summary
|
|
exit 1
|
|
fi
|
|
|
|
# Fallback group for non-component rows: host (podman) vs ssh (remote)
|
|
_access_grp=ssh
|
|
[ "$_use_local_podman" = "1" ] && _access_grp=host
|
|
|
|
_last_inside_grp=""
|
|
while IFS= read -r line; do
|
|
case "$line" in
|
|
E\|*)
|
|
IFS='|' read -r _ comp level key detail <<<"$line"
|
|
# Group IDs by component: inside.bank-02, inside.exchange-04; access = host|ssh
|
|
case "$comp" in
|
|
bank|exchange|merchant|caddy) _g="$comp" ;;
|
|
*) _g="$_access_grp" ;;
|
|
esac
|
|
if [ "$_g" != "$_last_inside_grp" ]; then
|
|
set_group "$_g"
|
|
_last_inside_grp="$_g"
|
|
fi
|
|
case "$level" in
|
|
OK) ok "[$comp] $key${detail:+ ($detail)}" ;;
|
|
ERROR) err "$comp" "$key" "$detail" ;;
|
|
WARN) warn "[$comp] $key" "$detail" ;;
|
|
INFO) info "[$comp] $key" "$detail" ;;
|
|
esac
|
|
;;
|
|
esac
|
|
done <<<"$RAW"
|
|
|
|
# Host loadavg + RAM + per-container RSS/CPU (same probe as e2e/ladder)
|
|
set_group load
|
|
section "inside · load / memory"
|
|
METRICS_DIR="${METRICS_DIR:-$(mktemp -d)}"
|
|
export METRICS_DIR
|
|
metrics_report_load "${METRICS_DIR}/load-inside.json" "inside" || true
|
|
|
|
# Disk free space: host + taler containers (WARN tight, ERROR full)
|
|
set_group disk
|
|
section "inside · disk free space"
|
|
_disk_ec=0
|
|
if [ "$_use_local_podman" = "1" ]; then
|
|
mon_disk_check_host "host" || _disk_ec=1
|
|
while read -r _cname; do
|
|
[ -n "$_cname" ] || continue
|
|
mon_disk_check_podman "$_cname" || _disk_ec=1
|
|
done < <(podman ps --format '{{.Names}}' 2>/dev/null | grep -iE 'hacktivism|taler-|stage-lfp|lfp-' || true)
|
|
else
|
|
# remote host via SSH: df on host + each container
|
|
_disk_raw=$(koopa_ssh_bash "${SSH_CMD_TIMEOUT:-20}" <<'DISK' || true
|
|
set +e
|
|
echo "###HOST###"
|
|
df -Pk / /var /home /tmp 2>/dev/null || df -Pk
|
|
echo "###CTRS###"
|
|
for c in $(podman ps --format '{{.Names}}' 2>/dev/null); do
|
|
echo "###CTR $c###"
|
|
podman exec "$c" df -Pk / /var /tmp 2>/dev/null || podman exec "$c" df -Pk 2>/dev/null
|
|
done
|
|
DISK
|
|
)
|
|
_host_df=$(printf '%s\n' "$_disk_raw" | sed -n '/^###HOST###$/,/^###CTRS###$/p' | sed '1d;$d')
|
|
mon_disk_check_remote_text "ssh:${KOOPA_SSH:-remote}" "$_host_df" || _disk_ec=1
|
|
_ctr=""
|
|
_buf=""
|
|
while IFS= read -r _line || [ -n "$_line" ]; do
|
|
case "$_line" in
|
|
'###CTR '*)
|
|
if [ -n "$_ctr" ]; then
|
|
mon_disk_check_remote_text "ctr:${_ctr}" "$_buf" || _disk_ec=1
|
|
fi
|
|
_ctr=${_line####CTR }
|
|
_ctr=${_ctr%###}
|
|
_buf=""
|
|
;;
|
|
'###CTRS###'|'###HOST###') ;;
|
|
*)
|
|
if [ -n "$_ctr" ]; then
|
|
_buf="${_buf}${_line}"$'\n'
|
|
fi
|
|
;;
|
|
esac
|
|
done <<<"$_disk_raw"
|
|
if [ -n "$_ctr" ]; then
|
|
mon_disk_check_remote_text "ctr:${_ctr}" "$_buf" || _disk_ec=1
|
|
fi
|
|
fi
|
|
unset _disk_raw _host_df _ctr _buf _line _cname _disk_ec
|
|
|
|
summary
|