monitoring: surface inventory and apt-deploy check phases
This commit is contained in:
Hernâni Marques
parent
f4b556c3c0
commit
0fef40fd63
12 changed files with 1657 additions and 46 deletions
|
|
@ -1,9 +1,14 @@
|
|||
#!/usr/bin/env bash
|
||||
# Inside status for bank / exchange / merchant containers.
|
||||
#
|
||||
# Access modes (shown in the log as flags):
|
||||
# host-podman — this host runs podman exec into containers (no SSH).
|
||||
# INSIDE_PODMAN=1 or INSIDE_MODE=local-podman, or auto on GOA host.
|
||||
# ssh — from a laptop/remote: SSH to KOOPA_SSH / INSIDE_SSH, then podman there.
|
||||
#
|
||||
# Profiles:
|
||||
# koopa (default LOCAL_STACK=1) — SSH KOOPA_SSH, containers taler-hacktivism*
|
||||
# stage-lfp (TESTPAYSAN) — SSH INSIDE_SSH (francpaysan-stage-user / stagepaysan),
|
||||
# low-priv podman only; ports 9030–9032; stats remain outside-in (urls).
|
||||
# koopa (default LOCAL_STACK=1) — taler-hacktivism* containers
|
||||
# stage-lfp (TESTPAYSAN) — SSH INSIDE_SSH (stagepaysan), low-priv podman
|
||||
set -euo pipefail
|
||||
ROOT=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=lib.sh
|
||||
|
|
@ -12,7 +17,6 @@ source "$ROOT/lib.sh"
|
|||
source "$ROOT/metrics.sh"
|
||||
|
||||
set_area inside
|
||||
set_group ssh
|
||||
|
||||
PROFILE="${INSIDE_PROFILE:-}"
|
||||
if [ -z "$PROFILE" ]; then
|
||||
|
|
@ -25,7 +29,27 @@ if [ -z "$PROFILE" ]; then
|
|||
fi
|
||||
fi
|
||||
|
||||
section "inside · collect (${PROFILE})"
|
||||
# Resolve host-podman vs ssh before first check ID
|
||||
_use_local_podman=0
|
||||
if [ "$PROFILE" != "stage-lfp" ]; then
|
||||
if [ "${INSIDE_PODMAN:-0}" = "1" ] || [ "${INSIDE_MODE:-}" = "local-podman" ]; then
|
||||
_use_local_podman=1
|
||||
elif command -v podman >/dev/null 2>&1 \
|
||||
&& podman ps --format '{{.Names}}' 2>/dev/null | grep -qE 'taler-hacktivism'; then
|
||||
_use_local_podman=1
|
||||
fi
|
||||
fi
|
||||
if [ "$_use_local_podman" = "1" ]; then
|
||||
INSIDE_ACCESS=host-podman
|
||||
set_group host
|
||||
else
|
||||
INSIDE_ACCESS=ssh
|
||||
set_group ssh
|
||||
fi
|
||||
export INSIDE_ACCESS
|
||||
|
||||
section "inside · collect (${PROFILE} · access=${INSIDE_ACCESS})"
|
||||
info "flags" "INSIDE_ACCESS=${INSIDE_ACCESS} INSIDE_PODMAN=${INSIDE_PODMAN:-0} INSIDE_MODE=${INSIDE_MODE:-} LOCAL_STACK=${LOCAL_STACK:-0} SKIP_SSH=${SKIP_SSH:-0} KOOPA_SSH=${KOOPA_SSH:-} INSIDE_SSH=${INSIDE_SSH:-}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# stage-lfp: low-priv stagepaysan on francpaysan-host
|
||||
|
|
@ -244,31 +268,70 @@ EOF
|
|||
info "stage host" "load probe empty"
|
||||
fi
|
||||
|
||||
set_group disk
|
||||
section "inside · disk free space (stage host + containers)"
|
||||
_disk_raw=$(mon_ssh_bash "$SSH_HOST" "${STAGE_SSH_T:-24}" <<'DISK' || true
|
||||
set +e
|
||||
echo "###HOST###"
|
||||
df -Pk / /var /home /tmp /mnt/data 2>/dev/null || df -Pk
|
||||
echo "###CTRS###"
|
||||
for c in $(podman ps --format '{{.Names}}' 2>/dev/null); do
|
||||
echo "###CTR $c###"
|
||||
podman exec "$c" df -Pk / /var /tmp 2>/dev/null || podman exec "$c" df -Pk 2>/dev/null
|
||||
done
|
||||
DISK
|
||||
)
|
||||
_host_df=$(printf '%s\n' "$_disk_raw" | sed -n '/^###HOST###$/,/^###CTRS###$/p' | sed '1d;$d')
|
||||
mon_disk_check_remote_text "ssh:${SSH_HOST}" "$_host_df" || true
|
||||
_ctr=""; _buf=""
|
||||
while IFS= read -r _line || [ -n "$_line" ]; do
|
||||
case "$_line" in
|
||||
'###CTR '*)
|
||||
if [ -n "$_ctr" ]; then mon_disk_check_remote_text "ctr:${_ctr}" "$_buf" || true; fi
|
||||
_ctr=${_line####CTR }; _ctr=${_ctr%###}; _buf=""
|
||||
;;
|
||||
'###CTRS###'|'###HOST###') ;;
|
||||
*) [ -n "$_ctr" ] && _buf="${_buf}${_line}"$'\n' ;;
|
||||
esac
|
||||
done <<<"$_disk_raw"
|
||||
[ -n "$_ctr" ] && mon_disk_check_remote_text "ctr:${_ctr}" "$_buf" || true
|
||||
|
||||
summary
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# koopa (default) — existing remote collect
|
||||
# koopa (default) — host-podman exec (on GOA host) or SSH to KOOPA_SSH (laptop)
|
||||
# ---------------------------------------------------------------------------
|
||||
section "inside · collect from koopa"
|
||||
section "inside · collect from koopa (access=${INSIDE_ACCESS})"
|
||||
|
||||
if [ "${SKIP_SSH}" = "1" ] && [ "${LOCAL_STACK:-0}" != "1" ]; then
|
||||
warn "ssh" "SKIP_SSH=1 and not local — skipped"
|
||||
if [ "$_use_local_podman" = "1" ]; then
|
||||
set_group host
|
||||
if ! command -v podman >/dev/null 2>&1; then
|
||||
err "host" "INSIDE_PODMAN/host-podman but podman missing"
|
||||
summary
|
||||
exit 1
|
||||
fi
|
||||
ok "host→container" "podman exec on this host (INSIDE_ACCESS=host-podman · no SSH)"
|
||||
elif [ "${SKIP_SSH}" = "1" ] && [ "${LOCAL_STACK:-0}" != "1" ]; then
|
||||
set_group ssh
|
||||
warn "ssh" "SKIP_SSH=1 and not host-podman — skipped"
|
||||
summary
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if ! koopa_ssh_ok; then
|
||||
err "ssh" "cannot reach ${KOOPA_SSH} in ${SSH_CONNECT_TIMEOUT}s — set SKIP_SSH=1 to skip inside"
|
||||
elif ! koopa_ssh_ok; then
|
||||
set_group ssh
|
||||
err "ssh" "cannot reach ${KOOPA_SSH} — from laptop use KOOPA_SSH=; on host set INSIDE_PODMAN=1"
|
||||
summary
|
||||
exit 1
|
||||
else
|
||||
set_group ssh
|
||||
ok "ssh ${KOOPA_SSH}" "remote host then podman (INSIDE_ACCESS=ssh · laptop/remote)"
|
||||
fi
|
||||
ok "ssh ${KOOPA_SSH}"
|
||||
|
||||
# One short remote script (≤ SSH_CMD_TIMEOUT). Every slow step is local curl -m 3 or quick pgrep.
|
||||
RAW=$(
|
||||
koopa_ssh_bash "${SSH_CMD_TIMEOUT}" <<'REMOTE' || true
|
||||
# Collect script (file → bash local or ssh bash -s)
|
||||
_INSIDE_SCRIPT=$(mktemp)
|
||||
trap 'rm -f "${_INSIDE_SCRIPT:-}"' RETURN
|
||||
cat >"$_INSIDE_SCRIPT" <<'REMOTE'
|
||||
set +e
|
||||
emit() { printf 'E|%s|%s|%s|%s\n' "$1" "$2" "$3" "$(printf '%s' "${4:-}" | tr '\n\r' ' ' | head -c 200)"; }
|
||||
# quick curl
|
||||
|
|
@ -376,23 +439,37 @@ else
|
|||
fi
|
||||
echo DONE
|
||||
REMOTE
|
||||
)
|
||||
|
||||
if [ "$_use_local_podman" = "1" ]; then
|
||||
RAW=$(bash "$_INSIDE_SCRIPT" || true)
|
||||
else
|
||||
RAW=$(koopa_ssh_bash "${SSH_CMD_TIMEOUT}" <"$_INSIDE_SCRIPT" || true)
|
||||
fi
|
||||
rm -f "$_INSIDE_SCRIPT"
|
||||
|
||||
if [ -z "$RAW" ] || ! echo "$RAW" | grep -q '^E|'; then
|
||||
err "ssh" "remote timed out or empty (cap ${SSH_CMD_TIMEOUT}s)"
|
||||
if [ "$_use_local_podman" = "1" ]; then
|
||||
err "host" "collect empty (INSIDE_ACCESS=host-podman · podman exec failed)"
|
||||
else
|
||||
err "ssh" "collect empty (INSIDE_ACCESS=ssh · cap ${SSH_CMD_TIMEOUT}s · host=${KOOPA_SSH})"
|
||||
fi
|
||||
summary
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Fallback group for non-component rows: host (podman) vs ssh (remote)
|
||||
_access_grp=ssh
|
||||
[ "$_use_local_podman" = "1" ] && _access_grp=host
|
||||
|
||||
_last_inside_grp=""
|
||||
while IFS= read -r line; do
|
||||
case "$line" in
|
||||
E\|*)
|
||||
IFS='|' read -r _ comp level key detail <<<"$line"
|
||||
# Group IDs by component so issues map cleanly: inside.bank-02, inside.exchange-04
|
||||
# Group IDs by component: inside.bank-02, inside.exchange-04; access = host|ssh
|
||||
case "$comp" in
|
||||
bank|exchange|merchant|caddy) _g="$comp" ;;
|
||||
*) _g="ssh" ;;
|
||||
*) _g="$_access_grp" ;;
|
||||
esac
|
||||
if [ "$_g" != "$_last_inside_grp" ]; then
|
||||
set_group "$_g"
|
||||
|
|
@ -415,4 +492,55 @@ METRICS_DIR="${METRICS_DIR:-$(mktemp -d)}"
|
|||
export METRICS_DIR
|
||||
metrics_report_load "${METRICS_DIR}/load-inside.json" "inside" || true
|
||||
|
||||
# Disk free space: host + taler containers (WARN tight, ERROR full)
|
||||
set_group disk
|
||||
section "inside · disk free space"
|
||||
_disk_ec=0
|
||||
if [ "$_use_local_podman" = "1" ]; then
|
||||
mon_disk_check_host "host" || _disk_ec=1
|
||||
while read -r _cname; do
|
||||
[ -n "$_cname" ] || continue
|
||||
mon_disk_check_podman "$_cname" || _disk_ec=1
|
||||
done < <(podman ps --format '{{.Names}}' 2>/dev/null | grep -iE 'hacktivism|taler-|stage-lfp|lfp-' || true)
|
||||
else
|
||||
# remote host via SSH: df on host + each container
|
||||
_disk_raw=$(koopa_ssh_bash "${SSH_CMD_TIMEOUT:-20}" <<'DISK' || true
|
||||
set +e
|
||||
echo "###HOST###"
|
||||
df -Pk / /var /home /tmp 2>/dev/null || df -Pk
|
||||
echo "###CTRS###"
|
||||
for c in $(podman ps --format '{{.Names}}' 2>/dev/null); do
|
||||
echo "###CTR $c###"
|
||||
podman exec "$c" df -Pk / /var /tmp 2>/dev/null || podman exec "$c" df -Pk 2>/dev/null
|
||||
done
|
||||
DISK
|
||||
)
|
||||
_host_df=$(printf '%s\n' "$_disk_raw" | sed -n '/^###HOST###$/,/^###CTRS###$/p' | sed '1d;$d')
|
||||
mon_disk_check_remote_text "ssh:${KOOPA_SSH:-remote}" "$_host_df" || _disk_ec=1
|
||||
_ctr=""
|
||||
_buf=""
|
||||
while IFS= read -r _line || [ -n "$_line" ]; do
|
||||
case "$_line" in
|
||||
'###CTR '*)
|
||||
if [ -n "$_ctr" ]; then
|
||||
mon_disk_check_remote_text "ctr:${_ctr}" "$_buf" || _disk_ec=1
|
||||
fi
|
||||
_ctr=${_line####CTR }
|
||||
_ctr=${_ctr%###}
|
||||
_buf=""
|
||||
;;
|
||||
'###CTRS###'|'###HOST###') ;;
|
||||
*)
|
||||
if [ -n "$_ctr" ]; then
|
||||
_buf="${_buf}${_line}"$'\n'
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done <<<"$_disk_raw"
|
||||
if [ -n "$_ctr" ]; then
|
||||
mon_disk_check_remote_text "ctr:${_ctr}" "$_buf" || _disk_ec=1
|
||||
fi
|
||||
fi
|
||||
unset _disk_raw _host_df _ctr _buf _line _cname _disk_ec
|
||||
|
||||
summary
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue