taler-monitoring v1.13.12
Multi-phase global SUMMARY, warn-filter dimmed context, SUMMARY chrome as meta; FP stage host-agent via francpaysan-stage-user (stagepaysan); monpages GOA + FP stage.
This commit is contained in:
Hernâni Marques
commit
5a5431bfa9
88 changed files with 21996 additions and 0 deletions
384
check_apt_deploy.sh
Executable file
384
check_apt_deploy.sh
Executable file
|
|
@ -0,0 +1,384 @@
|
|||
#!/usr/bin/env bash
|
||||
# check_apt_deploy.sh — podman apt-src merchant deploy tests on koopa.
|
||||
#
|
||||
# Containers (default):
|
||||
# Fresh (rebuild when repo versions change):
|
||||
# koopa-taler-deploy-test-apt-src-trixie
|
||||
# koopa-taler-deploy-test-apt-src-trixie-testing
|
||||
# Upgrade track (initial install, then mytops-style apt upgrade):
|
||||
# koopa-taler-deploy-test-apt-src-trixie-upgrade
|
||||
# koopa-taler-deploy-test-apt-src-trixie-testing-upgrade
|
||||
#
|
||||
# Emits a compact container board (board · key=value) for the HTML sticky/top
|
||||
# overview: which pods are active and which fail (httpd / ldd / …).
|
||||
#
|
||||
# Env:
|
||||
# APT_DEPLOY_CONTAINERS "name:suite:mode …" mode=fresh|upgrade (optional, default fresh)
|
||||
# APT_DEPLOY_SKIP=1
|
||||
# APT_DEPLOY_PODMAN=podman
|
||||
#
|
||||
set -euo pipefail
|
||||
ROOT=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=lib.sh
|
||||
source "$ROOT/lib.sh"
|
||||
|
||||
if [ "${APT_DEPLOY_SKIP:-0}" = "1" ]; then
|
||||
echo "[INFO] skip phase aptdeploy (APT_DEPLOY_SKIP=1)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
set_area aptdeploy
|
||||
section "aptdeploy · podman apt-src merchant (fresh + upgrade tracks)"
|
||||
|
||||
PODMAN_BIN="${APT_DEPLOY_PODMAN:-podman}"
|
||||
DEFAULT_LIST="\
|
||||
koopa-taler-deploy-test-apt-src-trixie:trixie:fresh \
|
||||
koopa-taler-deploy-test-apt-src-trixie-testing:trixie-testing:fresh \
|
||||
koopa-taler-deploy-test-apt-src-trixie-upgrade:trixie:upgrade \
|
||||
koopa-taler-deploy-test-apt-src-trixie-testing-upgrade:trixie-testing:upgrade"
|
||||
LIST="${APT_DEPLOY_CONTAINERS:-$DEFAULT_LIST}"
|
||||
|
||||
if ! command -v "$PODMAN_BIN" >/dev/null 2>&1; then
|
||||
err "podman" "podman binary missing ($PODMAN_BIN)"
|
||||
summary
|
||||
exit 1
|
||||
fi
|
||||
ok "podman" "$("$PODMAN_BIN" --version 2>/dev/null | head -1)"
|
||||
info "host" "$(hostname 2>/dev/null || echo unknown) · whoami=$(whoami)"
|
||||
|
||||
fail_any=0
|
||||
# board rows for end overview (human table + machine board · lines)
|
||||
BOARD_ROWS=()
|
||||
|
||||
_short_name() {
|
||||
# koopa-taler-deploy-test-apt-src-trixie-testing-upgrade → trixie-testing-upgrade
|
||||
local n="$1"
|
||||
n="${n#koopa-taler-deploy-test-apt-src-}"
|
||||
printf '%s' "$n"
|
||||
}
|
||||
|
||||
# Record one container for the top-of-page board (HTML parses "board · key=value").
|
||||
board_record() {
|
||||
local short="$1" name="$2" suite="$3" mode="$4" state="$5"
|
||||
local merchant="$6" httpd="$7" ldd="$8" verdict="$9" note="${10:-}"
|
||||
local kv
|
||||
kv="short=${short} name=${name} suite=${suite} mode=${mode} state=${state}"
|
||||
kv+=" merchant=${merchant} httpd=${httpd} ldd=${ldd} verdict=${verdict}"
|
||||
# note: no spaces (HTML/parser splits on whitespace)
|
||||
if [ -n "$note" ]; then
|
||||
note_safe=$(printf '%s' "$note" | tr ' ' '_' | tr -d '|')
|
||||
kv+=" note=${note_safe}"
|
||||
fi
|
||||
info "board" "$kv"
|
||||
BOARD_ROWS+=("$short|$suite|$mode|$state|$merchant|$httpd|$ldd|$verdict|$note")
|
||||
}
|
||||
|
||||
print_board_table() {
|
||||
section "aptdeploy · container overview (active / errors)"
|
||||
info "board-table" "| short | suite | mode | podman | merchant | httpd | ldd | verdict | note |"
|
||||
info "board-table" "|-------|-------|------|--------|----------|-------|-----|---------|------|"
|
||||
local r short suite mode state merchant httpd ldd verdict note
|
||||
local n_ok=0 n_err=0 n_miss=0
|
||||
for r in "${BOARD_ROWS[@]+"${BOARD_ROWS[@]}"}"; do
|
||||
IFS='|' read -r short suite mode state merchant httpd ldd verdict note <<<"$r"
|
||||
info "board-table" "| ${short} | ${suite} | ${mode} | ${state} | ${merchant} | ${httpd} | ${ldd} | ${verdict} | ${note} |"
|
||||
case "$verdict" in
|
||||
OK|ok) n_ok=$((n_ok + 1)) ;;
|
||||
MISSING|missing) n_miss=$((n_miss + 1)) ;;
|
||||
*) n_err=$((n_err + 1)) ;;
|
||||
esac
|
||||
done
|
||||
info "board-summary" "containers=${#BOARD_ROWS[@]} ok=${n_ok} error=${n_err} missing=${n_miss}"
|
||||
# one-line human skim (INFO/OK only — do not re-emit ERROR, details already counted)
|
||||
for r in "${BOARD_ROWS[@]+"${BOARD_ROWS[@]}"}"; do
|
||||
IFS='|' read -r short suite mode state merchant httpd ldd verdict note <<<"$r"
|
||||
if [ "$verdict" = "OK" ]; then
|
||||
ok "overview" "${short}: ${state} · merchant=${merchant} · httpd=${httpd} · ldd=${ldd}"
|
||||
else
|
||||
info "overview" "${short}: ${verdict} · state=${state} merchant=${merchant} httpd=${httpd} ldd=${ldd}${note:+ · ${note}}"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
print_pkg_table() {
|
||||
local name=$1
|
||||
local rows
|
||||
rows=$("$PODMAN_BIN" exec "$name" bash -lc '
|
||||
echo "| package | version |"
|
||||
echo "|---------|---------|"
|
||||
for p in taler-merchant libtalermerchant libtalerexchange libdonau libgnunet \
|
||||
taler-merchant-webui taler-merchant-typst taler-terms-generator; do
|
||||
v=$(dpkg-query -W -f="\${Version}" "$p" 2>/dev/null || echo missing)
|
||||
echo "| $p | $v |"
|
||||
done
|
||||
' 2>/dev/null || echo "| (query failed) | |")
|
||||
info "pkg-table" "$name"
|
||||
while IFS= read -r line; do
|
||||
[ -n "$line" ] || continue
|
||||
info "pkg" "$line"
|
||||
done <<<"$rows"
|
||||
}
|
||||
|
||||
# merchant package version only (for board)
|
||||
_merchant_ver() {
|
||||
local name=$1
|
||||
"$PODMAN_BIN" exec "$name" dpkg-query -W -f='${Version}' taler-merchant 2>/dev/null || echo "?"
|
||||
}
|
||||
|
||||
check_ldd_deep() {
|
||||
local name=$1
|
||||
local out
|
||||
out=$("$PODMAN_BIN" exec "$name" bash -lc '
|
||||
set +e
|
||||
bin=$(command -v taler-merchant-httpd)
|
||||
echo "=== ldd taler-merchant-httpd ==="
|
||||
ldd "$bin" 2>&1 | grep -E "libtalerutil|libdonau|libgnunet|not found" || true
|
||||
echo "=== ldd libdonau (if present) ==="
|
||||
for f in /usr/lib/*/libdonau.so* /usr/lib/*/libdonauutil.so*; do
|
||||
[ -e "$f" ] || continue
|
||||
echo "-- $f --"
|
||||
ldd "$f" 2>&1 | grep -E "libtalerutil|not found" || true
|
||||
done
|
||||
echo "=== libtalerutil files ==="
|
||||
ls -la /usr/lib/*/libtalerutil* 2>/dev/null || echo "(none)"
|
||||
' 2>/dev/null || echo "ldd probe failed")
|
||||
while IFS= read -r line; do
|
||||
[ -n "$line" ] || continue
|
||||
info "ldd" "$line"
|
||||
done <<<"$out"
|
||||
if echo "$out" | grep -q 'not found'; then
|
||||
err "ldd" "shared library missing in $name" "see ldd lines above"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
check_systemd() {
|
||||
local name=$1
|
||||
local out active_httpd
|
||||
out=$("$PODMAN_BIN" exec "$name" bash -lc '
|
||||
set +e
|
||||
echo "system: $(systemctl is-system-running 2>&1)"
|
||||
echo "target_enabled: $(systemctl is-enabled taler-merchant.target 2>&1)"
|
||||
echo "target_active: $(systemctl is-active taler-merchant.target 2>&1)"
|
||||
echo "httpd_active: $(systemctl is-active taler-merchant-httpd.service 2>&1)"
|
||||
echo "httpd_enabled: $(systemctl is-enabled taler-merchant-httpd.service 2>&1)"
|
||||
systemctl start taler-merchant.target 2>&1 | tail -5
|
||||
sleep 2
|
||||
echo "after_start_target: $(systemctl is-active taler-merchant.target 2>&1)"
|
||||
echo "after_start_httpd: $(systemctl is-active taler-merchant-httpd.service 2>&1)"
|
||||
systemctl --failed --no-legend 2>/dev/null | grep -i taler || echo "failed_taler: none"
|
||||
' 2>/dev/null || echo "systemd probe failed")
|
||||
while IFS= read -r line; do
|
||||
[ -n "$line" ] || continue
|
||||
info "systemd" "$line"
|
||||
done <<<"$out"
|
||||
|
||||
active_httpd=$(echo "$out" | sed -n 's/^after_start_httpd: //p' | tail -1)
|
||||
# export for caller board
|
||||
APT_LAST_HTTPD="${active_httpd:-?}"
|
||||
if [ "$active_httpd" = "active" ]; then
|
||||
ok "systemd httpd" "taler-merchant-httpd active after start taler-merchant.target"
|
||||
return 0
|
||||
fi
|
||||
err "systemd httpd" "taler-merchant-httpd not active" "after_start_httpd=${active_httpd:-?} · see systemd lines"
|
||||
return 1
|
||||
}
|
||||
|
||||
check_merchant_basics() {
|
||||
local name=$1
|
||||
local out
|
||||
out=$("$PODMAN_BIN" exec "$name" bash -lc '
|
||||
set +e
|
||||
echo "=== binaries ==="
|
||||
command -v taler-merchant-httpd taler-merchant-dbinit taler-config 2>/dev/null
|
||||
echo "=== --version ==="
|
||||
taler-merchant-httpd --version 2>&1
|
||||
echo "=== config probe ==="
|
||||
for u in \
|
||||
http://127.0.0.1:9966/config \
|
||||
http://127.0.0.1:8081/config \
|
||||
http://127.0.0.1/config
|
||||
do
|
||||
code=$(curl -sS -m 2 -o /tmp/mcfg -w "%{http_code}" "$u" 2>/dev/null || echo 000)
|
||||
echo "curl $u -> $code"
|
||||
[ "$code" = "200" ] && head -c 120 /tmp/mcfg && echo
|
||||
done
|
||||
sock=$(ls /run/taler-merchant/httpd/*.sock 2>/dev/null | head -1)
|
||||
if [ -n "$sock" ]; then
|
||||
echo "socket: $sock"
|
||||
code=$(curl -sS -m 2 --unix-socket "$sock" -o /tmp/mcfg -w "%{http_code}" http://localhost/config 2>/dev/null || echo 000)
|
||||
echo "curl --unix-socket -> $code"
|
||||
[ "$code" = "200" ] && head -c 160 /tmp/mcfg && echo
|
||||
else
|
||||
echo "socket: (none under /run/taler-merchant/httpd/)"
|
||||
fi
|
||||
' 2>/dev/null || echo "basics probe failed")
|
||||
|
||||
while IFS= read -r line; do
|
||||
[ -n "$line" ] || continue
|
||||
info "basics" "$line"
|
||||
done <<<"$out"
|
||||
|
||||
if echo "$out" | grep -qi 'error while loading shared libraries'; then
|
||||
err "httpd" "shared library error on --version" "see basics/ldd"
|
||||
return 1
|
||||
fi
|
||||
|
||||
set +e
|
||||
"$PODMAN_BIN" exec "$name" taler-merchant-httpd --version >/dev/null 2>&1
|
||||
local ec=$?
|
||||
set -e
|
||||
if [ "$ec" -eq 0 ]; then
|
||||
ok "httpd --version" "exit 0"
|
||||
else
|
||||
err "httpd --version" "exit $ec"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if echo "$out" | grep -qE 'curl .* -> 200'; then
|
||||
ok "merchant /config" "HTTP 200 (local)"
|
||||
else
|
||||
warn "merchant /config" "no local HTTP 200 (unit may need BASE_URL/db — still report version/ldd)"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
check_one() {
|
||||
local name="$1" expect_suite="$2" mode="${3:-fresh}"
|
||||
local st short merchant="-" httpd="-" ldd="-" verdict="OK" note="" c_fail=0
|
||||
|
||||
short="$(_short_name "$name")"
|
||||
APT_LAST_HTTPD="-"
|
||||
|
||||
# group ids: aptdeploy.trixie-01 / aptdeploy.trixie-testing-upgrade-01
|
||||
set_group "${expect_suite}${mode:+-$mode}"
|
||||
section "aptdeploy · $name (suite=$expect_suite mode=$mode)"
|
||||
|
||||
if ! "$PODMAN_BIN" container exists "$name" 2>/dev/null; then
|
||||
err "container" "$name missing" "run ensure-apt-deploy-test-containers.sh"
|
||||
board_record "$short" "$name" "$expect_suite" "$mode" "missing" "-" "-" "-" "MISSING" "ensure containers"
|
||||
fail_any=1
|
||||
return
|
||||
fi
|
||||
st=$("$PODMAN_BIN" inspect -f '{{.State.Status}}' "$name" 2>/dev/null || echo unknown)
|
||||
if [ "$st" != "running" ]; then
|
||||
err "container" "$name not running" "status=$st"
|
||||
board_record "$short" "$name" "$expect_suite" "$mode" "$st" "-" "-" "-" "ERROR" "not running"
|
||||
fail_any=1
|
||||
return
|
||||
fi
|
||||
ok "container" "$name running (mode=$mode)"
|
||||
|
||||
suite_line=$("$PODMAN_BIN" exec "$name" bash -lc \
|
||||
'grep -h "^Suites:" /etc/apt/sources.list.d/*taler* 2>/dev/null | head -1' 2>/dev/null || true)
|
||||
if echo "$suite_line" | grep -qE "Suites:[[:space:]]*${expect_suite}([[:space:]]|$)"; then
|
||||
ok "apt suite" "$expect_suite"
|
||||
else
|
||||
err "apt suite" "expected $expect_suite" "got: ${suite_line:-empty}"
|
||||
c_fail=1
|
||||
note="suite mismatch"
|
||||
fi
|
||||
|
||||
merchant="$(_merchant_ver "$name")"
|
||||
print_pkg_table "$name"
|
||||
|
||||
if ! check_merchant_basics "$name"; then
|
||||
c_fail=1
|
||||
if ! check_ldd_deep "$name"; then
|
||||
ldd="FAIL"
|
||||
c_fail=1
|
||||
else
|
||||
ldd="OK"
|
||||
fi
|
||||
else
|
||||
ldd_out=$("$PODMAN_BIN" exec "$name" bash -lc \
|
||||
'ldd "$(command -v taler-merchant-httpd)" 2>&1 | grep -E "libtalerutil|not found" || true' 2>/dev/null || true)
|
||||
if echo "$ldd_out" | grep -q 'not found'; then
|
||||
err "libtalerutil" "not found" "$ldd_out"
|
||||
check_ldd_deep "$name" || true
|
||||
ldd="FAIL"
|
||||
c_fail=1
|
||||
else
|
||||
ldd="OK"
|
||||
info "libtalerutil" "$(echo "$ldd_out" | tr '\n' ' ')"
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! check_systemd "$name"; then
|
||||
c_fail=1
|
||||
httpd="${APT_LAST_HTTPD:-FAIL}"
|
||||
# deeper ldd if httpd failed
|
||||
if [ "$ldd" != "FAIL" ]; then
|
||||
check_ldd_deep "$name" || { ldd="FAIL"; c_fail=1; }
|
||||
fi
|
||||
else
|
||||
httpd="active"
|
||||
fi
|
||||
# normalize httpd cell
|
||||
[ -z "$httpd" ] || [ "$httpd" = "-" ] && httpd="${APT_LAST_HTTPD:-?}"
|
||||
|
||||
# Disk free inside this deploy-test container (+ host once, first container only)
|
||||
if [ "${_APT_DEPLOY_HOST_DISK_DONE:-0}" != "1" ]; then
|
||||
set_group disk
|
||||
mon_disk_check_host "host" || fail_any=1
|
||||
_APT_DEPLOY_HOST_DISK_DONE=1
|
||||
fi
|
||||
set_group disk
|
||||
if ! mon_disk_check_podman "$name" "$PODMAN_BIN"; then
|
||||
c_fail=1
|
||||
note="${note:+$note; }disk"
|
||||
fi
|
||||
|
||||
if [ "$c_fail" -ne 0 ]; then
|
||||
verdict="ERROR"
|
||||
fail_any=1
|
||||
else
|
||||
verdict="OK"
|
||||
fi
|
||||
board_record "$short" "$name" "$expect_suite" "$mode" "running" "$merchant" "$httpd" "$ldd" "$verdict" "$note"
|
||||
}
|
||||
|
||||
# --- inventory first (what exists before checks) ---
|
||||
section "aptdeploy · inventory (podman)"
|
||||
info "inventory" "expected tracks: trixie/trixie-testing × fresh/upgrade (4 containers)"
|
||||
for entry in $LIST; do
|
||||
cname=${entry%%:*}
|
||||
rest=${entry#*:}
|
||||
case "$rest" in
|
||||
*:*) suite=${rest%%:*}; mode=${rest#*:} ;;
|
||||
*) suite=$rest; mode=fresh ;;
|
||||
esac
|
||||
short="$(_short_name "$cname")"
|
||||
if "$PODMAN_BIN" container exists "$cname" 2>/dev/null; then
|
||||
st=$("$PODMAN_BIN" inspect -f '{{.State.Status}}' "$cname" 2>/dev/null || echo unknown)
|
||||
ok "inventory" "${short}: ${st} (${suite}/${mode})"
|
||||
else
|
||||
warn "inventory" "${short}: missing (${suite}/${mode})" "will ERROR in check"
|
||||
fi
|
||||
done
|
||||
|
||||
for entry in $LIST; do
|
||||
cname=${entry%%:*}
|
||||
rest=${entry#*:}
|
||||
case "$rest" in
|
||||
*:*)
|
||||
suite=${rest%%:*}
|
||||
mode=${rest#*:}
|
||||
;;
|
||||
*)
|
||||
suite=$rest
|
||||
mode=fresh
|
||||
;;
|
||||
esac
|
||||
check_one "$cname" "$suite" "$mode"
|
||||
done
|
||||
|
||||
# Compact matrix + overview ERROR lines (jump targets for HTML)
|
||||
print_board_table
|
||||
|
||||
summary
|
||||
if [ "$fail_any" -ne 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
Loading…
Add table
Add a link
Reference in a new issue