monitoring: surface inventory and apt-deploy check phases
This commit is contained in:
Hernâni Marques
parent
f4b556c3c0
commit
0fef40fd63
12 changed files with 1657 additions and 46 deletions
|
|
@ -12,6 +12,12 @@
|
|||
set -euo pipefail
|
||||
ROOT=$(cd "$(dirname "$0")" && pwd)
|
||||
|
||||
# Line-buffered stdout/stderr so redirected logs (tee/host-agent) flush each line.
|
||||
if [ "${_TALER_MON_STDBUF:-0}" != "1" ] && command -v stdbuf >/dev/null 2>&1; then
|
||||
export _TALER_MON_STDBUF=1
|
||||
exec stdbuf -oL -eL bash "$0" "$@"
|
||||
fi
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
taler-monitoring — bank / exchange / merchant checks
|
||||
|
|
@ -29,8 +35,14 @@ Phases:
|
|||
e2e withdraw + pay (small amounts; remote aborts on login/KYC)
|
||||
ladder withdraw/pay amount ladder (GOA ceiling or stage TESTPAYSAN max_wire)
|
||||
auth401 merchant Basic-auth / case matrix (HTTP 401 paths; may create throwaway instance)
|
||||
aptdeploy koopa podman apt-src smoke: taler-merchant in
|
||||
koopa-taler-deploy-test-apt-src-trixie{,-testing}
|
||||
surface REMOTE-ONLY public inventory (NOT in default/all/full):
|
||||
ecosystem hosts (taler.net, gnunet.org, taler-systems.com, mattermost, …)
|
||||
or -d DOMAIN → that domain’s surface; port/protocol/TLS/CVE (OSV)
|
||||
all urls + inside + versions + sanity + e2e (SSH phases only on koopa)
|
||||
full all + server + ladder + auth401 (maximum; long-running, needs secrets)
|
||||
NOTE: surface is never included in all/full — pass it explicitly
|
||||
|
||||
Options:
|
||||
-d, --domain DOMAIN load profile from domains.conf (bank/exchange/merchant)
|
||||
|
|
@ -61,6 +73,10 @@ Env (same meaning):
|
|||
TALER_DOMAINS_CONF SKIP_SSH=1
|
||||
NO_COLOR=1 / CLICOLOR=0 disable green/yellow/red tags (default: coloured)
|
||||
SKIP_SSH=1 NO_COLOR=1
|
||||
RUN_TIMEOUT=600 whole-run wall clock seconds (default 600; 0=unlimited)
|
||||
long phases (ladder/full/e2e) need a higher value or 0
|
||||
DISK_WARN_USED_PCT=85 disk free: WARN when used ≥ this %
|
||||
DISK_ERR_USED_PCT=95 disk free: ERROR when used ≥ this % (or avail=0 / 100%)
|
||||
PERF_WARN_MS PERF_FAIL_MS (urls latency; default 8000 / 20000)
|
||||
QR_CHECK=0 skip QR form + qrencode/zbarimg (urls phase)
|
||||
QR_ECC=M qrencode ECC level (default M)
|
||||
|
|
@ -73,11 +89,23 @@ Env (same meaning):
|
|||
AUTH401_* see check_auth401.sh / secrets.env.example
|
||||
AUTH401_CONTINUE=1 / CONTINUE_ON_ERROR=1
|
||||
auth401: run all groups, collect every ERROR (no mid-run halt)
|
||||
APT_DEPLOY_SKIP=1 skip aptdeploy phase
|
||||
APT_DEPLOY_CONTAINERS="name:suite …" default trixie + trixie-testing deploy-test pods
|
||||
SURFACE_CVE=0 disable CVE queries in surface phase
|
||||
SURFACE_CVE_LEVEL=warn|error bare Server-header versions default warn
|
||||
(Debian package versions default error)
|
||||
SURFACE_CATALOG=path override surface-catalog.conf
|
||||
|
||||
Full load (GOA / hacktivism):
|
||||
./taler-monitoring.sh -d hacktivism.ch full
|
||||
# or explicit:
|
||||
./taler-monitoring.sh -d hacktivism.ch urls inside versions sanity server e2e ladder auth401
|
||||
# hacktivism host-agent also runs aptdeploy (apt-src merchant containers on koopa)
|
||||
|
||||
Remote surface / ecosystem (explicit only — never default):
|
||||
./taler-monitoring.sh surface
|
||||
./taler-monitoring.sh -d hacktivism.ch surface
|
||||
./taler-monitoring.sh -d lefrancpaysan.ch surface
|
||||
|
||||
SPA pin after selfbuild:
|
||||
EXPECT_WEBUI_VERSION=1.6.11 EXPECT_WEBUI_OVERLAY=selfbuild-v1.6.11 \\
|
||||
|
|
@ -111,6 +139,7 @@ while [ $# -gt 0 ]; do
|
|||
[ $# -ge 2 ] || { echo "missing arg for $1" >&2; exit 2; }
|
||||
apply_taler_domain "$2"
|
||||
DOMAIN_SET=1
|
||||
TALER_DOMAIN_FROM_CLI=1
|
||||
shift 2
|
||||
;;
|
||||
--bank)
|
||||
|
|
@ -130,12 +159,13 @@ while [ $# -gt 0 ]; do
|
|||
CURRENCY_OVERRIDE="$2"; shift 2
|
||||
;;
|
||||
--no-probe) NO_PROBE=1; shift ;;
|
||||
urls|inside|versions|sanity|server|e2e|ladder|goa-ladder|auth401|all|full) PHASES+=("$1"); shift ;;
|
||||
urls|inside|versions|sanity|server|e2e|ladder|goa-ladder|auth401|aptdeploy|apt-deploy|apt_src|surface|ecosystem|all|full) PHASES+=("$1"); shift ;;
|
||||
*)
|
||||
# bare domain shorthand: ./taler-monitoring.sh taler.net
|
||||
if [[ "$1" == *.* && "$1" != *://* && "$1" != -* ]]; then
|
||||
apply_taler_domain "$1"
|
||||
DOMAIN_SET=1
|
||||
TALER_DOMAIN_FROM_CLI=1
|
||||
shift
|
||||
else
|
||||
echo "Unknown: $1" >&2; usage >&2; exit 2
|
||||
|
|
@ -170,6 +200,8 @@ fi
|
|||
# Export so check_*.sh (re-source lib) see the same targets via env
|
||||
export TALER_DOMAIN BANK_PUBLIC EXCHANGE_PUBLIC MERCHANT_PUBLIC
|
||||
export EXPECT_CURRENCY SKIP_SSH LOCAL_STACK TALER_DOMAIN_PROBE CHECK_LANDING
|
||||
export TALER_DOMAIN_FROM_CLI="${TALER_DOMAIN_FROM_CLI:-0}"
|
||||
export DOMAIN_SET="${DOMAIN_SET:-0}"
|
||||
export WITHDRAW_AMT PAY_AMT CREDIT_AMT MERCHANT_INSTANCE
|
||||
export E2E_FAKE_INCOMING E2E_REMOTE E2E_VARIABLE E2E_ATM_MAX
|
||||
export E2E_WITHDRAW_VALUES E2E_PAY_VALUES E2E_USE_TEMPLATES E2E_TEMPLATE_MAP
|
||||
|
|
@ -288,33 +320,158 @@ if [ "${PROGRESS_TOTAL:-0}" = "0" ] || [ -z "${PROGRESS_TOTAL:-}" ]; then
|
|||
e2e) _pt=$((_pt + 240)) ;;
|
||||
ladder|goa-ladder) _pt=$((_pt + 120)) ;;
|
||||
auth401) _pt=$((_pt + 70)) ;;
|
||||
aptdeploy) _pt=$((_pt + 20)) ;;
|
||||
surface|ecosystem) _pt=$((_pt + 80)) ;;
|
||||
esac
|
||||
done
|
||||
set_progress_total "$_pt"
|
||||
unset _pt
|
||||
fi
|
||||
|
||||
# Whole-run wall clock (default 10 min). Override: RUN_TIMEOUT=0 (unlimited).
|
||||
: "${RUN_TIMEOUT:=600}"
|
||||
export RUN_TIMEOUT
|
||||
MON_T0=$(date +%s)
|
||||
export MON_T0
|
||||
RUN_TIMED_OUT=0
|
||||
RUN_TIMEOUT_AT_PHASE=""
|
||||
RUN_SKIPPED_PHASES=()
|
||||
|
||||
mon_seconds_left() {
|
||||
if [ "${RUN_TIMEOUT:-0}" -eq 0 ]; then
|
||||
printf '%s' "999999"
|
||||
return 0
|
||||
fi
|
||||
local now left
|
||||
now=$(date +%s)
|
||||
left=$((RUN_TIMEOUT - (now - MON_T0)))
|
||||
[ "$left" -lt 0 ] && left=0
|
||||
printf '%s' "$left"
|
||||
}
|
||||
|
||||
# Run one phase script under remaining RUN_TIMEOUT budget.
|
||||
# Exit 124 (timeout utility) → mark RUN_TIMED_OUT.
|
||||
run_phase() {
|
||||
local phase="$1" script="$2" left rc
|
||||
left=$(mon_seconds_left)
|
||||
if [ "${RUN_TIMEOUT:-0}" -gt 0 ] && [ "$left" -le 0 ]; then
|
||||
RUN_TIMED_OUT=1
|
||||
RUN_TIMEOUT_AT_PHASE="${RUN_TIMEOUT_AT_PHASE:-$phase}"
|
||||
return 1
|
||||
fi
|
||||
if [ "${RUN_TIMEOUT:-0}" -eq 0 ]; then
|
||||
"$script"
|
||||
return $?
|
||||
fi
|
||||
# Keep at least 2s so timeout(1) can start the child.
|
||||
[ "$left" -lt 2 ] && left=2
|
||||
set +e
|
||||
with_timeout "$left" "$script"
|
||||
rc=$?
|
||||
set -e
|
||||
if [ "$rc" -eq 124 ]; then
|
||||
RUN_TIMED_OUT=1
|
||||
RUN_TIMEOUT_AT_PHASE="$phase"
|
||||
return 1
|
||||
fi
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
# How inside/versions will reach containers (host-podman vs ssh)
|
||||
_INSIDE_ACCESS_HINT=ssh
|
||||
if [ "${INSIDE_PODMAN:-0}" = "1" ] || [ "${INSIDE_MODE:-}" = "local-podman" ]; then
|
||||
_INSIDE_ACCESS_HINT=host-podman
|
||||
elif [ "${SKIP_SSH:-0}" = "1" ]; then
|
||||
_INSIDE_ACCESS_HINT=skipped
|
||||
elif [ "${LOCAL_STACK:-0}" = "1" ] && command -v podman >/dev/null 2>&1 \
|
||||
&& podman ps --format '{{.Names}}' 2>/dev/null | grep -qE 'taler-hacktivism'; then
|
||||
_INSIDE_ACCESS_HINT=host-podman
|
||||
fi
|
||||
|
||||
printf 'target domain=%s\n' "${TALER_DOMAIN}"
|
||||
printf ' bank %s\n' "$BANK_PUBLIC"
|
||||
printf ' exchange %s\n' "$EXCHANGE_PUBLIC"
|
||||
printf ' merchant %s\n' "$MERCHANT_PUBLIC"
|
||||
printf ' currency expect=%s skip_ssh=%s\n' "${EXPECT_CURRENCY:-any}" "$SKIP_SSH"
|
||||
printf ' currency expect=%s\n' "${EXPECT_CURRENCY:-any}"
|
||||
printf ' phases %s\n' "${PHASES[*]}"
|
||||
printf ' flags LOCAL_STACK=%s SKIP_SSH=%s INSIDE_PODMAN=%s INSIDE_MODE=%s\n' \
|
||||
"${LOCAL_STACK:-}" "${SKIP_SSH:-0}" "${INSIDE_PODMAN:-0}" "${INSIDE_MODE:-}"
|
||||
printf ' flags KOOPA_SSH=%s INSIDE_SSH=%s INSIDE_PROFILE=%s\n' \
|
||||
"${KOOPA_SSH:-}" "${INSIDE_SSH:-}" "${INSIDE_PROFILE:-}"
|
||||
printf ' access inside/versions → %s' "$_INSIDE_ACCESS_HINT"
|
||||
case "$_INSIDE_ACCESS_HINT" in
|
||||
host-podman) printf ' (podman exec on this host; IDs inside.host-*)\n' ;;
|
||||
ssh) printf ' (SSH then podman; IDs inside.ssh-*)\n' ;;
|
||||
skipped) printf ' (SKIP_SSH=1)\n' ;;
|
||||
*) printf '\n' ;;
|
||||
esac
|
||||
if [ "${RUN_TIMEOUT:-0}" -eq 0 ]; then
|
||||
printf ' run_timeout unlimited (RUN_TIMEOUT=0)\n'
|
||||
else
|
||||
printf ' run_timeout %ss wall clock (RUN_TIMEOUT=; 0=unlimited)\n' "$RUN_TIMEOUT"
|
||||
fi
|
||||
printf ' progress total≈%s (set PROGRESS_TOTAL= to override; PROGRESS_OFF=1 to hide)\n' "${PROGRESS_TOTAL:-0}"
|
||||
unset _INSIDE_ACCESS_HINT
|
||||
|
||||
chmod +x "$ROOT"/check_*.sh 2>/dev/null || true
|
||||
|
||||
ec=0
|
||||
for p in "${PHASES[@]}"; do
|
||||
if [ "$RUN_TIMED_OUT" = "1" ]; then
|
||||
RUN_SKIPPED_PHASES+=("$p")
|
||||
continue
|
||||
fi
|
||||
left=$(mon_seconds_left)
|
||||
if [ "${RUN_TIMEOUT:-0}" -gt 0 ] && [ "$left" -le 0 ]; then
|
||||
RUN_TIMED_OUT=1
|
||||
RUN_TIMEOUT_AT_PHASE="${RUN_TIMEOUT_AT_PHASE:-$p}"
|
||||
RUN_SKIPPED_PHASES+=("$p")
|
||||
continue
|
||||
fi
|
||||
case "$p" in
|
||||
urls) "$ROOT/check_urls.sh" || ec=1 ;;
|
||||
inside) "$ROOT/check_inside.sh" || ec=1 ;;
|
||||
versions) "$ROOT/check_versions.sh" || ec=1 ;;
|
||||
sanity) "$ROOT/check_sanity.sh" || ec=1 ;;
|
||||
server) "$ROOT/check_server.sh" || ec=1 ;;
|
||||
e2e) "$ROOT/check_e2e.sh" || ec=1 ;;
|
||||
ladder|goa-ladder) "$ROOT/check_goa_ladder.sh" || ec=1 ;;
|
||||
auth401) "$ROOT/check_auth401.sh" || ec=1 ;;
|
||||
urls) run_phase urls "$ROOT/check_urls.sh" || ec=1 ;;
|
||||
inside) run_phase inside "$ROOT/check_inside.sh" || ec=1 ;;
|
||||
versions) run_phase versions "$ROOT/check_versions.sh" || ec=1 ;;
|
||||
sanity) run_phase sanity "$ROOT/check_sanity.sh" || ec=1 ;;
|
||||
server) run_phase server "$ROOT/check_server.sh" || ec=1 ;;
|
||||
e2e) run_phase e2e "$ROOT/check_e2e.sh" || ec=1 ;;
|
||||
ladder|goa-ladder) run_phase ladder "$ROOT/check_goa_ladder.sh" || ec=1 ;;
|
||||
auth401) run_phase auth401 "$ROOT/check_auth401.sh" || ec=1 ;;
|
||||
aptdeploy|apt-deploy|apt_src) run_phase aptdeploy "$ROOT/check_apt_deploy.sh" || ec=1 ;;
|
||||
surface|ecosystem) run_phase surface "$ROOT/check_surface.sh" || ec=1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Extraordinary run-budget failure: always report at end; HTML links top → here.
|
||||
if [ "$RUN_TIMED_OUT" = "1" ]; then
|
||||
ec=1
|
||||
elapsed=$(( $(date +%s) - MON_T0 ))
|
||||
skipped="${RUN_SKIPPED_PHASES[*]:-}"
|
||||
# Stable id for jump links (console HTML + err-top banner)
|
||||
printf '\n'
|
||||
printf '╔══════════════════════════════════════════════════════════╗\n'
|
||||
printf '║ RUN TIMEOUT · extraordinary (see jump target below) ║\n'
|
||||
printf '╚══════════════════════════════════════════════════════════╝\n'
|
||||
printf '┌ ERROR ┐ #run.timeout-01 RUN_TIMEOUT exceeded · budget %ss · elapsed %ss\n' \
|
||||
"$RUN_TIMEOUT" "$elapsed"
|
||||
printf ' detail: wall-clock limit hit'
|
||||
if [ -n "${RUN_TIMEOUT_AT_PHASE:-}" ]; then
|
||||
printf ' during/after phase "%s"' "$RUN_TIMEOUT_AT_PHASE"
|
||||
fi
|
||||
if [ -n "$skipped" ]; then
|
||||
printf ' · skipped: %s' "$skipped"
|
||||
fi
|
||||
printf '\n'
|
||||
printf ' id=run.timeout-01\n'
|
||||
printf ' hint: raise RUN_TIMEOUT= (seconds) or set RUN_TIMEOUT=0 for unlimited\n'
|
||||
printf -- '┌ RUN TIMEOUT · extraordinary ┐\n'
|
||||
printf -- ' • run.timeout-01 [run] RUN_TIMEOUT=%ss exceeded (elapsed %ss)%s%s\n' \
|
||||
"$RUN_TIMEOUT" "$elapsed" \
|
||||
"${RUN_TIMEOUT_AT_PHASE:+ · phase $RUN_TIMEOUT_AT_PHASE}" \
|
||||
"${skipped:+ · skipped $skipped}"
|
||||
printf -- '--- ERRORS (failed checks) ---\n'
|
||||
printf -- ' • run.timeout-01 [run] RUN_TIMEOUT=%ss exceeded (elapsed %ss)\n' \
|
||||
"$RUN_TIMEOUT" "$elapsed"
|
||||
fi
|
||||
|
||||
exit "$ec"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue