taler-monitoring/check_monitoring_pages.sh
Hernâni Marques 757090cbff
release 1.9.0: monpages top/bottom content markers + surface required
Require meaningful monitoring HTML: sticky-bar/generated at top, footer
or taler-mon:bottom at end; ERROR if body is progress-only. Mid-run
(incomplete meta or progress without complete foot) relaxes bottom only.
GOA always checks taler-monitoring-surface. Generator emits taler-mon
markers and taler-mon-page complete|incomplete meta.
2026-07-19 00:58:09 +02:00

515 lines
17 KiB
Bash
Executable file
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env bash
# check_monitoring_pages.sh — verify public monitoring HTML pages via FQDN
#
# Outside-in: curl https://<fqdn>/<path>/ and require a real HTML monitoring page.
# Catches: merchant JSON code 21 (Caddy handle missing), WP 404, empty stubs, deploy skip.
#
# Policy (v1.7.6+):
# • monpages is obligatory → missing/wrong pages are ERROR (exit 1).
# • GOA / hacktivism: full inventory of suite monitoring sites (catalog + on-disk discovery).
# • FP: only that stacks own monitoring hosts/paths (never GOA URLs).
# • Override soft mode only with MONPAGES_REQUIRE_PUBLIC=0 (emergency / staging).
#
# Env:
# MON_HOSTS space-separated FQDNs (default from TALER_DOMAIN)
# HTML_URL_OK path for OK page (default /monitoring/)
# HTML_URL_ERR err path; checked if present on disk or MONPAGES_CHECK_ERR=1
# MONITORING_PAGE_URLS optional full URL list (overrides inventory construction)
# MONPAGES_EXTRA_URLS additional absolute URLs (space-separated)
# MONPAGES_INVENTORY auto|full|job (default auto)
# auto/full = stack inventory; job = only MON_HOSTS+paths
# MONPAGES_REQUIRE_PUBLIC default 1 (ERROR). 0 = WARN only (escape hatch).
# MONPAGES_STAGING_BASE / HTML_OUT / DEPLOY_WWW_ROOT trees for discovery
# TIMEOUT curl timeout (from lib.sh)
#
set -euo pipefail
ROOT=$(cd "$(dirname "$0")" && pwd)
# shellcheck source=lib.sh
source "$ROOT/lib.sh"
set_area monpages
section "monpages · public monitoring HTML via FQDN (outside-in, obligatory)"
# Obligatory: public pages missing → ERROR. Soft only if explicitly disabled.
: "${MONPAGES_REQUIRE_PUBLIC:=1}"
: "${MONPAGES_INVENTORY:=auto}"
# Default hosts from domain (mirror run-host-report.sh)
if [ -z "${MON_HOSTS:-}" ]; then
case "${TALER_DOMAIN:-}" in
hacktivism.ch|koopa)
MON_HOSTS="bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch"
;;
stage.lefrancpaysan.ch|stage.*lefrancpaysan*)
MON_HOSTS="stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch"
;;
lefrancpaysan.ch)
MON_HOSTS="bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch"
;;
*)
MON_HOSTS="${TALER_DOMAIN:-}"
;;
esac
fi
HTML_URL_OK="${HTML_URL_OK:-/monitoring/}"
HTML_URL_ERR="${HTML_URL_ERR:-/monitoring_err/}"
# ensure leading + trailing slash
_norm_path() {
local p="$1"
case "$p" in
/*) ;;
*) p="/$p" ;;
esac
case "$p" in
*/) ;;
*) p="${p}/" ;;
esac
printf '%s' "$p"
}
HTML_URL_OK="$(_norm_path "$HTML_URL_OK")"
HTML_URL_ERR="$(_norm_path "$HTML_URL_ERR")"
# Stack family: goa | fp | other — FP never checks GOA and vice versa.
_monpages_family() {
case "${TALER_DOMAIN:-}" in
*lefrancpaysan*|*francpaysan*) printf 'fp' ;;
hacktivism.ch|koopa) printf 'goa' ;;
*)
# Infer from MON_HOSTS if domain ambiguous
case " ${MON_HOSTS:-} " in
*lefrancpaysan*) printf 'fp' ;;
*hacktivism*) printf 'goa' ;;
*) printf 'other' ;;
esac
;;
esac
}
_host_allowed() {
local h="$1" fam="$2"
case "$fam" in
goa)
case "$h" in *.hacktivism.ch) return 0 ;; *) return 1 ;; esac
;;
fp)
case "$h" in *lefrancpaysan*) return 0 ;; *) return 1 ;; esac
;;
*) return 0 ;;
esac
}
# Public page layout (v1.8.0+ simplified):
# • 9 landing stacks → /monitoring/ (+ /monitoring_err/ when present)
# • ecosystem software/versions inventory → only
# https://taler.hacktivism.ch/taler-monitoring-surface/
# https://taler.hacktivism.ch/taler-monitoring-surface_err/
# Legacy /taler-monitoring-{mail,mattermost,aptdeploy}* are NOT first-class pages.
_path_allowed() {
case "$1" in
monitoring|monitoring_err|taler-monitoring-surface|taler-monitoring-surface_err) return 0 ;;
*) return 1 ;;
esac
}
# Emit https://host/path/ for each …/host/path/index.html under root (max depth 3).
_discover_tree() {
local root="$1" fam="$2"
local f rel host path
[ -n "$root" ] && [ -d "$root" ] || return 0
# layout: $root/<fqdn>/<url-path>/index.html
while IFS= read -r -d '' f; do
rel="${f#"${root%/}"/}"
host="${rel%%/*}"
path="${rel#*/}"
path="${path%/index.html}"
[ -n "$host" ] && [ -n "$path" ] || continue
_host_allowed "$host" "$fam" || continue
_path_allowed "$path" || continue
printf 'https://%s/%s/\n' "$host" "$path"
done < <(find "${root%/}" -mindepth 3 -maxdepth 3 -type f -name index.html -print0 2>/dev/null)
}
# Nine landing stacks (GOA 3 + FP stage 3 + FP prod 3) with /monitoring/
_landing_nine() {
printf '%s\n' \
bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch \
stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch \
bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch
}
# Suite catalog of OK pages that must exist for this family.
_catalog_urls() {
local fam="$1" h
case "$fam" in
goa)
for h in bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch; do
printf 'https://%s/monitoring/\n' "$h"
done
# ecosystem surface only (versions / software found in Taler universe)
printf 'https://taler.hacktivism.ch/taler-monitoring-surface/\n'
;;
fp)
for h in $MON_HOSTS; do
[ -z "$h" ] && continue
_host_allowed "$h" fp || continue
printf 'https://%s/monitoring/\n' "$h"
done
;;
all|nine)
# optional: all nine landings + surface
while IFS= read -r h; do
[ -n "$h" ] || continue
printf 'https://%s/monitoring/\n' "$h"
done < <(_landing_nine)
printf 'https://taler.hacktivism.ch/taler-monitoring-surface/\n'
;;
esac
}
# Job-local URLs (this host-agent runs path).
_job_urls() {
local h p
for h in $MON_HOSTS; do
[ -z "$h" ] && continue
printf 'https://%s%s\n' "$h" "$HTML_URL_OK"
if [ "${MONPAGES_CHECK_ERR:-0}" = "1" ]; then
printf 'https://%s%s\n' "$h" "$HTML_URL_ERR"
fi
done
}
build_urls() {
local fam inv
fam="$(_monpages_family)"
inv="${MONPAGES_INVENTORY:-auto}"
if [ -n "${MONITORING_PAGE_URLS:-}" ]; then
# shellcheck disable=SC2086
printf '%s\n' $MONITORING_PAGE_URLS
else
case "$inv" in
job)
_job_urls
# v1.9.0: normal GOA runs always require surface page (content-checked)
if [ "$fam" = "goa" ]; then
printf 'https://taler.hacktivism.ch/taler-monitoring-surface/\n'
fi
;;
full|auto|*)
# Always include this jobs paths (host-agent post-check).
_job_urls
# Stack inventory: GOA = landings + surface; FP = only FP mon hosts.
if [ "$fam" = "goa" ] || [ "$fam" = "fp" ]; then
_catalog_urls "$fam"
# On-disk discovery (err pages only exist after failed runs; extras welcome)
_discover_tree "${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" "$fam"
_discover_tree "${MONPAGES_STAGING_BASE:-${HTML_OUT:-${HTML_BASE:-$HOME/monitoring-sites-staging}}}" "$fam"
fi
;;
esac
fi
if [ -n "${MONPAGES_EXTRA_URLS:-}" ]; then
# shellcheck disable=SC2086
printf '%s\n' $MONPAGES_EXTRA_URLS
fi
}
# True if body looks like our monitoring HTML (not merchant API / WP 404).
is_monitoring_html() {
local f="$1"
if grep -qE '"code"[[:space:]]*:[[:space:]]*21' "$f" 2>/dev/null; then
return 1
fi
if grep -qiE 'There is no endpoint defined for the URL' "$f" 2>/dev/null; then
return 1
fi
if grep -qiE 'wp-content|wordpress' "$f" 2>/dev/null \
&& grep -qiE '404|not found|page introuvable' "$f" 2>/dev/null; then
return 1
fi
if grep -qE 'sticky-bar|version-link|taler-monitoring|class="sticky|taler-mon:top' "$f" 2>/dev/null; then
return 0
fi
if head -c 4096 "$f" | grep -qiE '<!DOCTYPE html|<html' \
&& grep -qiE 'monitoring|mon ·|host-agent' "$f" 2>/dev/null; then
return 0
fi
if head -c 512 "$f" | grep -qiE '<!DOCTYPE html|<html'; then
return 0
fi
return 1
}
# Highest progress percent in page body (0 if none). Pattern: ┌ 42%┐ or ┌100%┐
_progress_max_pct() {
local f="$1"
local m
m=$(grep -oE '┌[[:space:]]*[0-9]+%┐' "$f" 2>/dev/null \
| grep -oE '[0-9]+' | sort -n | tail -1 || true)
printf '%s' "${m:-0}"
}
# Mid-run: incomplete meta / redirect stub, or progress never reached 100% and no complete foot.
_page_mid_run() {
local f="$1"
local pct
if grep -qE 'name="taler-mon-page"[[:space:]]+content="incomplete"' "$f" 2>/dev/null \
|| grep -q 'taler-mon:bottom:incomplete' "$f" 2>/dev/null \
|| grep -qE 'http-equiv="refresh".*monitoring_err|mode-redirect' "$f" 2>/dev/null; then
return 0
fi
# finished page always has complete foot or classic footer
if grep -q 'taler-mon:bottom:complete' "$f" 2>/dev/null \
|| grep -qE 'id="mon-footer"' "$f" 2>/dev/null; then
return 1
fi
pct=$(_progress_max_pct "$f")
# max progress seen < 100% and no complete footer → still running
if [ "${pct:-0}" -gt 0 ] && [ "${pct:-0}" -lt 100 ]; then
if ! grep -qE 'Console-style render of taler-monitoring|Rendu console de taler-monitoring' "$f" 2>/dev/null; then
return 0
fi
fi
return 1
}
# Validate top/bottom markers + meaningful body (v1.9.0).
# Sets _MON_CONTENT_MSG; returns 0=ok, 1=hard fail, 2=soft/mid-run bottom skip
validate_mon_content() {
local f="$1" url="$2"
local top_ok=1 bot_ok=1 body_ok=1 mid=0 pct
_MON_CONTENT_MSG=""
# --- top markers (always required) ---
if ! head -c 12000 "$f" | grep -qE 'sticky-bar|id="status-bar"|taler-mon:top'; then
top_ok=0
_MON_CONTENT_MSG="missing top marker (sticky-bar / status-bar / taler-mon:top)"
elif ! head -c 16000 "$f" | grep -qE 'data-generated-iso|name="generated"|version-link|taler-mon-page'; then
top_ok=0
_MON_CONTENT_MSG="missing top meta (generated / version-link / taler-mon-page)"
fi
# --- meaningful body: real checks, not only progress bars ---
if grep -qE 'class="line (ok|error|warn|blocker|info)"|┌ OK|┌ ERROR|┌ WARN|┌ BLOCKER|┌ INFO|class="box"' "$f" 2>/dev/null; then
body_ok=1
elif grep -qE 'taler-mon:bottom:incomplete|mode-redirect|has failures' "$f" 2>/dev/null; then
body_ok=1 # redirect stub still has sticky + box
else
# only progress / empty?
if grep -qE '┌[[:space:]]*[0-9]+%┐|█|░' "$f" 2>/dev/null \
&& ! grep -qE '┌ OK|┌ ERROR|class="line ok"' "$f" 2>/dev/null; then
body_ok=0
_MON_CONTENT_MSG="${_MON_CONTENT_MSG:+$_MON_CONTENT_MSG; }only progress bars / no check lines"
elif ! grep -qE 'console|host-agent|taler-monitoring|monitoring' "$f" 2>/dev/null; then
body_ok=0
_MON_CONTENT_MSG="${_MON_CONTENT_MSG:+$_MON_CONTENT_MSG; }no meaningful suite content"
fi
fi
# --- bottom markers ---
if grep -q 'taler-mon:bottom:complete' "$f" 2>/dev/null \
|| tail -c 8000 "$f" | grep -qE 'id="mon-footer"|</footer>|Console-style render of taler-monitoring'; then
bot_ok=1
elif grep -q 'taler-mon:bottom:incomplete' "$f" 2>/dev/null; then
bot_ok=1 # incomplete is an explicit bottom marker
mid=1
else
bot_ok=0
_MON_CONTENT_MSG="${_MON_CONTENT_MSG:+$_MON_CONTENT_MSG; }missing bottom marker (footer / taler-mon:bottom)"
fi
if _page_mid_run "$f"; then
mid=1
fi
pct=$(_progress_max_pct "$f")
if [ "$top_ok" != "1" ]; then
_MON_CONTENT_MSG="TOP: $_MON_CONTENT_MSG"
return 1
fi
if [ "$body_ok" != "1" ]; then
_MON_CONTENT_MSG="BODY: $_MON_CONTENT_MSG"
return 1
fi
if [ "$bot_ok" != "1" ]; then
if [ "$mid" = "1" ]; then
_MON_CONTENT_MSG="BOTTOM soft (mid-run progress ${pct}%): $_MON_CONTENT_MSG"
return 2
fi
_MON_CONTENT_MSG="BOTTOM: $_MON_CONTENT_MSG"
return 1
fi
if [ "$mid" = "1" ] && [ "${pct:-0}" -gt 0 ] && [ "${pct:-0}" -lt 100 ]; then
_MON_CONTENT_MSG="mid-run progress ${pct}% (bottom markers relaxed)"
return 2
fi
_MON_CONTENT_MSG="top+body+bottom markers OK"
return 0
}
_is_bare_url() {
local url="$1"
case " ${MONPAGES_BARE_URLS:-} " in
*" $url "*) return 0 ;;
esac
case "$url" in
*/) return 1 ;;
*) return 0 ;;
esac
}
_mon_fail_or_soft() {
local label="$1" detail="$2"
if [ "${MONPAGES_REQUIRE_PUBLIC:-1}" != "1" ]; then
warn "$label" "$detail (MONPAGES_REQUIRE_PUBLIC=0)"
return 0
fi
fail "$label" "$detail"
return 1
}
check_one() {
local url="$1"
local body code hint soft=0 vc
body=$(mktemp)
code=$(curl -skS -L --max-redirs 5 -m "${TIMEOUT}" -o "$body" -w '%{http_code}' "$url" 2>/dev/null || echo 000)
if _is_bare_url "$url" && [ "${MONPAGES_BARE_STRICT:-0}" != "1" ]; then
soft=1
fi
if grep -qE '"code"[[:space:]]*:[[:space:]]*21' "$body" 2>/dev/null \
|| grep -qiE 'There is no endpoint defined for the URL' "$body" 2>/dev/null; then
if [ "$soft" = "1" ]; then
warn "public mon page bare URL" \
"$url -> HTTP $code code 21 (prefer trailing slash; set MONPAGES_BARE_STRICT=1 to ERROR)"
rm -f "$body"
return 0
fi
_mon_fail_or_soft "public mon page missing" \
"$url -> HTTP $code merchant/API JSON code 21 (publish HTML + reverse-proxy handle /monitoring*)" || { rm -f "$body"; return 1; }
rm -f "$body"
return 0
fi
if [ "$code" != "200" ]; then
if [ "$soft" = "1" ]; then
warn "public mon page bare URL" "$url -> HTTP $code (prefer trailing slash form)"
rm -f "$body"
return 0
fi
_mon_fail_or_soft "public mon page" "$url -> HTTP $code (want 200 HTML)" || { rm -f "$body"; return 1; }
rm -f "$body"
return 0
fi
if ! is_monitoring_html "$body"; then
hint=$(head -c 120 "$body" | tr '\n' ' ' | tr -cd '[:print:] ')
_mon_fail_or_soft "public mon page not monitoring HTML" \
"$url -> HTTP 200 body!=suite (${hint}...)" || { rm -f "$body"; return 1; }
rm -f "$body"
return 0
fi
# v1.9.0: top + bottom markers + meaningful content
set +e
validate_mon_content "$body" "$url"
vc=$?
set -e
case "$vc" in
0)
ok "public mon page" "$url -> HTTP 200 · ${_MON_CONTENT_MSG}"
;;
2)
# mid-run: bottom may be incomplete — not ERROR
warn "public mon page mid-run" "$url -> ${_MON_CONTENT_MSG}"
ok "public mon page" "$url -> HTTP 200 suite HTML (mid-run bottom relaxed)"
;;
*)
if [ "$soft" = "1" ]; then
warn "public mon page content" "$url -> ${_MON_CONTENT_MSG}"
else
_mon_fail_or_soft "public mon page content" \
"$url -> ${_MON_CONTENT_MSG}" || { rm -f "$body"; return 1; }
fi
;;
esac
rm -f "$body"
return 0
}
URLS=()
while IFS= read -r line; do
[ -n "$line" ] || continue
# dedupe
skip=0
for e in "${URLS[@]+"${URLS[@]}"}"; do
[ "$e" = "$line" ] && skip=1 && break
done
[ "$skip" = "1" ] && continue
URLS+=("$line")
done < <(build_urls | sort -u)
# Bare URLs without trailing slash: optional soft check (default on).
# Many reverse proxies only redirect /monitoring → /monitoring/; bare may hit the app (code 21).
# MONPAGES_CHECK_BARE=0 → skip bare entirely
# MONPAGES_BARE_STRICT=1 → bare failures are ERROR (default: WARN if slash form exists)
if [ "${MONPAGES_CHECK_BARE:-1}" = "1" ]; then
_extra=()
for u in "${URLS[@]}"; do
case "$u" in
*/) _bare=${u%/}; [ -n "$_bare" ] && _extra+=("$_bare") ;;
esac
done
for u in "${_extra[@]+"${_extra[@]}"}"; do
skip=0
for e in "${URLS[@]+"${URLS[@]}"}"; do
[ "$e" = "$u" ] && skip=1 && break
done
[ "$skip" = "1" ] && continue
URLS+=("$u")
MONPAGES_BARE_URLS="${MONPAGES_BARE_URLS:-} $u"
done
export MONPAGES_BARE_URLS
fi
if [ "${#URLS[@]}" -eq 0 ]; then
fail "monpages" "no URLs to check (set MON_HOSTS or MONITORING_PAGE_URLS)"
exit 1
fi
_fam="$(_monpages_family)"
echo " family=${_fam} inventory=${MONPAGES_INVENTORY} require_public=${MONPAGES_REQUIRE_PUBLIC}"
echo " checking ${#URLS[@]} URL(s) via FQDN..."
ec=0
for u in "${URLS[@]}"; do
check_one "$u" || ec=1
done
if [ "$ec" -ne 0 ]; then
echo " hint: public monitoring HTML not served (404 / merchant code 21)"
echo " publish staging HTML + configure reverse-proxy/Caddy handle for mon paths"
echo " (on koopa: sudo ~/koopa-caddy/apply-monitoring-live.sh)"
if [ "$_fam" = "fp" ]; then
echo " FP: only FP mon hosts are checked — wire Infomaniak vhost for /monitoring*"
fi
if [ "$_fam" = "goa" ]; then
echo " GOA: full inventory (bank/exchange/taler /monitoring/ + surface/aptdeploy/mattermost/mail)"
fi
# Staging vs public diagnosis (host-agent sets MONPAGES_STAGING_BASE / HTML_OUT)
_stg="${MONPAGES_STAGING_BASE:-${HTML_OUT:-${HTML_BASE:-$HOME/monitoring-sites-staging}}}"
_ok_dir="${HTML_OK_DIR:-monitoring}"
if [ -n "${MON_HOSTS:-}" ] && [ -n "$_stg" ]; then
for _h in $MON_HOSTS; do
_f="$_stg/$_h/${_ok_dir}/index.html"
if [ -f "$_f" ]; then
echo " staging OK: $_f ($(wc -c <"$_f" | tr -d " ") bytes) — not published/served"
else
echo " staging missing: $_f"
fi
done
fi
_www="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}"
if [ -e "$_www" ] && [ ! -w "$_www" ]; then
echo " DEPLOY_WWW_ROOT=$_www not writable by $(id -un 2>/dev/null || echo user)"
fi
fi
exit "$ec"