c1a3cba28b
Hacktivism mon page is max-search only on bank/exchange/taler. Remove classic /monitoring-ladder units, monpages catalog, and Caddy bare redirs.
673 lines
24 KiB
Bash
Executable file
673 lines
24 KiB
Bash
Executable file
#!/usr/bin/env bash
|
||
# check_monitoring_pages.sh — verify public monitoring HTML pages via FQDN
|
||
#
|
||
# Outside-in: curl https://<fqdn>/<path>/ and require a real HTML monitoring page.
|
||
# Catches: merchant JSON code 21 (Caddy handle missing), WP 404, empty stubs, deploy skip.
|
||
#
|
||
# Policy (v1.7.6+):
|
||
# • monpages is obligatory → missing/wrong pages are ERROR (exit 1).
|
||
# • GOA / hacktivism: full inventory of suite monitoring sites (catalog + on-disk discovery).
|
||
# • FP: only that stack’s own monitoring hosts/paths (never GOA URLs).
|
||
# • Override soft mode only with MONPAGES_REQUIRE_PUBLIC=0 (emergency / staging).
|
||
#
|
||
# Env:
|
||
# MON_HOSTS space-separated FQDNs (default from TALER_DOMAIN)
|
||
# HTML_URL_OK path for OK page (default /monitoring/)
|
||
# HTML_URL_ERR err path; checked if present on disk or MONPAGES_CHECK_ERR=1
|
||
# MONITORING_PAGE_URLS optional full URL list (overrides inventory construction)
|
||
# MONPAGES_EXTRA_URLS additional absolute URLs (space-separated)
|
||
# MONPAGES_INVENTORY auto|full|job (default auto)
|
||
# auto/full = stack inventory; job = only MON_HOSTS+paths
|
||
# MONPAGES_REQUIRE_PUBLIC default 1 (ERROR). 0 = WARN only (escape hatch).
|
||
# MONPAGES_STAGING_BASE / HTML_OUT / DEPLOY_WWW_ROOT trees for discovery
|
||
# TIMEOUT curl timeout (from lib.sh)
|
||
#
|
||
set -euo pipefail
|
||
ROOT=$(cd "$(dirname "$0")" && pwd)
|
||
# shellcheck source=lib.sh
|
||
source "$ROOT/lib.sh"
|
||
|
||
set_area monpages
|
||
section "monpages · public monitoring HTML via FQDN (outside-in, obligatory)"
|
||
|
||
# Obligatory: public pages missing → ERROR. Soft only if explicitly disabled.
|
||
: "${MONPAGES_REQUIRE_PUBLIC:=1}"
|
||
: "${MONPAGES_INVENTORY:=auto}"
|
||
|
||
# Default hosts from domain (mirror run-host-report.sh)
|
||
if [ -z "${MON_HOSTS:-}" ]; then
|
||
case "${TALER_DOMAIN:-}" in
|
||
hacktivism.ch|koopa)
|
||
MON_HOSTS="bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch"
|
||
;;
|
||
stage.lefrancpaysan.ch|stage.*lefrancpaysan*)
|
||
MON_HOSTS="stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch"
|
||
;;
|
||
lefrancpaysan.ch)
|
||
MON_HOSTS="bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch"
|
||
;;
|
||
*)
|
||
MON_HOSTS="${TALER_DOMAIN:-}"
|
||
;;
|
||
esac
|
||
fi
|
||
|
||
HTML_URL_OK="${HTML_URL_OK:-/monitoring/}"
|
||
HTML_URL_ERR="${HTML_URL_ERR:-/monitoring_err/}"
|
||
# ensure leading + trailing slash
|
||
_norm_path() {
|
||
local p="$1"
|
||
case "$p" in
|
||
/*) ;;
|
||
*) p="/$p" ;;
|
||
esac
|
||
case "$p" in
|
||
*/) ;;
|
||
*) p="${p}/" ;;
|
||
esac
|
||
printf '%s' "$p"
|
||
}
|
||
HTML_URL_OK="$(_norm_path "$HTML_URL_OK")"
|
||
HTML_URL_ERR="$(_norm_path "$HTML_URL_ERR")"
|
||
|
||
# Stack family: goa | fp | other — FP never checks GOA and vice versa.
|
||
_monpages_family() {
|
||
case "${TALER_DOMAIN:-}" in
|
||
*lefrancpaysan*|*francpaysan*) printf 'fp' ;;
|
||
hacktivism.ch|koopa) printf 'goa' ;;
|
||
*)
|
||
# Infer from MON_HOSTS if domain ambiguous
|
||
case " ${MON_HOSTS:-} " in
|
||
*lefrancpaysan*) printf 'fp' ;;
|
||
*hacktivism*) printf 'goa' ;;
|
||
*) printf 'other' ;;
|
||
esac
|
||
;;
|
||
esac
|
||
}
|
||
|
||
_host_allowed() {
|
||
local h="$1" fam="$2"
|
||
case "$fam" in
|
||
goa)
|
||
case "$h" in *.hacktivism.ch) return 0 ;; *) return 1 ;; esac
|
||
;;
|
||
fp)
|
||
case "$h" in *lefrancpaysan*) return 0 ;; *) return 1 ;; esac
|
||
;;
|
||
*) return 0 ;;
|
||
esac
|
||
}
|
||
|
||
# Public mon URL families (suite name "taler-monitoring" ≠ a public path):
|
||
# 1) /monitoring(+_err) — landing hosts (bare + slash)
|
||
# 2) /monitoring-max-ladder(+_err) — GOA max-search amount ladder (daily, hacktivism)
|
||
# 3) /taler-monitoring-surface(+_err) — taler.hacktivism.ch only
|
||
# 4) /taler-monitoring-aptdeploy(+_err) — taler.hacktivism.ch only
|
||
# No classic /monitoring-ladder. No /taler-monitoring page.
|
||
|
||
_path_allowed() {
|
||
case "$1" in
|
||
monitoring|monitoring_err|\
|
||
monitoring-max-ladder|monitoring-max-ladder_err|\
|
||
taler-monitoring-surface|taler-monitoring-surface_err|\
|
||
taler-monitoring-aptdeploy|taler-monitoring-aptdeploy_err) return 0 ;;
|
||
# explicit deny (never invent a mon page here)
|
||
monitoring-ladder|monitoring-ladder_err|\
|
||
taler-monitoring|taler-monitoring-mail|taler-monitoring-mattermost) return 1 ;;
|
||
*) return 1 ;;
|
||
esac
|
||
}
|
||
|
||
# Emit https://host/path/ for each …/host/path/index.html under root (max depth 3).
|
||
_discover_tree() {
|
||
local root="$1" fam="$2"
|
||
local f rel host path
|
||
[ -n "$root" ] && [ -d "$root" ] || return 0
|
||
# layout: $root/<fqdn>/<url-path>/index.html
|
||
while IFS= read -r -d '' f; do
|
||
rel="${f#"${root%/}"/}"
|
||
host="${rel%%/*}"
|
||
path="${rel#*/}"
|
||
path="${path%/index.html}"
|
||
[ -n "$host" ] && [ -n "$path" ] || continue
|
||
_host_allowed "$host" "$fam" || continue
|
||
_path_allowed "$path" || continue
|
||
printf 'https://%s/%s/\n' "$host" "$path"
|
||
done < <(find "${root%/}" -mindepth 3 -maxdepth 3 -type f -name index.html -print0 2>/dev/null)
|
||
}
|
||
|
||
# Landing stacks (GOA 3 + FP stage 3) with /monitoring/
|
||
_landing_allowed() {
|
||
printf '%s\n' \
|
||
bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch \
|
||
stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch
|
||
}
|
||
|
||
# Suite catalog of OK pages that must exist for this family.
|
||
_catalog_urls() {
|
||
local fam="$1" h
|
||
case "$fam" in
|
||
goa)
|
||
for h in bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch; do
|
||
printf 'https://%s/monitoring/\n' "$h"
|
||
printf 'https://%s/monitoring-max-ladder/\n' "$h"
|
||
done
|
||
# ecosystem surface (versions / software) + apt-src deploy mon page
|
||
printf 'https://taler.hacktivism.ch/taler-monitoring-surface/\n'
|
||
printf 'https://taler.hacktivism.ch/taler-monitoring-aptdeploy/\n'
|
||
;;
|
||
fp)
|
||
for h in $MON_HOSTS; do
|
||
[ -z "$h" ] && continue
|
||
_host_allowed "$h" fp || continue
|
||
printf 'https://%s/monitoring/\n' "$h"
|
||
done
|
||
;;
|
||
all|nine)
|
||
# GOA + FP stage landings + surface + aptdeploy (nine kept as alias for inventory)
|
||
while IFS= read -r h; do
|
||
[ -n "$h" ] || continue
|
||
printf 'https://%s/monitoring/\n' "$h"
|
||
case "$h" in
|
||
*.hacktivism.ch)
|
||
printf 'https://%s/monitoring-max-ladder/\n' "$h"
|
||
;;
|
||
esac
|
||
done < <(_landing_allowed)
|
||
printf 'https://taler.hacktivism.ch/taler-monitoring-surface/\n'
|
||
printf 'https://taler.hacktivism.ch/taler-monitoring-aptdeploy/\n'
|
||
;;
|
||
esac
|
||
}
|
||
|
||
# Job-local URLs (this host-agent run’s path).
|
||
_job_urls() {
|
||
local h p
|
||
for h in $MON_HOSTS; do
|
||
[ -z "$h" ] && continue
|
||
printf 'https://%s%s\n' "$h" "$HTML_URL_OK"
|
||
if [ "${MONPAGES_CHECK_ERR:-0}" = "1" ]; then
|
||
printf 'https://%s%s\n' "$h" "$HTML_URL_ERR"
|
||
fi
|
||
done
|
||
}
|
||
|
||
build_urls() {
|
||
local fam inv
|
||
fam="$(_monpages_family)"
|
||
inv="${MONPAGES_INVENTORY:-auto}"
|
||
|
||
if [ -n "${MONITORING_PAGE_URLS:-}" ]; then
|
||
# shellcheck disable=SC2086
|
||
printf '%s\n' $MONITORING_PAGE_URLS
|
||
else
|
||
case "$inv" in
|
||
job)
|
||
# Only this host-agent job’s HTML paths (MON_HOSTS + HTML_URL_OK/_ERR).
|
||
# Specialized timers (aptdeploy, surface) must not inherit GOA full catalog
|
||
# (no bank/exchange /monitoring bare-URL noise on the aptdeploy page).
|
||
_job_urls
|
||
;;
|
||
full|auto|*)
|
||
# Always include this job’s paths (host-agent post-check).
|
||
_job_urls
|
||
# Stack inventory: GOA = landings + surface + aptdeploy; FP = only FP mon hosts.
|
||
# Full inventory belongs on the general stack run (run-hacktivism-monitoring).
|
||
if [ "$fam" = "goa" ] || [ "$fam" = "fp" ]; then
|
||
_catalog_urls "$fam"
|
||
# On-disk discovery (err pages only exist after failed runs; extras welcome)
|
||
[ -n "${DEPLOY_WWW_ROOT:-}" ] && _discover_tree "${DEPLOY_WWW_ROOT}" "$fam"
|
||
_discover_tree "${MONPAGES_STAGING_BASE:-${HTML_OUT:-${HTML_BASE:-$HOME/monitoring-sites-staging}}}" "$fam"
|
||
fi
|
||
;;
|
||
esac
|
||
fi
|
||
if [ -n "${MONPAGES_EXTRA_URLS:-}" ]; then
|
||
# shellcheck disable=SC2086
|
||
printf '%s\n' $MONPAGES_EXTRA_URLS
|
||
fi
|
||
}
|
||
|
||
# Install stub only (no console log) — not a real mon page.
|
||
is_bootstrap_html() {
|
||
local f="$1"
|
||
grep -qiE 'Monitoring page \(bootstrap\)|pill">STAGE</|install stub|bootstrap page' "$f" 2>/dev/null \
|
||
&& ! grep -qE 'taler-mon:top|id="mon-console"|class="console"' "$f" 2>/dev/null
|
||
}
|
||
|
||
# Stale mon HTML from pre-1.15.8 generators: box-drawing char-class regex in
|
||
# sticky filter JS (Python SyntaxWarning: invalid escape sequence '\-').
|
||
# Never accept these as "current" public assets (v1.18.2+).
|
||
is_stale_generator_html() {
|
||
local f="$1"
|
||
# Old isLowValue() JS (literal box glyphs + \- class)
|
||
if grep -qE 'if \(/\^\[[╔╗╚╝║═]' "$f" 2>/dev/null; then
|
||
return 0
|
||
fi
|
||
# Converter noise teed into old logs and re-rendered
|
||
if grep -qiE 'SyntaxWarning: invalid escape sequence' "$f" 2>/dev/null; then
|
||
return 0
|
||
fi
|
||
if grep -qiE 'console_to_html\.py:[0-9]+: SyntaxWarning' "$f" 2>/dev/null; then
|
||
return 0
|
||
fi
|
||
return 1
|
||
}
|
||
|
||
# True if body looks like our monitoring HTML (not merchant API / WP 404 / bootstrap).
|
||
is_monitoring_html() {
|
||
local f="$1"
|
||
if grep -qE '"code"[[:space:]]*:[[:space:]]*21' "$f" 2>/dev/null; then
|
||
return 1
|
||
fi
|
||
if grep -qiE 'There is no endpoint defined for the URL' "$f" 2>/dev/null; then
|
||
return 1
|
||
fi
|
||
if grep -qiE 'wp-content|wordpress' "$f" 2>/dev/null \
|
||
&& grep -qiE '404|not found|page introuvable' "$f" 2>/dev/null; then
|
||
return 1
|
||
fi
|
||
# Bootstrap sticky is not enough — need suite console output
|
||
if is_bootstrap_html "$f"; then
|
||
return 1
|
||
fi
|
||
if grep -qE 'sticky-bar|version-link|taler-monitoring|class="sticky|taler-mon:top' "$f" 2>/dev/null; then
|
||
return 0
|
||
fi
|
||
if head -c 4096 "$f" | grep -qiE '<!DOCTYPE html|<html' \
|
||
&& grep -qiE 'monitoring|mon ·|host-agent' "$f" 2>/dev/null; then
|
||
return 0
|
||
fi
|
||
if head -c 512 "$f" | grep -qiE '<!DOCTYPE html|<html'; then
|
||
return 0
|
||
fi
|
||
return 1
|
||
}
|
||
|
||
# Highest progress percent in page body (0 if none). Pattern: ┌ 42%┐ or ┌100%┐
|
||
_progress_max_pct() {
|
||
local f="$1"
|
||
local m
|
||
m=$(grep -oE '┌[[:space:]]*[0-9]+%┐' "$f" 2>/dev/null \
|
||
| grep -oE '[0-9]+' | sort -n | tail -1 || true)
|
||
printf '%s' "${m:-0}"
|
||
}
|
||
|
||
# Mid-run: incomplete meta / redirect stub, or progress never reached 100% and no complete foot.
|
||
_page_mid_run() {
|
||
local f="$1"
|
||
local pct
|
||
if grep -qE 'name="taler-mon-page"[[:space:]]+content="incomplete"' "$f" 2>/dev/null \
|
||
|| grep -q 'taler-mon:bottom:incomplete' "$f" 2>/dev/null \
|
||
|| grep -qE 'http-equiv="refresh".*monitoring_err|mode-redirect' "$f" 2>/dev/null; then
|
||
return 0
|
||
fi
|
||
# finished page always has complete foot or classic footer
|
||
if grep -q 'taler-mon:bottom:complete' "$f" 2>/dev/null \
|
||
|| grep -qE 'id="mon-footer"' "$f" 2>/dev/null; then
|
||
return 1
|
||
fi
|
||
pct=$(_progress_max_pct "$f")
|
||
# max progress seen < 100% and no complete footer → still running
|
||
if [ "${pct:-0}" -gt 0 ] && [ "${pct:-0}" -lt 100 ]; then
|
||
if ! grep -qE 'Console-style render of taler-monitoring|Rendu console de taler-monitoring' "$f" 2>/dev/null; then
|
||
return 0
|
||
fi
|
||
fi
|
||
return 1
|
||
}
|
||
|
||
# Validate top/bottom markers + meaningful body (v1.9.0).
|
||
# Sets _MON_CONTENT_MSG; returns 0=ok, 1=hard fail, 2=soft/mid-run bottom skip
|
||
#
|
||
# Head windows are large (v1.13.4+): sticky CSS grew past old 12–16 KiB limits and
|
||
# caused false TOP errors while publish races were also writing the file.
|
||
validate_mon_content() {
|
||
local f="$1" url="$2"
|
||
local top_ok=1 bot_ok=1 body_ok=1 mid=0 pct
|
||
local head_top="${MONPAGES_HEAD_TOP_BYTES:-49152}"
|
||
local head_meta="${MONPAGES_HEAD_META_BYTES:-65536}"
|
||
_MON_CONTENT_MSG=""
|
||
|
||
# Tiny / empty body = likely mid-publish race (partial rsync or curl)
|
||
if [ ! -s "$f" ] || [ "$(wc -c <"$f" 2>/dev/null || echo 0)" -lt 200 ]; then
|
||
top_ok=0
|
||
_MON_CONTENT_MSG="empty or truncated body (publish race?)"
|
||
# --- top markers (always required) ---
|
||
elif ! head -c "$head_top" "$f" | grep -qE 'sticky-bar|id="status-bar"|taler-mon:top'; then
|
||
# full-file fallback (small pages / unusual layout)
|
||
if ! grep -qE 'sticky-bar|id="status-bar"|taler-mon:top' "$f" 2>/dev/null; then
|
||
top_ok=0
|
||
_MON_CONTENT_MSG="missing top marker (sticky-bar / status-bar / taler-mon:top)"
|
||
fi
|
||
elif ! head -c "$head_meta" "$f" | grep -qE 'data-generated-iso|name="generated"|version-link|taler-mon-page'; then
|
||
if ! grep -qE 'data-generated-iso|name="generated"|version-link|taler-mon-page' "$f" 2>/dev/null; then
|
||
top_ok=0
|
||
_MON_CONTENT_MSG="missing top meta (generated / version-link / taler-mon-page)"
|
||
fi
|
||
fi
|
||
|
||
# --- meaningful body: real checks, not only progress bars ---
|
||
if grep -qE 'class="line (ok|error|warn|blocker|info)"|┌ OK|┌ ERROR|┌ WARN|┌ BLOCKER|┌ INFO|class="box"' "$f" 2>/dev/null; then
|
||
body_ok=1
|
||
elif grep -qE 'taler-mon:bottom:incomplete|mode-redirect|has failures' "$f" 2>/dev/null; then
|
||
body_ok=1 # redirect stub still has sticky + box
|
||
else
|
||
# only progress / empty?
|
||
if grep -qE '┌[[:space:]]*[0-9]+%┐|█|░' "$f" 2>/dev/null \
|
||
&& ! grep -qE '┌ OK|┌ ERROR|class="line ok"' "$f" 2>/dev/null; then
|
||
body_ok=0
|
||
_MON_CONTENT_MSG="${_MON_CONTENT_MSG:+$_MON_CONTENT_MSG; }only progress bars / no check lines"
|
||
elif ! grep -qE 'console|host-agent|taler-monitoring|monitoring' "$f" 2>/dev/null; then
|
||
body_ok=0
|
||
_MON_CONTENT_MSG="${_MON_CONTENT_MSG:+$_MON_CONTENT_MSG; }no meaningful suite content"
|
||
fi
|
||
fi
|
||
|
||
# --- bottom markers ---
|
||
if grep -q 'taler-mon:bottom:complete' "$f" 2>/dev/null \
|
||
|| tail -c 8000 "$f" | grep -qE 'id="mon-footer"|</footer>|Console-style render of taler-monitoring'; then
|
||
bot_ok=1
|
||
elif grep -q 'taler-mon:bottom:incomplete' "$f" 2>/dev/null; then
|
||
bot_ok=1 # incomplete is an explicit bottom marker
|
||
mid=1
|
||
else
|
||
bot_ok=0
|
||
_MON_CONTENT_MSG="${_MON_CONTENT_MSG:+$_MON_CONTENT_MSG; }missing bottom marker (footer / taler-mon:bottom)"
|
||
fi
|
||
|
||
if _page_mid_run "$f"; then
|
||
mid=1
|
||
fi
|
||
pct=$(_progress_max_pct "$f")
|
||
|
||
if [ "$top_ok" != "1" ]; then
|
||
# TOP missing is WARN-only (v1.13.8+): mid-run races and incomplete
|
||
# /monitoring stubs used to self-red the next HTML. Hard ERROR still for
|
||
# missing page / code 21 / non-suite body. Override: MONPAGES_TOP_STRICT=1.
|
||
_MON_CONTENT_MSG="TOP: $_MON_CONTENT_MSG"
|
||
if [ "${MONPAGES_TOP_STRICT:-0}" = "1" ]; then
|
||
return 1
|
||
fi
|
||
return 2
|
||
fi
|
||
if [ "$body_ok" != "1" ]; then
|
||
_MON_CONTENT_MSG="BODY: $_MON_CONTENT_MSG"
|
||
return 1
|
||
fi
|
||
if [ "$bot_ok" != "1" ]; then
|
||
if [ "$mid" = "1" ]; then
|
||
_MON_CONTENT_MSG="BOTTOM soft (mid-run progress ${pct}%): $_MON_CONTENT_MSG"
|
||
return 2
|
||
fi
|
||
_MON_CONTENT_MSG="BOTTOM: $_MON_CONTENT_MSG"
|
||
return 1
|
||
fi
|
||
if [ "$mid" = "1" ] && [ "${pct:-0}" -gt 0 ] && [ "${pct:-0}" -lt 100 ]; then
|
||
_MON_CONTENT_MSG="mid-run progress ${pct}% (bottom markers relaxed)"
|
||
return 2
|
||
fi
|
||
_MON_CONTENT_MSG="top+body+bottom markers OK"
|
||
return 0
|
||
}
|
||
|
||
_is_bare_url() {
|
||
local url="$1"
|
||
case " ${MONPAGES_BARE_URLS:-} " in
|
||
*" $url "*) return 0 ;;
|
||
esac
|
||
case "$url" in
|
||
*/) return 1 ;;
|
||
*) return 0 ;;
|
||
esac
|
||
}
|
||
|
||
_mon_fail_or_soft() {
|
||
local label="$1" detail="$2"
|
||
if [ "${MONPAGES_REQUIRE_PUBLIC:-1}" != "1" ]; then
|
||
warn "$label" "$detail (MONPAGES_REQUIRE_PUBLIC=0)"
|
||
return 0
|
||
fi
|
||
fail "$label" "$detail"
|
||
return 1
|
||
}
|
||
|
||
# Fetch + validate one URL. Retries content failures (publish/rsync races).
|
||
# MONPAGES_PRE_PUBLISH=1 → content failures are WARN only (suite still regenerates HTML after).
|
||
# MONPAGES_FETCH_RETRIES (default 3), MONPAGES_FETCH_SLEEP_S (default 0.5).
|
||
check_one() {
|
||
local url="$1"
|
||
local body code hint soft=0 vc attempt max_try sleep_s pre_pub
|
||
body=$(mktemp)
|
||
if _is_bare_url "$url" && [ "${MONPAGES_BARE_STRICT:-0}" != "1" ]; then
|
||
soft=1
|
||
fi
|
||
pre_pub=0
|
||
[ "${MONPAGES_PRE_PUBLISH:-0}" = "1" ] && pre_pub=1
|
||
max_try="${MONPAGES_FETCH_RETRIES:-3}"
|
||
sleep_s="${MONPAGES_FETCH_SLEEP_S:-0.5}"
|
||
[ "$max_try" -lt 1 ] && max_try=1
|
||
|
||
attempt=1
|
||
while [ "$attempt" -le "$max_try" ]; do
|
||
: >"$body"
|
||
code=$(curl -skS -L --max-redirs 5 -m "${TIMEOUT}" -o "$body" -w '%{http_code}' "$url" 2>/dev/null || echo 000)
|
||
|
||
if grep -qE '"code"[[:space:]]*:[[:space:]]*21' "$body" 2>/dev/null \
|
||
|| grep -qiE 'There is no endpoint defined for the URL' "$body" 2>/dev/null; then
|
||
# Bare without trailing slash often hits merchant until Caddy has redir * /path/ 302.
|
||
if [ "$soft" = "1" ]; then
|
||
warn "public mon page bare URL" \
|
||
"$url -> HTTP $code code 21 (prefer trailing slash; Caddy: redir * /path/ 302; set MONPAGES_BARE_STRICT=1 to ERROR)"
|
||
rm -f "$body"
|
||
return 0
|
||
fi
|
||
# hard 21: retry once more (rare brief outage) then fail
|
||
if [ "$attempt" -lt "$max_try" ]; then
|
||
attempt=$((attempt + 1))
|
||
sleep "$sleep_s" 2>/dev/null || sleep 1
|
||
continue
|
||
fi
|
||
_mon_fail_or_soft "public mon page missing" \
|
||
"$url -> HTTP $code merchant/API JSON code 21 (Caddy handle/redir; use redir * /path/ 302 inside handle${APPLY_MONITORING_LIVE:+; apply: sudo $APPLY_MONITORING_LIVE})" || { rm -f "$body"; return 1; }
|
||
rm -f "$body"
|
||
return 0
|
||
fi
|
||
|
||
if [ "$code" != "200" ]; then
|
||
if [ "$soft" = "1" ]; then
|
||
warn "public mon page bare URL" "$url -> HTTP $code (prefer trailing slash form)"
|
||
rm -f "$body"
|
||
return 0
|
||
fi
|
||
if [ "$attempt" -lt "$max_try" ]; then
|
||
attempt=$((attempt + 1))
|
||
sleep "$sleep_s" 2>/dev/null || sleep 1
|
||
continue
|
||
fi
|
||
_mon_fail_or_soft "public mon page" "$url -> HTTP $code (want 200 HTML)" || { rm -f "$body"; return 1; }
|
||
rm -f "$body"
|
||
return 0
|
||
fi
|
||
|
||
if is_bootstrap_html "$body"; then
|
||
if [ "$attempt" -lt "$max_try" ]; then
|
||
attempt=$((attempt + 1))
|
||
sleep "$sleep_s" 2>/dev/null || sleep 1
|
||
continue
|
||
fi
|
||
_mon_fail_or_soft "public mon page bootstrap stub" \
|
||
"$url -> install bootstrap HTML still live (run host-agent; suite auto-update must replace stub)" || { rm -f "$body"; return 1; }
|
||
rm -f "$body"
|
||
return 0
|
||
fi
|
||
|
||
# Stale generator JS / SyntaxWarning in public body (pre-1.15.8 assets)
|
||
if is_stale_generator_html "$body"; then
|
||
if [ "$attempt" -lt "$max_try" ]; then
|
||
attempt=$((attempt + 1))
|
||
sleep "$sleep_s" 2>/dev/null || sleep 1
|
||
continue
|
||
fi
|
||
_mon_fail_or_soft "public mon page stale generator" \
|
||
"$url -> old mon HTML (box-drawing JS regex and/or SyntaxWarning in body) — re-run host-agent on suite ≥v1.15.8 / v1.18.2; do not leave pre-update assets live" || { rm -f "$body"; return 1; }
|
||
rm -f "$body"
|
||
return 0
|
||
fi
|
||
|
||
if ! is_monitoring_html "$body"; then
|
||
if [ "$attempt" -lt "$max_try" ]; then
|
||
attempt=$((attempt + 1))
|
||
sleep "$sleep_s" 2>/dev/null || sleep 1
|
||
continue
|
||
fi
|
||
hint=$(head -c 120 "$body" | tr '\n' ' ' | tr -cd '[:print:] ')
|
||
_mon_fail_or_soft "public mon page not monitoring HTML" \
|
||
"$url -> HTTP 200 body!=suite (${hint}...)" || { rm -f "$body"; return 1; }
|
||
rm -f "$body"
|
||
return 0
|
||
fi
|
||
|
||
# v1.9.0: top + bottom markers + meaningful content
|
||
set +e
|
||
validate_mon_content "$body" "$url"
|
||
vc=$?
|
||
set -e
|
||
case "$vc" in
|
||
0)
|
||
ok "public mon page" "$url -> HTTP 200 · ${_MON_CONTENT_MSG}"
|
||
rm -f "$body"
|
||
return 0
|
||
;;
|
||
2)
|
||
# soft: TOP incomplete / mid-run bottom — WARN, never hard fail
|
||
case "${_MON_CONTENT_MSG}" in
|
||
TOP:*)
|
||
warn "public mon page content" \
|
||
"$url -> ${_MON_CONTENT_MSG} (WARN only; set MONPAGES_TOP_STRICT=1 to ERROR)"
|
||
;;
|
||
*)
|
||
warn "public mon page mid-run" "$url -> ${_MON_CONTENT_MSG}"
|
||
ok "public mon page" "$url -> HTTP 200 suite HTML (mid-run bottom relaxed)"
|
||
;;
|
||
esac
|
||
rm -f "$body"
|
||
return 0
|
||
;;
|
||
*)
|
||
# Content race (truncated HTML while another agent publishes): retry
|
||
if [ "$attempt" -lt "$max_try" ]; then
|
||
attempt=$((attempt + 1))
|
||
sleep "$sleep_s" 2>/dev/null || sleep 1
|
||
continue
|
||
fi
|
||
# TOP failures should already be soft (return 2); belt-and-braces:
|
||
case "${_MON_CONTENT_MSG}" in
|
||
TOP:*)
|
||
if [ "${MONPAGES_TOP_STRICT:-0}" != "1" ]; then
|
||
warn "public mon page content" \
|
||
"$url -> ${_MON_CONTENT_MSG} (WARN only; set MONPAGES_TOP_STRICT=1 to ERROR)"
|
||
rm -f "$body"
|
||
return 0
|
||
fi
|
||
;;
|
||
esac
|
||
# Pre-publish inventory in host-agent PHASES: WARN only (post-check is hard)
|
||
if [ "$pre_pub" = "1" ] || [ "$soft" = "1" ]; then
|
||
warn "public mon page content" \
|
||
"$url -> ${_MON_CONTENT_MSG}${pre_pub:+ (pre-publish; post-check is authoritative)}"
|
||
rm -f "$body"
|
||
return 0
|
||
fi
|
||
_mon_fail_or_soft "public mon page content" \
|
||
"$url -> ${_MON_CONTENT_MSG}" || { rm -f "$body"; return 1; }
|
||
rm -f "$body"
|
||
return 0
|
||
;;
|
||
esac
|
||
done
|
||
rm -f "$body"
|
||
return 0
|
||
}
|
||
|
||
URLS=()
|
||
while IFS= read -r line; do
|
||
[ -n "$line" ] || continue
|
||
# dedupe
|
||
skip=0
|
||
for e in "${URLS[@]+"${URLS[@]}"}"; do
|
||
[ "$e" = "$line" ] && skip=1 && break
|
||
done
|
||
[ "$skip" = "1" ] && continue
|
||
URLS+=("$line")
|
||
done < <(build_urls | sort -u)
|
||
|
||
# Bare URLs without trailing slash: checked by default.
|
||
# Merchant JSON code 21 is always ERROR (Caddy/proxy not serving mon path).
|
||
# Other bare failures: WARN unless MONPAGES_BARE_STRICT=1 (then ERROR).
|
||
# MONPAGES_CHECK_BARE=0 → skip bare entirely
|
||
# MONPAGES_BARE_STRICT=1 → non-21 bare failures are ERROR too
|
||
if [ "${MONPAGES_CHECK_BARE:-1}" = "1" ]; then
|
||
_extra=()
|
||
for u in "${URLS[@]}"; do
|
||
case "$u" in
|
||
*/) _bare=${u%/}; [ -n "$_bare" ] && _extra+=("$_bare") ;;
|
||
esac
|
||
done
|
||
for u in "${_extra[@]+"${_extra[@]}"}"; do
|
||
skip=0
|
||
for e in "${URLS[@]+"${URLS[@]}"}"; do
|
||
[ "$e" = "$u" ] && skip=1 && break
|
||
done
|
||
[ "$skip" = "1" ] && continue
|
||
URLS+=("$u")
|
||
MONPAGES_BARE_URLS="${MONPAGES_BARE_URLS:-} $u"
|
||
done
|
||
export MONPAGES_BARE_URLS
|
||
fi
|
||
|
||
if [ "${#URLS[@]}" -eq 0 ]; then
|
||
fail "monpages" "no URLs to check (set MON_HOSTS or MONITORING_PAGE_URLS)"
|
||
exit 1
|
||
fi
|
||
|
||
_fam="$(_monpages_family)"
|
||
echo " family=${_fam} inventory=${MONPAGES_INVENTORY} require_public=${MONPAGES_REQUIRE_PUBLIC}"
|
||
echo " checking ${#URLS[@]} URL(s) via FQDN..."
|
||
ec=0
|
||
for u in "${URLS[@]}"; do
|
||
check_one "$u" || ec=1
|
||
done
|
||
|
||
if [ "$ec" -ne 0 ]; then
|
||
echo " hint: public monitoring HTML not served (404 / merchant code 21)"
|
||
echo " publish staging HTML + configure reverse-proxy/Caddy handle for mon paths"
|
||
echo " (set APPLY_MONITORING_LIVE in env; sudo "$APPLY_MONITORING_LIVE")"
|
||
echo " bare code 21: Caddy redir inside handle must be: redir * /path/ 302 (not redir /path/ 302)"
|
||
if [ "$_fam" = "fp" ]; then
|
||
echo " FP: only FP mon hosts are checked — wire Infomaniak vhost for /monitoring*"
|
||
fi
|
||
if [ "$_fam" = "goa" ]; then
|
||
echo " GOA: bank/exchange/taler /monitoring/ + surface + aptdeploy (v1.9.4+ layout)"
|
||
fi
|
||
# Staging vs public diagnosis (host-agent sets MONPAGES_STAGING_BASE / HTML_OUT)
|
||
_stg="${MONPAGES_STAGING_BASE:-${HTML_OUT:-${HTML_BASE:-$HOME/monitoring-sites-staging}}}"
|
||
_ok_dir="${HTML_OK_DIR:-monitoring}"
|
||
if [ -n "${MON_HOSTS:-}" ] && [ -n "$_stg" ]; then
|
||
for _h in $MON_HOSTS; do
|
||
_f="$_stg/$_h/${_ok_dir}/index.html"
|
||
if [ -f "$_f" ]; then
|
||
echo " staging OK: $_f ($(wc -c <"$_f" | tr -d " ") bytes) — not published/served"
|
||
else
|
||
echo " staging missing: $_f"
|
||
fi
|
||
done
|
||
fi
|
||
_www="${DEPLOY_WWW_ROOT:-}"
|
||
if [ -e "$_www" ] && [ ! -w "$_www" ]; then
|
||
echo " DEPLOY_WWW_ROOT=$_www not writable by $(id -un 2>/dev/null || echo user)"
|
||
fi
|
||
fi
|
||
|
||
exit "$ec"
|