taler-monitoring/check_monitoring_pages.sh

#!/usr/bin/env bash
# check_monitoring_pages.sh — verify public monitoring HTML pages via FQDN
#
# Outside-in: curl https://<fqdn>/<path>/ and require a real HTML monitoring page.
# Catches: merchant JSON code 21 (Caddy handle missing), WP 404, empty stubs, deploy skip.
#
# Policy (v1.7.6+):
#   • monpages is obligatory → missing/wrong pages are ERROR (exit 1).
#   • GOA / hacktivism: full inventory of suite monitoring sites (catalog + on-disk discovery).
#   • FP: only that stack’s own monitoring hosts/paths (never GOA URLs).
#   • Override soft mode only with MONPAGES_REQUIRE_PUBLIC=0 (emergency / staging).
#
# Env:
#   MON_HOSTS             space-separated FQDNs (default from TALER_DOMAIN)
#   HTML_URL_OK           path for OK page (default /monitoring/)
#   HTML_URL_ERR          err path; checked if present on disk or MONPAGES_CHECK_ERR=1
#   MONITORING_PAGE_URLS  optional full URL list (overrides inventory construction)
#   MONPAGES_EXTRA_URLS   additional absolute URLs (space-separated)
#   MONPAGES_INVENTORY    auto|full|job  (default auto)
#                         auto/full = stack inventory; job = only MON_HOSTS+paths
#   MONPAGES_REQUIRE_PUBLIC  default 1 (ERROR). 0 = WARN only (escape hatch).
#   MONPAGES_STAGING_BASE / HTML_OUT / DEPLOY_WWW_ROOT  trees for discovery
#   TIMEOUT               curl timeout (from lib.sh)
#
set -euo pipefail
ROOT=$(cd "$(dirname "$0")" && pwd)
# shellcheck source=lib.sh
source "$ROOT/lib.sh"

set_area monpages
section "monpages · public monitoring HTML via FQDN (outside-in, obligatory)"

# Obligatory: public pages missing → ERROR. Soft only if explicitly disabled.
: "${MONPAGES_REQUIRE_PUBLIC:=1}"
: "${MONPAGES_INVENTORY:=auto}"

# Default hosts from domain (mirror run-host-report.sh)
if [ -z "${MON_HOSTS:-}" ]; then
  case "${TALER_DOMAIN:-}" in
    hacktivism.ch|koopa)
      MON_HOSTS="bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch"
      ;;
    stage.lefrancpaysan.ch|stage.*lefrancpaysan*)
      MON_HOSTS="stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch"
      ;;
    lefrancpaysan.ch)
      MON_HOSTS="bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch"
      ;;
    *)
      MON_HOSTS="${TALER_DOMAIN:-}"
      ;;
  esac
fi

HTML_URL_OK="${HTML_URL_OK:-/monitoring/}"
HTML_URL_ERR="${HTML_URL_ERR:-/monitoring_err/}"
# ensure leading + trailing slash
_norm_path() {
  local p="$1"
  case "$p" in
    /*) ;;
    *) p="/$p" ;;
  esac
  case "$p" in
    */) ;;
    *) p="${p}/" ;;
  esac
  printf '%s' "$p"
}
HTML_URL_OK="$(_norm_path "$HTML_URL_OK")"
HTML_URL_ERR="$(_norm_path "$HTML_URL_ERR")"

# Stack family: goa | fp | other — FP never checks GOA and vice versa.
_monpages_family() {
  case "${TALER_DOMAIN:-}" in
    *lefrancpaysan*|*francpaysan*) printf 'fp' ;;
    hacktivism.ch|koopa) printf 'goa' ;;
    *)
      # Infer from MON_HOSTS if domain ambiguous
      case " ${MON_HOSTS:-} " in
        *lefrancpaysan*) printf 'fp' ;;
        *hacktivism*) printf 'goa' ;;
        *) printf 'other' ;;
      esac
      ;;
  esac
}

_host_allowed() {
  local h="$1" fam="$2"
  case "$fam" in
    goa)
      case "$h" in *.hacktivism.ch) return 0 ;; *) return 1 ;; esac
      ;;
    fp)
      case "$h" in *lefrancpaysan*) return 0 ;; *) return 1 ;; esac
      ;;
    *) return 0 ;;
  esac
}

# Emit https://host/path/ for each …/host/path/index.html under root (max depth 3).
_discover_tree() {
  local root="$1" fam="$2"
  local f rel host path
  [ -n "$root" ] && [ -d "$root" ] || return 0
  # layout: $root/<fqdn>/<url-path>/index.html
  while IFS= read -r -d '' f; do
    rel="${f#"${root%/}"/}"
    host="${rel%%/*}"
    path="${rel#*/}"
    path="${path%/index.html}"
    [ -n "$host" ] && [ -n "$path" ] || continue
    _host_allowed "$host" "$fam" || continue
    printf 'https://%s/%s/\n' "$host" "$path"
  done < <(find "${root%/}" -mindepth 3 -maxdepth 3 -type f -name index.html -print0 2>/dev/null)
}

# Suite catalog of OK pages that must exist for this family.
_catalog_urls() {
  local fam="$1" h
  case "$fam" in
    goa)
      for h in bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch; do
        printf 'https://%s/monitoring/\n' "$h"
      done
      for h in \
        taler-monitoring-surface \
        taler-monitoring-aptdeploy \
        taler-monitoring-mattermost \
        taler-monitoring-mail
      do
        printf 'https://taler.hacktivism.ch/%s/\n' "$h"
      done
      ;;
    fp)
      for h in $MON_HOSTS; do
        [ -z "$h" ] && continue
        _host_allowed "$h" fp || continue
        printf 'https://%s/monitoring/\n' "$h"
      done
      ;;
  esac
}

# Job-local URLs (this host-agent run’s path).
_job_urls() {
  local h p
  for h in $MON_HOSTS; do
    [ -z "$h" ] && continue
    printf 'https://%s%s\n' "$h" "$HTML_URL_OK"
    if [ "${MONPAGES_CHECK_ERR:-0}" = "1" ]; then
      printf 'https://%s%s\n' "$h" "$HTML_URL_ERR"
    fi
  done
}

build_urls() {
  local fam inv
  fam="$(_monpages_family)"
  inv="${MONPAGES_INVENTORY:-auto}"

  if [ -n "${MONITORING_PAGE_URLS:-}" ]; then
    # shellcheck disable=SC2086
    printf '%s\n' $MONITORING_PAGE_URLS
  else
    case "$inv" in
      job)
        _job_urls
        ;;
      full|auto|*)
        # Always include this job’s paths (host-agent post-check).
        _job_urls
        # Stack inventory: GOA = all suite mon sites; FP = only FP mon hosts.
        if [ "$fam" = "goa" ] || [ "$fam" = "fp" ]; then
          _catalog_urls "$fam"
          # On-disk discovery (err pages only exist after failed runs; extras welcome)
          _discover_tree "${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" "$fam"
          _discover_tree "${MONPAGES_STAGING_BASE:-${HTML_OUT:-${HTML_BASE:-$HOME/monitoring-sites-staging}}}" "$fam"
        fi
        ;;
    esac
  fi
  if [ -n "${MONPAGES_EXTRA_URLS:-}" ]; then
    # shellcheck disable=SC2086
    printf '%s\n' $MONPAGES_EXTRA_URLS
  fi
}

# True if body looks like our monitoring HTML (not merchant API / WP 404).
is_monitoring_html() {
  local f="$1"
  if grep -qE '"code"[[:space:]]*:[[:space:]]*21' "$f" 2>/dev/null; then
    return 1
  fi
  if grep -qiE 'There is no endpoint defined for the URL' "$f" 2>/dev/null; then
    return 1
  fi
  if grep -qiE 'wp-content|wordpress' "$f" 2>/dev/null \
    && grep -qiE '404|not found|page introuvable' "$f" 2>/dev/null; then
    return 1
  fi
  if grep -qE 'sticky-bar|version-link|taler-monitoring|class="sticky' "$f" 2>/dev/null; then
    return 0
  fi
  if head -c 4096 "$f" | grep -qiE '<!DOCTYPE html|<html' \
    && grep -qiE 'monitoring|mon ·|host-agent' "$f" 2>/dev/null; then
    return 0
  fi
  if head -c 512 "$f" | grep -qiE '<!DOCTYPE html|<html'; then
    return 0
  fi
  return 1
}

_is_bare_url() {
  local url="$1"
  case " ${MONPAGES_BARE_URLS:-} " in
    *" $url "*) return 0 ;;
  esac
  case "$url" in
    */) return 1 ;;
    *) return 0 ;;
  esac
}

_mon_fail_or_soft() {
  local label="$1" detail="$2"
  if [ "${MONPAGES_REQUIRE_PUBLIC:-1}" != "1" ]; then
    warn "$label" "$detail (MONPAGES_REQUIRE_PUBLIC=0)"
    return 0
  fi
  fail "$label" "$detail"
  return 1
}

check_one() {
  local url="$1"
  local body code hint soft=0
  body=$(mktemp)
  code=$(curl -skS -L --max-redirs 5 -m "${TIMEOUT}" -o "$body" -w '%{http_code}' "$url" 2>/dev/null || echo 000)
  if _is_bare_url "$url" && [ "${MONPAGES_BARE_STRICT:-0}" != "1" ]; then
    soft=1
  fi
  if grep -qE '"code"[[:space:]]*:[[:space:]]*21' "$body" 2>/dev/null \
    || grep -qiE 'There is no endpoint defined for the URL' "$body" 2>/dev/null; then
    if [ "$soft" = "1" ]; then
      warn "public mon page bare URL" \
        "$url -> HTTP $code code 21 (prefer trailing slash; set MONPAGES_BARE_STRICT=1 to ERROR)"
      rm -f "$body"
      return 0
    fi
    _mon_fail_or_soft "public mon page missing" \
      "$url -> HTTP $code merchant/API JSON code 21 (publish HTML + reverse-proxy handle /monitoring*)" || { rm -f "$body"; return 1; }
    rm -f "$body"
    return 0
  fi
  if [ "$code" != "200" ]; then
    if [ "$soft" = "1" ]; then
      warn "public mon page bare URL" "$url -> HTTP $code (prefer trailing slash form)"
      rm -f "$body"
      return 0
    fi
    _mon_fail_or_soft "public mon page" "$url -> HTTP $code (want 200 HTML)" || { rm -f "$body"; return 1; }
    rm -f "$body"
    return 0
  fi
  if is_monitoring_html "$body"; then
    if grep -qE 'sticky-bar|version-link' "$body" 2>/dev/null; then
      ok "public mon page" "$url -> HTTP 200 monitoring HTML"
    else
      warn "public mon page" "$url -> HTTP 200 HTML but weak markers (stub/bootstrap?)"
    fi
    rm -f "$body"
    return 0
  fi
  hint=$(head -c 120 "$body" | tr '\n' ' ' | tr -cd '[:print:] ')
  _mon_fail_or_soft "public mon page not monitoring HTML" "$url -> HTTP 200 body!=suite (${hint}...)" || { rm -f "$body"; return 1; }
  rm -f "$body"
  return 0
}

URLS=()
while IFS= read -r line; do
  [ -n "$line" ] || continue
  # dedupe
  skip=0
  for e in "${URLS[@]+"${URLS[@]}"}"; do
    [ "$e" = "$line" ] && skip=1 && break
  done
  [ "$skip" = "1" ] && continue
  URLS+=("$line")
done < <(build_urls | sort -u)

# Bare URLs without trailing slash: optional soft check (default on).
# Many reverse proxies only redirect /monitoring → /monitoring/; bare may hit the app (code 21).
# MONPAGES_CHECK_BARE=0  → skip bare entirely
# MONPAGES_BARE_STRICT=1 → bare failures are ERROR (default: WARN if slash form exists)
if [ "${MONPAGES_CHECK_BARE:-1}" = "1" ]; then
  _extra=()
  for u in "${URLS[@]}"; do
    case "$u" in
      */) _bare=${u%/}; [ -n "$_bare" ] && _extra+=("$_bare") ;;
    esac
  done
  for u in "${_extra[@]+"${_extra[@]}"}"; do
    skip=0
    for e in "${URLS[@]+"${URLS[@]}"}"; do
      [ "$e" = "$u" ] && skip=1 && break
    done
    [ "$skip" = "1" ] && continue
    URLS+=("$u")
    MONPAGES_BARE_URLS="${MONPAGES_BARE_URLS:-} $u"
  done
  export MONPAGES_BARE_URLS
fi

if [ "${#URLS[@]}" -eq 0 ]; then
  fail "monpages" "no URLs to check (set MON_HOSTS or MONITORING_PAGE_URLS)"
  exit 1
fi

_fam="$(_monpages_family)"
echo "  family=${_fam} inventory=${MONPAGES_INVENTORY} require_public=${MONPAGES_REQUIRE_PUBLIC}"
echo "  checking ${#URLS[@]} URL(s) via FQDN..."
ec=0
for u in "${URLS[@]}"; do
  check_one "$u" || ec=1
done

if [ "$ec" -ne 0 ]; then
  echo "  hint: public monitoring HTML not served (404 / merchant code 21)"
  echo "        publish staging HTML + configure reverse-proxy/Caddy handle for mon paths"
  echo "        (on koopa: sudo ~/koopa-caddy/apply-monitoring-live.sh)"
  if [ "$_fam" = "fp" ]; then
    echo "        FP: only FP mon hosts are checked — wire Infomaniak vhost for /monitoring*"
  fi
  if [ "$_fam" = "goa" ]; then
    echo "        GOA: full inventory (bank/exchange/taler /monitoring/ + surface/aptdeploy/mattermost/mail)"
  fi
  # Staging vs public diagnosis (host-agent sets MONPAGES_STAGING_BASE / HTML_OUT)
  _stg="${MONPAGES_STAGING_BASE:-${HTML_OUT:-${HTML_BASE:-$HOME/monitoring-sites-staging}}}"
  _ok_dir="${HTML_OK_DIR:-monitoring}"
  if [ -n "${MON_HOSTS:-}" ] && [ -n "$_stg" ]; then
    for _h in $MON_HOSTS; do
      _f="$_stg/$_h/${_ok_dir}/index.html"
      if [ -f "$_f" ]; then
        echo "        staging OK: $_f ($(wc -c <"$_f" | tr -d " ") bytes) — not published/served"
      else
        echo "        staging missing: $_f"
      fi
    done
  fi
  _www="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}"
  if [ -e "$_www" ] && [ ! -w "$_www" ]; then
    echo "        DEPLOY_WWW_ROOT=$_www not writable by $(id -un 2>/dev/null || echo user)"
  fi
fi

exit "$ec"