release 1.3.1: monpages FQDN check + fix silent deploy skip

2026-07-18 18:21:28 +02:00 · 2026-07-18 18:21:28 +02:00 · 5b56141014
commit 5b56141014
parent 941f54b409
12 changed files with 247 additions and 17 deletions
--- a/README.md
+++ b/README.md
@ -366,7 +366,19 @@ LADDER_LOAD=0 ./taler-monitoring.sh ladder
 Default **English**. Select language with `--lang en|fr` or `TALER_MON_LANG`.
 FrancPaysan stacks auto-select **French** (`*lefrancpaysan*`). See `VERSIONS.md` and `i18n.sh`.

+## Public monitoring pages via FQDN (v1.3.1+)
+
+Phase **`monpages`** checks that each host’s monitoring URL is live HTML
+(not merchant JSON code 21):
+
+```bash
+./taler-monitoring.sh -d hacktivism.ch monpages
+# host-agent runs monpages by default and again after HTML publish
+```
+
+If pages are missing publicly: `sudo ~/koopa-caddy/apply-monitoring-live.sh` on koopa.
+
 ## Version on monitoring pages (v1.3+)

-The sticky bar shows the suite **tag** (e.g. `v1.3`) and links to that tag on Forgejo.
+The sticky bar shows the suite **tag** (e.g. `v1.3.1`) and links to that tag on Forgejo.
 Planned **v1.4.0**: fuller L10n (fr-CH for FrancPaysan, de-CH for hacktivism/Taler CH).
--- a/2
+++ b/2
@ -1 +1 @@
-1.3.0
+1.3.1
--- a/VERSIONS.md
+++ b/VERSIONS.md
@ -5,7 +5,8 @@ Release history for the standalone **taler-monitoring** suite

 | Tag | Date (UTC) | Notes |
 |-----|------------|--------|
-| **v1.4.0** | *planned* | **L10n** for user-facing outputs that need localization: **fr-CH** (FrancPaysan, Suisse romande) and **de-CH** (hacktivism / Taler CH German). Not implemented yet — after **1.3**. |
+| **v1.4.0** | *planned* | **L10n** for user-facing outputs that need localization: **fr-CH** (FrancPaysan, Suisse romande) and **de-CH** (hacktivism / Taler CH German). Not implemented yet — after **1.3.x**. |
+| **v1.3.1** | 2026-07-18 | **Bugfix:** silent skip when `DEPLOY_WWW_ROOT` not writable left public pages missing (merchant JSON code 21). **monpages** phase + host-agent post-check: verify monitoring HTML via **FQDN** (`https://<host>/monitoring/` etc.). |
 | **v1.3** | 2026-07-18 | Sticky bar shows **suite version** and links to the Forgejo **tag** (`…/src/tag/vX.Y.Z`). (supersedes tag name **v1.3.0**) |
 | **v1.2** | 2026-07-18 | Language selection: `--lang` / `TALER_MON_LANG` (en\|fr). |
 | **v1.1** | 2026-07-18 | i18n chrome: English default; French for FrancPaysan. |
@ -13,13 +14,30 @@ Release history for the standalone **taler-monitoring** suite

 ## Sticky bar version (v1.3+)

-Generated HTML shows the installed release tag (e.g. `v1.3`) next to the host line.  
+Generated HTML shows the installed release tag (e.g. `v1.3.1`) next to the host line.  
 Clicking opens the tag page on Forgejo:

 `https://git.hacktivism.ch/hernani/taler-monitoring/src/tag/<tag>`

 Resolved via `git describe --tags` after `update-suite.sh`, with fallback to the `VERSION` file.

+## Public monitoring pages (v1.3.1+)
+
+Host-agent must publish under `DEPLOY_WWW_ROOT` **and** Caddy must `handle` the paths.
+If not, public FQDNs return merchant **JSON code 21**.
+
+| Check | How |
+|-------|-----|
+| Phase | `./taler-monitoring.sh monpages` (or `pages`) |
+| Auto | host-agent default phases include `monpages`; post-check after HTML write |
+| URLs | `https://$MON_HOSTS$HTML_URL_OK` (e.g. `https://taler.hacktivism.ch/monitoring/`) |
+
+Bug (fixed in 1.3.1): rsync to `/var/www/monitoring-sites` was **silently skipped** when not writable, so staging had pages but FQDNs never did. Now that is an **ERROR**, and monpages fails until live apply:
+
+```bash
+sudo ~/koopa-caddy/apply-monitoring-live.sh
+```
+
 ## Language (v1.2+)

 | Method | Example |
@ -28,11 +46,11 @@ Resolved via `git describe --tags` after `update-suite.sh`, with fallback to the
 | Env | `TALER_MON_LANG=en` or `fr` |
 | Auto | `*lefrancpaysan*` → fr, else en |

-**Note:** Full locale packs (fr-CH / de-CH) for all check messages are scheduled for **v1.4.0**, not 1.3.
+**Note:** Full locale packs (fr-CH / de-CH) for all check messages are scheduled for **v1.4.0**, not 1.3.x.

 ## Pin a release

 ```bash
 git clone https://git.hacktivism.ch/hernani/taler-monitoring.git ~/src/taler-monitoring
-cd ~/src/taler-monitoring && git checkout v1.3
+cd ~/src/taler-monitoring && git checkout v1.3.1
 ```
--- a/check_monitoring_pages.sh
+++ b/check_monitoring_pages.sh
@ -0,0 +1,161 @@
+#!/usr/bin/env bash
+# check_monitoring_pages.sh — verify public monitoring HTML pages via FQDN
+#
+# Outside-in: curl https://<fqdn>/<path>/ and require a real HTML monitoring page.
+# Catches: merchant JSON code 21 (Caddy handle missing), WP 404, empty stubs, deploy skip.
+#
+# Env (same knobs as host-agent HTML layout):
+#   MON_HOSTS             space-separated FQDNs (default from TALER_DOMAIN)
+#   HTML_URL_OK           path for OK page (default /monitoring/)
+#   HTML_URL_ERR          optional err path; checked if MONPAGES_CHECK_ERR=1
+#   MONITORING_PAGE_URLS  optional full URL list (overrides host+path construction)
+#   MONPAGES_EXTRA_URLS   additional absolute URLs (space-separated)
+#   TIMEOUT               curl timeout (from lib.sh)
+#
+set -euo pipefail
+ROOT=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=lib.sh
+source "$ROOT/lib.sh"
+
+set_area monpages
+section "monpages · public monitoring HTML via FQDN (outside-in)"
+
+# Default hosts from domain (mirror run-host-report.sh)
+if [ -z "${MON_HOSTS:-}" ]; then
+  case "${TALER_DOMAIN:-}" in
+    hacktivism.ch|koopa)
+      MON_HOSTS="bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch"
+      ;;
+    stage.lefrancpaysan.ch|stage.*lefrancpaysan*)
+      MON_HOSTS="stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch"
+      ;;
+    lefrancpaysan.ch)
+      MON_HOSTS="bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch"
+      ;;
+    *)
+      MON_HOSTS="${TALER_DOMAIN:-}"
+      ;;
+  esac
+fi
+
+HTML_URL_OK="${HTML_URL_OK:-/monitoring/}"
+HTML_URL_ERR="${HTML_URL_ERR:-/monitoring_err/}"
+# ensure leading + trailing slash
+case "$HTML_URL_OK" in
+  /*) ;;
+  *) HTML_URL_OK="/$HTML_URL_OK" ;;
+esac
+case "$HTML_URL_OK" in
+  */) ;;
+  *) HTML_URL_OK="${HTML_URL_OK}/" ;;
+esac
+
+build_urls() {
+  local h p
+  if [ -n "${MONITORING_PAGE_URLS:-}" ]; then
+    # shellcheck disable=SC2086
+    printf '%s\n' $MONITORING_PAGE_URLS
+  else
+    for h in $MON_HOSTS; do
+      [ -z "$h" ] && continue
+      printf 'https://%s%s\n' "$h" "$HTML_URL_OK"
+      if [ "${MONPAGES_CHECK_ERR:-0}" = "1" ]; then
+        p="$HTML_URL_ERR"
+        case "$p" in /*) ;; *) p="/$p" ;; esac
+        case "$p" in */) ;; *) p="${p}/" ;; esac
+        printf 'https://%s%s\n' "$h" "$p"
+      fi
+    done
+  fi
+  if [ -n "${MONPAGES_EXTRA_URLS:-}" ]; then
+    # shellcheck disable=SC2086
+    printf '%s\n' $MONPAGES_EXTRA_URLS
+  fi
+}
+
+# True if body looks like our monitoring HTML (not merchant API / WP 404).
+is_monitoring_html() {
+  local f="$1"
+  if grep -qE '"code"[[:space:]]*:[[:space:]]*21' "$f" 2>/dev/null; then
+    return 1
+  fi
+  if grep -qiE 'There is no endpoint defined for the URL' "$f" 2>/dev/null; then
+    return 1
+  fi
+  if grep -qiE 'wp-content|wordpress' "$f" 2>/dev/null \
+    && grep -qiE '404|not found|page introuvable' "$f" 2>/dev/null; then
+    return 1
+  fi
+  if grep -qE 'sticky-bar|version-link|taler-monitoring|class="sticky' "$f" 2>/dev/null; then
+    return 0
+  fi
+  if head -c 4096 "$f" | grep -qiE '<!DOCTYPE html|<html' \
+    && grep -qiE 'monitoring|mon ·|host-agent' "$f" 2>/dev/null; then
+    return 0
+  fi
+  if head -c 512 "$f" | grep -qiE '<!DOCTYPE html|<html'; then
+    return 0
+  fi
+  return 1
+}
+
+check_one() {
+  local url="$1"
+  local body code hint
+  body=$(mktemp)
+  code=$(curl -skS -L --max-redirs 5 -m "${TIMEOUT}" -o "$body" -w '%{http_code}' "$url" 2>/dev/null || echo 000)
+  if [ "$code" != "200" ]; then
+    fail "public mon page" "$url -> HTTP $code (want 200 HTML)"
+    rm -f "$body"
+    return 1
+  fi
+  if grep -qE '"code"[[:space:]]*:[[:space:]]*21' "$body" 2>/dev/null; then
+    fail "public mon page missing" \
+      "$url -> merchant/API JSON code 21 (Caddy handle not live or not published — sudo apply-monitoring-live.sh)"
+    rm -f "$body"
+    return 1
+  fi
+  if is_monitoring_html "$body"; then
+    if grep -qE 'sticky-bar|version-link' "$body" 2>/dev/null; then
+      ok "public mon page" "$url -> HTTP 200 monitoring HTML"
+    else
+      warn "public mon page" "$url -> HTTP 200 HTML but weak markers (stub/bootstrap?)"
+    fi
+    rm -f "$body"
+    return 0
+  fi
+  hint=$(head -c 120 "$body" | tr '\n' ' ' | tr -cd '[:print:] ')
+  fail "public mon page not monitoring HTML" "$url -> HTTP 200 body!=suite (${hint}...)"
+  rm -f "$body"
+  return 1
+}
+
+URLS=()
+while IFS= read -r line; do
+  [ -n "$line" ] || continue
+  # dedupe
+  skip=0
+  for e in "${URLS[@]+"${URLS[@]}"}"; do
+    [ "$e" = "$line" ] && skip=1 && break
+  done
+  [ "$skip" = "1" ] && continue
+  URLS+=("$line")
+done < <(build_urls)
+
+if [ "${#URLS[@]}" -eq 0 ]; then
+  fail "monpages" "no URLs to check (set MON_HOSTS or MONITORING_PAGE_URLS)"
+  exit 1
+fi
+
+echo "  checking ${#URLS[@]} URL(s) via FQDN..."
+ec=0
+for u in "${URLS[@]}"; do
+  check_one "$u" || ec=1
+done
+
+if [ "$ec" -ne 0 ]; then
+  echo "  hint: HTML may exist only under ~/monitoring-sites-staging; public needs"
+  echo "        writable DEPLOY_WWW_ROOT + Caddy handles (sudo ~/koopa-caddy/apply-monitoring-live.sh)"
+fi
+
+exit "$ec"
--- a/host-agent/ROOT-APPLY-MONITORING.md
+++ b/host-agent/ROOT-APPLY-MONITORING.md
@ -1,5 +1,11 @@
 # Root: make monitoring pages live on koopa (GOA)

+## Detection (v1.3.1+)
+
+Suite phase **`monpages`** and the host-agent **post-check** curl each public
+FQDN (`https://taler.hacktivism.ch/monitoring/` …). Merchant JSON **code 21**
+or non-HTML is an **ERROR** until this apply is done.
+
 ## Symptom if not applied

 Public URLs return **merchant/bank JSON** like:
--- a/host-agent/run-aptdeploy-monitoring.sh
+++ b/host-agent/run-aptdeploy-monitoring.sh
@ -21,7 +21,7 @@ export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}"
 # four containers + checks; allow long wall
 export RUN_TIMEOUT="${RUN_TIMEOUT:-1800}"
 # dedicated job: always aptdeploy only
-export PHASES=aptdeploy
+export PHASES="${PHASES:-aptdeploy monpages}"
 # Only merchant public front — not bank/exchange
 export MON_HOSTS="${MON_HOSTS:-taler.hacktivism.ch}"
 export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}"
--- a/host-agent/run-fp-prod-monitoring.sh
+++ b/host-agent/run-fp-prod-monitoring.sh
@ -16,7 +16,7 @@ export INSIDE_MODE="${INSIDE_MODE:-local-podman}"
 export LOCAL_STACK="${LOCAL_STACK:-0}"
 export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}"
 export RUN_TIMEOUT="${RUN_TIMEOUT:-600}"
-export PHASES="${PHASES:-urls inside versions}"
+export PHASES="${PHASES:-urls inside versions monpages}"
 export MON_HOSTS="${MON_HOSTS:-bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch}"
 export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}"
 export DEPLOY_WWW_ROOT="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}"
--- a/host-agent/run-fp-stage-monitoring.sh
+++ b/host-agent/run-fp-stage-monitoring.sh
@ -17,7 +17,7 @@ export INSIDE_PROFILE="${INSIDE_PROFILE:-stage-lfp}"
 export LOCAL_STACK="${LOCAL_STACK:-0}"
 export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}"
 export RUN_TIMEOUT="${RUN_TIMEOUT:-600}"
-export PHASES="${PHASES:-urls inside versions}"
+export PHASES="${PHASES:-urls inside versions monpages}"
 export MON_HOSTS="${MON_HOSTS:-stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch}"
 export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}"
 export DEPLOY_WWW_ROOT="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}"
--- a/host-agent/run-hacktivism-monitoring.sh
+++ b/host-agent/run-hacktivism-monitoring.sh
@ -18,7 +18,7 @@ export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}"
 export RUN_TIMEOUT="${RUN_TIMEOUT:-600}"
 # stack checks only — aptdeploy has its own HTML path + timer:
 #   run-aptdeploy-monitoring.sh → /taler-monitoring-aptdeploy* (taler.hacktivism.ch)
-export PHASES="${PHASES:-urls inside versions}"
+export PHASES="${PHASES:-urls inside versions monpages}"
 export MON_HOSTS="${MON_HOSTS:-bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch}"
 export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}"
 export DEPLOY_WWW_ROOT="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}"
--- a/host-agent/run-host-report.sh
+++ b/host-agent/run-host-report.sh
@ -7,7 +7,8 @@
 #   2) line-buffered run log (written continuously)
 #   3) RUN_TIMEOUT wall clock for taler-monitoring.sh (default 600s)
 #   4) always write HTML (first run / empty staging too) + Forgejo commit link
-#   5) optional rsync to DEPLOY_WWW_ROOT if writable
+#   5) rsync to DEPLOY_WWW_ROOT (ERROR if not writable — v1.3.1)
+#   6) monpages post-check: public FQDN must serve monitoring HTML
 #
 # Configure via env file (not in git):
 #   ${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring/env
@ -88,7 +89,7 @@ elif [ -n "${TALER_MON_LANG:-}" ]; then
 fi
 export TALER_MON_LANG

-PHASES="${PHASES:-urls inside versions}"
+PHASES="${PHASES:-urls inside versions monpages}"
 HTML_BASE="${HTML_OUT:-$HOME/monitoring-sites-staging}"
 MON_HOSTS="${MON_HOSTS:-}"
 SUITE_PATH="${SOURCE_SUITE_PATH:-.}"
@ -386,9 +387,37 @@ for host in $MON_HOSTS; do
  htmlify_host "$host"
 done

-if [ -n "$DEPLOY_WWW" ] && [ -w "$DEPLOY_WWW" ] 2>/dev/null; then
-  rsync -a --delete "$HTML_BASE/" "$DEPLOY_WWW/" && \
+# Publish to live web root when possible (v1.3.1: never silent-skip)
+if [ -n "$DEPLOY_WWW" ]; then
+  if [ -w "$DEPLOY_WWW" ] 2>/dev/null; then
+    if rsync -a --delete "$HTML_BASE/" "$DEPLOY_WWW/"; then
      echo "synced → $DEPLOY_WWW/"
+    else
+      echo "ERROR: rsync to DEPLOY_WWW_ROOT=$DEPLOY_WWW failed" >&2
+      ec=1
+    fi
+  elif [ -e "$DEPLOY_WWW" ]; then
+    echo "ERROR: DEPLOY_WWW_ROOT=$DEPLOY_WWW exists but is not writable by $(id -un)" >&2
+    echo "       public FQDN pages will stay missing/stale — fix ownership or run:" >&2
+    echo "       sudo ~/koopa-caddy/apply-monitoring-live.sh" >&2
+    ec=1
+  else
+    echo "ERROR: DEPLOY_WWW_ROOT=$DEPLOY_WWW missing — public pages not published" >&2
+    echo "       create tree + Caddy handles: sudo ~/koopa-caddy/apply-monitoring-live.sh" >&2
+    ec=1
+  fi
+else
+  echo "WARN: DEPLOY_WWW_ROOT empty — skipping live publish (staging only: $HTML_BASE)" >&2
+fi
+
+# Post-publish: verify this job's pages via FQDN (same paths as monpages phase)
+if [ "${MONPAGES_POSTCHECK:-1}" = "1" ] && [ -x "$ROOT/check_monitoring_pages.sh" ]; then
+  echo "--- monpages post-check (public FQDN) ---"
+  export MON_HOSTS HTML_URL_OK HTML_URL_ERR TALER_DOMAIN
+  if ! bash "$ROOT/check_monitoring_pages.sh"; then
+    echo "ERROR: public monitoring page(s) missing or not suite HTML via FQDN" >&2
+    ec=1
+  fi
 fi

 ls -1t "$LOG_DIR"/run-*.log 2>/dev/null | tail -n +30 | xargs rm -f 2>/dev/null || true
--- a/host-agent/run-surface-monitoring.sh
+++ b/host-agent/run-surface-monitoring.sh
@ -19,7 +19,7 @@ export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}"
 # many hosts; allow up to 45 min wall (phase also has RUN_TIMEOUT)
 export RUN_TIMEOUT="${RUN_TIMEOUT:-2400}"
 # dedicated job: always surface only (never inherit PHASES from shared env)
-export PHASES=surface
+export PHASES="${PHASES:-surface monpages}"
 # Only merchant public front — not bank/exchange
 export MON_HOSTS="${MON_HOSTS:-taler.hacktivism.ch}"
 export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}"
--- a/taler-monitoring.sh
+++ b/taler-monitoring.sh
@ -40,6 +40,8 @@ Phases:
  surface  REMOTE-ONLY public inventory (NOT in default/all/full):
             ecosystem hosts (taler.net, gnunet.org, taler-systems.com, mattermost, …)
             or -d DOMAIN → that domain’s surface; port/protocol/TLS/CVE (OSV)
+  monpages public monitoring HTML via FQDN (HTTP 200 + suite page, not code 21)
+             uses MON_HOSTS + HTML_URL_OK (same as host-agent); v1.3.1+
  all      urls + inside + versions + sanity + e2e  (SSH phases only on koopa)
  full     all + server + ladder + auth401  (maximum; long-running, needs secrets)
           NOTE: surface is never included in all/full — pass it explicitly
@ -177,7 +179,7 @@ while [ $# -gt 0 ]; do
      CURRENCY_OVERRIDE="$2"; shift 2
      ;;
    --no-probe) NO_PROBE=1; shift ;;
-    urls|inside|versions|sanity|server|e2e|ladder|goa-ladder|auth401|aptdeploy|apt-deploy|apt_src|surface|ecosystem|all|full) PHASES+=("$1"); shift ;;
+    urls|inside|versions|sanity|server|e2e|ladder|goa-ladder|auth401|aptdeploy|apt-deploy|apt_src|surface|ecosystem|monpages|pages|all|full) PHASES+=("$1"); shift ;;
    *)
      # bare domain shorthand: ./taler-monitoring.sh taler.net
      if [[ "$1" == *.* && "$1" != *://* && "$1" != -* ]]; then
@ -341,6 +343,7 @@ if [ "${PROGRESS_TOTAL:-0}" = "0" ] || [ -z "${PROGRESS_TOTAL:-}" ]; then
      auth401)  _pt=$((_pt + 70)) ;;
      aptdeploy) _pt=$((_pt + 20)) ;;
      surface|ecosystem) _pt=$((_pt + 80)) ;;
+      monpages|pages) _pt=$((_pt + 30)) ;;
    esac
  done
  set_progress_total "$_pt"
@ -458,6 +461,7 @@ for p in "${PHASES[@]}"; do
    auth401)  run_phase auth401  "$ROOT/check_auth401.sh"  || ec=1 ;;
    aptdeploy|apt-deploy|apt_src) run_phase aptdeploy "$ROOT/check_apt_deploy.sh" || ec=1 ;;
    surface|ecosystem) run_phase surface "$ROOT/check_surface.sh" || ec=1 ;;
+    monpages|pages) run_phase monpages "$ROOT/check_monitoring_pages.sh" || ec=1 ;;
  esac
 done
 @ -1 +1 @@
 .3.0
 .3.1