release 1.13.4: monpages race-proof publish and content checks

Avoid false TOP/content ERRORs from mid-publish races: atomic HTML write, rsync --delay-updates + settle, monpages fetch retries with larger head windows, pre-publish soft content, bootstrap markers.
2026-07-19 02:55:56 +02:00 · 2026-07-19 02:55:56 +02:00 · e450470429
commit e450470429
parent d5213adb7a
5 changed files with 179 additions and 73 deletions
--- a/2
+++ b/2
@ -1 +1 @@
-1.13.3
+1.13.4
--- a/VERSIONS.md
+++ b/VERSIONS.md
@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre

 | Tag | Date (UTC) | Notes |
 |-----|------------|--------|
+| **v1.13.4** | 2026-07-19 | **Bugfix:** monpages race-proof — atomic HTML write; rsync `--delay-updates` + settle; fetch retries; larger head windows; pre-publish soft content (`MONPAGES_PRE_PUBLISH`); bootstrap markers. |
 | **v1.13.3** | 2026-07-19 | **Bugfix:** sticky **generated** time in **Europe/Zurich** (CEST/CET · MESZ/MEZ), not UTC `…Z`; ISO keeps offset for age JS. |
 | **v1.13.2** | 2026-07-19 | **Bugfix:** sticky version link uses **commit** URL (`/src/commit/<sha>`) not `/src/tag/v…` (tags not yet pushed → Forgejo fat 404); tags v1.11–v1.13.1 pushed to origin. |
 | **v1.13.1** | 2026-07-19 | **Bugfix:** progress bar **global** done/total — phase `summary()` no longer snaps total to phase-local N (fixes 34/34 then 38/47); final `progress_finish` snaps once; header shows 0/N. |
--- a/check_monitoring_pages.sh
+++ b/check_monitoring_pages.sh
@ -293,18 +293,32 @@ _page_mid_run() {

 # Validate top/bottom markers + meaningful body (v1.9.0).
 # Sets _MON_CONTENT_MSG; returns 0=ok, 1=hard fail, 2=soft/mid-run bottom skip
+#
+# Head windows are large (v1.13.4+): sticky CSS grew past old 12–16 KiB limits and
+# caused false TOP errors while publish races were also writing the file.
 validate_mon_content() {
  local f="$1" url="$2"
  local top_ok=1 bot_ok=1 body_ok=1 mid=0 pct
+  local head_top="${MONPAGES_HEAD_TOP_BYTES:-49152}"
+  local head_meta="${MONPAGES_HEAD_META_BYTES:-65536}"
  _MON_CONTENT_MSG=""

+  # Tiny / empty body = likely mid-publish race (partial rsync or curl)
+  if [ ! -s "$f" ] || [ "$(wc -c <"$f" 2>/dev/null || echo 0)" -lt 200 ]; then
+    top_ok=0
+    _MON_CONTENT_MSG="empty or truncated body (publish race?)"
  # --- top markers (always required) ---
-  if ! head -c 12000 "$f" | grep -qE 'sticky-bar|id="status-bar"|taler-mon:top'; then
-    top_ok=0
-    _MON_CONTENT_MSG="missing top marker (sticky-bar / status-bar / taler-mon:top)"
-  elif ! head -c 16000 "$f" | grep -qE 'data-generated-iso|name="generated"|version-link|taler-mon-page'; then
-    top_ok=0
-    _MON_CONTENT_MSG="missing top meta (generated / version-link / taler-mon-page)"
+  elif ! head -c "$head_top" "$f" | grep -qE 'sticky-bar|id="status-bar"|taler-mon:top'; then
+    # full-file fallback (small pages / unusual layout)
+    if ! grep -qE 'sticky-bar|id="status-bar"|taler-mon:top' "$f" 2>/dev/null; then
+      top_ok=0
+      _MON_CONTENT_MSG="missing top marker (sticky-bar / status-bar / taler-mon:top)"
+    fi
+  elif ! head -c "$head_meta" "$f" | grep -qE 'data-generated-iso|name="generated"|version-link|taler-mon-page'; then
+    if ! grep -qE 'data-generated-iso|name="generated"|version-link|taler-mon-page' "$f" 2>/dev/null; then
+      top_ok=0
+      _MON_CONTENT_MSG="missing top meta (generated / version-link / taler-mon-page)"
+    fi
  fi

  # --- meaningful body: real checks, not only progress bars ---
@ -386,76 +400,128 @@ _mon_fail_or_soft() {
  return 1
 }

+# Fetch + validate one URL. Retries content failures (publish/rsync races).
+# MONPAGES_PRE_PUBLISH=1 → content failures are WARN only (suite still regenerates HTML after).
+# MONPAGES_FETCH_RETRIES (default 3), MONPAGES_FETCH_SLEEP_S (default 0.5).
 check_one() {
  local url="$1"
-  local body code hint soft=0 vc
+  local body code hint soft=0 vc attempt max_try sleep_s pre_pub
  body=$(mktemp)
-  code=$(curl -skS -L --max-redirs 5 -m "${TIMEOUT}" -o "$body" -w '%{http_code}' "$url" 2>/dev/null || echo 000)
  if _is_bare_url "$url" && [ "${MONPAGES_BARE_STRICT:-0}" != "1" ]; then
    soft=1
  fi
-  if grep -qE '"code"[[:space:]]*:[[:space:]]*21' "$body" 2>/dev/null \
-    || grep -qiE 'There is no endpoint defined for the URL' "$body" 2>/dev/null; then
-    # Bare without trailing slash often hits merchant until Caddy has redir * /path/ 302.
-    # Default: WARN on bare (soft) so slash-OK stacks still publish green/yellow pages.
-    # MONPAGES_BARE_STRICT=1 → ERROR (enforce bare redir live).
-    if [ "$soft" = "1" ]; then
-      warn "public mon page bare URL" \
-        "$url -> HTTP $code code 21 (prefer trailing slash; Caddy: redir * /path/ 302; set MONPAGES_BARE_STRICT=1 to ERROR)"
-      rm -f "$body"
-      return 0
-    fi
-    _mon_fail_or_soft "public mon page missing" \
-      "$url -> HTTP $code merchant/API JSON code 21 (Caddy handle/redir; use redir * /path/ 302 inside handle; apply: sudo /home/hernani/koopa-caddy/apply-monitoring-live.sh)" || { rm -f "$body"; return 1; }
-    rm -f "$body"
-    return 0
-  fi
-  if [ "$code" != "200" ]; then
-    if [ "$soft" = "1" ]; then
-      warn "public mon page bare URL" "$url -> HTTP $code (prefer trailing slash form)"
-      rm -f "$body"
-      return 0
-    fi
-    _mon_fail_or_soft "public mon page" "$url -> HTTP $code (want 200 HTML)" || { rm -f "$body"; return 1; }
-    rm -f "$body"
-    return 0
-  fi
-  if is_bootstrap_html "$body"; then
-    _mon_fail_or_soft "public mon page bootstrap stub" \
-      "$url -> install bootstrap HTML still live (run host-agent; suite auto-update must replace stub)" || { rm -f "$body"; return 1; }
-    rm -f "$body"
-    return 0
-  fi
-  if ! is_monitoring_html "$body"; then
-    hint=$(head -c 120 "$body" | tr '\n' ' ' | tr -cd '[:print:] ')
-    _mon_fail_or_soft "public mon page not monitoring HTML" \
-      "$url -> HTTP 200 body!=suite (${hint}...)" || { rm -f "$body"; return 1; }
-    rm -f "$body"
-    return 0
-  fi
-  # v1.9.0: top + bottom markers + meaningful content
-  set +e
-  validate_mon_content "$body" "$url"
-  vc=$?
-  set -e
-  case "$vc" in
-    0)
-      ok "public mon page" "$url -> HTTP 200 · ${_MON_CONTENT_MSG}"
-      ;;
-    2)
-      # mid-run: bottom may be incomplete — not ERROR
-      warn "public mon page mid-run" "$url -> ${_MON_CONTENT_MSG}"
-      ok "public mon page" "$url -> HTTP 200 suite HTML (mid-run bottom relaxed)"
-      ;;
-    *)
+  pre_pub=0
+  [ "${MONPAGES_PRE_PUBLISH:-0}" = "1" ] && pre_pub=1
+  max_try="${MONPAGES_FETCH_RETRIES:-3}"
+  sleep_s="${MONPAGES_FETCH_SLEEP_S:-0.5}"
+  [ "$max_try" -lt 1 ] && max_try=1
+
+  attempt=1
+  while [ "$attempt" -le "$max_try" ]; do
+    : >"$body"
+    code=$(curl -skS -L --max-redirs 5 -m "${TIMEOUT}" -o "$body" -w '%{http_code}' "$url" 2>/dev/null || echo 000)
+
+    if grep -qE '"code"[[:space:]]*:[[:space:]]*21' "$body" 2>/dev/null \
+      || grep -qiE 'There is no endpoint defined for the URL' "$body" 2>/dev/null; then
+      # Bare without trailing slash often hits merchant until Caddy has redir * /path/ 302.
      if [ "$soft" = "1" ]; then
-        warn "public mon page content" "$url -> ${_MON_CONTENT_MSG}"
-      else
+        warn "public mon page bare URL" \
+          "$url -> HTTP $code code 21 (prefer trailing slash; Caddy: redir * /path/ 302; set MONPAGES_BARE_STRICT=1 to ERROR)"
+        rm -f "$body"
+        return 0
+      fi
+      # hard 21: retry once more (rare brief outage) then fail
+      if [ "$attempt" -lt "$max_try" ]; then
+        attempt=$((attempt + 1))
+        sleep "$sleep_s" 2>/dev/null || sleep 1
+        continue
+      fi
+      _mon_fail_or_soft "public mon page missing" \
+        "$url -> HTTP $code merchant/API JSON code 21 (Caddy handle/redir; use redir * /path/ 302 inside handle; apply: sudo /home/hernani/koopa-caddy/apply-monitoring-live.sh)" || { rm -f "$body"; return 1; }
+      rm -f "$body"
+      return 0
+    fi
+
+    if [ "$code" != "200" ]; then
+      if [ "$soft" = "1" ]; then
+        warn "public mon page bare URL" "$url -> HTTP $code (prefer trailing slash form)"
+        rm -f "$body"
+        return 0
+      fi
+      if [ "$attempt" -lt "$max_try" ]; then
+        attempt=$((attempt + 1))
+        sleep "$sleep_s" 2>/dev/null || sleep 1
+        continue
+      fi
+      _mon_fail_or_soft "public mon page" "$url -> HTTP $code (want 200 HTML)" || { rm -f "$body"; return 1; }
+      rm -f "$body"
+      return 0
+    fi
+
+    if is_bootstrap_html "$body"; then
+      if [ "$attempt" -lt "$max_try" ]; then
+        attempt=$((attempt + 1))
+        sleep "$sleep_s" 2>/dev/null || sleep 1
+        continue
+      fi
+      _mon_fail_or_soft "public mon page bootstrap stub" \
+        "$url -> install bootstrap HTML still live (run host-agent; suite auto-update must replace stub)" || { rm -f "$body"; return 1; }
+      rm -f "$body"
+      return 0
+    fi
+
+    if ! is_monitoring_html "$body"; then
+      if [ "$attempt" -lt "$max_try" ]; then
+        attempt=$((attempt + 1))
+        sleep "$sleep_s" 2>/dev/null || sleep 1
+        continue
+      fi
+      hint=$(head -c 120 "$body" | tr '\n' ' ' | tr -cd '[:print:] ')
+      _mon_fail_or_soft "public mon page not monitoring HTML" \
+        "$url -> HTTP 200 body!=suite (${hint}...)" || { rm -f "$body"; return 1; }
+      rm -f "$body"
+      return 0
+    fi
+
+    # v1.9.0: top + bottom markers + meaningful content
+    set +e
+    validate_mon_content "$body" "$url"
+    vc=$?
+    set -e
+    case "$vc" in
+      0)
+        ok "public mon page" "$url -> HTTP 200 · ${_MON_CONTENT_MSG}"
+        rm -f "$body"
+        return 0
+        ;;
+      2)
+        # mid-run: bottom may be incomplete — not ERROR
+        warn "public mon page mid-run" "$url -> ${_MON_CONTENT_MSG}"
+        ok "public mon page" "$url -> HTTP 200 suite HTML (mid-run bottom relaxed)"
+        rm -f "$body"
+        return 0
+        ;;
+      *)
+        # Content race (truncated HTML while another agent publishes): retry
+        if [ "$attempt" -lt "$max_try" ]; then
+          attempt=$((attempt + 1))
+          sleep "$sleep_s" 2>/dev/null || sleep 1
+          continue
+        fi
+        # Pre-publish inventory in host-agent PHASES: WARN only (post-check is hard)
+        if [ "$pre_pub" = "1" ] || [ "$soft" = "1" ]; then
+          warn "public mon page content" \
+            "$url -> ${_MON_CONTENT_MSG}${pre_pub:+ (pre-publish; post-check is authoritative)}"
+          rm -f "$body"
+          return 0
+        fi
        _mon_fail_or_soft "public mon page content" \
          "$url -> ${_MON_CONTENT_MSG}" || { rm -f "$body"; return 1; }
-      fi
-      ;;
-  esac
+        rm -f "$body"
+        return 0
+        ;;
+    esac
+  done
  rm -f "$body"
  return 0
 }
--- a/host-agent/run-host-report.sh
+++ b/host-agent/run-host-report.sh
@ -280,9 +280,17 @@ if [ -x "${ROOT:-}/taler-monitoring.sh" ]; then
    fi
  done

+  # monpages inside PHASES runs *before* htmlify/publish — content failures would
+  # race with concurrent publishes or reflect last run's incomplete redirect stub.
+  # Soft-content there; post-check after rsync stays hard (authoritative).
+  case " $PHASES " in
+    *" monpages "*|*" pages "*) export MONPAGES_PRE_PUBLISH=1 ;;
+    *) unset MONPAGES_PRE_PUBLISH 2>/dev/null || true ;;
+  esac
  # shellcheck disable=SC2086
  ./taler-monitoring.sh -d "$TALER_DOMAIN" $PHASES
  ec=$?
+  unset MONPAGES_PRE_PUBLISH 2>/dev/null || true
 fi

 # --- always generate HTML (first run / missing pages / every run) ---
@ -304,21 +312,27 @@ write_bootstrap_html() {
  if [ -n "$curl" ]; then
    clink="<a href=\"$(printf '%s' "$curl" | sed 's/&/\&amp;/g')\">$cshort</a>"
  fi
-  cat >"$out" <<EOF
+  local tmp
+  tmp=$(mktemp "${out}.XXXXXX")
+  cat >"$tmp" <<EOF
 <!DOCTYPE html>
 <html lang="en"><head><meta charset="utf-8"/>
 <meta name="generated" content="${iso}"/>
+<meta name="taler-mon-page" content="incomplete"/>
 <title>monitoring ${mode} · ${host}</title>
 <style>
 body{margin:0;background:#0c0c0c;color:#c8c8c8;font-family:ui-monospace,monospace}
 a{color:#61afef}.err{color:#ff5c5c}
 .sticky-bar{position:sticky;top:0;z-index:100;padding:10px 14px;background:#2a1010f2;border-bottom:2px solid #a33}
 .sticky-bar .pill{font-weight:800;color:#ff5c5c;margin-right:8px}
+.version-link{color:#61afef}
 main{padding:1.5rem}
 </style></head><body>
+<!-- taler-mon:top -->
 <div class="sticky-bar" id="status-bar">
  <span class="pill">BOOTSTRAP</span>
  ${PAGE_LABEL:-monitoring} · ${host} ·
+  <span class="version-link muted">bootstrap</span> ·
  <span data-generated-iso="${iso}">generated ${gen}</span>
  · source ${clink}
 </div>
@ -326,8 +340,10 @@ main{padding:1.5rem}
 <p class="err">Bootstrap page (converter missing or first install). See host-agent log.</p>
 <pre style="white-space:pre-wrap;font-size:12px">$(tail -n 80 "$LOG" 2>/dev/null | sed 's/&/\&amp;/g;s/</\&lt;/g' || true)</pre>
 </main>
+<!-- taler-mon:bottom:incomplete -->
 </body></html>
 EOF
+  mv -f "$tmp" "$out"
  echo "bootstrap html → $out"
 }

@ -411,8 +427,11 @@ done
 # Publish to live web root when possible (v1.3.1+: never silent-skip)
 if [ -n "$DEPLOY_WWW" ]; then
  if [ -w "$DEPLOY_WWW" ] 2>/dev/null; then
-    if rsync -a --delete "$HTML_BASE/" "$DEPLOY_WWW/"; then
-      echo "synced → $DEPLOY_WWW/"
+    # --delay-updates: write to temp names then rename (less partial-file exposure)
+    if rsync -a --delete --delay-updates "$HTML_BASE/" "$DEPLOY_WWW/"; then
+      echo "synced → $DEPLOY_WWW/ (delay-updates)"
+      # brief settle so concurrent monpages curls do not hit mid-rename FS views
+      sleep "${MONPAGES_PUBLISH_SETTLE_S:-0.3}" 2>/dev/null || sleep 1
    else
      echo "ERROR: rsync to DEPLOY_WWW_ROOT=$DEPLOY_WWW failed" >&2
      ec=1
@ -441,12 +460,16 @@ if [ "${MONPAGES_POSTCHECK:-1}" = "1" ] && [ -x "$ROOT/check_monitoring_pages.sh
  export MONPAGES_REQUIRE_PUBLIC="${MONPAGES_REQUIRE_PUBLIC:-1}"
  # auto: GOA = all suite mon sites; FP = only FP mon hosts; job paths always included
  export MONPAGES_INVENTORY="${MONPAGES_INVENTORY:-auto}"
+  # Hard check (not pre-publish soft)
+  unset MONPAGES_PRE_PUBLISH 2>/dev/null || true
+  export MONPAGES_PRE_PUBLISH=0
  if ! bash "$ROOT/check_monitoring_pages.sh"; then
    echo "ERROR: public monitoring page(s) missing or not suite HTML via FQDN" >&2
    echo "       staging HTML_OUT=${HTML_BASE} · DEPLOY_WWW=${DEPLOY_WWW:-}" >&2
    ec=1
    _monpages_failed=1
  fi
+  unset MONPAGES_PRE_PUBLISH 2>/dev/null || true
 fi

 # If public check failed after first HTML write, rewrite pages so sticky bar shows monpages ERRORs
--- a/site-gen/console_to_html.py
+++ b/site-gen/console_to_html.py
@ -5,6 +5,7 @@ from __future__ import annotations
 import argparse
 import html
 import re
+import os
 from datetime import datetime, timezone
 from pathlib import Path

@ -27,6 +28,22 @@ def format_generated_now() -> tuple[str, str]:
    generated_at = now.strftime("%Y-%m-%d %H:%M:%S") + f" {tz_name}"
    return generated_at, generated_iso

+
+def atomic_write_text(path: Path, text: str, encoding: str = "utf-8") -> None:
+    """Write file atomically (tmp + replace) so monpages never curls a half-written page."""
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_name(f".{path.name}.tmp.{os.getpid()}")
+    try:
+        tmp.write_text(text, encoding=encoding)
+        os.replace(tmp, path)
+    finally:
+        if tmp.exists():
+            try:
+                tmp.unlink()
+            except OSError:
+                pass
+
 def ui_lang(explicit: str = "") -> str:
    """en (default) or fr. Env: TALER_MON_LANG; auto fr for lefrancpaysan hosts."""
    import os
@ -1901,8 +1918,7 @@ def main() -> None:
            suite_version_url=getattr(args, "suite_version_url", "") or "",
        )

-    args.out.parent.mkdir(parents=True, exist_ok=True)
-    args.out.write_text(html_out, encoding="utf-8")
+    atomic_write_text(args.out, html_out, encoding="utf-8")
    print(f"wrote {args.out} (errors={n_err} warnings={n_warn} level={status_level(n_err, n_warn)})")
 @ -1 +1 @@
 .13.3
 .13.4