From a38992099eb9642ae016ce9bcc018dd0aa4dcf4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 02:55:56 +0200 Subject: [PATCH] release 1.13.4: monpages race-proof publish and content checks Avoid false TOP/content ERRORs from mid-publish races: atomic HTML write, rsync --delay-updates + settle, monpages fetch retries with larger head windows, pre-publish soft content, bootstrap markers. --- VERSION | 2 +- VERSIONS.md | 1 + check_monitoring_pages.sh | 200 ++++++++++++++++++++++------------ host-agent/run-host-report.sh | 29 ++++- site-gen/console_to_html.py | 20 +++- 5 files changed, 179 insertions(+), 73 deletions(-) diff --git a/VERSION b/VERSION index 01b7568..80138e7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.13.3 +1.13.4 diff --git a/VERSIONS.md b/VERSIONS.md index e31d1e0..d78d636 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| +| **v1.13.4** | 2026-07-19 | **Bugfix:** monpages race-proof — atomic HTML write; rsync `--delay-updates` + settle; fetch retries; larger head windows; pre-publish soft content (`MONPAGES_PRE_PUBLISH`); bootstrap markers. | | **v1.13.3** | 2026-07-19 | **Bugfix:** sticky **generated** time in **Europe/Zurich** (CEST/CET · MESZ/MEZ), not UTC `…Z`; ISO keeps offset for age JS. | | **v1.13.2** | 2026-07-19 | **Bugfix:** sticky version link uses **commit** URL (`/src/commit/`) not `/src/tag/v…` (tags not yet pushed → Forgejo fat 404); tags v1.11–v1.13.1 pushed to origin. | | **v1.13.1** | 2026-07-19 | **Bugfix:** progress bar **global** done/total — phase `summary()` no longer snaps total to phase-local N (fixes 34/34 then 38/47); final `progress_finish` snaps once; header shows 0/N. | diff --git a/check_monitoring_pages.sh b/check_monitoring_pages.sh index 0a66bd1..ad1ead2 100755 --- a/check_monitoring_pages.sh +++ b/check_monitoring_pages.sh @@ -293,18 +293,32 @@ _page_mid_run() { # Validate top/bottom markers + meaningful body (v1.9.0). # Sets _MON_CONTENT_MSG; returns 0=ok, 1=hard fail, 2=soft/mid-run bottom skip +# +# Head windows are large (v1.13.4+): sticky CSS grew past old 12–16 KiB limits and +# caused false TOP errors while publish races were also writing the file. validate_mon_content() { local f="$1" url="$2" local top_ok=1 bot_ok=1 body_ok=1 mid=0 pct + local head_top="${MONPAGES_HEAD_TOP_BYTES:-49152}" + local head_meta="${MONPAGES_HEAD_META_BYTES:-65536}" _MON_CONTENT_MSG="" + # Tiny / empty body = likely mid-publish race (partial rsync or curl) + if [ ! -s "$f" ] || [ "$(wc -c <"$f" 2>/dev/null || echo 0)" -lt 200 ]; then + top_ok=0 + _MON_CONTENT_MSG="empty or truncated body (publish race?)" # --- top markers (always required) --- - if ! head -c 12000 "$f" | grep -qE 'sticky-bar|id="status-bar"|taler-mon:top'; then - top_ok=0 - _MON_CONTENT_MSG="missing top marker (sticky-bar / status-bar / taler-mon:top)" - elif ! head -c 16000 "$f" | grep -qE 'data-generated-iso|name="generated"|version-link|taler-mon-page'; then - top_ok=0 - _MON_CONTENT_MSG="missing top meta (generated / version-link / taler-mon-page)" + elif ! head -c "$head_top" "$f" | grep -qE 'sticky-bar|id="status-bar"|taler-mon:top'; then + # full-file fallback (small pages / unusual layout) + if ! grep -qE 'sticky-bar|id="status-bar"|taler-mon:top' "$f" 2>/dev/null; then + top_ok=0 + _MON_CONTENT_MSG="missing top marker (sticky-bar / status-bar / taler-mon:top)" + fi + elif ! head -c "$head_meta" "$f" | grep -qE 'data-generated-iso|name="generated"|version-link|taler-mon-page'; then + if ! grep -qE 'data-generated-iso|name="generated"|version-link|taler-mon-page' "$f" 2>/dev/null; then + top_ok=0 + _MON_CONTENT_MSG="missing top meta (generated / version-link / taler-mon-page)" + fi fi # --- meaningful body: real checks, not only progress bars --- @@ -386,76 +400,128 @@ _mon_fail_or_soft() { return 1 } +# Fetch + validate one URL. Retries content failures (publish/rsync races). +# MONPAGES_PRE_PUBLISH=1 → content failures are WARN only (suite still regenerates HTML after). +# MONPAGES_FETCH_RETRIES (default 3), MONPAGES_FETCH_SLEEP_S (default 0.5). check_one() { local url="$1" - local body code hint soft=0 vc + local body code hint soft=0 vc attempt max_try sleep_s pre_pub body=$(mktemp) - code=$(curl -skS -L --max-redirs 5 -m "${TIMEOUT}" -o "$body" -w '%{http_code}' "$url" 2>/dev/null || echo 000) if _is_bare_url "$url" && [ "${MONPAGES_BARE_STRICT:-0}" != "1" ]; then soft=1 fi - if grep -qE '"code"[[:space:]]*:[[:space:]]*21' "$body" 2>/dev/null \ - || grep -qiE 'There is no endpoint defined for the URL' "$body" 2>/dev/null; then - # Bare without trailing slash often hits merchant until Caddy has redir * /path/ 302. - # Default: WARN on bare (soft) so slash-OK stacks still publish green/yellow pages. - # MONPAGES_BARE_STRICT=1 → ERROR (enforce bare redir live). - if [ "$soft" = "1" ]; then - warn "public mon page bare URL" \ - "$url -> HTTP $code code 21 (prefer trailing slash; Caddy: redir * /path/ 302; set MONPAGES_BARE_STRICT=1 to ERROR)" - rm -f "$body" - return 0 - fi - _mon_fail_or_soft "public mon page missing" \ - "$url -> HTTP $code merchant/API JSON code 21 (Caddy handle/redir; use redir * /path/ 302 inside handle; apply: sudo /home/hernani/koopa-caddy/apply-monitoring-live.sh)" || { rm -f "$body"; return 1; } - rm -f "$body" - return 0 - fi - if [ "$code" != "200" ]; then - if [ "$soft" = "1" ]; then - warn "public mon page bare URL" "$url -> HTTP $code (prefer trailing slash form)" - rm -f "$body" - return 0 - fi - _mon_fail_or_soft "public mon page" "$url -> HTTP $code (want 200 HTML)" || { rm -f "$body"; return 1; } - rm -f "$body" - return 0 - fi - if is_bootstrap_html "$body"; then - _mon_fail_or_soft "public mon page bootstrap stub" \ - "$url -> install bootstrap HTML still live (run host-agent; suite auto-update must replace stub)" || { rm -f "$body"; return 1; } - rm -f "$body" - return 0 - fi - if ! is_monitoring_html "$body"; then - hint=$(head -c 120 "$body" | tr '\n' ' ' | tr -cd '[:print:] ') - _mon_fail_or_soft "public mon page not monitoring HTML" \ - "$url -> HTTP 200 body!=suite (${hint}...)" || { rm -f "$body"; return 1; } - rm -f "$body" - return 0 - fi - # v1.9.0: top + bottom markers + meaningful content - set +e - validate_mon_content "$body" "$url" - vc=$? - set -e - case "$vc" in - 0) - ok "public mon page" "$url -> HTTP 200 · ${_MON_CONTENT_MSG}" - ;; - 2) - # mid-run: bottom may be incomplete — not ERROR - warn "public mon page mid-run" "$url -> ${_MON_CONTENT_MSG}" - ok "public mon page" "$url -> HTTP 200 suite HTML (mid-run bottom relaxed)" - ;; - *) + pre_pub=0 + [ "${MONPAGES_PRE_PUBLISH:-0}" = "1" ] && pre_pub=1 + max_try="${MONPAGES_FETCH_RETRIES:-3}" + sleep_s="${MONPAGES_FETCH_SLEEP_S:-0.5}" + [ "$max_try" -lt 1 ] && max_try=1 + + attempt=1 + while [ "$attempt" -le "$max_try" ]; do + : >"$body" + code=$(curl -skS -L --max-redirs 5 -m "${TIMEOUT}" -o "$body" -w '%{http_code}' "$url" 2>/dev/null || echo 000) + + if grep -qE '"code"[[:space:]]*:[[:space:]]*21' "$body" 2>/dev/null \ + || grep -qiE 'There is no endpoint defined for the URL' "$body" 2>/dev/null; then + # Bare without trailing slash often hits merchant until Caddy has redir * /path/ 302. if [ "$soft" = "1" ]; then - warn "public mon page content" "$url -> ${_MON_CONTENT_MSG}" - else + warn "public mon page bare URL" \ + "$url -> HTTP $code code 21 (prefer trailing slash; Caddy: redir * /path/ 302; set MONPAGES_BARE_STRICT=1 to ERROR)" + rm -f "$body" + return 0 + fi + # hard 21: retry once more (rare brief outage) then fail + if [ "$attempt" -lt "$max_try" ]; then + attempt=$((attempt + 1)) + sleep "$sleep_s" 2>/dev/null || sleep 1 + continue + fi + _mon_fail_or_soft "public mon page missing" \ + "$url -> HTTP $code merchant/API JSON code 21 (Caddy handle/redir; use redir * /path/ 302 inside handle; apply: sudo /home/hernani/koopa-caddy/apply-monitoring-live.sh)" || { rm -f "$body"; return 1; } + rm -f "$body" + return 0 + fi + + if [ "$code" != "200" ]; then + if [ "$soft" = "1" ]; then + warn "public mon page bare URL" "$url -> HTTP $code (prefer trailing slash form)" + rm -f "$body" + return 0 + fi + if [ "$attempt" -lt "$max_try" ]; then + attempt=$((attempt + 1)) + sleep "$sleep_s" 2>/dev/null || sleep 1 + continue + fi + _mon_fail_or_soft "public mon page" "$url -> HTTP $code (want 200 HTML)" || { rm -f "$body"; return 1; } + rm -f "$body" + return 0 + fi + + if is_bootstrap_html "$body"; then + if [ "$attempt" -lt "$max_try" ]; then + attempt=$((attempt + 1)) + sleep "$sleep_s" 2>/dev/null || sleep 1 + continue + fi + _mon_fail_or_soft "public mon page bootstrap stub" \ + "$url -> install bootstrap HTML still live (run host-agent; suite auto-update must replace stub)" || { rm -f "$body"; return 1; } + rm -f "$body" + return 0 + fi + + if ! is_monitoring_html "$body"; then + if [ "$attempt" -lt "$max_try" ]; then + attempt=$((attempt + 1)) + sleep "$sleep_s" 2>/dev/null || sleep 1 + continue + fi + hint=$(head -c 120 "$body" | tr '\n' ' ' | tr -cd '[:print:] ') + _mon_fail_or_soft "public mon page not monitoring HTML" \ + "$url -> HTTP 200 body!=suite (${hint}...)" || { rm -f "$body"; return 1; } + rm -f "$body" + return 0 + fi + + # v1.9.0: top + bottom markers + meaningful content + set +e + validate_mon_content "$body" "$url" + vc=$? + set -e + case "$vc" in + 0) + ok "public mon page" "$url -> HTTP 200 · ${_MON_CONTENT_MSG}" + rm -f "$body" + return 0 + ;; + 2) + # mid-run: bottom may be incomplete — not ERROR + warn "public mon page mid-run" "$url -> ${_MON_CONTENT_MSG}" + ok "public mon page" "$url -> HTTP 200 suite HTML (mid-run bottom relaxed)" + rm -f "$body" + return 0 + ;; + *) + # Content race (truncated HTML while another agent publishes): retry + if [ "$attempt" -lt "$max_try" ]; then + attempt=$((attempt + 1)) + sleep "$sleep_s" 2>/dev/null || sleep 1 + continue + fi + # Pre-publish inventory in host-agent PHASES: WARN only (post-check is hard) + if [ "$pre_pub" = "1" ] || [ "$soft" = "1" ]; then + warn "public mon page content" \ + "$url -> ${_MON_CONTENT_MSG}${pre_pub:+ (pre-publish; post-check is authoritative)}" + rm -f "$body" + return 0 + fi _mon_fail_or_soft "public mon page content" \ "$url -> ${_MON_CONTENT_MSG}" || { rm -f "$body"; return 1; } - fi - ;; - esac + rm -f "$body" + return 0 + ;; + esac + done rm -f "$body" return 0 } diff --git a/host-agent/run-host-report.sh b/host-agent/run-host-report.sh index a29a08f..fa1d6dd 100755 --- a/host-agent/run-host-report.sh +++ b/host-agent/run-host-report.sh @@ -280,9 +280,17 @@ if [ -x "${ROOT:-}/taler-monitoring.sh" ]; then fi done + # monpages inside PHASES runs *before* htmlify/publish — content failures would + # race with concurrent publishes or reflect last run's incomplete redirect stub. + # Soft-content there; post-check after rsync stays hard (authoritative). + case " $PHASES " in + *" monpages "*|*" pages "*) export MONPAGES_PRE_PUBLISH=1 ;; + *) unset MONPAGES_PRE_PUBLISH 2>/dev/null || true ;; + esac # shellcheck disable=SC2086 ./taler-monitoring.sh -d "$TALER_DOMAIN" $PHASES ec=$? + unset MONPAGES_PRE_PUBLISH 2>/dev/null || true fi # --- always generate HTML (first run / missing pages / every run) --- @@ -304,21 +312,27 @@ write_bootstrap_html() { if [ -n "$curl" ]; then clink="$cshort" fi - cat >"$out" <"$tmp" < + monitoring ${mode} · ${host} + @@ -326,8 +340,10 @@ main{padding:1.5rem}

Bootstrap page (converter missing or first install). See host-agent log.

$(tail -n 80 "$LOG" 2>/dev/null | sed 's/&/\&/g;s/
 
+
 
 EOF
+  mv -f "$tmp" "$out"
   echo "bootstrap html → $out"
 }
 
@@ -411,8 +427,11 @@ done
 # Publish to live web root when possible (v1.3.1+: never silent-skip)
 if [ -n "$DEPLOY_WWW" ]; then
   if [ -w "$DEPLOY_WWW" ] 2>/dev/null; then
-    if rsync -a --delete "$HTML_BASE/" "$DEPLOY_WWW/"; then
-      echo "synced → $DEPLOY_WWW/"
+    # --delay-updates: write to temp names then rename (less partial-file exposure)
+    if rsync -a --delete --delay-updates "$HTML_BASE/" "$DEPLOY_WWW/"; then
+      echo "synced → $DEPLOY_WWW/ (delay-updates)"
+      # brief settle so concurrent monpages curls do not hit mid-rename FS views
+      sleep "${MONPAGES_PUBLISH_SETTLE_S:-0.3}" 2>/dev/null || sleep 1
     else
       echo "ERROR: rsync to DEPLOY_WWW_ROOT=$DEPLOY_WWW failed" >&2
       ec=1
@@ -441,12 +460,16 @@ if [ "${MONPAGES_POSTCHECK:-1}" = "1" ] && [ -x "$ROOT/check_monitoring_pages.sh
   export MONPAGES_REQUIRE_PUBLIC="${MONPAGES_REQUIRE_PUBLIC:-1}"
   # auto: GOA = all suite mon sites; FP = only FP mon hosts; job paths always included
   export MONPAGES_INVENTORY="${MONPAGES_INVENTORY:-auto}"
+  # Hard check (not pre-publish soft)
+  unset MONPAGES_PRE_PUBLISH 2>/dev/null || true
+  export MONPAGES_PRE_PUBLISH=0
   if ! bash "$ROOT/check_monitoring_pages.sh"; then
     echo "ERROR: public monitoring page(s) missing or not suite HTML via FQDN" >&2
     echo "       staging HTML_OUT=${HTML_BASE} · DEPLOY_WWW=${DEPLOY_WWW:-}" >&2
     ec=1
     _monpages_failed=1
   fi
+  unset MONPAGES_PRE_PUBLISH 2>/dev/null || true
 fi
 
 # If public check failed after first HTML write, rewrite pages so sticky bar shows monpages ERRORs
diff --git a/site-gen/console_to_html.py b/site-gen/console_to_html.py
index eae74bc..bb574ba 100755
--- a/site-gen/console_to_html.py
+++ b/site-gen/console_to_html.py
@@ -5,6 +5,7 @@ from __future__ import annotations
 import argparse
 import html
 import re
+import os
 from datetime import datetime, timezone
 from pathlib import Path
 
@@ -27,6 +28,22 @@ def format_generated_now() -> tuple[str, str]:
     generated_at = now.strftime("%Y-%m-%d %H:%M:%S") + f" {tz_name}"
     return generated_at, generated_iso
 
+
+def atomic_write_text(path: Path, text: str, encoding: str = "utf-8") -> None:
+    """Write file atomically (tmp + replace) so monpages never curls a half-written page."""
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_name(f".{path.name}.tmp.{os.getpid()}")
+    try:
+        tmp.write_text(text, encoding=encoding)
+        os.replace(tmp, path)
+    finally:
+        if tmp.exists():
+            try:
+                tmp.unlink()
+            except OSError:
+                pass
+
 def ui_lang(explicit: str = "") -> str:
     """en (default) or fr. Env: TALER_MON_LANG; auto fr for lefrancpaysan hosts."""
     import os
@@ -1901,8 +1918,7 @@ def main() -> None:
             suite_version_url=getattr(args, "suite_version_url", "") or "",
         )
 
-    args.out.parent.mkdir(parents=True, exist_ok=True)
-    args.out.write_text(html_out, encoding="utf-8")
+    atomic_write_text(args.out, html_out, encoding="utf-8")
     print(f"wrote {args.out} (errors={n_err} warnings={n_warn} level={status_level(n_err, n_warn)})")