release 1.13.4: monpages race-proof publish and content checks
Avoid false TOP/content ERRORs from mid-publish races: atomic HTML write, rsync --delay-updates + settle, monpages fetch retries with larger head windows, pre-publish soft content, bootstrap markers.
This commit is contained in:
Hernâni Marques
parent
d5213adb7a
commit
01cd4496d3
5 changed files with 179 additions and 73 deletions
2
VERSION
2
VERSION
|
|
@ -1 +1 @@
|
|||
1.13.3
|
||||
1.13.4
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre
|
|||
|
||||
| Tag | Date (UTC) | Notes |
|
||||
|-----|------------|--------|
|
||||
| **v1.13.4** | 2026-07-19 | **Bugfix:** monpages race-proof — atomic HTML write; rsync `--delay-updates` + settle; fetch retries; larger head windows; pre-publish soft content (`MONPAGES_PRE_PUBLISH`); bootstrap markers. |
|
||||
| **v1.13.3** | 2026-07-19 | **Bugfix:** sticky **generated** time in **Europe/Zurich** (CEST/CET · MESZ/MEZ), not UTC `…Z`; ISO keeps offset for age JS. |
|
||||
| **v1.13.2** | 2026-07-19 | **Bugfix:** sticky version link uses **commit** URL (`/src/commit/<sha>`) not `/src/tag/v…` (tags not yet pushed → Forgejo fat 404); tags v1.11–v1.13.1 pushed to origin. |
|
||||
| **v1.13.1** | 2026-07-19 | **Bugfix:** progress bar **global** done/total — phase `summary()` no longer snaps total to phase-local N (fixes 34/34 then 38/47); final `progress_finish` snaps once; header shows 0/N. |
|
||||
|
|
|
|||
|
|
@ -293,19 +293,33 @@ _page_mid_run() {
|
|||
|
||||
# Validate top/bottom markers + meaningful body (v1.9.0).
|
||||
# Sets _MON_CONTENT_MSG; returns 0=ok, 1=hard fail, 2=soft/mid-run bottom skip
|
||||
#
|
||||
# Head windows are large (v1.13.4+): sticky CSS grew past old 12–16 KiB limits and
|
||||
# caused false TOP errors while publish races were also writing the file.
|
||||
validate_mon_content() {
|
||||
local f="$1" url="$2"
|
||||
local top_ok=1 bot_ok=1 body_ok=1 mid=0 pct
|
||||
local head_top="${MONPAGES_HEAD_TOP_BYTES:-49152}"
|
||||
local head_meta="${MONPAGES_HEAD_META_BYTES:-65536}"
|
||||
_MON_CONTENT_MSG=""
|
||||
|
||||
# Tiny / empty body = likely mid-publish race (partial rsync or curl)
|
||||
if [ ! -s "$f" ] || [ "$(wc -c <"$f" 2>/dev/null || echo 0)" -lt 200 ]; then
|
||||
top_ok=0
|
||||
_MON_CONTENT_MSG="empty or truncated body (publish race?)"
|
||||
# --- top markers (always required) ---
|
||||
if ! head -c 12000 "$f" | grep -qE 'sticky-bar|id="status-bar"|taler-mon:top'; then
|
||||
elif ! head -c "$head_top" "$f" | grep -qE 'sticky-bar|id="status-bar"|taler-mon:top'; then
|
||||
# full-file fallback (small pages / unusual layout)
|
||||
if ! grep -qE 'sticky-bar|id="status-bar"|taler-mon:top' "$f" 2>/dev/null; then
|
||||
top_ok=0
|
||||
_MON_CONTENT_MSG="missing top marker (sticky-bar / status-bar / taler-mon:top)"
|
||||
elif ! head -c 16000 "$f" | grep -qE 'data-generated-iso|name="generated"|version-link|taler-mon-page'; then
|
||||
fi
|
||||
elif ! head -c "$head_meta" "$f" | grep -qE 'data-generated-iso|name="generated"|version-link|taler-mon-page'; then
|
||||
if ! grep -qE 'data-generated-iso|name="generated"|version-link|taler-mon-page' "$f" 2>/dev/null; then
|
||||
top_ok=0
|
||||
_MON_CONTENT_MSG="missing top meta (generated / version-link / taler-mon-page)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- meaningful body: real checks, not only progress bars ---
|
||||
if grep -qE 'class="line (ok|error|warn|blocker|info)"|┌ OK|┌ ERROR|┌ WARN|┌ BLOCKER|┌ INFO|class="box"' "$f" 2>/dev/null; then
|
||||
|
|
@ -386,53 +400,89 @@ _mon_fail_or_soft() {
|
|||
return 1
|
||||
}
|
||||
|
||||
# Fetch + validate one URL. Retries content failures (publish/rsync races).
|
||||
# MONPAGES_PRE_PUBLISH=1 → content failures are WARN only (suite still regenerates HTML after).
|
||||
# MONPAGES_FETCH_RETRIES (default 3), MONPAGES_FETCH_SLEEP_S (default 0.5).
|
||||
check_one() {
|
||||
local url="$1"
|
||||
local body code hint soft=0 vc
|
||||
local body code hint soft=0 vc attempt max_try sleep_s pre_pub
|
||||
body=$(mktemp)
|
||||
code=$(curl -skS -L --max-redirs 5 -m "${TIMEOUT}" -o "$body" -w '%{http_code}' "$url" 2>/dev/null || echo 000)
|
||||
if _is_bare_url "$url" && [ "${MONPAGES_BARE_STRICT:-0}" != "1" ]; then
|
||||
soft=1
|
||||
fi
|
||||
pre_pub=0
|
||||
[ "${MONPAGES_PRE_PUBLISH:-0}" = "1" ] && pre_pub=1
|
||||
max_try="${MONPAGES_FETCH_RETRIES:-3}"
|
||||
sleep_s="${MONPAGES_FETCH_SLEEP_S:-0.5}"
|
||||
[ "$max_try" -lt 1 ] && max_try=1
|
||||
|
||||
attempt=1
|
||||
while [ "$attempt" -le "$max_try" ]; do
|
||||
: >"$body"
|
||||
code=$(curl -skS -L --max-redirs 5 -m "${TIMEOUT}" -o "$body" -w '%{http_code}' "$url" 2>/dev/null || echo 000)
|
||||
|
||||
if grep -qE '"code"[[:space:]]*:[[:space:]]*21' "$body" 2>/dev/null \
|
||||
|| grep -qiE 'There is no endpoint defined for the URL' "$body" 2>/dev/null; then
|
||||
# Bare without trailing slash often hits merchant until Caddy has redir * /path/ 302.
|
||||
# Default: WARN on bare (soft) so slash-OK stacks still publish green/yellow pages.
|
||||
# MONPAGES_BARE_STRICT=1 → ERROR (enforce bare redir live).
|
||||
if [ "$soft" = "1" ]; then
|
||||
warn "public mon page bare URL" \
|
||||
"$url -> HTTP $code code 21 (prefer trailing slash; Caddy: redir * /path/ 302; set MONPAGES_BARE_STRICT=1 to ERROR)"
|
||||
rm -f "$body"
|
||||
return 0
|
||||
fi
|
||||
# hard 21: retry once more (rare brief outage) then fail
|
||||
if [ "$attempt" -lt "$max_try" ]; then
|
||||
attempt=$((attempt + 1))
|
||||
sleep "$sleep_s" 2>/dev/null || sleep 1
|
||||
continue
|
||||
fi
|
||||
_mon_fail_or_soft "public mon page missing" \
|
||||
"$url -> HTTP $code merchant/API JSON code 21 (Caddy handle/redir; use redir * /path/ 302 inside handle; apply: sudo /home/hernani/koopa-caddy/apply-monitoring-live.sh)" || { rm -f "$body"; return 1; }
|
||||
rm -f "$body"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ "$code" != "200" ]; then
|
||||
if [ "$soft" = "1" ]; then
|
||||
warn "public mon page bare URL" "$url -> HTTP $code (prefer trailing slash form)"
|
||||
rm -f "$body"
|
||||
return 0
|
||||
fi
|
||||
if [ "$attempt" -lt "$max_try" ]; then
|
||||
attempt=$((attempt + 1))
|
||||
sleep "$sleep_s" 2>/dev/null || sleep 1
|
||||
continue
|
||||
fi
|
||||
_mon_fail_or_soft "public mon page" "$url -> HTTP $code (want 200 HTML)" || { rm -f "$body"; return 1; }
|
||||
rm -f "$body"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if is_bootstrap_html "$body"; then
|
||||
if [ "$attempt" -lt "$max_try" ]; then
|
||||
attempt=$((attempt + 1))
|
||||
sleep "$sleep_s" 2>/dev/null || sleep 1
|
||||
continue
|
||||
fi
|
||||
_mon_fail_or_soft "public mon page bootstrap stub" \
|
||||
"$url -> install bootstrap HTML still live (run host-agent; suite auto-update must replace stub)" || { rm -f "$body"; return 1; }
|
||||
rm -f "$body"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! is_monitoring_html "$body"; then
|
||||
if [ "$attempt" -lt "$max_try" ]; then
|
||||
attempt=$((attempt + 1))
|
||||
sleep "$sleep_s" 2>/dev/null || sleep 1
|
||||
continue
|
||||
fi
|
||||
hint=$(head -c 120 "$body" | tr '\n' ' ' | tr -cd '[:print:] ')
|
||||
_mon_fail_or_soft "public mon page not monitoring HTML" \
|
||||
"$url -> HTTP 200 body!=suite (${hint}...)" || { rm -f "$body"; return 1; }
|
||||
rm -f "$body"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# v1.9.0: top + bottom markers + meaningful content
|
||||
set +e
|
||||
validate_mon_content "$body" "$url"
|
||||
|
|
@ -441,21 +491,37 @@ check_one() {
|
|||
case "$vc" in
|
||||
0)
|
||||
ok "public mon page" "$url -> HTTP 200 · ${_MON_CONTENT_MSG}"
|
||||
rm -f "$body"
|
||||
return 0
|
||||
;;
|
||||
2)
|
||||
# mid-run: bottom may be incomplete — not ERROR
|
||||
warn "public mon page mid-run" "$url -> ${_MON_CONTENT_MSG}"
|
||||
ok "public mon page" "$url -> HTTP 200 suite HTML (mid-run bottom relaxed)"
|
||||
rm -f "$body"
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
if [ "$soft" = "1" ]; then
|
||||
warn "public mon page content" "$url -> ${_MON_CONTENT_MSG}"
|
||||
else
|
||||
# Content race (truncated HTML while another agent publishes): retry
|
||||
if [ "$attempt" -lt "$max_try" ]; then
|
||||
attempt=$((attempt + 1))
|
||||
sleep "$sleep_s" 2>/dev/null || sleep 1
|
||||
continue
|
||||
fi
|
||||
# Pre-publish inventory in host-agent PHASES: WARN only (post-check is hard)
|
||||
if [ "$pre_pub" = "1" ] || [ "$soft" = "1" ]; then
|
||||
warn "public mon page content" \
|
||||
"$url -> ${_MON_CONTENT_MSG}${pre_pub:+ (pre-publish; post-check is authoritative)}"
|
||||
rm -f "$body"
|
||||
return 0
|
||||
fi
|
||||
_mon_fail_or_soft "public mon page content" \
|
||||
"$url -> ${_MON_CONTENT_MSG}" || { rm -f "$body"; return 1; }
|
||||
fi
|
||||
rm -f "$body"
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
done
|
||||
rm -f "$body"
|
||||
return 0
|
||||
}
|
||||
|
|
|
|||
|
|
@ -280,9 +280,17 @@ if [ -x "${ROOT:-}/taler-monitoring.sh" ]; then
|
|||
fi
|
||||
done
|
||||
|
||||
# monpages inside PHASES runs *before* htmlify/publish — content failures would
|
||||
# race with concurrent publishes or reflect last run's incomplete redirect stub.
|
||||
# Soft-content there; post-check after rsync stays hard (authoritative).
|
||||
case " $PHASES " in
|
||||
*" monpages "*|*" pages "*) export MONPAGES_PRE_PUBLISH=1 ;;
|
||||
*) unset MONPAGES_PRE_PUBLISH 2>/dev/null || true ;;
|
||||
esac
|
||||
# shellcheck disable=SC2086
|
||||
./taler-monitoring.sh -d "$TALER_DOMAIN" $PHASES
|
||||
ec=$?
|
||||
unset MONPAGES_PRE_PUBLISH 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# --- always generate HTML (first run / missing pages / every run) ---
|
||||
|
|
@ -304,21 +312,27 @@ write_bootstrap_html() {
|
|||
if [ -n "$curl" ]; then
|
||||
clink="<a href=\"$(printf '%s' "$curl" | sed 's/&/\&/g')\">$cshort</a>"
|
||||
fi
|
||||
cat >"$out" <<EOF
|
||||
local tmp
|
||||
tmp=$(mktemp "${out}.XXXXXX")
|
||||
cat >"$tmp" <<EOF
|
||||
<!DOCTYPE html>
|
||||
<html lang="en"><head><meta charset="utf-8"/>
|
||||
<meta name="generated" content="${iso}"/>
|
||||
<meta name="taler-mon-page" content="incomplete"/>
|
||||
<title>monitoring ${mode} · ${host}</title>
|
||||
<style>
|
||||
body{margin:0;background:#0c0c0c;color:#c8c8c8;font-family:ui-monospace,monospace}
|
||||
a{color:#61afef}.err{color:#ff5c5c}
|
||||
.sticky-bar{position:sticky;top:0;z-index:100;padding:10px 14px;background:#2a1010f2;border-bottom:2px solid #a33}
|
||||
.sticky-bar .pill{font-weight:800;color:#ff5c5c;margin-right:8px}
|
||||
.version-link{color:#61afef}
|
||||
main{padding:1.5rem}
|
||||
</style></head><body>
|
||||
<!-- taler-mon:top -->
|
||||
<div class="sticky-bar" id="status-bar">
|
||||
<span class="pill">BOOTSTRAP</span>
|
||||
${PAGE_LABEL:-monitoring} · ${host} ·
|
||||
<span class="version-link muted">bootstrap</span> ·
|
||||
<span data-generated-iso="${iso}">generated ${gen}</span>
|
||||
· source ${clink}
|
||||
</div>
|
||||
|
|
@ -326,8 +340,10 @@ main{padding:1.5rem}
|
|||
<p class="err">Bootstrap page (converter missing or first install). See host-agent log.</p>
|
||||
<pre style="white-space:pre-wrap;font-size:12px">$(tail -n 80 "$LOG" 2>/dev/null | sed 's/&/\&/g;s/</\</g' || true)</pre>
|
||||
</main>
|
||||
<!-- taler-mon:bottom:incomplete -->
|
||||
</body></html>
|
||||
EOF
|
||||
mv -f "$tmp" "$out"
|
||||
echo "bootstrap html → $out"
|
||||
}
|
||||
|
||||
|
|
@ -411,8 +427,11 @@ done
|
|||
# Publish to live web root when possible (v1.3.1+: never silent-skip)
|
||||
if [ -n "$DEPLOY_WWW" ]; then
|
||||
if [ -w "$DEPLOY_WWW" ] 2>/dev/null; then
|
||||
if rsync -a --delete "$HTML_BASE/" "$DEPLOY_WWW/"; then
|
||||
echo "synced → $DEPLOY_WWW/"
|
||||
# --delay-updates: write to temp names then rename (less partial-file exposure)
|
||||
if rsync -a --delete --delay-updates "$HTML_BASE/" "$DEPLOY_WWW/"; then
|
||||
echo "synced → $DEPLOY_WWW/ (delay-updates)"
|
||||
# brief settle so concurrent monpages curls do not hit mid-rename FS views
|
||||
sleep "${MONPAGES_PUBLISH_SETTLE_S:-0.3}" 2>/dev/null || sleep 1
|
||||
else
|
||||
echo "ERROR: rsync to DEPLOY_WWW_ROOT=$DEPLOY_WWW failed" >&2
|
||||
ec=1
|
||||
|
|
@ -441,12 +460,16 @@ if [ "${MONPAGES_POSTCHECK:-1}" = "1" ] && [ -x "$ROOT/check_monitoring_pages.sh
|
|||
export MONPAGES_REQUIRE_PUBLIC="${MONPAGES_REQUIRE_PUBLIC:-1}"
|
||||
# auto: GOA = all suite mon sites; FP = only FP mon hosts; job paths always included
|
||||
export MONPAGES_INVENTORY="${MONPAGES_INVENTORY:-auto}"
|
||||
# Hard check (not pre-publish soft)
|
||||
unset MONPAGES_PRE_PUBLISH 2>/dev/null || true
|
||||
export MONPAGES_PRE_PUBLISH=0
|
||||
if ! bash "$ROOT/check_monitoring_pages.sh"; then
|
||||
echo "ERROR: public monitoring page(s) missing or not suite HTML via FQDN" >&2
|
||||
echo " staging HTML_OUT=${HTML_BASE} · DEPLOY_WWW=${DEPLOY_WWW:-}" >&2
|
||||
ec=1
|
||||
_monpages_failed=1
|
||||
fi
|
||||
unset MONPAGES_PRE_PUBLISH 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# If public check failed after first HTML write, rewrite pages so sticky bar shows monpages ERRORs
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from __future__ import annotations
|
|||
import argparse
|
||||
import html
|
||||
import re
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
|
|
@ -27,6 +28,22 @@ def format_generated_now() -> tuple[str, str]:
|
|||
generated_at = now.strftime("%Y-%m-%d %H:%M:%S") + f" {tz_name}"
|
||||
return generated_at, generated_iso
|
||||
|
||||
|
||||
def atomic_write_text(path: Path, text: str, encoding: str = "utf-8") -> None:
|
||||
"""Write file atomically (tmp + replace) so monpages never curls a half-written page."""
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_name(f".{path.name}.tmp.{os.getpid()}")
|
||||
try:
|
||||
tmp.write_text(text, encoding=encoding)
|
||||
os.replace(tmp, path)
|
||||
finally:
|
||||
if tmp.exists():
|
||||
try:
|
||||
tmp.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def ui_lang(explicit: str = "") -> str:
|
||||
"""en (default) or fr. Env: TALER_MON_LANG; auto fr for lefrancpaysan hosts."""
|
||||
import os
|
||||
|
|
@ -1901,8 +1918,7 @@ def main() -> None:
|
|||
suite_version_url=getattr(args, "suite_version_url", "") or "",
|
||||
)
|
||||
|
||||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.out.write_text(html_out, encoding="utf-8")
|
||||
atomic_write_text(args.out, html_out, encoding="utf-8")
|
||||
print(f"wrote {args.out} (errors={n_err} warnings={n_warn} level={status_level(n_err, n_warn)})")
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue