release 1.13.4: monpages race-proof publish and content checks
Avoid false TOP/content ERRORs from mid-publish races: atomic HTML write, rsync --delay-updates + settle, monpages fetch retries with larger head windows, pre-publish soft content, bootstrap markers.
This commit is contained in:
Hernâni Marques
parent
d5213adb7a
commit
e450470429
5 changed files with 179 additions and 73 deletions
|
|
@ -293,18 +293,32 @@ _page_mid_run() {
|
|||
|
||||
# Validate top/bottom markers + meaningful body (v1.9.0).
|
||||
# Sets _MON_CONTENT_MSG; returns 0=ok, 1=hard fail, 2=soft/mid-run bottom skip
|
||||
#
|
||||
# Head windows are large (v1.13.4+): sticky CSS grew past old 12–16 KiB limits and
|
||||
# caused false TOP errors while publish races were also writing the file.
|
||||
validate_mon_content() {
|
||||
local f="$1" url="$2"
|
||||
local top_ok=1 bot_ok=1 body_ok=1 mid=0 pct
|
||||
local head_top="${MONPAGES_HEAD_TOP_BYTES:-49152}"
|
||||
local head_meta="${MONPAGES_HEAD_META_BYTES:-65536}"
|
||||
_MON_CONTENT_MSG=""
|
||||
|
||||
# Tiny / empty body = likely mid-publish race (partial rsync or curl)
|
||||
if [ ! -s "$f" ] || [ "$(wc -c <"$f" 2>/dev/null || echo 0)" -lt 200 ]; then
|
||||
top_ok=0
|
||||
_MON_CONTENT_MSG="empty or truncated body (publish race?)"
|
||||
# --- top markers (always required) ---
|
||||
if ! head -c 12000 "$f" | grep -qE 'sticky-bar|id="status-bar"|taler-mon:top'; then
|
||||
top_ok=0
|
||||
_MON_CONTENT_MSG="missing top marker (sticky-bar / status-bar / taler-mon:top)"
|
||||
elif ! head -c 16000 "$f" | grep -qE 'data-generated-iso|name="generated"|version-link|taler-mon-page'; then
|
||||
top_ok=0
|
||||
_MON_CONTENT_MSG="missing top meta (generated / version-link / taler-mon-page)"
|
||||
elif ! head -c "$head_top" "$f" | grep -qE 'sticky-bar|id="status-bar"|taler-mon:top'; then
|
||||
# full-file fallback (small pages / unusual layout)
|
||||
if ! grep -qE 'sticky-bar|id="status-bar"|taler-mon:top' "$f" 2>/dev/null; then
|
||||
top_ok=0
|
||||
_MON_CONTENT_MSG="missing top marker (sticky-bar / status-bar / taler-mon:top)"
|
||||
fi
|
||||
elif ! head -c "$head_meta" "$f" | grep -qE 'data-generated-iso|name="generated"|version-link|taler-mon-page'; then
|
||||
if ! grep -qE 'data-generated-iso|name="generated"|version-link|taler-mon-page' "$f" 2>/dev/null; then
|
||||
top_ok=0
|
||||
_MON_CONTENT_MSG="missing top meta (generated / version-link / taler-mon-page)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- meaningful body: real checks, not only progress bars ---
|
||||
|
|
@ -386,76 +400,128 @@ _mon_fail_or_soft() {
|
|||
return 1
|
||||
}
|
||||
|
||||
# Fetch + validate one URL. Retries content failures (publish/rsync races).
|
||||
# MONPAGES_PRE_PUBLISH=1 → content failures are WARN only (suite still regenerates HTML after).
|
||||
# MONPAGES_FETCH_RETRIES (default 3), MONPAGES_FETCH_SLEEP_S (default 0.5).
|
||||
check_one() {
|
||||
local url="$1"
|
||||
local body code hint soft=0 vc
|
||||
local body code hint soft=0 vc attempt max_try sleep_s pre_pub
|
||||
body=$(mktemp)
|
||||
code=$(curl -skS -L --max-redirs 5 -m "${TIMEOUT}" -o "$body" -w '%{http_code}' "$url" 2>/dev/null || echo 000)
|
||||
if _is_bare_url "$url" && [ "${MONPAGES_BARE_STRICT:-0}" != "1" ]; then
|
||||
soft=1
|
||||
fi
|
||||
if grep -qE '"code"[[:space:]]*:[[:space:]]*21' "$body" 2>/dev/null \
|
||||
|| grep -qiE 'There is no endpoint defined for the URL' "$body" 2>/dev/null; then
|
||||
# Bare without trailing slash often hits merchant until Caddy has redir * /path/ 302.
|
||||
# Default: WARN on bare (soft) so slash-OK stacks still publish green/yellow pages.
|
||||
# MONPAGES_BARE_STRICT=1 → ERROR (enforce bare redir live).
|
||||
if [ "$soft" = "1" ]; then
|
||||
warn "public mon page bare URL" \
|
||||
"$url -> HTTP $code code 21 (prefer trailing slash; Caddy: redir * /path/ 302; set MONPAGES_BARE_STRICT=1 to ERROR)"
|
||||
rm -f "$body"
|
||||
return 0
|
||||
fi
|
||||
_mon_fail_or_soft "public mon page missing" \
|
||||
"$url -> HTTP $code merchant/API JSON code 21 (Caddy handle/redir; use redir * /path/ 302 inside handle; apply: sudo /home/hernani/koopa-caddy/apply-monitoring-live.sh)" || { rm -f "$body"; return 1; }
|
||||
rm -f "$body"
|
||||
return 0
|
||||
fi
|
||||
if [ "$code" != "200" ]; then
|
||||
if [ "$soft" = "1" ]; then
|
||||
warn "public mon page bare URL" "$url -> HTTP $code (prefer trailing slash form)"
|
||||
rm -f "$body"
|
||||
return 0
|
||||
fi
|
||||
_mon_fail_or_soft "public mon page" "$url -> HTTP $code (want 200 HTML)" || { rm -f "$body"; return 1; }
|
||||
rm -f "$body"
|
||||
return 0
|
||||
fi
|
||||
if is_bootstrap_html "$body"; then
|
||||
_mon_fail_or_soft "public mon page bootstrap stub" \
|
||||
"$url -> install bootstrap HTML still live (run host-agent; suite auto-update must replace stub)" || { rm -f "$body"; return 1; }
|
||||
rm -f "$body"
|
||||
return 0
|
||||
fi
|
||||
if ! is_monitoring_html "$body"; then
|
||||
hint=$(head -c 120 "$body" | tr '\n' ' ' | tr -cd '[:print:] ')
|
||||
_mon_fail_or_soft "public mon page not monitoring HTML" \
|
||||
"$url -> HTTP 200 body!=suite (${hint}...)" || { rm -f "$body"; return 1; }
|
||||
rm -f "$body"
|
||||
return 0
|
||||
fi
|
||||
# v1.9.0: top + bottom markers + meaningful content
|
||||
set +e
|
||||
validate_mon_content "$body" "$url"
|
||||
vc=$?
|
||||
set -e
|
||||
case "$vc" in
|
||||
0)
|
||||
ok "public mon page" "$url -> HTTP 200 · ${_MON_CONTENT_MSG}"
|
||||
;;
|
||||
2)
|
||||
# mid-run: bottom may be incomplete — not ERROR
|
||||
warn "public mon page mid-run" "$url -> ${_MON_CONTENT_MSG}"
|
||||
ok "public mon page" "$url -> HTTP 200 suite HTML (mid-run bottom relaxed)"
|
||||
;;
|
||||
*)
|
||||
pre_pub=0
|
||||
[ "${MONPAGES_PRE_PUBLISH:-0}" = "1" ] && pre_pub=1
|
||||
max_try="${MONPAGES_FETCH_RETRIES:-3}"
|
||||
sleep_s="${MONPAGES_FETCH_SLEEP_S:-0.5}"
|
||||
[ "$max_try" -lt 1 ] && max_try=1
|
||||
|
||||
attempt=1
|
||||
while [ "$attempt" -le "$max_try" ]; do
|
||||
: >"$body"
|
||||
code=$(curl -skS -L --max-redirs 5 -m "${TIMEOUT}" -o "$body" -w '%{http_code}' "$url" 2>/dev/null || echo 000)
|
||||
|
||||
if grep -qE '"code"[[:space:]]*:[[:space:]]*21' "$body" 2>/dev/null \
|
||||
|| grep -qiE 'There is no endpoint defined for the URL' "$body" 2>/dev/null; then
|
||||
# Bare without trailing slash often hits merchant until Caddy has redir * /path/ 302.
|
||||
if [ "$soft" = "1" ]; then
|
||||
warn "public mon page content" "$url -> ${_MON_CONTENT_MSG}"
|
||||
else
|
||||
warn "public mon page bare URL" \
|
||||
"$url -> HTTP $code code 21 (prefer trailing slash; Caddy: redir * /path/ 302; set MONPAGES_BARE_STRICT=1 to ERROR)"
|
||||
rm -f "$body"
|
||||
return 0
|
||||
fi
|
||||
# hard 21: retry once more (rare brief outage) then fail
|
||||
if [ "$attempt" -lt "$max_try" ]; then
|
||||
attempt=$((attempt + 1))
|
||||
sleep "$sleep_s" 2>/dev/null || sleep 1
|
||||
continue
|
||||
fi
|
||||
_mon_fail_or_soft "public mon page missing" \
|
||||
"$url -> HTTP $code merchant/API JSON code 21 (Caddy handle/redir; use redir * /path/ 302 inside handle; apply: sudo /home/hernani/koopa-caddy/apply-monitoring-live.sh)" || { rm -f "$body"; return 1; }
|
||||
rm -f "$body"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ "$code" != "200" ]; then
|
||||
if [ "$soft" = "1" ]; then
|
||||
warn "public mon page bare URL" "$url -> HTTP $code (prefer trailing slash form)"
|
||||
rm -f "$body"
|
||||
return 0
|
||||
fi
|
||||
if [ "$attempt" -lt "$max_try" ]; then
|
||||
attempt=$((attempt + 1))
|
||||
sleep "$sleep_s" 2>/dev/null || sleep 1
|
||||
continue
|
||||
fi
|
||||
_mon_fail_or_soft "public mon page" "$url -> HTTP $code (want 200 HTML)" || { rm -f "$body"; return 1; }
|
||||
rm -f "$body"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if is_bootstrap_html "$body"; then
|
||||
if [ "$attempt" -lt "$max_try" ]; then
|
||||
attempt=$((attempt + 1))
|
||||
sleep "$sleep_s" 2>/dev/null || sleep 1
|
||||
continue
|
||||
fi
|
||||
_mon_fail_or_soft "public mon page bootstrap stub" \
|
||||
"$url -> install bootstrap HTML still live (run host-agent; suite auto-update must replace stub)" || { rm -f "$body"; return 1; }
|
||||
rm -f "$body"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! is_monitoring_html "$body"; then
|
||||
if [ "$attempt" -lt "$max_try" ]; then
|
||||
attempt=$((attempt + 1))
|
||||
sleep "$sleep_s" 2>/dev/null || sleep 1
|
||||
continue
|
||||
fi
|
||||
hint=$(head -c 120 "$body" | tr '\n' ' ' | tr -cd '[:print:] ')
|
||||
_mon_fail_or_soft "public mon page not monitoring HTML" \
|
||||
"$url -> HTTP 200 body!=suite (${hint}...)" || { rm -f "$body"; return 1; }
|
||||
rm -f "$body"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# v1.9.0: top + bottom markers + meaningful content
|
||||
set +e
|
||||
validate_mon_content "$body" "$url"
|
||||
vc=$?
|
||||
set -e
|
||||
case "$vc" in
|
||||
0)
|
||||
ok "public mon page" "$url -> HTTP 200 · ${_MON_CONTENT_MSG}"
|
||||
rm -f "$body"
|
||||
return 0
|
||||
;;
|
||||
2)
|
||||
# mid-run: bottom may be incomplete — not ERROR
|
||||
warn "public mon page mid-run" "$url -> ${_MON_CONTENT_MSG}"
|
||||
ok "public mon page" "$url -> HTTP 200 suite HTML (mid-run bottom relaxed)"
|
||||
rm -f "$body"
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
# Content race (truncated HTML while another agent publishes): retry
|
||||
if [ "$attempt" -lt "$max_try" ]; then
|
||||
attempt=$((attempt + 1))
|
||||
sleep "$sleep_s" 2>/dev/null || sleep 1
|
||||
continue
|
||||
fi
|
||||
# Pre-publish inventory in host-agent PHASES: WARN only (post-check is hard)
|
||||
if [ "$pre_pub" = "1" ] || [ "$soft" = "1" ]; then
|
||||
warn "public mon page content" \
|
||||
"$url -> ${_MON_CONTENT_MSG}${pre_pub:+ (pre-publish; post-check is authoritative)}"
|
||||
rm -f "$body"
|
||||
return 0
|
||||
fi
|
||||
_mon_fail_or_soft "public mon page content" \
|
||||
"$url -> ${_MON_CONTENT_MSG}" || { rm -f "$body"; return 1; }
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
rm -f "$body"
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
done
|
||||
rm -f "$body"
|
||||
return 0
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue