taler-monitoring/host-agent/run-host-report.sh

#!/usr/bin/env bash
# run-host-report.sh — global reporting generation for all stacks
# (GOA/hacktivism, FP stage, FP prod, …).
#
# Shared behaviour (always):
#   1) update-suite → latest git.hacktivism.ch (local ~/.config never overwritten)
#   2) line-buffered run log (written continuously)
#   3) RUN_TIMEOUT wall clock for taler-monitoring.sh (default 600s)
#   4) always write HTML (first run / empty staging too) + Forgejo commit link
#   5) rsync to DEPLOY_WWW_ROOT (ERROR if not writable — v1.3.1)
#   6) monpages post-check: public FQDN must serve monitoring HTML
#   7) exit code: STRICT_EXIT default 1 (v1.7.1) — unit fails if publish/public pages fail
#
# Configure via env file (not in git):
#   ${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring/env
# or TALER_MONITORING_ENV=/path/to/env
#
# Thin wrappers only set stack defaults then exec this script:
#   run-hacktivism-monitoring.sh
#   run-fp-stage-monitoring.sh
#   run-fp-prod-monitoring.sh
#
set -uo pipefail

export PATH="${HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin:${PATH:-}"
export PYTHONUNBUFFERED=1

AGENT_DIR=$(cd "$(dirname "$0")" && pwd)
AGENT_MON=$(cd "$AGENT_DIR/.." && pwd)
CFG_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring"
ENV_FILE="${TALER_MONITORING_ENV:-$CFG_DIR/env}"
mkdir -p "$CFG_DIR"

# Thin wrappers (surface, aptdeploy-only, …) export stack knobs before exec.
# Snapshot them so ~/.config/taler-monitoring/env cannot clobber dedicated jobs
# (env often has PHASES="urls inside versions aptdeploy" for the main timer).
_wrap_keys=(
  PHASES HTML_OUT HTML_OK_DIR HTML_ERR_DIR HTML_URL_OK HTML_URL_ERR PAGE_LABEL
  RUN_TIMEOUT STATE_NAME AGENT_LABEL MON_HOSTS TALER_DOMAIN DEPLOY_WWW_ROOT
  INSIDE_PODMAN INSIDE_MODE LOCAL_STACK SKIP_SSH CONTINUE_ON_ERROR AUTH401_CONTINUE
  APT_DEPLOY_ENSURE SURFACE_SCOPE TALER_DOMAIN_FROM_CLI STRICT_EXIT SOURCE_SUITE_PATH
  MONPAGES_INVENTORY MONPAGES_CHECK_BARE MONPAGES_BARE_STRICT MONPAGES_REQUIRE_PUBLIC
  CHECK_BANK MERCHANT_REQUIRED CHECK_LANDING
)
for _k in "${_wrap_keys[@]}"; do
  if [ -n "${!_k+x}" ]; then
    printf -v "_WRAP_SET_${_k}" '%s' 1
    printf -v "_WRAP_VAL_${_k}" '%s' "${!_k}"
  fi
done
unset _k

if [ -f "$ENV_FILE" ]; then
  # shellcheck disable=SC1090
  set -a
  # shellcheck source=/dev/null
  source "$ENV_FILE"
  set +a
fi
# Tell update-suite.sh not to re-source env (would clobber wrapper PHASES etc.)
export _TALER_MON_ENV_LOADED=1

# Restore wrapper overrides (dedicated jobs win over shared env file)
for _k in "${_wrap_keys[@]}"; do
  _set_var="_WRAP_SET_${_k}"
  _val_var="_WRAP_VAL_${_k}"
  if [ "${!_set_var:-}" = "1" ]; then
    printf -v "$_k" '%s' "${!_val_var}"
    export "$_k"
  fi
done
unset _k _set_var _val_var

# --- global report defaults (env file / wrapper wins if already set) ---
export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}"
export AUTH401_CONTINUE="${AUTH401_CONTINUE:-${CONTINUE_ON_ERROR}}"
export RUN_TIMEOUT="${RUN_TIMEOUT:-600}"
export TALER_DOMAIN="${TALER_DOMAIN:-hacktivism.ch}"
# UI language: selectable via TALER_MON_LANG / --lang (en|fr).
# If unset, auto: FrancPaysan domains → fr, else en.
if [ "${TALER_MON_LANG_SET:-0}" != "1" ] && [ -z "${TALER_MON_LANG:-}" ]; then
  case "$TALER_DOMAIN" in
    *lefrancpaysan*|*francpaysan*) export TALER_MON_LANG=fr ;;
    *) export TALER_MON_LANG=en ;;
  esac
elif [ -n "${TALER_MON_LANG:-}" ]; then
  case "$TALER_MON_LANG" in
    fr|FR|fra|french) export TALER_MON_LANG=fr ;;
    *) export TALER_MON_LANG=en ;;
  esac
  export TALER_MON_LANG_SET=1
fi
export TALER_MON_LANG

PHASES="${PHASES:-urls inside versions monpages}"
HTML_BASE="${HTML_OUT:-$HOME/monitoring-sites-staging}"
MON_HOSTS="${MON_HOSTS:-}"
SUITE_PATH="${SOURCE_SUITE_PATH:-.}"
AGENT_LABEL="${AGENT_LABEL:-host-report}"
STATE_NAME="${STATE_NAME:-taler-monitoring-${TALER_DOMAIN}}"
STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/${STATE_NAME}"
LOG_DIR="$STATE_DIR/logs"
DEPLOY_WWW="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}"
# HTML path layout under each host (monitoring / surface / …)
HTML_OK_DIR="${HTML_OK_DIR:-monitoring}"
HTML_ERR_DIR="${HTML_ERR_DIR:-monitoring_err}"
HTML_URL_OK="${HTML_URL_OK:-/monitoring/}"
HTML_URL_ERR="${HTML_URL_ERR:-/monitoring_err/}"
PAGE_LABEL="${PAGE_LABEL:-monitoring}"
SUITE_VERSION="${SUITE_VERSION:-}"
SUITE_VERSION_URL="${SUITE_VERSION_URL:-}"
if [ -z "$SUITE_VERSION" ] && [ -d "${ROOT:-$AGENT_MON}/.git" ]; then
  _vd="${ROOT:-$AGENT_MON}"
  SUITE_VERSION=$(git -C "$_vd" describe --tags --exact-match 2>/dev/null     || git -C "$_vd" describe --tags --abbrev=0 2>/dev/null || true)
fi
if [ -z "$SUITE_VERSION" ] && [ -f "${ROOT:-$AGENT_MON}/VERSION" ]; then
  SUITE_VERSION="v$(tr -d '[:space:]' <"${ROOT:-$AGENT_MON}/VERSION" | sed 's/^v//')"
fi
[ -z "$SUITE_VERSION" ] && SUITE_VERSION="unknown"
case "$SUITE_VERSION" in unknown|v*) ;; *) SUITE_VERSION="v$SUITE_VERSION" ;; esac
# Prefer commit URL over /src/tag/… (tags lag push → sticky version 404 on Forgejo).
if [ -z "$SUITE_VERSION_URL" ]; then
  _web="${SOURCE_REPO_WEB:-https://git.hacktivism.ch/hernani/taler-monitoring}"
  if [ -n "${COMMIT:-}" ] && [ "$COMMIT" != "unknown" ]; then
    SUITE_VERSION_URL="${_web}/src/commit/${COMMIT}"
  elif [ "$SUITE_VERSION" != "unknown" ]; then
    SUITE_VERSION_URL="${_web}/src/tag/${SUITE_VERSION}"
  fi
fi
export SUITE_VERSION SUITE_VERSION_URL


# Sensible MON_HOSTS from domain if not set
if [ -z "$MON_HOSTS" ]; then
  case "$TALER_DOMAIN" in
    hacktivism.ch|koopa)
      MON_HOSTS="bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch"
      ;;
    stage.lefrancpaysan.ch|stage.*lefrancpaysan*)
      MON_HOSTS="stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch"
      ;;
    lefrancpaysan.ch)
      MON_HOSTS="bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch"
      ;;
    *)
      MON_HOSTS="${TALER_DOMAIN}"
      ;;
  esac
fi

mkdir -p "$LOG_DIR" "$HTML_BASE"

STAMP=$(date +%Y%m%d-%H%M%S)
LOG="$LOG_DIR/run-${STAMP}.log"
UPD_LOG="$LOG_DIR/suite-update-${STAMP}.log"
: >"$LOG"

# --- suite auto-upgrade BEFORE the HTML log (git noise must not pollute mon pages) ---
OVERLAY_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring/suite-overlay"
if [ -f "$AGENT_MON/taler-monitoring.sh" ]; then
  mkdir -p "$OVERLAY_DIR"
  rsync -a \
    --exclude secrets.env --exclude 'android-test/artifacts' --exclude 'android-test/apks' \
    --exclude 'android-test/out*' --exclude '__pycache__' \
    "$AGENT_MON/" "$OVERLAY_DIR/" 2>/dev/null || true
fi

# shellcheck source=update-suite.sh
if ! source "$AGENT_DIR/update-suite.sh" >"$UPD_LOG" 2>&1; then
  echo "ERROR: suite auto-update failed — refusing to run with stale tree" >&2
  echo "       detail: $UPD_LOG" >&2
  tail -n 30 "$UPD_LOG" >&2 || true
  echo "       fix git/network or set SUITE_UPDATE_STRICT=0 only as emergency" >&2
  exit 1
fi
if [ -d "${SUITE_DIR:-}/host-agent" ]; then
  AGENT_DIR="${SUITE_DIR}/host-agent"
fi

# Now open the public HTML log (clean agent header + suite summary only)
if command -v stdbuf >/dev/null 2>&1; then
  exec > >(stdbuf -oL -eL tee -a "$LOG") 2>&1
else
  exec > >(tee -a "$LOG") 2>&1
fi

echo "======== $(date -Iseconds 2>/dev/null || date) ${AGENT_LABEL} ========"
echo "env_file=$ENV_FILE"
echo "domain=$TALER_DOMAIN · phases=$PHASES · RUN_TIMEOUT=${RUN_TIMEOUT}s"
echo "hosts=$MON_HOSTS"
echo "flags INSIDE_PODMAN=${INSIDE_PODMAN:-} INSIDE_MODE=${INSIDE_MODE:-} LOCAL_STACK=${LOCAL_STACK:-} SKIP_SSH=${SKIP_SSH:-} INSIDE_PROFILE=${INSIDE_PROFILE:-}"
# One-line suite pin (no git M/checkout noise)
echo "suite ${SUITE_VERSION:-?} @ ${COMMIT_SHORT:-?}  ${COMMIT_URL:-}"
echo "  dir=${SUITE_DIR:-}"

SUITE_MON="${SUITE_DIR}"
ROOT="$SUITE_MON"
_need_overlay=0
# Detect missing features in Forgejo tree (untracked helper files alone are not enough —
# taler-monitoring.sh must accept the phase name).
case " $PHASES " in
  *" aptdeploy "*|*" apt-deploy "*|*" apt_src "*)
    if ! grep -qE 'aptdeploy\|apt-deploy' "$SUITE_MON/taler-monitoring.sh" 2>/dev/null; then
      _need_overlay=1
    fi
    ;;
esac
case " $PHASES " in
  *" surface "*|*" ecosystem "*)
    if ! grep -qE 'surface\|ecosystem' "$SUITE_MON/taler-monitoring.sh" 2>/dev/null \
      || [ ! -f "$SUITE_MON/check_surface.sh" ]; then
      _need_overlay=1
    fi
    ;;
esac
# Wrapper-preserve + HTML path knobs may live only in local overlay until pushed
if [ -f "$OVERLAY_DIR/host-agent/run-host-report.sh" ] \
  && ! grep -q '_wrap_keys\|_WRAP_SET_' "$SUITE_MON/host-agent/run-host-report.sh" 2>/dev/null \
  && grep -q '_wrap_keys\|_WRAP_SET_' "$OVERLAY_DIR/host-agent/run-host-report.sh" 2>/dev/null; then
  _need_overlay=1
fi
if [ "$_need_overlay" = "1" ] && [ -f "$OVERLAY_DIR/taler-monitoring.sh" ]; then
  echo "WARN: Forgejo @ ${COMMIT_SHORT:-?} lagging local suite-overlay (phases=[$PHASES])" >&2
  echo "      re-applying suite-overlay → $SUITE_MON" >&2
  mkdir -p "$SUITE_MON"
  rsync -a \
    --exclude secrets.env --exclude 'android-test/artifacts' --exclude 'android-test/apks' \
    --exclude 'android-test/out*' --exclude '__pycache__' \
    "$OVERLAY_DIR/" "$SUITE_MON/"
  ROOT="$SUITE_MON"
elif [ "$_need_overlay" = "1" ]; then
  echo "WARN: Forgejo missing features and no suite-overlay — using agent mon $AGENT_MON" >&2
  ROOT="$AGENT_MON"
else
  echo "using suite mon @ ${COMMIT_SHORT:-?} · $ROOT"
  # Do NOT rsync suite-overlay over a fresh Forgejo tree — that reintroduced
  # stale site-gen/host-agent (bootstrap sticky, old monpages) on stage/betel.
fi
unset _need_overlay

if [ -f "$ROOT/site-gen/console_to_html.py" ]; then
  SITE_GEN="$ROOT/site-gen"
elif [ -f "$AGENT_MON/site-gen/console_to_html.py" ]; then
  SITE_GEN="$AGENT_MON/site-gen"
  echo "note: using agent site-gen at $SITE_GEN"
elif [ -f "$SUITE_MON/site-gen/console_to_html.py" ]; then
  SITE_GEN="$SUITE_MON/site-gen"
else
  SITE_GEN=""
fi

if [ ! -x "$ROOT/taler-monitoring.sh" ]; then
  if [ -x "$AGENT_MON/taler-monitoring.sh" ]; then
    ROOT="$AGENT_MON"
    echo "note: using agent mon tree $ROOT"
  else
    echo "ERROR: suite missing at $ROOT" >&2
    COMMIT="${COMMIT:-unknown}"
    COMMIT_SHORT="${COMMIT_SHORT:-unknown}"
    COMMIT_URL="${COMMIT_URL:-}"
  fi
fi

echo "commit=${COMMIT_SHORT:-?} · ${COMMIT_URL:-}"

ec=1
if [ -x "${ROOT:-}/taler-monitoring.sh" ]; then
  cd "$ROOT" || exit 1
  chmod +x taler-monitoring.sh check_*.sh host-agent/*.sh site-gen/*.sh site-gen/*.py 2>/dev/null || true

  # Overlay newer inside/versions host-podman bits if suite clone is stale
  for f in check_inside.sh check_versions.sh taler-monitoring.sh lib.sh; do
    if [ -f "$AGENT_MON/$f" ] && [ -f "$ROOT/$f" ]; then
      if ! grep -q 'host-podman\|INSIDE_ACCESS\|RUN_TIMEOUT' "$ROOT/$f" 2>/dev/null \
        && grep -q 'host-podman\|INSIDE_ACCESS\|RUN_TIMEOUT' "$AGENT_MON/$f" 2>/dev/null; then
        echo "note: overlay $f from agent install"
        cp -f "$AGENT_MON/$f" "$ROOT/$f"
        chmod +x "$ROOT/$f" 2>/dev/null || true
      fi
    fi
  done

  # monpages inside PHASES runs *before* htmlify/publish — content failures would
  # race with concurrent publishes or reflect last run's incomplete redirect stub.
  # Soft-content there; post-check after rsync stays hard (authoritative).
  case " $PHASES " in
    *" monpages "*|*" pages "*) export MONPAGES_PRE_PUBLISH=1 ;;
    *) unset MONPAGES_PRE_PUBLISH 2>/dev/null || true ;;
  esac
  # shellcheck disable=SC2086
  ./taler-monitoring.sh -d "$TALER_DOMAIN" $PHASES
  ec=$?
  unset MONPAGES_PRE_PUBLISH 2>/dev/null || true
fi

# --- always generate HTML (first run / missing pages / every run) ---
write_bootstrap_html() {
  local out="$1" host="$2" mode="$3"
  mkdir -p "$(dirname "$out")"
  local cshort="${COMMIT_SHORT:-unknown}"
  local curl="${COMMIT_URL:-}"
  local clink="$cshort"
  local gen iso
  # Local wall clock (CEST/CET · MESZ/MEZ on CH/DE hosts), not UTC Z
  if TZ=Europe/Zurich date +"%Y-%m-%d %H:%M:%S %Z" >/dev/null 2>&1; then
    gen=$(TZ=Europe/Zurich date +"%Y-%m-%d %H:%M:%S %Z")
    iso=$(TZ=Europe/Zurich date +"%Y-%m-%dT%H:%M:%S%z" | sed 's/\([+-][0-9][0-9]\)\([0-9][0-9]\)$/\1:\2/')
  else
    gen=$(date +"%Y-%m-%d %H:%M:%S %Z" 2>/dev/null || date)
    iso=$(date -Iseconds 2>/dev/null || date)
  fi
  if [ -n "$curl" ]; then
    clink="<a href=\"$(printf '%s' "$curl" | sed 's/&/\&amp;/g')\">$cshort</a>"
  fi
  local tmp
  tmp=$(mktemp "${out}.XXXXXX")
  cat >"$tmp" <<EOF
<!DOCTYPE html>
<html lang="en"><head><meta charset="utf-8"/>
<meta name="generated" content="${iso}"/>
<meta name="taler-mon-page" content="incomplete"/>
<title>monitoring ${mode} · ${host}</title>
<style>
body{margin:0;background:#0c0c0c;color:#c8c8c8;font-family:ui-monospace,monospace}
a{color:#61afef}.err{color:#ff5c5c}
.sticky-bar{position:sticky;top:0;z-index:100;padding:10px 14px;background:#2a1010f2;border-bottom:2px solid #a33}
.sticky-bar .pill{font-weight:800;color:#ff5c5c;margin-right:8px}
.version-link{color:#61afef}
main{padding:1.5rem}
</style></head><body>
<!-- taler-mon:top -->
<div class="sticky-bar" id="status-bar">
  <span class="pill">BOOTSTRAP</span>
  ${PAGE_LABEL:-monitoring} · ${host} ·
  <span class="version-link muted">bootstrap</span> ·
  <span data-generated-iso="${iso}">generated ${gen}</span>
  · source ${clink}
</div>
<main>
<p class="err">Bootstrap page (converter missing or first install). See host-agent log.</p>
<pre style="white-space:pre-wrap;font-size:12px">$(tail -n 80 "$LOG" 2>/dev/null | sed 's/&/\&amp;/g;s/</\&lt;/g' || true)</pre>
</main>
<!-- taler-mon:bottom:incomplete -->
</body></html>
EOF
  mv -f "$tmp" "$out"
  echo "bootstrap html → $out"
}

htmlify_host() {
  local host="$1"
  local mon="$HTML_BASE/$host/${HTML_OK_DIR}/index.html"
  local mon_err="$HTML_BASE/$host/${HTML_ERR_DIR}/index.html"
  mkdir -p "$HTML_BASE/$host/${HTML_OK_DIR}" "$HTML_BASE/$host/${HTML_ERR_DIR}"

  if [ -n "$SITE_GEN" ] && [ -f "$SITE_GEN/console_to_html.py" ]; then
    python3 "$SITE_GEN/console_to_html.py" \
      --lang "${TALER_MON_LANG:-en}" \
      --log "$LOG" \
      --out "$mon_err" \
      --hostname "$host" \
      --mode err \
      --commit "${COMMIT:-}" \
      --commit-url "${COMMIT_URL:-}" \
      --suite-version "${SUITE_VERSION:-unknown}" \
      --suite-version-url "${SUITE_VERSION_URL:-}" \
      --suite-path "$SUITE_PATH" \
      --page-label "$PAGE_LABEL" \
      --path-err "$HTML_URL_ERR" \
      --link-other "$HTML_URL_OK"
    if [ "$ec" -eq 0 ]; then
      python3 "$SITE_GEN/console_to_html.py" \
      --lang "${TALER_MON_LANG:-en}" \
        --log "$LOG" \
        --out "$mon" \
        --hostname "$host" \
        --mode ok \
        --commit "${COMMIT:-}" \
        --commit-url "${COMMIT_URL:-}" \
      --suite-version "${SUITE_VERSION:-unknown}" \
      --suite-version-url "${SUITE_VERSION_URL:-}" \
        --suite-path "$SUITE_PATH" \
        --page-label "$PAGE_LABEL" \
        --path-err "$HTML_URL_ERR" \
        --link-other ""
      rm -rf "$HTML_BASE/$host/${HTML_ERR_DIR}"
      echo "html $host → ${HTML_URL_OK} only (clean · ${COMMIT_SHORT:-?})"
    else
      python3 "$SITE_GEN/console_to_html.py" \
      --lang "${TALER_MON_LANG:-en}" \
        --log "$LOG" \
        --out "$mon" \
        --hostname "$host" \
        --mode redirect \
        --commit "${COMMIT:-}" \
        --commit-url "${COMMIT_URL:-}" \
      --suite-version "${SUITE_VERSION:-unknown}" \
      --suite-version-url "${SUITE_VERSION_URL:-}" \
        --suite-path "$SUITE_PATH" \
        --page-label "$PAGE_LABEL" \
        --path-err "$HTML_URL_ERR"
      echo "html $host → ${HTML_URL_OK} stub + ${HTML_URL_ERR} (ec=$ec · ${COMMIT_SHORT:-?})"
    fi
  else
    echo "WARN: console_to_html.py missing — bootstrap pages"
    write_bootstrap_html "$mon_err" "$host" "err"
    if [ "$ec" -eq 0 ]; then
      write_bootstrap_html "$mon" "$host" "ok"
      rm -rf "$HTML_BASE/$host/${HTML_ERR_DIR}"
    else
      write_bootstrap_html "$mon" "$host" "redirect"
    fi
  fi

  if [ ! -f "$mon" ]; then
    write_bootstrap_html "$mon" "$host" "err"
  fi

  printf '%s\n' "${COMMIT:-unknown}" >"$HTML_BASE/$host/COMMIT"
  printf '%s\n' "${COMMIT_URL:-}" >"$HTML_BASE/$host/COMMIT_URL"
}
for host in $MON_HOSTS; do
  htmlify_host "$host"
  echo "public URL (expected): https://${host}${HTML_URL_OK}"
done

# Publish to live web root when possible (v1.3.1+: never silent-skip)
if [ -n "$DEPLOY_WWW" ]; then
  if [ -w "$DEPLOY_WWW" ] 2>/dev/null; then
    # --delay-updates: write to temp names then rename (less partial-file exposure)
    if rsync -a --delete --delay-updates "$HTML_BASE/" "$DEPLOY_WWW/"; then
      echo "synced → $DEPLOY_WWW/ (delay-updates)"
      # brief settle so concurrent monpages curls do not hit mid-rename FS views
      sleep "${MONPAGES_PUBLISH_SETTLE_S:-0.3}" 2>/dev/null || sleep 1
    else
      echo "ERROR: rsync to DEPLOY_WWW_ROOT=$DEPLOY_WWW failed" >&2
      ec=1
    fi
  elif [ -e "$DEPLOY_WWW" ]; then
    echo "ERROR: DEPLOY_WWW_ROOT=$DEPLOY_WWW exists but is not writable by $(id -un)" >&2
    echo "       public FQDN pages will stay missing/stale — fix ownership or run:" >&2
    echo "       sudo /home/hernani/koopa-caddy/apply-monitoring-live.sh" >&2
    ec=1
  else
    echo "ERROR: DEPLOY_WWW_ROOT=$DEPLOY_WWW missing — public pages not published" >&2
    echo "       create tree + Caddy handles: sudo /home/hernani/koopa-caddy/apply-monitoring-live.sh" >&2
    ec=1
  fi
else
  echo "WARN: DEPLOY_WWW_ROOT empty — skipping live publish (staging only: $HTML_BASE)" >&2
fi

# Post-publish: monpages obligatory (v1.7.6+) — GOA full inventory / FP only FP hosts
_monpages_failed=0
if [ "${MONPAGES_POSTCHECK:-1}" = "1" ] && [ -x "$ROOT/check_monitoring_pages.sh" ]; then
  echo "--- monpages post-check (public FQDN, obligatory) ---"
  export MON_HOSTS HTML_URL_OK HTML_URL_ERR TALER_DOMAIN HTML_OUT HTML_BASE
  export MONPAGES_STAGING_BASE="${HTML_BASE}"
  export DEPLOY_WWW_ROOT="${DEPLOY_WWW:-${DEPLOY_WWW_ROOT:-}}"
  export MONPAGES_REQUIRE_PUBLIC="${MONPAGES_REQUIRE_PUBLIC:-1}"
  # auto: GOA = all suite mon sites; FP = only FP mon hosts; job paths always included
  export MONPAGES_INVENTORY="${MONPAGES_INVENTORY:-auto}"
  # Hard check (not pre-publish soft)
  unset MONPAGES_PRE_PUBLISH 2>/dev/null || true
  export MONPAGES_PRE_PUBLISH=0
  if ! bash "$ROOT/check_monitoring_pages.sh"; then
    echo "ERROR: public monitoring page(s) missing or not suite HTML via FQDN" >&2
    echo "       staging HTML_OUT=${HTML_BASE} · DEPLOY_WWW=${DEPLOY_WWW:-}" >&2
    ec=1
    _monpages_failed=1
  fi
  unset MONPAGES_PRE_PUBLISH 2>/dev/null || true
fi

# If public check failed after first HTML write, rewrite pages so sticky bar shows monpages ERRORs
if [ "$_monpages_failed" = "1" ]; then
  echo "--- re-htmlify after monpages failure (include public outage in report) ---"
  # append marker to log so converter sees ERROR lines if monpages was post-only
  {
    echo ""
    echo "ERROR monpages: public FQDN monitoring HTML missing or merchant JSON code 21"
    echo "ERROR monpages: fix with sudo /home/hernani/koopa-caddy/apply-monitoring-live.sh (www + Caddy handles)"
  } >>"$LOG"
  for host in $MON_HOSTS; do
    htmlify_host "$host"
  done
fi

ls -1t "$LOG_DIR"/run-*.log 2>/dev/null | tail -n +30 | xargs rm -f 2>/dev/null || true

echo "======== done ec=$ec · log=$LOG · ${COMMIT_URL:-} ========"
# v1.7.1: default STRICT_EXIT=1 so systemd/timer reflects publish/public failures
if [ "${STRICT_EXIT:-1}" = "1" ]; then
  exit "$ec"
fi
exit 0