release 1.10.4: strict suite auto-upgrade; bootstrap monpages ERROR; stage timer units

VERSION

@@ -1 +1 @@
-1.10.3
+1.10.4

VERSIONS.md

@@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre
 
 | Tag | Date (UTC) | Notes |
 |-----|------------|--------|
+| **v1.10.4** | 2026-07-19 | **Fix:** suite **auto-upgrade strict** (`SUITE_UPDATE_STRICT=1` default — abort on fetch fail); monpages ERROR on **bootstrap** HTML; ship mytops-stage timer/service units; host-agent refuses stale tree. |
 | **v1.10.3** | 2026-07-19 | **Bugfix:** sticky ERROR only from check badges (`┌ ERROR`) / `ERROR monpages`; ignore git `HEAD is now at …` commit subjects that mention ERROR. |
 | **v1.10.2** | 2026-07-19 | **Bugfix:** sticky ERROR count — classify with word-boundary `\bERROR\b` (not substring); avoids false red from git “wrote-errors” commit lines and similar. |
 | **v1.10.1** | 2026-07-19 | **Bugfix:** HTML converter must not treat teed `wrote … (errors=N)` lines as ERROR (false sticky red after clean host-agent runs). |

check_monitoring_pages.sh

@@ -220,7 +220,14 @@ build_urls() {
   fi
 }
 
-# True if body looks like our monitoring HTML (not merchant API / WP 404).
+# Install stub only (no console log) — not a real mon page.
+is_bootstrap_html() {
+  local f="$1"
+  grep -qiE 'Monitoring page \(bootstrap\)|pill">STAGE</|install stub|bootstrap page' "$f" 2>/dev/null \
+    && ! grep -qE 'taler-mon:top|id="mon-console"|class="console"' "$f" 2>/dev/null
+}
+
+# True if body looks like our monitoring HTML (not merchant API / WP 404 / bootstrap).
 is_monitoring_html() {
   local f="$1"
   if grep -qE '"code"[[:space:]]*:[[:space:]]*21' "$f" 2>/dev/null; then
@@ -233,6 +240,10 @@ is_monitoring_html() {
     && grep -qiE '404|not found|page introuvable' "$f" 2>/dev/null; then
     return 1
   fi
+  # Bootstrap sticky is not enough — need suite console output
+  if is_bootstrap_html "$f"; then
+    return 1
+  fi
   if grep -qE 'sticky-bar|version-link|taler-monitoring|class="sticky|taler-mon:top' "$f" 2>/dev/null; then
     return 0
   fi
@@ -401,6 +412,12 @@ check_one() {
     rm -f "$body"
     return 0
   fi
+  if is_bootstrap_html "$body"; then
+    _mon_fail_or_soft "public mon page bootstrap stub" \
+      "$url -> install bootstrap HTML still live (run host-agent; suite auto-update must replace stub)" || { rm -f "$body"; return 1; }
+    rm -f "$body"
+    return 0
+  fi
   if ! is_monitoring_html "$body"; then
     hint=$(head -c 120 "$body" | tr '\n' ' ' | tr -cd '[:print:] ')
     _mon_fail_or_soft "public mon page not monitoring HTML" \

host-agent/README.md

@@ -4,12 +4,15 @@
 
 | Behaviour | Default |
 |-----------|---------|
-| Suite refresh | `update-suite.sh` → `origin/main` |
+| Suite refresh | **`update-suite.sh` every run** → `git fetch --tags` + hard reset to `origin/main` |
+| Update strict | `SUITE_UPDATE_STRICT=1` (default): **abort** if fetch/reset fails — no silent old tree |
 | Wall clock | `RUN_TIMEOUT=600` |
 | Log | line-buffered `run-*.log` (continuous write) |
-| HTML | always (first run / empty tree too) + commit URL |
+| HTML | always (first run / empty tree too) + commit URL; **bootstrap install stubs are monpages ERROR** |
 | Timeout notice | end of log + top jump on `/monitoring_err/` |
 
+**Rule:** public mon pages must show a **current suite tag/commit**, not install bootstrap or months-old trees. Timers exist so this keeps working without manual SSH.
+
 Thin wrappers only set stack defaults:
 
 | Wrapper | Stack | Typical user |
@@ -17,8 +20,9 @@ Thin wrappers only set stack defaults:
 | `run-hacktivism-monitoring.sh` | GOA `hacktivism.ch` | `hernani` @ koopa |
 | `run-fp-stage-monitoring.sh` | FP stage | `stagepaysan` |
 | `run-fp-prod-monitoring.sh` | FP prod | `francpaysan` |
+| `run-host-report.sh` + `taler-monitoring-mytops-stage.{service,timer}` | mytops **stage / betel** only | `taler-monitoring` @ betel (4h) |
 | `run-aptdeploy.sh` | CLI: ensure + **aptdeploy** (no HTML) | local if hostname=koopa, else **ssh koopa-external** |
-| `run-aptdeploy-monitoring.sh` | optional apt-src tests (not a public mon-page type) | `hernani` @ koopa (4h) |
+| `run-aptdeploy-monitoring.sh` | apt-src tests → `/taler-monitoring-aptdeploy*` | `hernani` @ koopa (4h) |
 | `run-surface-monitoring.sh` | **surface** → only `/taler-monitoring-surface*` (ecosystem + mail + mattermost + versions) | `hernani` @ koopa (hourly) |
 | `run-mail-monitoring.sh` / `run-mattermost-monitoring.sh` | **deprecated v1.8.0** — aliases of surface | disabled timers |
 

host-agent/run-host-report.sh

@@ -170,7 +170,17 @@ if [ -f "$AGENT_MON/taler-monitoring.sh" ]; then
 fi
 
 # shellcheck source=update-suite.sh
-source "$AGENT_DIR/update-suite.sh"
+# Strict suite auto-upgrade (default): abort if Forgejo fetch/reset fails so
+# hosts never keep publishing old / bootstrap HTML indefinitely.
+if ! source "$AGENT_DIR/update-suite.sh"; then
+  echo "ERROR: suite auto-update failed — refusing to run with stale tree" >&2
+  echo "       fix git/network or set SUITE_UPDATE_STRICT=0 only as emergency" >&2
+  exit 1
+fi
+# Prefer host-agent scripts from the *updated* suite tree
+if [ -d "${SUITE_DIR:-}/host-agent" ]; then
+  AGENT_DIR="${SUITE_DIR}/host-agent"
+fi
 
 SUITE_MON="${SUITE_DIR}"
 ROOT="$SUITE_MON"

host-agent/taler-monitoring-mytops-stage.service

@@ -0,0 +1,20 @@
+[Unit]
+Description=taler-monitoring mytops stage (betel) — auto-update suite + urls/monpages
+Documentation=file:%h/src/taler-monitoring/host-agent/README.md
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+# Env: ~/.config/taler-monitoring/env (SUITE_GIT_*, MON_HOSTS, PHASES=urls monpages, …)
+EnvironmentFile=-%h/.config/taler-monitoring/env
+Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin
+Environment=SUITE_UPDATE_STRICT=1
+WorkingDirectory=%h/src/taler-monitoring
+# run-host-report sources update-suite.sh first (strict fetch/reset to main)
+ExecStart=%h/src/taler-monitoring/host-agent/run-host-report.sh
+Nice=10
+TimeoutStartSec=20min
+
+[Install]
+WantedBy=default.target

host-agent/taler-monitoring-mytops-stage.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=taler-monitoring mytops stage (betel) every 4h — suite auto-upgrade + mon HTML
+
+[Timer]
+OnBootSec=5min
+OnUnitActiveSec=4h
+Persistent=true
+Unit=taler-monitoring-mytops-stage.service
+
+[Install]
+WantedBy=timers.target

host-agent/update-suite.sh

@@ -1,5 +1,17 @@
 #!/usr/bin/env bash
-# update-suite.sh — latest taler-monitoring from Forgejo (standalone repo root).
+# update-suite.sh — pull latest taler-monitoring from Forgejo (standalone repo root).
+#
+# Sourced by run-host-report.sh / wrappers. Every host-agent run must land on
+# current origin/$SUITE_GIT_REF (default main). Stale clones are not acceptable.
+#
+# Env:
+#   SUITE_GIT_URL          default https://git.hacktivism.ch/hernani/taler-monitoring.git
+#   SUITE_GIT_REF          default main
+#   SUITE_DIR              default ~/src/taler-monitoring
+#   SUITE_UPDATE_MODE       reset (default) | pull
+#   SUITE_UPDATE_STRICT     1 (default) = fetch/reset failure aborts the agent run
+#                          0 = warn and continue on old tree (emergency only)
+#
 set -uo pipefail
 CFG_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring"
 ENV_FILE="${TALER_MONITORING_ENV:-$CFG_DIR/env}"
@@ -9,31 +21,64 @@ fi
 SUITE_GIT_URL="${SUITE_GIT_URL:-https://git.hacktivism.ch/hernani/taler-monitoring.git}"
 SUITE_GIT_REF="${SUITE_GIT_REF:-main}"
 SUITE_UPDATE_MODE="${SUITE_UPDATE_MODE:-reset}"
+SUITE_UPDATE_STRICT="${SUITE_UPDATE_STRICT:-1}"
 if [ -z "${SUITE_DIR:-}" ]; then
   if [ -d "$HOME/src/taler-monitoring/.git" ]; then SUITE_DIR="$HOME/src/taler-monitoring"
   elif [ -d "$HOME/taler-monitoring/.git" ]; then SUITE_DIR="$HOME/taler-monitoring"
   else SUITE_DIR="$HOME/src/taler-monitoring"; fi
 fi
 export SUITE_DIR
+
+_upd_die() {
+  echo "ERROR suite-update: $*" >&2
+  if [ "${SUITE_UPDATE_STRICT}" = "1" ]; then
+    return 1 2>/dev/null || exit 1
+  fi
+  echo "WARN suite-update: continuing on existing tree (SUITE_UPDATE_STRICT=0)" >&2
+  return 0
+}
+
 mkdir -p "$(dirname "$SUITE_DIR")" "$CFG_DIR"
 if [ ! -d "$SUITE_DIR/.git" ]; then
   echo "clone $SUITE_GIT_URL → $SUITE_DIR"
-  git clone --branch "$SUITE_GIT_REF" "$SUITE_GIT_URL" "$SUITE_DIR" || git clone "$SUITE_GIT_URL" "$SUITE_DIR"
+  if ! git clone --branch "$SUITE_GIT_REF" "$SUITE_GIT_URL" "$SUITE_DIR" \
+    && ! git clone "$SUITE_GIT_URL" "$SUITE_DIR"; then
+    _upd_die "git clone failed"
+    return 1 2>/dev/null || exit 1
+  fi
 fi
-cd "$SUITE_DIR" || exit 1
+cd "$SUITE_DIR" || { _upd_die "cd $SUITE_DIR failed"; return 1 2>/dev/null || exit 1; }
 git remote set-url origin "$SUITE_GIT_URL" 2>/dev/null || true
-echo "fetch origin ($SUITE_GIT_REF) …"
-if ! git fetch --tags origin 2>&1; then
-  echo "WARN: git fetch failed — using existing tree" >&2
-  return 0 2>/dev/null || exit 0
-fi
-case "$SUITE_UPDATE_MODE" in
+
+_before=$(git rev-parse --short=12 HEAD 2>/dev/null || echo none)
+_before_ver=$(git describe --tags --exact-match 2>/dev/null \
+  || git describe --tags --abbrev=0 2>/dev/null \
+  || echo unknown)
+
+echo "fetch origin ($SUITE_GIT_REF) + tags …"
+if ! git fetch --tags --force --prune origin 2>&1; then
+  _upd_die "git fetch failed — suite would stay at ${_before} (${_before_ver})"
+  # fall through only if STRICT=0
+else
+  case "$SUITE_UPDATE_MODE" in
     reset)
-    git checkout -B "$SUITE_GIT_REF" "origin/$SUITE_GIT_REF" 2>/dev/null || git checkout -B "$SUITE_GIT_REF" FETCH_HEAD
-    git reset --hard "origin/$SUITE_GIT_REF" 2>/dev/null || git reset --hard FETCH_HEAD
+      if ! git checkout -B "$SUITE_GIT_REF" "origin/$SUITE_GIT_REF" 2>/dev/null \
+        && ! git checkout -B "$SUITE_GIT_REF" FETCH_HEAD; then
+        _upd_die "git checkout $SUITE_GIT_REF failed"
+      fi
+      if ! git reset --hard "origin/$SUITE_GIT_REF" 2>/dev/null \
+        && ! git reset --hard FETCH_HEAD; then
+        _upd_die "git reset --hard origin/$SUITE_GIT_REF failed"
+      fi
       ;;
-  *) git pull --ff-only origin "$SUITE_GIT_REF" 2>&1 || true ;;
-esac
+    *)
+      if ! git pull --ff-only origin "$SUITE_GIT_REF" 2>&1; then
+        _upd_die "git pull --ff-only failed"
+      fi
+      ;;
+  esac
+fi
+
 COMMIT=$(git rev-parse HEAD)
 COMMIT_SHORT=$(git rev-parse --short=12 HEAD)
 SOURCE_REPO_WEB="${SOURCE_REPO_WEB:-https://git.hacktivism.ch/hernani/taler-monitoring}"
@@ -50,13 +95,10 @@ if command -v git >/dev/null 2>&1; then
 fi
 if [ -z "$SUITE_VERSION" ] && [ -f "$SUITE_DIR/VERSION" ]; then
   SUITE_VERSION="v$(tr -d '[:space:]' <"$SUITE_DIR/VERSION" | sed 's/^v//')"
-  case "$SUITE_VERSION" in v*) ;; *) SUITE_VERSION="v${SUITE_VERSION#v}" ;; esac
 fi
 [ -z "$SUITE_VERSION" ] && SUITE_VERSION="unknown"
-# normalize: ensure leading v for tags
 case "$SUITE_VERSION" in
-  unknown) ;;
-  v*) ;;
+  unknown|v*) ;;
   *) SUITE_VERSION="v${SUITE_VERSION}" ;;
 esac
 SUITE_VERSION_URL=""
@@ -64,9 +106,22 @@ if [ -n "$SOURCE_REPO_WEB" ] && [ "$SUITE_VERSION" != "unknown" ]; then
   SUITE_VERSION_URL="${SOURCE_REPO_WEB}/src/tag/${SUITE_VERSION}"
 fi
 export SUITE_VERSION SUITE_VERSION_URL
-echo "  version=$SUITE_VERSION ${SUITE_VERSION_URL:+· $SUITE_VERSION_URL}"
 
+if [ "$_before" != "$COMMIT_SHORT" ]; then
+  echo "  upgraded ${_before} (${_before_ver}) → ${COMMIT_SHORT} (${SUITE_VERSION})"
+else
+  echo "  already at ${COMMIT_SHORT} (${SUITE_VERSION})"
+fi
+echo "  version=$SUITE_VERSION ${SUITE_VERSION_URL:+· $SUITE_VERSION_URL}"
 echo "suite @ $COMMIT_SHORT  $COMMIT_URL"
 echo "  dir=$SUITE_DIR"
+
+# Sanity: must have suite entrypoint after update
+if [ ! -x "$SUITE_DIR/taler-monitoring.sh" ] && [ ! -f "$SUITE_DIR/taler-monitoring.sh" ]; then
+  _upd_die "taler-monitoring.sh missing after update"
+  return 1 2>/dev/null || exit 1
+fi
+
 ln -sfn "$SUITE_DIR" "$HOME/taler-monitoring" 2>/dev/null || true
 chmod +x taler-monitoring.sh check_*.sh host-agent/*.sh site-gen/*.sh site-gen/*.py 2>/dev/null || true
+return 0 2>/dev/null || true