diff --git a/VERSION b/VERSION index 587c5f0..18b3114 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.10.3 +1.10.4 diff --git a/VERSIONS.md b/VERSIONS.md index 8d14e6f..7b0bad9 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| +| **v1.10.4** | 2026-07-19 | **Fix:** suite **auto-upgrade strict** (`SUITE_UPDATE_STRICT=1` default — abort on fetch fail); monpages ERROR on **bootstrap** HTML; ship mytops-stage timer/service units; host-agent refuses stale tree. | | **v1.10.3** | 2026-07-19 | **Bugfix:** sticky ERROR only from check badges (`┌ ERROR`) / `ERROR monpages`; ignore git `HEAD is now at …` commit subjects that mention ERROR. | | **v1.10.2** | 2026-07-19 | **Bugfix:** sticky ERROR count — classify with word-boundary `\bERROR\b` (not substring); avoids false red from git “wrote-errors” commit lines and similar. | | **v1.10.1** | 2026-07-19 | **Bugfix:** HTML converter must not treat teed `wrote … (errors=N)` lines as ERROR (false sticky red after clean host-agent runs). | diff --git a/check_monitoring_pages.sh b/check_monitoring_pages.sh index 35b7314..3a0e7aa 100755 --- a/check_monitoring_pages.sh +++ b/check_monitoring_pages.sh @@ -220,7 +220,14 @@ build_urls() { fi } -# True if body looks like our monitoring HTML (not merchant API / WP 404). +# Install stub only (no console log) — not a real mon page. +is_bootstrap_html() { + local f="$1" + grep -qiE 'Monitoring page \(bootstrap\)|pill">STAGE/dev/null \ + && ! grep -qE 'taler-mon:top|id="mon-console"|class="console"' "$f" 2>/dev/null +} + +# True if body looks like our monitoring HTML (not merchant API / WP 404 / bootstrap). is_monitoring_html() { local f="$1" if grep -qE '"code"[[:space:]]*:[[:space:]]*21' "$f" 2>/dev/null; then @@ -233,6 +240,10 @@ is_monitoring_html() { && grep -qiE '404|not found|page introuvable' "$f" 2>/dev/null; then return 1 fi + # Bootstrap sticky is not enough — need suite console output + if is_bootstrap_html "$f"; then + return 1 + fi if grep -qE 'sticky-bar|version-link|taler-monitoring|class="sticky|taler-mon:top' "$f" 2>/dev/null; then return 0 fi @@ -401,6 +412,12 @@ check_one() { rm -f "$body" return 0 fi + if is_bootstrap_html "$body"; then + _mon_fail_or_soft "public mon page bootstrap stub" \ + "$url -> install bootstrap HTML still live (run host-agent; suite auto-update must replace stub)" || { rm -f "$body"; return 1; } + rm -f "$body" + return 0 + fi if ! is_monitoring_html "$body"; then hint=$(head -c 120 "$body" | tr '\n' ' ' | tr -cd '[:print:] ') _mon_fail_or_soft "public mon page not monitoring HTML" \ diff --git a/host-agent/README.md b/host-agent/README.md index 4ee1768..e838ebf 100644 --- a/host-agent/README.md +++ b/host-agent/README.md @@ -4,12 +4,15 @@ | Behaviour | Default | |-----------|---------| -| Suite refresh | `update-suite.sh` → `origin/main` | +| Suite refresh | **`update-suite.sh` every run** → `git fetch --tags` + hard reset to `origin/main` | +| Update strict | `SUITE_UPDATE_STRICT=1` (default): **abort** if fetch/reset fails — no silent old tree | | Wall clock | `RUN_TIMEOUT=600` | | Log | line-buffered `run-*.log` (continuous write) | -| HTML | always (first run / empty tree too) + commit URL | +| HTML | always (first run / empty tree too) + commit URL; **bootstrap install stubs are monpages ERROR** | | Timeout notice | end of log + top jump on `/monitoring_err/` | +**Rule:** public mon pages must show a **current suite tag/commit**, not install bootstrap or months-old trees. Timers exist so this keeps working without manual SSH. + Thin wrappers only set stack defaults: | Wrapper | Stack | Typical user | @@ -17,8 +20,9 @@ Thin wrappers only set stack defaults: | `run-hacktivism-monitoring.sh` | GOA `hacktivism.ch` | `hernani` @ koopa | | `run-fp-stage-monitoring.sh` | FP stage | `stagepaysan` | | `run-fp-prod-monitoring.sh` | FP prod | `francpaysan` | +| `run-host-report.sh` + `taler-monitoring-mytops-stage.{service,timer}` | mytops **stage / betel** only | `taler-monitoring` @ betel (4h) | | `run-aptdeploy.sh` | CLI: ensure + **aptdeploy** (no HTML) | local if hostname=koopa, else **ssh koopa-external** | -| `run-aptdeploy-monitoring.sh` | optional apt-src tests (not a public mon-page type) | `hernani` @ koopa (4h) | +| `run-aptdeploy-monitoring.sh` | apt-src tests → `/taler-monitoring-aptdeploy*` | `hernani` @ koopa (4h) | | `run-surface-monitoring.sh` | **surface** → only `/taler-monitoring-surface*` (ecosystem + mail + mattermost + versions) | `hernani` @ koopa (hourly) | | `run-mail-monitoring.sh` / `run-mattermost-monitoring.sh` | **deprecated v1.8.0** — aliases of surface | disabled timers | diff --git a/host-agent/run-host-report.sh b/host-agent/run-host-report.sh index 5fa79e6..5168268 100755 --- a/host-agent/run-host-report.sh +++ b/host-agent/run-host-report.sh @@ -170,7 +170,17 @@ if [ -f "$AGENT_MON/taler-monitoring.sh" ]; then fi # shellcheck source=update-suite.sh -source "$AGENT_DIR/update-suite.sh" +# Strict suite auto-upgrade (default): abort if Forgejo fetch/reset fails so +# hosts never keep publishing old / bootstrap HTML indefinitely. +if ! source "$AGENT_DIR/update-suite.sh"; then + echo "ERROR: suite auto-update failed — refusing to run with stale tree" >&2 + echo " fix git/network or set SUITE_UPDATE_STRICT=0 only as emergency" >&2 + exit 1 +fi +# Prefer host-agent scripts from the *updated* suite tree +if [ -d "${SUITE_DIR:-}/host-agent" ]; then + AGENT_DIR="${SUITE_DIR}/host-agent" +fi SUITE_MON="${SUITE_DIR}" ROOT="$SUITE_MON" diff --git a/host-agent/taler-monitoring-mytops-stage.service b/host-agent/taler-monitoring-mytops-stage.service new file mode 100644 index 0000000..0c07bfb --- /dev/null +++ b/host-agent/taler-monitoring-mytops-stage.service @@ -0,0 +1,20 @@ +[Unit] +Description=taler-monitoring mytops stage (betel) — auto-update suite + urls/monpages +Documentation=file:%h/src/taler-monitoring/host-agent/README.md +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +# Env: ~/.config/taler-monitoring/env (SUITE_GIT_*, MON_HOSTS, PHASES=urls monpages, …) +EnvironmentFile=-%h/.config/taler-monitoring/env +Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin +Environment=SUITE_UPDATE_STRICT=1 +WorkingDirectory=%h/src/taler-monitoring +# run-host-report sources update-suite.sh first (strict fetch/reset to main) +ExecStart=%h/src/taler-monitoring/host-agent/run-host-report.sh +Nice=10 +TimeoutStartSec=20min + +[Install] +WantedBy=default.target diff --git a/host-agent/taler-monitoring-mytops-stage.timer b/host-agent/taler-monitoring-mytops-stage.timer new file mode 100644 index 0000000..73a09fa --- /dev/null +++ b/host-agent/taler-monitoring-mytops-stage.timer @@ -0,0 +1,11 @@ +[Unit] +Description=taler-monitoring mytops stage (betel) every 4h — suite auto-upgrade + mon HTML + +[Timer] +OnBootSec=5min +OnUnitActiveSec=4h +Persistent=true +Unit=taler-monitoring-mytops-stage.service + +[Install] +WantedBy=timers.target diff --git a/host-agent/update-suite.sh b/host-agent/update-suite.sh index 37393cb..095b1f9 100755 --- a/host-agent/update-suite.sh +++ b/host-agent/update-suite.sh @@ -1,5 +1,17 @@ #!/usr/bin/env bash -# update-suite.sh — latest taler-monitoring from Forgejo (standalone repo root). +# update-suite.sh — pull latest taler-monitoring from Forgejo (standalone repo root). +# +# Sourced by run-host-report.sh / wrappers. Every host-agent run must land on +# current origin/$SUITE_GIT_REF (default main). Stale clones are not acceptable. +# +# Env: +# SUITE_GIT_URL default https://git.hacktivism.ch/hernani/taler-monitoring.git +# SUITE_GIT_REF default main +# SUITE_DIR default ~/src/taler-monitoring +# SUITE_UPDATE_MODE reset (default) | pull +# SUITE_UPDATE_STRICT 1 (default) = fetch/reset failure aborts the agent run +# 0 = warn and continue on old tree (emergency only) +# set -uo pipefail CFG_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring" ENV_FILE="${TALER_MONITORING_ENV:-$CFG_DIR/env}" @@ -9,31 +21,64 @@ fi SUITE_GIT_URL="${SUITE_GIT_URL:-https://git.hacktivism.ch/hernani/taler-monitoring.git}" SUITE_GIT_REF="${SUITE_GIT_REF:-main}" SUITE_UPDATE_MODE="${SUITE_UPDATE_MODE:-reset}" +SUITE_UPDATE_STRICT="${SUITE_UPDATE_STRICT:-1}" if [ -z "${SUITE_DIR:-}" ]; then if [ -d "$HOME/src/taler-monitoring/.git" ]; then SUITE_DIR="$HOME/src/taler-monitoring" elif [ -d "$HOME/taler-monitoring/.git" ]; then SUITE_DIR="$HOME/taler-monitoring" else SUITE_DIR="$HOME/src/taler-monitoring"; fi fi export SUITE_DIR + +_upd_die() { + echo "ERROR suite-update: $*" >&2 + if [ "${SUITE_UPDATE_STRICT}" = "1" ]; then + return 1 2>/dev/null || exit 1 + fi + echo "WARN suite-update: continuing on existing tree (SUITE_UPDATE_STRICT=0)" >&2 + return 0 +} + mkdir -p "$(dirname "$SUITE_DIR")" "$CFG_DIR" if [ ! -d "$SUITE_DIR/.git" ]; then echo "clone $SUITE_GIT_URL → $SUITE_DIR" - git clone --branch "$SUITE_GIT_REF" "$SUITE_GIT_URL" "$SUITE_DIR" || git clone "$SUITE_GIT_URL" "$SUITE_DIR" + if ! git clone --branch "$SUITE_GIT_REF" "$SUITE_GIT_URL" "$SUITE_DIR" \ + && ! git clone "$SUITE_GIT_URL" "$SUITE_DIR"; then + _upd_die "git clone failed" + return 1 2>/dev/null || exit 1 + fi fi -cd "$SUITE_DIR" || exit 1 +cd "$SUITE_DIR" || { _upd_die "cd $SUITE_DIR failed"; return 1 2>/dev/null || exit 1; } git remote set-url origin "$SUITE_GIT_URL" 2>/dev/null || true -echo "fetch origin ($SUITE_GIT_REF) …" -if ! git fetch --tags origin 2>&1; then - echo "WARN: git fetch failed — using existing tree" >&2 - return 0 2>/dev/null || exit 0 + +_before=$(git rev-parse --short=12 HEAD 2>/dev/null || echo none) +_before_ver=$(git describe --tags --exact-match 2>/dev/null \ + || git describe --tags --abbrev=0 2>/dev/null \ + || echo unknown) + +echo "fetch origin ($SUITE_GIT_REF) + tags …" +if ! git fetch --tags --force --prune origin 2>&1; then + _upd_die "git fetch failed — suite would stay at ${_before} (${_before_ver})" + # fall through only if STRICT=0 +else + case "$SUITE_UPDATE_MODE" in + reset) + if ! git checkout -B "$SUITE_GIT_REF" "origin/$SUITE_GIT_REF" 2>/dev/null \ + && ! git checkout -B "$SUITE_GIT_REF" FETCH_HEAD; then + _upd_die "git checkout $SUITE_GIT_REF failed" + fi + if ! git reset --hard "origin/$SUITE_GIT_REF" 2>/dev/null \ + && ! git reset --hard FETCH_HEAD; then + _upd_die "git reset --hard origin/$SUITE_GIT_REF failed" + fi + ;; + *) + if ! git pull --ff-only origin "$SUITE_GIT_REF" 2>&1; then + _upd_die "git pull --ff-only failed" + fi + ;; + esac fi -case "$SUITE_UPDATE_MODE" in - reset) - git checkout -B "$SUITE_GIT_REF" "origin/$SUITE_GIT_REF" 2>/dev/null || git checkout -B "$SUITE_GIT_REF" FETCH_HEAD - git reset --hard "origin/$SUITE_GIT_REF" 2>/dev/null || git reset --hard FETCH_HEAD - ;; - *) git pull --ff-only origin "$SUITE_GIT_REF" 2>&1 || true ;; -esac + COMMIT=$(git rev-parse HEAD) COMMIT_SHORT=$(git rev-parse --short=12 HEAD) SOURCE_REPO_WEB="${SOURCE_REPO_WEB:-https://git.hacktivism.ch/hernani/taler-monitoring}" @@ -50,13 +95,10 @@ if command -v git >/dev/null 2>&1; then fi if [ -z "$SUITE_VERSION" ] && [ -f "$SUITE_DIR/VERSION" ]; then SUITE_VERSION="v$(tr -d '[:space:]' <"$SUITE_DIR/VERSION" | sed 's/^v//')" - case "$SUITE_VERSION" in v*) ;; *) SUITE_VERSION="v${SUITE_VERSION#v}" ;; esac fi [ -z "$SUITE_VERSION" ] && SUITE_VERSION="unknown" -# normalize: ensure leading v for tags case "$SUITE_VERSION" in - unknown) ;; - v*) ;; + unknown|v*) ;; *) SUITE_VERSION="v${SUITE_VERSION}" ;; esac SUITE_VERSION_URL="" @@ -64,9 +106,22 @@ if [ -n "$SOURCE_REPO_WEB" ] && [ "$SUITE_VERSION" != "unknown" ]; then SUITE_VERSION_URL="${SOURCE_REPO_WEB}/src/tag/${SUITE_VERSION}" fi export SUITE_VERSION SUITE_VERSION_URL -echo " version=$SUITE_VERSION ${SUITE_VERSION_URL:+· $SUITE_VERSION_URL}" +if [ "$_before" != "$COMMIT_SHORT" ]; then + echo " upgraded ${_before} (${_before_ver}) → ${COMMIT_SHORT} (${SUITE_VERSION})" +else + echo " already at ${COMMIT_SHORT} (${SUITE_VERSION})" +fi +echo " version=$SUITE_VERSION ${SUITE_VERSION_URL:+· $SUITE_VERSION_URL}" echo "suite @ $COMMIT_SHORT $COMMIT_URL" echo " dir=$SUITE_DIR" + +# Sanity: must have suite entrypoint after update +if [ ! -x "$SUITE_DIR/taler-monitoring.sh" ] && [ ! -f "$SUITE_DIR/taler-monitoring.sh" ]; then + _upd_die "taler-monitoring.sh missing after update" + return 1 2>/dev/null || exit 1 +fi + ln -sfn "$SUITE_DIR" "$HOME/taler-monitoring" 2>/dev/null || true chmod +x taler-monitoring.sh check_*.sh host-agent/*.sh site-gen/*.sh site-gen/*.py 2>/dev/null || true +return 0 2>/dev/null || true