taler-monitoring v1.13.12

Multi-phase global SUMMARY, warn-filter dimmed context, SUMMARY chrome as meta; FP stage host-agent via francpaysan-stage-user (stagepaysan); monpages GOA + FP stage.
2026-07-19 04:21:16 +02:00 · 2026-07-19 04:21:16 +02:00 · 5a5431bfa9
commit 5a5431bfa9
88 changed files with 21996 additions and 0 deletions
--- a/meta/README.md
+++ b/meta/README.md
@ -0,0 +1,21 @@
+# meta/ — optional & deprecated tooling
+
+Not required for the normal **host-agent** monitoring pipeline
+(`host-agent/run-*.sh` + `site-gen/console_to_html.py`).
+
+| Subdir | Contents |
+|--------|----------|
+| **`deprecated/`** | Pre-v1.8 mail/mattermost host-agent wrappers + user systemd units (folded into **surface**) |
+| **`firecuda/`** | Outside-runner install (launchd) + pull helpers — optional; prefer koopa host-agent |
+| **`site-gen-legacy/`** | Multi-host `generate-monitoring-sites` / `deploy-monitoring-sites` (older laptop/outside path) |
+
+## Production path (keep using)
+
+```text
+host-agent/run-host-report.sh
+host-agent/run-{hacktivism,surface,aptdeploy,fp-*}.sh
+host-agent/update-suite.sh
+site-gen/console_to_html.py
+```
+
+Machine-local paths/SSH: **`taler-monitoring-env`** → `~/.config/taler-monitoring/env`.
--- a/meta/deprecated/run-mail-monitoring.sh
+++ b/meta/deprecated/run-mail-monitoring.sh
@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# DEPRECATED (v1.8.0): mail is part of taler-monitoring-surface.
+# Kept as a thin alias so old timers/docs do not break.
+#
+# Prefer: run-surface-monitoring.sh
+# Public page: https://taler.hacktivism.ch/taler-monitoring-surface/
+#
+set -uo pipefail
+AGENT_DIR=$(cd "$(dirname "$0")" && pwd)
+echo "NOTE: run-mail-monitoring.sh is deprecated (v1.8.0) — folding into surface" >&2
+export PHASES="${PHASES:-mail surface monpages}"
+# force surface HTML paths (no /taler-monitoring-mail*)
+export HTML_OK_DIR="taler-monitoring-surface"
+export HTML_ERR_DIR="taler-monitoring-surface_err"
+export HTML_URL_OK="/taler-monitoring-surface/"
+export HTML_URL_ERR="/taler-monitoring-surface_err/"
+export PAGE_LABEL="taler-monitoring-surface"
+export MON_HOSTS="${MON_HOSTS:-taler.hacktivism.ch}"
+export MONPAGES_INVENTORY=job
+exec bash "$AGENT_DIR/run-surface-monitoring.sh" "$@"
--- a/meta/deprecated/run-mattermost-monitoring.sh
+++ b/meta/deprecated/run-mattermost-monitoring.sh
@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# DEPRECATED (v1.8.0): mattermost is part of taler-monitoring-surface.
+# Kept as a thin alias so old timers/docs do not break.
+#
+# Prefer: run-surface-monitoring.sh
+# Public page: https://taler.hacktivism.ch/taler-monitoring-surface/
+#
+set -uo pipefail
+AGENT_DIR=$(cd "$(dirname "$0")" && pwd)
+echo "NOTE: run-mattermost-monitoring.sh is deprecated (v1.8.0) — folding into surface" >&2
+export PHASES="${PHASES:-mattermost surface monpages}"
+export HTML_OK_DIR="taler-monitoring-surface"
+export HTML_ERR_DIR="taler-monitoring-surface_err"
+export HTML_URL_OK="/taler-monitoring-surface/"
+export HTML_URL_ERR="/taler-monitoring-surface_err/"
+export PAGE_LABEL="taler-monitoring-surface"
+export MON_HOSTS="${MON_HOSTS:-taler.hacktivism.ch}"
+export MONPAGES_INVENTORY=job
+exec bash "$AGENT_DIR/run-surface-monitoring.sh" "$@"
--- a/meta/deprecated/taler-monitoring-mail.service
+++ b/meta/deprecated/taler-monitoring-mail.service
@ -0,0 +1,18 @@
+[Unit]
+Description=Taler monitoring mail (firefly/pixel MX SMTP IMAP)
+Documentation=file:%h/src/taler-monitoring/host-agent/README.md
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin
+Environment=CONTINUE_ON_ERROR=1
+Environment=RUN_TIMEOUT=600
+WorkingDirectory=%h/src/taler-monitoring
+ExecStart=%h/src/taler-monitoring/meta/deprecated/run-mail-monitoring.sh
+Nice=10
+TimeoutStartSec=20min
+
+[Install]
+WantedBy=default.target
--- a/meta/deprecated/taler-monitoring-mail.timer
+++ b/meta/deprecated/taler-monitoring-mail.timer
@ -0,0 +1,14 @@
+[Unit]
+Description=Periodic taler-monitoring mail (MX/SMTP/IMAP)
+Documentation=file:%h/src/taler-monitoring/host-agent/README.md
+
+[Timer]
+OnBootSec=30min
+OnUnitActiveSec=4h
+AccuracySec=5min
+Persistent=true
+RandomizedDelaySec=10min
+Unit=taler-monitoring-mail.service
+
+[Install]
+WantedBy=timers.target
--- a/meta/deprecated/taler-monitoring-mattermost.service
+++ b/meta/deprecated/taler-monitoring-mattermost.service
@ -0,0 +1,18 @@
+[Unit]
+Description=Taler monitoring Mattermost (mattermost.taler.net outside-in)
+Documentation=file:%h/src/taler-monitoring/host-agent/README.md
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin
+Environment=CONTINUE_ON_ERROR=1
+Environment=RUN_TIMEOUT=300
+WorkingDirectory=%h/src/taler-monitoring
+ExecStart=%h/src/taler-monitoring/meta/deprecated/run-mattermost-monitoring.sh
+Nice=10
+TimeoutStartSec=15min
+
+[Install]
+WantedBy=default.target
--- a/meta/deprecated/taler-monitoring-mattermost.timer
+++ b/meta/deprecated/taler-monitoring-mattermost.timer
@ -0,0 +1,14 @@
+[Unit]
+Description=Periodic taler-monitoring Mattermost (chat health)
+Documentation=file:%h/src/taler-monitoring/host-agent/README.md
+
+[Timer]
+OnBootSec=25min
+OnUnitActiveSec=1h
+AccuracySec=2min
+Persistent=true
+RandomizedDelaySec=5min
+Unit=taler-monitoring-mattermost.service
+
+[Install]
+WantedBy=timers.target
--- a/meta/firecuda/com.hacktivism.taler-monitoring-sites.plist
+++ b/meta/firecuda/com.hacktivism.taler-monitoring-sites.plist
@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<!--
+  launchd user agent — firecuda (max): every 4 hours
+  Install: ./install-firecuda-timer.sh
+-->
+<plist version="1.0">
+<dict>
+  <key>Label</key>
+  <string>com.hacktivism.taler-monitoring-sites</string>
+  <key>ProgramArguments</key>
+  <array>
+    <string>/bin/bash</string>
+    <string>__SITE_GEN__/run-on-firecuda.sh</string>
+  </array>
+  <key>StartInterval</key>
+  <integer>14400</integer>
+  <key>RunAtLoad</key>
+  <true/>
+  <key>StandardOutPath</key>
+  <string>__HOME__/Library/Logs/taler-monitoring-sites/launchd.out.log</string>
+  <key>StandardErrorPath</key>
+  <string>__HOME__/Library/Logs/taler-monitoring-sites/launchd.err.log</string>
+  <key>EnvironmentVariables</key>
+  <dict>
+    <key>PATH</key>
+    <string>/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin</string>
+    <key>SKIP_SSH</key>
+    <string>1</string>
+    <key>CONTINUE_ON_ERROR</key>
+    <string>1</string>
+  </dict>
+</dict>
+</plist>
--- a/meta/firecuda/install-firecuda-timer.sh
+++ b/meta/firecuda/install-firecuda-timer.sh
@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+# install-firecuda-timer.sh — OPTIONAL: rsync suite to firecuda + launchd every 4h
+#
+# POLICY (2026-07): firecuda launchd is **disabled**. Prefer koopa host-agent.
+# This script remains for documentation / future re-enable only.
+# Do not run unless you intentionally want a second outside-only scheduler.
+#
+#   ./install-firecuda-timer.sh
+#   ./install-firecuda-timer.sh --run-now
+#
+set -euo pipefail
+
+ROOT=$(cd "$(dirname "$0")" && pwd)
+MON_ROOT=$(cd "$ROOT/../.." && pwd)
+FIRECUDA="${FIRECUDA_SSH:-}"
+if [ -z "$FIRECUDA" ]; then
+  echo "Set FIRECUDA_SSH to your outside-runner SSH host alias" >&2
+  exit 2
+fi
+REMOTE_BASE="${FIRECUDA_INSTALL_DIR:-taler-monitoring-site-gen}"
+RUN_NOW=0
+[ "${1:-}" = "--run-now" ] && RUN_NOW=1
+
+echo "install → $FIRECUDA:~/$REMOTE_BASE (outside-only, every 4h)"
+
+ssh -o BatchMode=yes -o ConnectTimeout=20 "$FIRECUDA" \
+  "mkdir -p \"\$HOME/$REMOTE_BASE\" \"\$HOME/Library/Logs/taler-monitoring-sites\" \"\$HOME/var/taler-monitoring-sites-work\" \"\$HOME/Library/LaunchAgents\""
+
+# Sync full taler-monitoring suite (no secrets.env)
+rsync -az --delete \
+  --exclude secrets.env \
+  --exclude 'android-test/artifacts' \
+  --exclude '.git' \
+  --exclude 'site-gen/settings.conf' \
+  "$MON_ROOT/" \
+  "${FIRECUDA}:$REMOTE_BASE/"
+
+# settings on firecuda: outside only, run locally on firecuda, deploy to koopa
+ssh -o BatchMode=yes "$FIRECUDA" "cat > \"\$HOME/$REMOTE_BASE/site-gen/settings.conf\" <<'EOF'
+# auto-installed by install-firecuda-timer.sh — no secrets
+RUNNER_SSH=
+SKIP_SSH=1
+CONTINUE_ON_ERROR=1
+PHASES_ERR=urls versions
+PHASES_OK=urls versions
+DEPLOY_SSH=${DEPLOY_SSH:-}
+DEPLOY_STAGING=${DEPLOY_STAGING:-$HOME/monitoring-sites-staging}
+WORK_ROOT=
+MONITORING_ROOT=..
+SOURCE_REPO_WEB=https://git.hacktivism.ch/hernani/taler-monitoring
+SOURCE_COMMIT_URL_TMPL={repo}/src/commit/{commit}
+SOURCE_SUITE_PATH=.
+EOF
+# WORK_ROOT expanded on remote
+sed -i.bak \"s|^WORK_ROOT=.*|WORK_ROOT=\$HOME/var/taler-monitoring-sites-work|\" \"\$HOME/$REMOTE_BASE/site-gen/settings.conf\" 2>/dev/null || \
+  sed -i '' \"s|^WORK_ROOT=.*|WORK_ROOT=\$HOME/var/taler-monitoring-sites-work|\" \"\$HOME/$REMOTE_BASE/site-gen/settings.conf\"
+"
+
+# launchd plist with absolute paths
+ssh -o BatchMode=yes "$FIRECUDA" bash -s <<REMOTE
+set -euo pipefail
+HOME_DIR=\$HOME
+FIRECUDA_META="\$HOME_DIR/$REMOTE_BASE/meta/firecuda"
+SITE_GEN="\$HOME_DIR/$REMOTE_BASE/site-gen"
+PLIST_SRC="\$FIRECUDA_META/com.hacktivism.taler-monitoring-sites.plist"
+PLIST_DST="\$HOME_DIR/Library/LaunchAgents/com.hacktivism.taler-monitoring-sites.plist"
+sed -e "s|__SITE_GEN__|\${FIRECUDA_META}|g" -e "s|__HOME__|\${HOME_DIR}|g" "\$PLIST_SRC" > "\$PLIST_DST"
+chmod +x "\$FIRECUDA_META"/*.sh "\$SITE_GEN"/console_to_html.py 2>/dev/null || true
+chmod +x "\$HOME_DIR/$REMOTE_BASE/meta/site-gen-legacy"/*.sh 2>/dev/null || true
+chmod +x "\$FIRECUDA_META/run-on-firecuda.sh"
+launchctl bootout "gui/\$(id -u)/com.hacktivism.taler-monitoring-sites" 2>/dev/null || true
+launchctl unload "\$PLIST_DST" 2>/dev/null || true
+launchctl bootstrap "gui/\$(id -u)" "\$PLIST_DST" 2>/dev/null || launchctl load "\$PLIST_DST"
+launchctl enable "gui/\$(id -u)/com.hacktivism.taler-monitoring-sites" 2>/dev/null || true
+launchctl print "gui/\$(id -u)/com.hacktivism.taler-monitoring-sites" 2>/dev/null | head -20 || launchctl list | grep taler-monitoring || true
+echo "OK: launchd com.hacktivism.taler-monitoring-sites (StartInterval=14400)"
+REMOTE
+
+if [ "$RUN_NOW" = "1" ]; then
+  echo "run once now…"
+  ssh -o BatchMode=yes "$FIRECUDA" "\"\$HOME/$REMOTE_BASE/meta/firecuda/run-on-firecuda.sh\"" || true
+fi
+
+echo "done. logs: $FIRECUDA:~/Library/Logs/taler-monitoring-sites/"
+echo "manual: ssh $FIRECUDA '~/taler-monitoring-site-gen/meta/firecuda/run-on-firecuda.sh'"
--- a/meta/firecuda/pull-firecuda-to-koopa.sh
+++ b/meta/firecuda/pull-firecuda-to-koopa.sh
@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# pull-firecuda-to-koopa.sh — copy HTML from firecuda work dir → koopa staging
+# Run on laptop (hernani) where both SSH aliases work.
+set -euo pipefail
+FIRECUDA="${FIRECUDA_SSH:-}"
+if [ -z "$FIRECUDA" ]; then
+  echo "Set FIRECUDA_SSH to your outside-runner SSH host alias" >&2
+  exit 2
+fi
+KOOPA="${DEPLOY_SSH:?set DEPLOY_SSH in env}"
+REMOTE_HTML="${FIRECUDA_HTML:-var/taler-monitoring-sites-work/html/}"
+STAGE="${DEPLOY_STAGING:-${DEPLOY_STAGING:-$HOME/monitoring-sites-staging}}"
+
+echo "firecuda:$REMOTE_HTML → $KOOPA:$STAGE"
+ssh -o BatchMode=yes -o ConnectTimeout=20 "$KOOPA" "mkdir -p '$STAGE'"
+rsync -az --delete \
+  "${FIRECUDA}:${REMOTE_HTML}" \
+  "${KOOPA}:${STAGE}/"
+echo "OK staged. Root: rsync to  (see ROOT-ON-KOOPA.md)"
--- a/meta/firecuda/run-on-firecuda.sh
+++ b/meta/firecuda/run-on-firecuda.sh
@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# run-on-firecuda.sh — entrypoint for launchd/cron on outside-runner.
+# Outside-only monitoring (SKIP_SSH=1), all 9 sites, then stage HTML to koopa.
+#
+# Lives in meta/firecuda/ (optional). Prefer koopa host-agent for production mon.
+#
+set -uo pipefail
+
+export PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:${PATH:-}"
+
+HERE=$(cd "$(dirname "$0")" && pwd)
+SUITE_ROOT=$(cd "$HERE/../.." && pwd)
+LEGACY="$SUITE_ROOT/meta/site-gen-legacy"
+SITE_GEN="$SUITE_ROOT/site-gen"
+
+LOG_DIR="${FIRECUDA_LOG_DIR:-$HOME/Library/Logs/taler-monitoring-sites}"
+mkdir -p "$LOG_DIR"
+STAMP=$(date +%Y%m%d-%H%M%S)
+LOG="$LOG_DIR/run-${STAMP}.log"
+exec >>"$LOG" 2>&1
+
+echo "======== $(date -Iseconds 2>/dev/null || date) outside-runner monitoring sites ========"
+
+export SKIP_SSH=1
+export CONTINUE_ON_ERROR=1
+export AUTH401_CONTINUE=1
+export RUNNER_SSH=
+export PHASES_ERR="${PHASES_ERR:-urls versions}"
+export PHASES_OK="${PHASES_OK:-urls versions}"
+export WORK_ROOT="${WORK_ROOT:-$HOME/var/taler-monitoring-sites-work}"
+export MONITORING_ROOT="${MONITORING_ROOT:-$SUITE_ROOT}"
+export DEPLOY_SSH="${DEPLOY_SSH:-}"
+export DEPLOY_STAGING="${DEPLOY_STAGING:-$HOME/monitoring-sites-staging}"
+export SITE_GEN_DIR="$SITE_GEN"
+
+unset ONLY_HOSTS
+
+cd "$LEGACY" || exit 1
+chmod +x generate-monitoring-sites.sh deploy-monitoring-sites.sh 2>/dev/null || true
+chmod +x "$SITE_GEN/console_to_html.py" 2>/dev/null || true
+
+./generate-monitoring-sites.sh
+ec=$?
+
+if [ -n "${DEPLOY_SSH:-}" ] && [ "${DEPLOY_FROM_FIRECUDA:-0}" = "1" ]; then
+  if [ -x ./deploy-monitoring-sites.sh ]; then
+    ./deploy-monitoring-sites.sh || echo "WARN: deploy staging failed"
+  fi
+else
+  echo "note: HTML at ${WORK_ROOT}/html — pull from laptop:"
+  echo "  rsync -az outside-runner:var/taler-monitoring-sites-work/html/ \\"
+  echo "    \${DEPLOY_SSH}:monitoring-sites-staging/"
+  echo "  # or: $HERE/pull-firecuda-to-koopa.sh"
+fi
+
+ls -1t "$LOG_DIR"/run-*.log 2>/dev/null | tail -n +21 | xargs rm -f 2>/dev/null || true
+
+echo "======== done ec=$ec ========"
+exit "$ec"
--- a/meta/site-gen-legacy/deploy-monitoring-sites.sh
+++ b/meta/site-gen-legacy/deploy-monitoring-sites.sh
@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# deploy-monitoring-sites.sh — rsync generated HTML to DEPLOY_WWW_ROOT on koopa.
+# Does NOT reload Caddy (needs root — see ROOT-ON-KOOPA.md).
+set -uo pipefail
+
+HERE=$(cd "$(dirname "$0")" && pwd)
+SUITE_ROOT=$(cd "$HERE/../.." && pwd)
+SITE_GEN="${SITE_GEN_DIR:-$SUITE_ROOT/site-gen}"
+ROOT="$HERE"
+SETTINGS="${SITE_GEN_SETTINGS:-$SITE_GEN/settings.conf}"
+[ -f "$SETTINGS" ] || SETTINGS="$SITE_GEN/settings.conf.example"
+# shellcheck disable=SC1090
+set -a
+source <(grep -v '^\s*#' "$SETTINGS" | grep -v '^\s*$' | sed 's/\r$//')
+set +a
+
+WORK_ROOT="${WORK_ROOT:-/tmp/taler-monitoring-sites-work}"
+HTML_DIR="$WORK_ROOT/html"
+DEPLOY_SSH="${DEPLOY_SSH:-}"
+DEPLOY_WWW_ROOT="${DEPLOY_WWW_ROOT:-}"
+
+if [ ! -d "$HTML_DIR" ]; then
+  echo "no HTML_DIR $HTML_DIR — run generate-monitoring-sites.sh first" >&2
+  exit 1
+fi
+
+echo "deploy $HTML_DIR → ${DEPLOY_SSH}:${DEPLOY_WWW_ROOT}/"
+# Prefer writing to a hernani-owned staging dir if /var/www not writable
+STAGE_REMOTE="${DEPLOY_STAGING:-${DEPLOY_STAGING:-$HOME/monitoring-sites-staging}}"
+
+ssh -o BatchMode=yes -o ConnectTimeout=20 "$DEPLOY_SSH" \
+  "mkdir -p '$STAGE_REMOTE' '$DEPLOY_WWW_ROOT' 2>/dev/null || mkdir -p '$STAGE_REMOTE'"
+
+rsync -az --delete "$HTML_DIR/" "${DEPLOY_SSH}:${STAGE_REMOTE}/"
+echo "staged on $DEPLOY_SSH:$STAGE_REMOTE"
+echo
+echo "If $DEPLOY_WWW_ROOT is root-owned, as root on koopa:"
+echo "  mkdir -p $DEPLOY_WWW_ROOT"
+echo "  rsync -a --delete $STAGE_REMOTE/ $DEPLOY_WWW_ROOT/"
+echo "  chown -R caddy:caddy $DEPLOY_WWW_ROOT   # or root:caddy — match other www"
+echo "  # then Caddyfile handles + reload — see ROOT-ON-KOOPA.md"
+echo
+# try direct rsync to DEPLOY_WWW_ROOT if writable
+if ssh -o BatchMode=yes "$DEPLOY_SSH" "test -w '$DEPLOY_WWW_ROOT' 2>/dev/null || test -w \"\$(dirname '$DEPLOY_WWW_ROOT')\""; then
+  rsync -az --delete "$HTML_DIR/" "${DEPLOY_SSH}:${DEPLOY_WWW_ROOT}/" && \
+    echo "also synced directly to $DEPLOY_WWW_ROOT"
+fi
--- a/meta/site-gen-legacy/generate-monitoring-sites.sh
+++ b/meta/site-gen-legacy/generate-monitoring-sites.sh
@ -0,0 +1,277 @@
+#!/usr/bin/env bash
+# generate-monitoring-sites.sh — run taler-monitoring, build console HTML for
+# /monitoring and /monitoring_err (static files for host Caddy later).
+#
+# Usage:
+#   ./generate-monitoring-sites.sh              # all SITES (or ONLY_HOSTS)
+#   ONLY_HOSTS="bank.hacktivism.ch taler.hacktivism.ch" ./generate-monitoring-sites.sh
+#   ./generate-monitoring-sites.sh --dry-run    # HTML from empty/missing logs only
+#   ./generate-monitoring-sites.sh --skip-run   # only convert existing logs
+#
+# Settings: settings.conf (from settings.conf.example). No secrets in settings.
+# Secrets for e2e: ../secrets.env (optional; default phases are urls versions).
+#
+set -uo pipefail
+
+export PYTHONUNBUFFERED=1
+# Global reporting defaults (same as host-agent run-host-report.sh)
+: "${RUN_TIMEOUT:=600}"
+export RUN_TIMEOUT
+
+HERE=$(cd "$(dirname "$0")" && pwd)
+SUITE_ROOT=$(cd "$HERE/../.." && pwd)
+SITE_GEN="${SITE_GEN_DIR:-$SUITE_ROOT/site-gen}"
+ROOT="$HERE"
+SETTINGS="${SITE_GEN_SETTINGS:-$SITE_GEN/settings.conf}"
+EXAMPLE="${SITE_GEN}/settings.conf.example"
+[ -f "$EXAMPLE" ] || EXAMPLE="$HERE/settings.conf.example"
+
+if [ ! -f "$SETTINGS" ]; then
+  if [ -f "$EXAMPLE" ]; then
+    echo "note: no $SETTINGS — using example defaults (copy to settings.conf to customize)"
+    SETTINGS="$EXAMPLE"
+  else
+    echo "missing settings: $EXAMPLE" >&2
+    exit 2
+  fi
+fi
+
+# Load KEY=value lines; allow SITES already set in environment to win.
+# Multiline SITES in the conf file: use SITES_FILE= or env SITES= instead.
+while IFS= read -r _line || [ -n "$_line" ]; do
+  _line=${_line%%#*}
+  _line=$(echo "$_line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
+  [ -z "$_line" ] && continue
+  case "$_line" in
+    SITES=*|SITES_FILE=*) continue ;; # handled below / env
+    *=*)
+      _k=${_line%%=*}
+      _v=${_line#*=}
+      # do not override pre-set env
+      if [ -z "${!_k+x}" ] 2>/dev/null || [ -z "${!_k}" ]; then
+        export "$_k=$_v"
+      fi
+      ;;
+  esac
+done < "$SETTINGS"
+
+# SITES from dedicated file or keep env; else parse heredoc-style from example list
+if [ -n "${SITES_FILE:-}" ] && [ -f "$SITES_FILE" ]; then
+  SITES=$(grep -v '^\s*#' "$SITES_FILE" | grep -v '^\s*$')
+  export SITES
+elif [ -z "${SITES:-}" ]; then
+  # default 9 fronts
+  SITES="bank.hacktivism.ch|hacktivism.ch
+exchange.hacktivism.ch|hacktivism.ch
+taler.hacktivism.ch|hacktivism.ch
+stage.bank.lefrancpaysan.ch|stage.lefrancpaysan.ch
+stage.exchange.lefrancpaysan.ch|stage.lefrancpaysan.ch
+stage.monnaie.lefrancpaysan.ch|stage.lefrancpaysan.ch
+bank.lefrancpaysan.ch|lefrancpaysan.ch
+exchange.lefrancpaysan.ch|lefrancpaysan.ch
+monnaie.lefrancpaysan.ch|lefrancpaysan.ch"
+  export SITES
+fi
+
+SKIP_RUN=0
+DRY=0
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --skip-run) SKIP_RUN=1; shift ;;
+    --dry-run) DRY=1; SKIP_RUN=1; shift ;;
+    -h|--help) sed -n '1,20p' "$0"; exit 0 ;;
+    *) echo "unknown arg: $1" >&2; exit 2 ;;
+  esac
+done
+
+MONITORING_ROOT=$(cd "${MONITORING_ROOT:-$SUITE_ROOT}" && pwd)
+WORK_ROOT="${WORK_ROOT:-/tmp/taler-monitoring-sites-work}"
+LOG_DIR="$WORK_ROOT/logs"
+HTML_DIR="$WORK_ROOT/html"
+mkdir -p "$LOG_DIR" "$HTML_DIR"
+
+# commit of monitoring suite used for this run
+if git -C "$MONITORING_ROOT/../.." rev-parse HEAD >/dev/null 2>&1; then
+  REPO_ROOT=$(cd "$MONITORING_ROOT/../.." && pwd)
+elif git -C "$MONITORING_ROOT" rev-parse HEAD >/dev/null 2>&1; then
+  REPO_ROOT=$(cd "$MONITORING_ROOT" && pwd)
+else
+  REPO_ROOT=$(cd "$ROOT/../../.." && pwd)
+fi
+COMMIT=$(git -C "$REPO_ROOT" rev-parse HEAD 2>/dev/null || echo unknown)
+COMMIT_SHORT=$(git -C "$REPO_ROOT" rev-parse --short=12 HEAD 2>/dev/null || echo unknown)
+SOURCE_REPO_WEB="${SOURCE_REPO_WEB:-https://git.hacktivism.ch/hernani/taler-monitoring}"
+TMPL="${SOURCE_COMMIT_URL_TMPL:-\{repo\}/src/commit/\{commit\}}"
+COMMIT_URL=$(printf '%s' "$TMPL" | sed "s|{repo}|$SOURCE_REPO_WEB|g; s|{commit}|$COMMIT|g")
+SUITE_PATH="${SOURCE_SUITE_PATH:-.}"
+
+# Outside-only public checks (no SSH into koopa/stage containers)
+PHASES_ERR="${PHASES_ERR:-urls versions}"
+PHASES_OK="${PHASES_OK:-urls versions}"
+CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}"
+SKIP_SSH="${SKIP_SSH:-1}"
+# Drop SSH phases if someone passes "all" / inside / server / e2e by mistake
+filter_phases_outside() {
+  local out=() p
+  for p in "$@"; do
+    case "$p" in
+      inside|server|e2e|ladder|goa-ladder|auth401) continue ;;
+      all) out+=(urls versions);;
+      *) out+=("$p") ;;
+    esac
+  done
+  printf '%s\n' "${out[@]}"
+}
+
+run_mon() {
+  local domain="$1" phases="$2" logfile="$3" cont="$4"
+  # shellcheck disable=SC2206
+  local ph=( $phases )
+  mapfile -t ph < <(filter_phases_outside "${ph[@]}")
+  [ "${#ph[@]}" -gt 0 ] || ph=(urls versions)
+
+  : >"$logfile"
+  echo "+ -d $domain · phases=${ph[*]} · CONTINUE=$cont · SKIP_SSH=$SKIP_SSH · RUN_TIMEOUT=$RUN_TIMEOUT"
+
+  if [ -n "${RUNNER_SSH:-}" ]; then
+    echo "+ via ssh $RUNNER_SSH"
+    rsync -az --delete \
+      --exclude secrets.env \
+      --exclude 'android-test/artifacts' \
+      --exclude '.git' \
+      "$MONITORING_ROOT/" \
+      "${RUNNER_SSH}:${RUNNER_REMOTE_WORKDIR}/" \
+      || return 1
+    # shellcheck disable=SC2029
+    if command -v stdbuf >/dev/null 2>&1; then
+      ssh -o BatchMode=yes -o ConnectTimeout=30 "$RUNNER_SSH" \
+        "cd '${RUNNER_REMOTE_WORKDIR}' && chmod +x taler-monitoring.sh check_*.sh 2>/dev/null; \
+         export CONTINUE_ON_ERROR='$cont' AUTH401_CONTINUE='$cont' SKIP_SSH='$SKIP_SSH' RUN_TIMEOUT='$RUN_TIMEOUT'; \
+         stdbuf -oL -eL ./taler-monitoring.sh -d '$domain' ${ph[*]}" \
+        2>&1 | stdbuf -oL -eL tee -a "$logfile"
+    else
+      ssh -o BatchMode=yes -o ConnectTimeout=30 "$RUNNER_SSH" \
+        "cd '${RUNNER_REMOTE_WORKDIR}' && chmod +x taler-monitoring.sh check_*.sh 2>/dev/null; \
+         export CONTINUE_ON_ERROR='$cont' AUTH401_CONTINUE='$cont' SKIP_SSH='$SKIP_SSH' RUN_TIMEOUT='$RUN_TIMEOUT'; \
+         ./taler-monitoring.sh -d '$domain' ${ph[*]}" \
+        2>&1 | tee -a "$logfile"
+    fi
+    return "${PIPESTATUS[0]}"
+  fi
+
+  if command -v stdbuf >/dev/null 2>&1; then
+    ( cd "$MONITORING_ROOT" && \
+        CONTINUE_ON_ERROR="$cont" AUTH401_CONTINUE="$cont" SKIP_SSH="$SKIP_SSH" \
+        RUN_TIMEOUT="$RUN_TIMEOUT" \
+        stdbuf -oL -eL ./taler-monitoring.sh -d "$domain" "${ph[@]}" ) \
+      2>&1 | stdbuf -oL -eL tee -a "$logfile"
+  else
+    ( cd "$MONITORING_ROOT" && \
+        CONTINUE_ON_ERROR="$cont" AUTH401_CONTINUE="$cont" SKIP_SSH="$SKIP_SSH" \
+        RUN_TIMEOUT="$RUN_TIMEOUT" \
+        ./taler-monitoring.sh -d "$domain" "${ph[@]}" ) \
+      2>&1 | tee -a "$logfile"
+  fi
+  return "${PIPESTATUS[0]}"
+}
+
+htmlify() {
+  local host="$1" mode="$2" log="$3" out="$4" other="$5"
+  mkdir -p "$(dirname "$out")"
+  if [ ! -f "$SITE_GEN/console_to_html.py" ]; then
+    echo "WARN: console_to_html.py missing — bootstrap $out"
+    {
+      echo "<!DOCTYPE html><html><head><meta charset=utf-8><title>$host $mode</title></head>"
+      echo "<body><h1>$host · $mode</h1><p>commit <a href=\"$COMMIT_URL\">$COMMIT_SHORT</a></p>"
+      echo "<pre>"
+      tail -n 100 "$log" 2>/dev/null | sed 's/&/\&amp;/g;s/</\&lt;/g' || true
+      echo "</pre></body></html>"
+    } >"$out"
+    return 0
+  fi
+  python3 "$SITE_GEN/console_to_html.py" \
+    --log "$log" \
+    --out "$out" \
+    --hostname "$host" \
+    --mode "$mode" \
+    --commit "$COMMIT" \
+    --commit-url "$COMMIT_URL" \
+    --suite-path "$SUITE_PATH" \
+    --link-other "$other"
+}
+
+# Parse SITES
+mapfile -t SITE_LINES < <(printf '%s\n' "$SITES" | sed '/^\s*$/d' | sed 's/#.*//')
+ONLY="${ONLY_HOSTS:-}"
+
+for line in "${SITE_LINES[@]}"; do
+  line=$(echo "$line" | tr -d ' \t')
+  [ -z "$line" ] && continue
+  host=${line%%|*}
+  domain=${line#*|}
+  if [ -n "$ONLY" ]; then
+    echo " $ONLY " | grep -q " $host " || continue
+  fi
+
+  echo "======== $host  (domain=$domain)  commit=$COMMIT_SHORT ========"
+  base_html="$HTML_DIR/$host"
+  mkdir -p "$base_html/monitoring" "$base_html/monitoring_err"
+  log_err="$LOG_DIR/${host}.err.log"
+  log_ok="$LOG_DIR/${host}.ok.log"
+  ec_err=1
+  ec_ok=1
+
+  if [ "$SKIP_RUN" != "1" ]; then
+    run_mon "$domain" "$PHASES_ERR" "$log_err" "$CONTINUE_ON_ERROR"
+    ec_err=$?
+    # ok-run without continue (strict)
+    run_mon "$domain" "$PHASES_OK" "$log_ok" "0"
+    ec_ok=$?
+  else
+    [ -f "$log_err" ] || : >"$log_err"
+    [ -f "$log_ok" ] || : >"$log_ok"
+    # infer exit from log if present
+    if grep -q '\[ ERROR \]' "$log_err" 2>/dev/null || grep -q 'ERROR' "$log_err" 2>/dev/null; then
+      ec_err=1
+    else
+      ec_err=0
+    fi
+    ec_ok=$ec_err
+  fi
+
+  # Always write pages (first run / empty logs too)
+  [ -f "$log_err" ] || : >"$log_err"
+  [ -f "$log_ok" ] || : >"$log_ok"
+
+  htmlify "$host" "err" "$log_err" \
+    "$base_html/monitoring_err/index.html" "/monitoring/"
+
+  if [ "$ec_ok" -eq 0 ] && [ "$ec_err" -eq 0 ]; then
+    htmlify "$host" "ok" "$log_ok" \
+      "$base_html/monitoring/index.html" "/monitoring_err/"
+    echo "  → /monitoring (clean) + /monitoring_err"
+  else
+    htmlify "$host" "redirect" "$log_err" \
+      "$base_html/monitoring/index.html" "/monitoring_err/"
+    echo "  → /monitoring (redirect stub) + /monitoring_err (errors)  ec_err=$ec_err ec_ok=$ec_ok"
+  fi
+
+  # first-run safety
+  if [ ! -f "$base_html/monitoring/index.html" ]; then
+    htmlify "$host" "err" "$log_err" \
+      "$base_html/monitoring/index.html" "/monitoring_err/"
+  fi
+
+  printf '%s\n' "$COMMIT" >"$base_html/COMMIT"
+  printf '%s\n' "$COMMIT_URL" >"$base_html/COMMIT_URL"
+  printf 'host=%s domain=%s ec_err=%s ec_ok=%s commit=%s run_timeout=%s\n' \
+    "$host" "$domain" "$ec_err" "$ec_ok" "$COMMIT" "$RUN_TIMEOUT" | tee "$base_html/STATUS.txt"
+done
+
+echo
+echo "HTML under $HTML_DIR"
+echo "commit $COMMIT_SHORT  $COMMIT_URL"
+echo "RUN_TIMEOUT=${RUN_TIMEOUT}s (global; 0=unlimited)"
+echo "Deploy (as hernani, may need sudo for /var/www):"
+echo "  $HERE/deploy-monitoring-sites.sh"
+echo "Root on koopa: see ROOT-ON-KOOPA.md"