diff --git a/.gitignore b/.gitignore index 822d63a..1fa6cfa 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ scripts/taler-monitoring/settings.conf # Local e2e/overrides — never commit (use secrets.env.example as template) scripts/taler-monitoring/secrets.env scripts/taler-monitoring/secrets.env.local +scripts/taler-monitoring/site-gen/settings.conf diff --git a/configs/caddy/Caddyfile b/configs/caddy/Caddyfile index 77b2f34..7b960b9 100644 --- a/configs/caddy/Caddyfile +++ b/configs/caddy/Caddyfile @@ -37,6 +37,22 @@ taler.hacktivism.ch { } } + # taler-monitoring console HTML (host static — not in app containers) + handle /monitoring_err { + redir /monitoring_err/ 302 + } + handle /monitoring_err/ { + root * /var/www/monitoring-sites/{host}/monitoring_err + file_server + } + handle /monitoring { + redir /monitoring/ 302 + } + handle /monitoring/ { + root * /var/www/monitoring-sites/{host}/monitoring + file_server + } + # SPA: /webui → /webui/ redir /webui /webui/ 302 @@ -62,6 +78,22 @@ exchange.hacktivism.ch { } } + # taler-monitoring console HTML (host static — not in app containers) + handle /monitoring_err { + redir /monitoring_err/ 302 + } + handle /monitoring_err/ { + root * /var/www/monitoring-sites/{host}/monitoring_err + file_server + } + handle /monitoring { + redir /monitoring/ 302 + } + handle /monitoring/ { + root * /var/www/monitoring-sites/{host}/monitoring + file_server + } + reverse_proxy 127.0.0.1:9011 { import proxy_public } @@ -79,6 +111,22 @@ bank.hacktivism.ch { } } + # taler-monitoring console HTML (host static — not in app containers) + handle /monitoring_err { + redir /monitoring_err/ 302 + } + handle /monitoring_err/ { + root * /var/www/monitoring-sites/{host}/monitoring_err + file_server + } + handle /monitoring { + redir /monitoring/ 302 + } + handle /monitoring/ { + root * /var/www/monitoring-sites/{host}/monitoring + file_server + } + # Static terms/privacy on landing nginx :9013 handle /terms* { reverse_proxy 127.0.0.1:9013 { diff --git a/host/caddy/Caddyfile b/host/caddy/Caddyfile index 77b2f34..7b960b9 100644 --- a/host/caddy/Caddyfile +++ b/host/caddy/Caddyfile @@ -37,6 +37,22 @@ taler.hacktivism.ch { } } + # taler-monitoring console HTML (host static — not in app containers) + handle /monitoring_err { + redir /monitoring_err/ 302 + } + handle /monitoring_err/ { + root * /var/www/monitoring-sites/{host}/monitoring_err + file_server + } + handle /monitoring { + redir /monitoring/ 302 + } + handle /monitoring/ { + root * /var/www/monitoring-sites/{host}/monitoring + file_server + } + # SPA: /webui → /webui/ redir /webui /webui/ 302 @@ -62,6 +78,22 @@ exchange.hacktivism.ch { } } + # taler-monitoring console HTML (host static — not in app containers) + handle /monitoring_err { + redir /monitoring_err/ 302 + } + handle /monitoring_err/ { + root * /var/www/monitoring-sites/{host}/monitoring_err + file_server + } + handle /monitoring { + redir /monitoring/ 302 + } + handle /monitoring/ { + root * /var/www/monitoring-sites/{host}/monitoring + file_server + } + reverse_proxy 127.0.0.1:9011 { import proxy_public } @@ -79,6 +111,22 @@ bank.hacktivism.ch { } } + # taler-monitoring console HTML (host static — not in app containers) + handle /monitoring_err { + redir /monitoring_err/ 302 + } + handle /monitoring_err/ { + root * /var/www/monitoring-sites/{host}/monitoring_err + file_server + } + handle /monitoring { + redir /monitoring/ 302 + } + handle /monitoring/ { + root * /var/www/monitoring-sites/{host}/monitoring + file_server + } + # Static terms/privacy on landing nginx :9013 handle /terms* { reverse_proxy 127.0.0.1:9013 { diff --git a/scripts/taler-monitoring/.gitignore b/scripts/taler-monitoring/.gitignore new file mode 100644 index 0000000..754bb8e --- /dev/null +++ b/scripts/taler-monitoring/.gitignore @@ -0,0 +1,6 @@ +android-test/apks/ +android-test/out*/ +android-test/artifacts/ +__pycache__/ +secrets.env +*.pyc diff --git a/scripts/taler-monitoring/DEPENDENCIES.md b/scripts/taler-monitoring/DEPENDENCIES.md new file mode 100644 index 0000000..3e7db4c --- /dev/null +++ b/scripts/taler-monitoring/DEPENDENCIES.md @@ -0,0 +1,70 @@ +# taler-monitoring — host package dependencies + +## koopa (openSUSE Tumbleweed, user `hernani`) + +### Required (host-agent / GOA primary: `urls inside versions`) + +| Binary | Package (zypper) | Role | +|--------|------------------|------| +| `bash` | (base) | runner | +| `curl` | `curl` | HTTPS/API probes | +| `python3` | `python313-base` (provides `/usr/bin/python3`) | JSON, HTML site-gen, metrics | +| `podman` | `podman` | **inside**: `podman exec` into taler-hacktivism* | +| `git` | `git` | pin commit SHA in monitoring HTML footer | +| `rsync` | `rsync` | optional: deploy/sync suite | +| `timeout` / `mktemp` / `getent` | coreutils / glibc | helpers | + +**Status on koopa (2026-07-18):** all of the above present; rootless podman owns GOA containers. + +### Recommended (full `urls` QR group) + +| Binary | Package | Role | +|--------|---------|------| +| `qrencode` | **`qrencode`** | encode taler:// / payto:// → PNG | +| `zbarimg` | **`zbar`** | decode PNG; exact payload match | + +Without these, QR checks **WARN** and skip encode/decode (`apt install` messages in code are Debian-oriented; on Tumbleweed use zypper). + +```bash +# as root on koopa +zypper refresh +zypper in qrencode zbar +# verify as hernani: +command -v qrencode zbarimg +``` + +### Optional (not needed for host-agent default) + +| Binary / tool | When | +|---------------|------| +| `jq` | nice-to-have; suite uses python3 for JSON | +| `dig` / `bind-utils` | debug DNS; not required for default phases | +| `ssh` | only for access mode **ssh** (laptop → `KOOPA_SSH` / stage `INSIDE_SSH`); not needed for **host-podman** | +| `node` / `pnpm` + wallet-cli | **e2e** / ladder only (+ secrets) | +| `secrets.env` / admin passwords | e2e only — never system packages | + +### systemd user (no extra packages) + +- `systemd` user session + **`loginctl enable-linger hernani`** (already **Linger=yes** on koopa) +- Units: `scripts/taler-monitoring/host-agent/*.service|path|timer` + +### Caddy HTML publish (separate from monitoring run) + +- Host Caddy already installed (root service). +- Writing `/var/www/monitoring-sites`: either root `rsync` or directory owned/writable by `hernani` (you set this up). + +--- + +## firecuda-external (macOS, outside-only timer) + +| Need | Notes | +|------|--------| +| `bash`, `curl`, `python3`, `rsync`, `git` (optional) | usually Homebrew / Xcode CLT | +| **No** `podman` | phases are `urls versions` only (`SKIP_SSH=1`) | +| `qrencode` / `zbarimg` | optional for QR; `brew install qrencode zbar` if desired | + +--- + +## Franc Paysan hosts (francpaysan / stagepaysan) + +Document package lists under **`francpaysan-admin-log`** when those host-agents are installed (same idea: curl, python3, podman if inside, qrencode/zbar for full urls). diff --git a/scripts/taler-monitoring/README.md b/scripts/taler-monitoring/README.md index 578b683..7d3ae2f 100644 --- a/scripts/taler-monitoring/README.md +++ b/scripts/taler-monitoring/README.md @@ -1,5 +1,10 @@ # taler-monitoring +Host packages (koopa / runners): **[DEPENDENCIES.md](./DEPENDENCIES.md)**. +Host-agents (GOA + FP; shared report pipeline): **[host-agent/README.md](./host-agent/README.md)**. +Public HTML site-gen (same timeout/log/HTML defaults): **[site-gen/README.md](./site-gen/README.md)**. +FP install notes: `francpaysan-admin-log/docs/taler-monitoring-host-agents.md`. + Report for the **GOA** stack with boxed severity badges and **grouped test IDs** `area.group-NN` (e.g. `www.exchange-01`, `e2e.pay-03`): diff --git a/scripts/taler-monitoring/TESTS.md b/scripts/taler-monitoring/TESTS.md index fb829d0..7b667f4 100644 --- a/scripts/taler-monitoring/TESTS.md +++ b/scripts/taler-monitoring/TESTS.md @@ -18,6 +18,8 @@ Every check line has a **global** run number and a **grouped** id: | **www** | `check_urls.sh` | `exchange` `perf` `stats` `bank` `merchant` `paivana` `landing` `qr` | | **inside** | `check_inside.sh` | `ssh` `bank` `exchange` `merchant` `caddy` `load` | | **versions** | `check_versions.sh` | `outside` `inside` `compare` | +| **aptdeploy** | `check_apt_deploy.sh` | `trixie` `trixie-testing` (koopa podman apt-src merchant smoke) | +| **surface** | `check_surface.sh` | **remote-only** ecosystem / `-d` domain inventory (not in default/all/full) | | **sanity** | `check_sanity.sh` | `bank` `exchange` `merchant` | | **server** | `check_server.sh` | (flat `server-NN` or host groups) | | **e2e** | `check_e2e.sh` | `prereq` `load` `bank` `wallet` `atm` `settle` `pay` `shop` `paivana` `dig` `report` | @@ -73,12 +75,14 @@ Numbering follows **executed** checks (early skip may shift later NN inside the | Group | Checks | |-------|--------| -| **inside.ssh-** | ssh reachability / remote collect | +| **inside.host-** | **host-podman**: reachability via `podman exec` on this machine (GOA host / `INSIDE_PODMAN=1`) — not SSH | +| **inside.ssh-** | **ssh**: from laptop/remote — SSH to `KOOPA_SSH` / `INSIDE_SSH`, then podman there | | **inside.bank-** | container, libeufin, postgres, local `/config`, nginx, DNS pins | | **inside.exchange-** | container, httpd, wirewatch, aggregator, transfer, local keys, DNS | | **inside.merchant-** | container, httpd, wirewatch, depositcheck, DNS | | **inside.caddy-** | host reverse-proxy process | | **inside.load-** | host loadavg + RAM; per-container RSS/CPU | +| **inside.disk-** / **aptdeploy.disk-** / **versions.disk-** / **server.disk-** | free space via `df`: **WARN** if use ≥ `DISK_WARN_USED_PCT` (85), **ERROR** if full or use ≥ `DISK_ERR_USED_PCT` (95); host + containers | Remote lines `E|comp|LEVEL|key|detail` each become one numbered result under that component group. @@ -112,8 +116,28 @@ When filing an issue, quote the full id + label, e.g. | Area.group | Meaning | |------------|---------| | **versions.outside-** | deb.taler.net suite index | -| **versions.inside-** | packages in containers | +| **versions.host-** | host-podman: packages via local `podman exec` (`INSIDE_PODMAN=1`) | +| **versions.ssh-** | ssh: packages via `KOOPA_SSH` / `INSIDE_SSH` from laptop/remote | | **versions.compare-** | installed vs suite | +| **aptdeploy.trixie-fresh-** | fresh rebuild container `…-apt-src-trixie` | +| **aptdeploy.trixie-testing-fresh-** | fresh rebuild `…-trixie-testing` | +| **aptdeploy.trixie-upgrade-** | upgrade-track `…-trixie-upgrade` (install once, then mytops-style upgrade) | +| **aptdeploy.trixie-testing-upgrade-** | upgrade-track `…-trixie-testing-upgrade` | + +Standalone: `host-agent/run-aptdeploy.sh` (on koopa local; else `ssh koopa-external`). + +### surface (remote-only public inventory) + +```bash +./taler-monitoring.sh surface # taler.net / gnunet / taler-systems / mattermost / … +./taler-monitoring.sh -d hacktivism.ch surface # all things under that domain +``` + +- Catalog: `surface-catalog.conf` (extend as hosts are found) +- DNS, optional ICMP, TCP ports, then **protocol** checks (HTTPS/HTTP/SSH) before ERROR +- TLS cert expiry; Server-header / `/config` version hints; else **WARN: version unknown** +- OSV CVE query when package+version known → **ERROR** on hits +- Never SSH / never podman on the *targets* | **sanity.bank-** / **.exchange-** / **.merchant-** | public + optional server-side per component | | **ladder.plan-** / **.load-** / **.withdraw-** / **.pay-** / **.report-** | amount ladder (GOA ceiling or stage TESTPAYSAN `max_wire`) | | **server-** | SSH host ports / processes (flat unless grouped later) | diff --git a/scripts/taler-monitoring/check_apt_deploy.sh b/scripts/taler-monitoring/check_apt_deploy.sh new file mode 100755 index 0000000..5240e82 --- /dev/null +++ b/scripts/taler-monitoring/check_apt_deploy.sh @@ -0,0 +1,279 @@ +#!/usr/bin/env bash +# check_apt_deploy.sh — podman apt-src merchant deploy tests on koopa. +# +# Containers (default): +# Fresh (rebuild when repo versions change): +# koopa-taler-deploy-test-apt-src-trixie +# koopa-taler-deploy-test-apt-src-trixie-testing +# Upgrade track (initial install, then mytops-style apt upgrade): +# koopa-taler-deploy-test-apt-src-trixie-upgrade +# koopa-taler-deploy-test-apt-src-trixie-testing-upgrade +# +# Env: +# APT_DEPLOY_CONTAINERS "name:suite:mode …" mode=fresh|upgrade (optional, default fresh) +# APT_DEPLOY_SKIP=1 +# APT_DEPLOY_PODMAN=podman +# +set -euo pipefail +ROOT=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=lib.sh +source "$ROOT/lib.sh" + +if [ "${APT_DEPLOY_SKIP:-0}" = "1" ]; then + echo "[INFO] skip phase aptdeploy (APT_DEPLOY_SKIP=1)" + exit 0 +fi + +set_area aptdeploy +section "aptdeploy · podman apt-src merchant (fresh + upgrade tracks)" + +PODMAN_BIN="${APT_DEPLOY_PODMAN:-podman}" +DEFAULT_LIST="\ +koopa-taler-deploy-test-apt-src-trixie:trixie:fresh \ +koopa-taler-deploy-test-apt-src-trixie-testing:trixie-testing:fresh \ +koopa-taler-deploy-test-apt-src-trixie-upgrade:trixie:upgrade \ +koopa-taler-deploy-test-apt-src-trixie-testing-upgrade:trixie-testing:upgrade" +LIST="${APT_DEPLOY_CONTAINERS:-$DEFAULT_LIST}" + +if ! command -v "$PODMAN_BIN" >/dev/null 2>&1; then + err "podman" "podman binary missing ($PODMAN_BIN)" + summary + exit 1 +fi +ok "podman" "$("$PODMAN_BIN" --version 2>/dev/null | head -1)" +info "host" "$(hostname 2>/dev/null || echo unknown) · whoami=$(whoami)" + +fail_any=0 + +print_pkg_table() { + local name=$1 + local rows + rows=$("$PODMAN_BIN" exec "$name" bash -lc ' + echo "| package | version |" + echo "|---------|---------|" + for p in taler-merchant libtalermerchant libtalerexchange libdonau libgnunet \ + taler-merchant-webui taler-merchant-typst taler-terms-generator; do + v=$(dpkg-query -W -f="\${Version}" "$p" 2>/dev/null || echo missing) + echo "| $p | $v |" + done + ' 2>/dev/null || echo "| (query failed) | |") + info "pkg-table" "$name" + # emit as plain lines so HTML log keeps the table + while IFS= read -r line; do + [ -n "$line" ] || continue + info "pkg" "$line" + done <<<"$rows" +} + +check_ldd_deep() { + local name=$1 + local out + out=$("$PODMAN_BIN" exec "$name" bash -lc ' + set +e + bin=$(command -v taler-merchant-httpd) + echo "=== ldd taler-merchant-httpd ===" + ldd "$bin" 2>&1 | grep -E "libtalerutil|libdonau|libgnunet|not found" || true + echo "=== ldd libdonau (if present) ===" + for f in /usr/lib/*/libdonau.so* /usr/lib/*/libdonauutil.so*; do + [ -e "$f" ] || continue + echo "-- $f --" + ldd "$f" 2>&1 | grep -E "libtalerutil|not found" || true + done + echo "=== libtalerutil files ===" + ls -la /usr/lib/*/libtalerutil* 2>/dev/null || echo "(none)" + ' 2>/dev/null || echo "ldd probe failed") + while IFS= read -r line; do + [ -n "$line" ] || continue + info "ldd" "$line" + done <<<"$out" + if echo "$out" | grep -q 'not found'; then + err "ldd" "shared library missing in $name" "see ldd lines above" + return 1 + fi + return 0 +} + +check_systemd() { + local name=$1 + local out active_target active_httpd enabled + out=$("$PODMAN_BIN" exec "$name" bash -lc ' + set +e + echo "system: $(systemctl is-system-running 2>&1)" + echo "target_enabled: $(systemctl is-enabled taler-merchant.target 2>&1)" + echo "target_active: $(systemctl is-active taler-merchant.target 2>&1)" + echo "httpd_active: $(systemctl is-active taler-merchant-httpd.service 2>&1)" + echo "httpd_enabled: $(systemctl is-enabled taler-merchant-httpd.service 2>&1)" + systemctl start taler-merchant.target 2>&1 | tail -5 + sleep 2 + echo "after_start_target: $(systemctl is-active taler-merchant.target 2>&1)" + echo "after_start_httpd: $(systemctl is-active taler-merchant-httpd.service 2>&1)" + systemctl --failed --no-legend 2>/dev/null | grep -i taler || echo "failed_taler: none" + ' 2>/dev/null || echo "systemd probe failed") + while IFS= read -r line; do + [ -n "$line" ] || continue + info "systemd" "$line" + done <<<"$out" + + active_httpd=$(echo "$out" | sed -n 's/^after_start_httpd: //p' | tail -1) + if [ "$active_httpd" = "active" ]; then + ok "systemd httpd" "taler-merchant-httpd active after start taler-merchant.target" + return 0 + fi + # soft if target not shipped as enabled in container — still ERROR for deploy smoke + err "systemd httpd" "taler-merchant-httpd not active" "after_start_httpd=${active_httpd:-?} · see systemd lines" + return 1 +} + +check_merchant_basics() { + local name=$1 + local out code + # unix socket or curl localhost if httpd listens + out=$("$PODMAN_BIN" exec "$name" bash -lc ' + set +e + echo "=== binaries ===" + command -v taler-merchant-httpd taler-merchant-dbinit taler-config 2>/dev/null + echo "=== --version ===" + taler-merchant-httpd --version 2>&1 + echo "=== config probe ===" + # try common paths without TLS + for u in \ + http://127.0.0.1:9966/config \ + http://127.0.0.1:8081/config \ + http://127.0.0.1/config + do + code=$(curl -sS -m 2 -o /tmp/mcfg -w "%{http_code}" "$u" 2>/dev/null || echo 000) + echo "curl $u -> $code" + [ "$code" = "200" ] && head -c 120 /tmp/mcfg && echo + done + sock=$(ls /run/taler-merchant/httpd/*.sock 2>/dev/null | head -1) + if [ -n "$sock" ]; then + echo "socket: $sock" + code=$(curl -sS -m 2 --unix-socket "$sock" -o /tmp/mcfg -w "%{http_code}" http://localhost/config 2>/dev/null || echo 000) + echo "curl --unix-socket -> $code" + [ "$code" = "200" ] && head -c 160 /tmp/mcfg && echo + else + echo "socket: (none under /run/taler-merchant/httpd/)" + fi + ' 2>/dev/null || echo "basics probe failed") + + while IFS= read -r line; do + [ -n "$line" ] || continue + info "basics" "$line" + done <<<"$out" + + if echo "$out" | grep -qi 'error while loading shared libraries'; then + err "httpd" "shared library error on --version" "see basics/ldd" + return 1 + fi + if ! echo "$out" | grep -q 'taler-merchant-httpd --version\|v\.\|taler-merchant-httpd'; then + # version line varies; success if exit was ok — check via separate exec + : + fi + + set +e + "$PODMAN_BIN" exec "$name" taler-merchant-httpd --version >/dev/null 2>&1 + local ec=$? + set -e + if [ "$ec" -eq 0 ]; then + ok "httpd --version" "exit 0" + else + err "httpd --version" "exit $ec" + return 1 + fi + + # /config is nice-to-have if stack is fully configured + if echo "$out" | grep -qE 'curl .* -> 200'; then + ok "merchant /config" "HTTP 200 (local)" + else + warn "merchant /config" "no local HTTP 200 (unit may need BASE_URL/db — still report version/ldd)" + fi + return 0 +} + +check_one() { + local name="$1" expect_suite="$2" mode="${3:-fresh}" + local st suite_line pkgs + + # group ids: aptdeploy.trixie-01 / aptdeploy.trixie-testing-upgrade-01 + set_group "${expect_suite}${mode:+-$mode}" + section "aptdeploy · $name (suite=$expect_suite mode=$mode)" + + if ! "$PODMAN_BIN" container exists "$name" 2>/dev/null; then + err "container" "$name missing" "run ensure-apt-deploy-test-containers.sh" + fail_any=1 + return + fi + st=$("$PODMAN_BIN" inspect -f '{{.State.Status}}' "$name" 2>/dev/null || echo unknown) + if [ "$st" != "running" ]; then + err "container" "$name not running" "status=$st" + fail_any=1 + return + fi + ok "container" "$name running (mode=$mode)" + + suite_line=$("$PODMAN_BIN" exec "$name" bash -lc \ + 'grep -h "^Suites:" /etc/apt/sources.list.d/*taler* 2>/dev/null | head -1' 2>/dev/null || true) + if echo "$suite_line" | grep -qE "Suites:[[:space:]]*${expect_suite}([[:space:]]|$)"; then + ok "apt suite" "$expect_suite" + else + err "apt suite" "expected $expect_suite" "got: ${suite_line:-empty}" + fail_any=1 + fi + + print_pkg_table "$name" + + if ! check_merchant_basics "$name"; then + fail_any=1 + check_ldd_deep "$name" || fail_any=1 + else + # still show ldd summary as info when ok + ldd_out=$("$PODMAN_BIN" exec "$name" bash -lc \ + 'ldd "$(command -v taler-merchant-httpd)" 2>&1 | grep -E "libtalerutil|not found" || true' 2>/dev/null || true) + if echo "$ldd_out" | grep -q 'not found'; then + err "libtalerutil" "not found" "$ldd_out" + check_ldd_deep "$name" || true + fail_any=1 + else + info "libtalerutil" "$(echo "$ldd_out" | tr '\n' ' ')" + fi + fi + + if ! check_systemd "$name"; then + fail_any=1 + check_ldd_deep "$name" || true + fi + + # Disk free inside this deploy-test container (+ host once, first container only) + if [ "${_APT_DEPLOY_HOST_DISK_DONE:-0}" != "1" ]; then + set_group disk + mon_disk_check_host "host" || fail_any=1 + _APT_DEPLOY_HOST_DISK_DONE=1 + fi + set_group disk + if ! mon_disk_check_podman "$name" "$PODMAN_BIN"; then + fail_any=1 + fi +} + +for entry in $LIST; do + # name:suite or name:suite:mode + cname=${entry%%:*} + rest=${entry#*:} + case "$rest" in + *:*) + suite=${rest%%:*} + mode=${rest#*:} + ;; + *) + suite=$rest + mode=fresh + ;; + esac + check_one "$cname" "$suite" "$mode" +done + +summary +if [ "$fail_any" -ne 0 ]; then + exit 1 +fi +exit 0 diff --git a/scripts/taler-monitoring/check_inside.sh b/scripts/taler-monitoring/check_inside.sh index 0584c9b..7f62880 100755 --- a/scripts/taler-monitoring/check_inside.sh +++ b/scripts/taler-monitoring/check_inside.sh @@ -1,9 +1,14 @@ #!/usr/bin/env bash # Inside status for bank / exchange / merchant containers. +# +# Access modes (shown in the log as flags): +# host-podman — this host runs podman exec into containers (no SSH). +# INSIDE_PODMAN=1 or INSIDE_MODE=local-podman, or auto on GOA host. +# ssh — from a laptop/remote: SSH to KOOPA_SSH / INSIDE_SSH, then podman there. +# # Profiles: -# koopa (default LOCAL_STACK=1) — SSH KOOPA_SSH, containers taler-hacktivism* -# stage-lfp (TESTPAYSAN) — SSH INSIDE_SSH (francpaysan-stage-user / stagepaysan), -# low-priv podman only; ports 9030–9032; stats remain outside-in (urls). +# koopa (default LOCAL_STACK=1) — taler-hacktivism* containers +# stage-lfp (TESTPAYSAN) — SSH INSIDE_SSH (stagepaysan), low-priv podman set -euo pipefail ROOT=$(cd "$(dirname "$0")" && pwd) # shellcheck source=lib.sh @@ -12,7 +17,6 @@ source "$ROOT/lib.sh" source "$ROOT/metrics.sh" set_area inside -set_group ssh PROFILE="${INSIDE_PROFILE:-}" if [ -z "$PROFILE" ]; then @@ -25,7 +29,27 @@ if [ -z "$PROFILE" ]; then fi fi -section "inside · collect (${PROFILE})" +# Resolve host-podman vs ssh before first check ID +_use_local_podman=0 +if [ "$PROFILE" != "stage-lfp" ]; then + if [ "${INSIDE_PODMAN:-0}" = "1" ] || [ "${INSIDE_MODE:-}" = "local-podman" ]; then + _use_local_podman=1 + elif command -v podman >/dev/null 2>&1 \ + && podman ps --format '{{.Names}}' 2>/dev/null | grep -qE 'taler-hacktivism'; then + _use_local_podman=1 + fi +fi +if [ "$_use_local_podman" = "1" ]; then + INSIDE_ACCESS=host-podman + set_group host +else + INSIDE_ACCESS=ssh + set_group ssh +fi +export INSIDE_ACCESS + +section "inside · collect (${PROFILE} · access=${INSIDE_ACCESS})" +info "flags" "INSIDE_ACCESS=${INSIDE_ACCESS} INSIDE_PODMAN=${INSIDE_PODMAN:-0} INSIDE_MODE=${INSIDE_MODE:-} LOCAL_STACK=${LOCAL_STACK:-0} SKIP_SSH=${SKIP_SSH:-0} KOOPA_SSH=${KOOPA_SSH:-} INSIDE_SSH=${INSIDE_SSH:-}" # --------------------------------------------------------------------------- # stage-lfp: low-priv stagepaysan on francpaysan-host @@ -244,31 +268,70 @@ EOF info "stage host" "load probe empty" fi + set_group disk + section "inside · disk free space (stage host + containers)" + _disk_raw=$(mon_ssh_bash "$SSH_HOST" "${STAGE_SSH_T:-24}" <<'DISK' || true +set +e +echo "###HOST###" +df -Pk / /var /home /tmp /mnt/data 2>/dev/null || df -Pk +echo "###CTRS###" +for c in $(podman ps --format '{{.Names}}' 2>/dev/null); do + echo "###CTR $c###" + podman exec "$c" df -Pk / /var /tmp 2>/dev/null || podman exec "$c" df -Pk 2>/dev/null +done +DISK +) + _host_df=$(printf '%s\n' "$_disk_raw" | sed -n '/^###HOST###$/,/^###CTRS###$/p' | sed '1d;$d') + mon_disk_check_remote_text "ssh:${SSH_HOST}" "$_host_df" || true + _ctr=""; _buf="" + while IFS= read -r _line || [ -n "$_line" ]; do + case "$_line" in + '###CTR '*) + if [ -n "$_ctr" ]; then mon_disk_check_remote_text "ctr:${_ctr}" "$_buf" || true; fi + _ctr=${_line####CTR }; _ctr=${_ctr%###}; _buf="" + ;; + '###CTRS###'|'###HOST###') ;; + *) [ -n "$_ctr" ] && _buf="${_buf}${_line}"$'\n' ;; + esac + done <<<"$_disk_raw" + [ -n "$_ctr" ] && mon_disk_check_remote_text "ctr:${_ctr}" "$_buf" || true + summary exit 0 fi # --------------------------------------------------------------------------- -# koopa (default) — existing remote collect +# koopa (default) — host-podman exec (on GOA host) or SSH to KOOPA_SSH (laptop) # --------------------------------------------------------------------------- -section "inside · collect from koopa" +section "inside · collect from koopa (access=${INSIDE_ACCESS})" -if [ "${SKIP_SSH}" = "1" ] && [ "${LOCAL_STACK:-0}" != "1" ]; then - warn "ssh" "SKIP_SSH=1 and not local — skipped" +if [ "$_use_local_podman" = "1" ]; then + set_group host + if ! command -v podman >/dev/null 2>&1; then + err "host" "INSIDE_PODMAN/host-podman but podman missing" + summary + exit 1 + fi + ok "host→container" "podman exec on this host (INSIDE_ACCESS=host-podman · no SSH)" +elif [ "${SKIP_SSH}" = "1" ] && [ "${LOCAL_STACK:-0}" != "1" ]; then + set_group ssh + warn "ssh" "SKIP_SSH=1 and not host-podman — skipped" summary exit 0 -fi - -if ! koopa_ssh_ok; then - err "ssh" "cannot reach ${KOOPA_SSH} in ${SSH_CONNECT_TIMEOUT}s — set SKIP_SSH=1 to skip inside" +elif ! koopa_ssh_ok; then + set_group ssh + err "ssh" "cannot reach ${KOOPA_SSH} — from laptop use KOOPA_SSH=; on host set INSIDE_PODMAN=1" summary exit 1 +else + set_group ssh + ok "ssh ${KOOPA_SSH}" "remote host then podman (INSIDE_ACCESS=ssh · laptop/remote)" fi -ok "ssh ${KOOPA_SSH}" -# One short remote script (≤ SSH_CMD_TIMEOUT). Every slow step is local curl -m 3 or quick pgrep. -RAW=$( - koopa_ssh_bash "${SSH_CMD_TIMEOUT}" <<'REMOTE' || true +# Collect script (file → bash local or ssh bash -s) +_INSIDE_SCRIPT=$(mktemp) +trap 'rm -f "${_INSIDE_SCRIPT:-}"' RETURN +cat >"$_INSIDE_SCRIPT" <<'REMOTE' set +e emit() { printf 'E|%s|%s|%s|%s\n' "$1" "$2" "$3" "$(printf '%s' "${4:-}" | tr '\n\r' ' ' | head -c 200)"; } # quick curl @@ -376,23 +439,37 @@ else fi echo DONE REMOTE -) + +if [ "$_use_local_podman" = "1" ]; then + RAW=$(bash "$_INSIDE_SCRIPT" || true) +else + RAW=$(koopa_ssh_bash "${SSH_CMD_TIMEOUT}" <"$_INSIDE_SCRIPT" || true) +fi +rm -f "$_INSIDE_SCRIPT" if [ -z "$RAW" ] || ! echo "$RAW" | grep -q '^E|'; then - err "ssh" "remote timed out or empty (cap ${SSH_CMD_TIMEOUT}s)" + if [ "$_use_local_podman" = "1" ]; then + err "host" "collect empty (INSIDE_ACCESS=host-podman · podman exec failed)" + else + err "ssh" "collect empty (INSIDE_ACCESS=ssh · cap ${SSH_CMD_TIMEOUT}s · host=${KOOPA_SSH})" + fi summary exit 1 fi +# Fallback group for non-component rows: host (podman) vs ssh (remote) +_access_grp=ssh +[ "$_use_local_podman" = "1" ] && _access_grp=host + _last_inside_grp="" while IFS= read -r line; do case "$line" in E\|*) IFS='|' read -r _ comp level key detail <<<"$line" - # Group IDs by component so issues map cleanly: inside.bank-02, inside.exchange-04 + # Group IDs by component: inside.bank-02, inside.exchange-04; access = host|ssh case "$comp" in bank|exchange|merchant|caddy) _g="$comp" ;; - *) _g="ssh" ;; + *) _g="$_access_grp" ;; esac if [ "$_g" != "$_last_inside_grp" ]; then set_group "$_g" @@ -415,4 +492,55 @@ METRICS_DIR="${METRICS_DIR:-$(mktemp -d)}" export METRICS_DIR metrics_report_load "${METRICS_DIR}/load-inside.json" "inside" || true +# Disk free space: host + taler containers (WARN tight, ERROR full) +set_group disk +section "inside · disk free space" +_disk_ec=0 +if [ "$_use_local_podman" = "1" ]; then + mon_disk_check_host "host" || _disk_ec=1 + while read -r _cname; do + [ -n "$_cname" ] || continue + mon_disk_check_podman "$_cname" || _disk_ec=1 + done < <(podman ps --format '{{.Names}}' 2>/dev/null | grep -iE 'hacktivism|taler-|stage-lfp|lfp-' || true) +else + # remote host via SSH: df on host + each container + _disk_raw=$(koopa_ssh_bash "${SSH_CMD_TIMEOUT:-20}" <<'DISK' || true +set +e +echo "###HOST###" +df -Pk / /var /home /tmp 2>/dev/null || df -Pk +echo "###CTRS###" +for c in $(podman ps --format '{{.Names}}' 2>/dev/null); do + echo "###CTR $c###" + podman exec "$c" df -Pk / /var /tmp 2>/dev/null || podman exec "$c" df -Pk 2>/dev/null +done +DISK +) + _host_df=$(printf '%s\n' "$_disk_raw" | sed -n '/^###HOST###$/,/^###CTRS###$/p' | sed '1d;$d') + mon_disk_check_remote_text "ssh:${KOOPA_SSH:-remote}" "$_host_df" || _disk_ec=1 + _ctr="" + _buf="" + while IFS= read -r _line || [ -n "$_line" ]; do + case "$_line" in + '###CTR '*) + if [ -n "$_ctr" ]; then + mon_disk_check_remote_text "ctr:${_ctr}" "$_buf" || _disk_ec=1 + fi + _ctr=${_line####CTR } + _ctr=${_ctr%###} + _buf="" + ;; + '###CTRS###'|'###HOST###') ;; + *) + if [ -n "$_ctr" ]; then + _buf="${_buf}${_line}"$'\n' + fi + ;; + esac + done <<<"$_disk_raw" + if [ -n "$_ctr" ]; then + mon_disk_check_remote_text "ctr:${_ctr}" "$_buf" || _disk_ec=1 + fi +fi +unset _disk_raw _host_df _ctr _buf _line _cname _disk_ec + summary diff --git a/scripts/taler-monitoring/check_server.sh b/scripts/taler-monitoring/check_server.sh index 8412a12..3a9618a 100755 --- a/scripts/taler-monitoring/check_server.sh +++ b/scripts/taler-monitoring/check_server.sh @@ -110,6 +110,15 @@ if systemctl is-active caddy >/dev/null 2>&1 || pgrep -x caddy >/dev/null 2>&1; else report WARN "caddy" "not detected as active" fi + +# disk free (host + containers) — lines for mon_disk_check_remote_text on laptop side +echo "###DISK_HOST###" +df -Pk / /var /home /tmp 2>/dev/null || df -Pk +echo "###DISK_CTRS###" +for c in $(podman ps --format '{{.Names}}' 2>/dev/null); do + echo "###CTR $c###" + podman exec "$c" df -Pk / /var /tmp 2>/dev/null || podman exec "$c" df -Pk 2>/dev/null +done REMOTE ) @@ -126,4 +135,24 @@ while IFS= read -r line; do esac done <<<"$REMOTE" +set_group disk +section "server · disk free space" +_host_df=$(printf '%s\n' "$REMOTE" | sed -n '/^###DISK_HOST###$/,/^###DISK_CTRS###$/p' | sed '1d;$d') +mon_disk_check_remote_text "ssh:${KOOPA_SSH}" "$_host_df" || true +_ctr=""; _buf="" +while IFS= read -r _line || [ -n "$_line" ]; do + case "$_line" in + '###CTR '*) + if [ -n "$_ctr" ]; then mon_disk_check_remote_text "ctr:${_ctr}" "$_buf" || true; fi + _ctr=${_line####CTR }; _ctr=${_ctr%###}; _buf="" + ;; + '###DISK_CTRS###'|'###DISK_HOST###') ;; + *) + # only buffer after we entered CTR section + if [ -n "$_ctr" ]; then _buf="${_buf}${_line}"$'\n'; fi + ;; + esac +done <<<"$REMOTE" +[ -n "$_ctr" ] && mon_disk_check_remote_text "ctr:${_ctr}" "$_buf" || true + summary diff --git a/scripts/taler-monitoring/check_surface.sh b/scripts/taler-monitoring/check_surface.sh new file mode 100755 index 0000000..1452d3b --- /dev/null +++ b/scripts/taler-monitoring/check_surface.sh @@ -0,0 +1,692 @@ +#!/usr/bin/env bash +# check_surface.sh — REMOTE-ONLY public surface / ecosystem scan. +# +# NOT in default phases. Explicit: +# ./taler-monitoring.sh surface +# ./taler-monitoring.sh -d hacktivism.ch surface +# +# Without -d / with generic ecosystem: catalog of taler.net, gnunet.org, +# taler-systems.com, taler-ops, mattermost, … +# With -d DOMAIN: expand domains.conf stack hosts + catalog entries matching +# that domain + common subdomain guesses; port/protocol probes only remote. +# +# Rules: +# - no SSH into targets, no local podman on targets +# - ICMP/port alone never proves "down" — confirm via expected protocol +# - catalogued host unreachable via protocol → ERROR +# - version unknown after probes → WARN +# - CVE hit (OSV) for identified software+version → ERROR +# +set -euo pipefail +ROOT=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=lib.sh +source "$ROOT/lib.sh" + +set_area surface +section "surface · remote ecosystem / domain inventory (outside-in only)" + +CATALOG="${SURFACE_CATALOG:-$ROOT/surface-catalog.conf}" +# Common ports when scanning a host discovered without explicit list +DEFAULT_SCAN_PORTS="${SURFACE_SCAN_PORTS:-22,80,443,993,8443,9418}" +PORT_TIMEOUT="${SURFACE_PORT_TIMEOUT:-2}" +HTTP_TIMEOUT="${SURFACE_HTTP_TIMEOUT:-12}" +# CVE via OSV (public, no key). Disable: SURFACE_CVE=0 +: "${SURFACE_CVE:=1}" +# Extra ports when -d domain mode +DOMAIN_EXTRA_PORTS="${SURFACE_DOMAIN_PORTS:-22,80,443,8443}" + +declare -A HOST_PORTS # host -> comma ports +declare -A HOST_PROTO # host -> expect proto +declare -A HOST_LABEL # host -> short label +declare -A HOST_EXPECT # host -> 1 if catalogued (must respond) +ORDERED_HOSTS=() + +add_host() { + local h="$1" ports="${2:-}" proto="${3:-https}" label="${4:-}" expect="${5:-1}" + h=$(printf '%s' "$h" | tr 'A-Z' 'a-z' | sed 's#^https\?://##;s#/.*##;s/\.$//') + [ -n "$h" ] || return 0 + if [ -z "${HOST_PORTS[$h]+x}" ]; then + ORDERED_HOSTS+=("$h") + HOST_PORTS[$h]="${ports:-443}" + HOST_PROTO[$h]="${proto:-https}" + HOST_LABEL[$h]="${label:-$h}" + HOST_EXPECT[$h]="$expect" + else + # merge ports + local p + for p in ${ports//,/ }; do + case ",${HOST_PORTS[$h]}," in + *",$p,"*) ;; + *) HOST_PORTS[$h]="${HOST_PORTS[$h]},$p" ;; + esac + done + [ "${HOST_EXPECT[$h]}" = "1" ] || HOST_EXPECT[$h]="$expect" + fi +} + +load_catalog() { + local line h ports proto label + [ -f "$CATALOG" ] || { + warn "catalog" "missing $CATALOG — using empty base list" + return 0 + } + while IFS= read -r line || [ -n "$line" ]; do + line=${line%%#*} + line=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + [ -z "$line" ] && continue + # host ports proto label + h=$(echo "$line" | awk '{print $1}') + ports=$(echo "$line" | awk '{print $2}') + proto=$(echo "$line" | awk '{print $3}') + label=$(echo "$line" | awk '{print $4}') + [ "$ports" = "-" ] && ports="" + add_host "$h" "$ports" "${proto:-https}" "${label:-}" 1 + done <"$CATALOG" + info "catalog" "loaded $CATALOG · ${#ORDERED_HOSTS[@]} hosts" +} + +# Clear inventory (used when switching to domain-only scope) +clear_hosts() { + ORDERED_HOSTS=() + unset HOST_PORTS HOST_PROTO HOST_LABEL HOST_EXPECT + declare -gA HOST_PORTS HOST_PROTO HOST_LABEL HOST_EXPECT +} + +# Expand for -d domain: ONLY that domain (remote). No whole-ecosystem catalog. +expand_domain_scope() { + local d="$1" h + d=${d#https://}; d=${d%/} + d=${d#http://} + info "scope" "domain mode −d $d (remote only · not full ecosystem catalog)" + + clear_hosts + + # stack endpoints from applied profile (always expected) + for h in \ + "${BANK_PUBLIC:-}" \ + "${EXCHANGE_PUBLIC:-}" \ + "${MERCHANT_PUBLIC:-}" \ + "${PAIVANA_PUBLIC:-}" + do + [ -n "$h" ] || continue + add_host "$h" "80,443,$DOMAIN_EXTRA_PORTS" https "stack" 1 + done + + # apex + common subdomains (expected if DNS exists — set expect after DNS in scan, + # but catalogued stack already expected; guesses start optional) + for sub in "" www bank exchange taler merchant backend shop shops stage \ + git docs www2 api static media landing mon401 + do + if [ -z "$sub" ]; then + h="$d" + else + h="${sub}.${d}" + fi + # stack hosts already added as expect=1; guesses optional until we promote + if [ -z "${HOST_PORTS[$h]+x}" ]; then + add_host "$h" "$DOMAIN_EXTRA_PORTS" https "guess" 0 + fi + done + + # also pick catalog lines that belong to this domain only + if [ -f "$CATALOG" ]; then + local line ports proto label + while IFS= read -r line || [ -n "$line" ]; do + line=${line%%#*} + line=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + [ -z "$line" ] && continue + h=$(echo "$line" | awk '{print $1}') + h=$(printf '%s' "$h" | tr 'A-Z' 'a-z' | sed 's#^https\?://##;s#/.*##') + if [ "$h" = "$d" ] || [[ "$h" == *".$d" ]]; then + ports=$(echo "$line" | awk '{print $2}') + proto=$(echo "$line" | awk '{print $3}') + label=$(echo "$line" | awk '{print $4}') + [ "$ports" = "-" ] && ports="" + add_host "$h" "$ports" "${proto:-https}" "${label:-dom}" 1 + fi + done <"$CATALOG" + fi +} + +# Ecosystem mode: full catalog; all entries expected +expand_ecosystem_scope() { + info "scope" "ecosystem mode (taler.net / gnunet.org / taler-systems.com / taler-ops / mattermost, … · remote only)" + local h + for h in "${ORDERED_HOSTS[@]}"; do + HOST_EXPECT[$h]=1 + done +} + +# --- remote probes (no SSH) --- +dns_ok() { + local h="$1" + getent ahosts "$h" >/dev/null 2>&1 || getent hosts "$h" >/dev/null 2>&1 +} + +# TCP connect only (not proof of service) +tcp_open() { + local h="$1" port="$2" + if command -v timeout >/dev/null 2>&1; then + timeout "$PORT_TIMEOUT" bash -c "echo >/dev/tcp/${h}/${port}" 2>/dev/null + else + bash -c "echo >/dev/tcp/${h}/${port}" 2>/dev/null + fi +} + +# ICMP optional — never sole grounds for ERROR +ping_host() { + local h="$1" + ping -c 1 -W 2 "$h" >/dev/null 2>&1 || ping -c 1 -w 2 "$h" >/dev/null 2>&1 +} + +# HTTPS probe: code, server header, version hints, cert subject/dates +probe_https() { + local h="$1" port="${2:-443}" + local url hdr body code server via cert_end subj san + url="https://${h}/" + [ "$port" != "443" ] && url="https://${h}:${port}/" + body=$(mktemp) + hdr=$(mktemp) + code=$(curl -skS -m "$HTTP_TIMEOUT" -D "$hdr" -o "$body" -w '%{http_code}' \ + --connect-timeout "$PORT_TIMEOUT" "$url" 2>/dev/null || echo 000) + server=$(awk 'BEGIN{IGNORECASE=1} /^server:/{sub(/\r$/,""); sub(/^server:[[:space:]]*/,""); print; exit}' "$hdr" 2>/dev/null || true) + via=$(awk 'BEGIN{IGNORECASE=1} /^x-powered-by:/{sub(/\r$/,""); sub(/^[^:]+:[[:space:]]*/,""); print; exit}' "$hdr" 2>/dev/null || true) + # cert + cert_end=$(echo | openssl s_client -servername "$h" -connect "${h}:${port}" 2>/dev/null \ + | openssl x509 -noout -enddate 2>/dev/null | sed 's/notAfter=//') + subj=$(echo | openssl s_client -servername "$h" -connect "${h}:${port}" 2>/dev/null \ + | openssl x509 -noout -subject 2>/dev/null | head -1) + # taler version hints in body + local taler_hint + taler_hint=$(grep -oE 'taler[^"[:space:]]{0,40}|GNU Taler|libeufin|gnunet' "$body" 2>/dev/null | head -3 | tr '\n' ' ' || true) + # config JSON version if /config works + local cfg_ver="" + local ccode + ccode=$(curl -skS -m "$HTTP_TIMEOUT" -o "$body" -w '%{http_code}' \ + "https://${h}/config" 2>/dev/null || echo 000) + if [ "$ccode" = "200" ]; then + cfg_ver=$(python3 -c 'import json,sys +try: + d=json.load(open(sys.argv[1])) + print(d.get("version") or d.get("name") or d.get("currency") or "config-json") +except Exception: + print("")' "$body" 2>/dev/null || true) + fi + rm -f "$hdr" "$body" + printf 'code=%s server=%s powered=%s cert_end=%s cfg=%s hint=%s' \ + "$code" "${server:-}" "${via:-}" "${cert_end:-}" "${cfg_ver:-}" "${taler_hint:-}" + # return 0 if HTTP answered (any 2xx/3xx/4xx — service is up) + case "$code" in + 2??|3??|4??) return 0 ;; + *) return 1 ;; + esac +} + +probe_http() { + local h="$1" port="${2:-80}" + local url code + url="http://${h}/" + [ "$port" != "80" ] && url="http://${h}:${port}/" + code=$(curl -sS -m "$HTTP_TIMEOUT" -o /dev/null -w '%{http_code}' \ + --connect-timeout "$PORT_TIMEOUT" "$url" 2>/dev/null || echo 000) + printf 'code=%s' "$code" + case "$code" in 2??|3??|4??) return 0 ;; *) return 1 ;; esac +} + +probe_ssh_banner() { + local h="$1" port="${2:-22}" + local ban + ban=$(timeout "$PORT_TIMEOUT" bash -c "exec 3<>/dev/tcp/${h}/${port}; dd bs=256 count=1 <&3 2>/dev/null" 2>/dev/null \ + | tr -d '\r' | head -1 || true) + printf 'banner=%s' "${ban:-}" + [ -n "$ban" ] +} + +# Map Server header → package name guess for OSV +guess_package() { + local server="$1" + local s + s=$(printf '%s' "$server" | tr 'A-Z' 'a-z') + case "$s" in + *nginx*) echo "nginx" ;; + *apache*|*httpd*) echo "apache" ;; + *caddy*) echo "caddy" ;; + *openbsd\ httpd*) echo "openbsd-httpd" ;; + *) echo "" ;; + esac +} + +extract_version() { + local server="$1" + # nginx/1.22.1 → 1.22.1 + printf '%s' "$server" | sed -n 's/.*\/\([0-9][0-9.]*\).*/\1/p' | head -1 +} + +# OSV / CVE check for software version taken from Server headers. +# +# Important: headers only expose upstream versions (nginx/1.26.3), never the +# Debian package revision (1.26.3-3+deb13u7). Querying OSV ecosystem=Debian +# with the bare version yields massive false positives (ancient DEBIAN-CVE-* +# with introduced:0 and no fixed event, plus every package revision that +# merely *starts with* 1.26.3). +# +# Policy: +# - bare upstream version → upstream SEMVER signal only (WARN by default) +# - full Debian package version (contains '-') → Debian OSV, filtered +# - SURFACE_CVE=0 disables; SURFACE_CVE_LEVEL=error|warn (default warn for bare) +check_cves() { + local pkg="$1" ver="$2" host="$3" + [ "$SURFACE_CVE" = "1" ] || return 0 + [ -n "$pkg" ] && [ -n "$ver" ] || return 0 + local level out rc + # bare header versions are approximate → WARN unless overridden + if [[ "$ver" == *-* ]] || [[ "$ver" == *+* ]]; then + level="${SURFACE_CVE_LEVEL:-error}" + else + level="${SURFACE_CVE_LEVEL:-warn}" + fi + out=$( + SURFACE_CVE_PKG="$pkg" SURFACE_CVE_VER="$ver" SURFACE_CVE_HOST="$host" python3 - <<'PY' 2>/dev/null || true +import json, os, re, urllib.request + +pkg = os.environ.get("SURFACE_CVE_PKG", "") +ver = os.environ.get("SURFACE_CVE_VER", "") +host = os.environ.get("SURFACE_CVE_HOST", "") + +def ver_tuple(s: str): + nums = [int(x) for x in re.findall(r"\d+", s or "")[:5]] + return tuple(nums) if nums else () + +def vt_cmp(a, b): + n = max(len(a), len(b)) + a = a + (0,) * (n - len(a)) + b = b + (0,) * (n - len(b)) + return (a > b) - (a < b) + +def in_semver_events(ver, events): + """True if ver is in [introduced, fixed) for SEMVER-like event list.""" + vt = ver_tuple(ver) + if not vt: + return False + # process sequential introduced/fixed pairs + intro = None + for e in events or []: + if "introduced" in e: + intro = e.get("introduced") + elif "fixed" in e or "last_affected" in e: + fixed = e.get("fixed") + last = e.get("last_affected") + lo = ver_tuple("0" if intro in (None, "0") else str(intro)) + if vt_cmp(vt, lo) < 0: + intro = None + continue + if fixed is not None: + if vt_cmp(vt, ver_tuple(str(fixed))) < 0: + return True + elif last is not None: + if vt_cmp(vt, ver_tuple(str(last))) <= 0: + return True + intro = None + # open-ended introduced without fixed → ignore (never clears; Debian noise) + return False + +def osv_post(body): + req = urllib.request.Request( + "https://api.osv.dev/v1/query", + data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=20) as r: + return json.load(r) + except Exception: + return {} + +def osv_get(vid): + try: + with urllib.request.urlopen(f"https://api.osv.dev/v1/vulns/{vid}", timeout=15) as r: + return json.load(r) + except Exception: + return {} + +bare = not (("-" in ver) or ("+" in ver)) +hits = [] + +if bare: + # Upstream signal via CVE records that carry extracted_events for this package + # (Server header has no Debian revision — do NOT use ecosystem=Debian). + seed = { + "nginx": [ + "CVE-2025-23419", + "CVE-2025-53859", + "CVE-2024-7347", + "CVE-2024-34161", + "CVE-2024-32760", + "CVE-2024-31079", + "CVE-2024-24989", + "CVE-2024-24990", + ], + "apache": [ + "CVE-2024-38474", + "CVE-2024-38476", + "CVE-2024-38477", + "CVE-2023-31122", + "CVE-2023-43622", + ], + "caddy": [ + "CVE-2022-29718", + "CVE-2023-50463", + ], + }.get(pkg, []) + for vid in seed: + doc = osv_get(vid) + if not doc: + continue + ok_hit = False + for a in doc.get("affected") or []: + # prefer extracted_events (human SEMVER) on GIT/nginx ranges + for rg in a.get("ranges") or []: + db = rg.get("database_specific") or {} + extracted = db.get("extracted_events") or [] + if extracted and in_semver_events(ver, extracted): + # package name filter when present + cpes = db.get("cpe") or [] + if isinstance(cpes, str): + cpes = [cpes] + blob = json.dumps(a).lower() + json.dumps(cpes).lower() + if pkg == "nginx" and "nginx" not in blob and "f5" not in blob: + continue + ok_hit = True + break + if rg.get("type") == "SEMVER" and in_semver_events(ver, rg.get("events") or []): + ok_hit = True + break + if ok_hit: + break + if ok_hit: + hits.append(vid) +else: + # Full package version — Debian ecosystem is meaningful + data = osv_post({"package": {"name": pkg, "ecosystem": "Debian"}, "version": ver}) + for v in data.get("vulns") or []: + vid = v.get("id") or "?" + actionable = False + for a in v.get("affected") or []: + for rg in a.get("ranges") or []: + events = rg.get("events") or [] + has_end = any(("fixed" in e) or ("last_affected" in e) for e in events) + if not has_end: + continue # open-ended Debian noise + if rg.get("type") in ("ECOSYSTEM", "SEMVER"): + # version string is full deb version; trust OSV query match + # but only if a fixed event exists (actionable) + actionable = True + break + if actionable: + break + # exact version listed + versions = a.get("versions") or [] + if ver in versions: + actionable = True + break + if actionable: + hits.append(vid) + +# de-dup preserve order +seen = set() +uniq = [] +for h in hits: + if h not in seen: + seen.add(h) + uniq.append(h) + +if uniq: + print(f"HIT {len(uniq)} " + ",".join(uniq[:12])) +else: + print("CLEAN") +PY + ) + rc=0 + case "$out" in + HIT\ *) + local n ids + n=$(printf '%s' "$out" | awk '{print $2}') + ids=$(printf '%s' "$out" | cut -d' ' -f3-) + if [ "$level" = "error" ]; then + err "cve" "$host $pkg $ver — $n actionable vuln(s)" "$ids" + rc=1 + else + warn "cve" "$host $pkg $ver — $n actionable vuln(s) (header version · not Debian pkg)" "$ids" + rc=0 + fi + ;; + CLEAN) + if [[ "$ver" == *-* ]] || [[ "$ver" == *+* ]]; then + info "cve" "$host $pkg $ver — OSV clean (Debian package version)" + else + info "cve" "$host $pkg $ver — no actionable upstream CVE for bare Server-header version" + fi + ;; + *) + info "cve" "$host $pkg $ver — CVE probe skipped/unavailable" + ;; + esac + return "$rc" +} + +cert_expiry_check() { + local h="$1" end="$2" + [ -n "$end" ] || return 0 + local end_epoch now left days + end_epoch=$(date -d "$end" +%s 2>/dev/null || date -j -f "%b %e %T %Y %Z" "$end" +%s 2>/dev/null || echo 0) + now=$(date +%s) + [ "$end_epoch" -gt 0 ] || return 0 + left=$((end_epoch - now)) + days=$((left / 86400)) + if [ "$left" -le 0 ]; then + err "tls" "$h certificate EXPIRED" "notAfter=$end" + return 1 + fi + if [ "$days" -le 14 ]; then + warn "tls" "$h certificate expires in ${days}d" "notAfter=$end" + else + info "tls" "$h cert ok · ${days}d left · notAfter=$end" + fi + return 0 +} + +scan_host() { + local h="$1" + local ports proto label expect + local p open_ports=() any_proto_ok=0 ping_ok=0 dns=0 + local probe_detail server_hdr pkg ver + + ports=${HOST_PORTS[$h]:-443} + proto=${HOST_PROTO[$h]:-https} + label=${HOST_LABEL[$h]:-$h} + expect=${HOST_EXPECT[$h]:-0} + + set_group "$label" + + if dns_ok "$h"; then + dns=1 + ok "dns" "$h resolves" + else + if [ "$expect" = "1" ]; then + err "dns" "$h does not resolve (catalogued)" + else + info "dns" "$h no resolve (optional guess) — skip" + fi + return 0 + fi + + if ping_host "$h"; then + ping_ok=1 + info "ping" "$h ICMP ok" + else + info "ping" "$h ICMP no reply (not decisive)" + fi + + # port scan (unique ports) + local _seen_ports=" " + for p in ${ports//,/ }; do + [ -n "$p" ] || continue + case "$_seen_ports" in *" $p "*) continue ;; esac + _seen_ports="$_seen_ports$p " + if tcp_open "$h" "$p"; then + open_ports+=("$p") + info "port" "$h:$p open (TCP)" + else + info "port" "$h:$p closed/filtered (TCP)" + fi + done + + # Protocol verification on open ports (and always try 443/80 for https/http expect) + local try_ports=("${open_ports[@]}") + if [ ${#try_ports[@]} -eq 0 ]; then + # still try expected protocol ports even if scan said closed (scan false negatives) + case "$proto" in + https) try_ports=(443) ;; + http) try_ports=(80) ;; + ssh) try_ports=(22) ;; + *) try_ports=(443 80) ;; + esac + fi + + server_hdr="" + ver="" + for p in "${try_ports[@]}"; do + case "$p" in + 443|8443) + if probe_detail=$(probe_https "$h" "$p"); then + any_proto_ok=1 + ok "https" "$h:$p up · $probe_detail" + server_hdr=$(printf '%s' "$probe_detail" | sed -n 's/.*server=\([^ ]*\).*/\1/p') + # cert + local cend + cend=$(printf '%s' "$probe_detail" | sed -n 's/.*cert_end=\([^ ]*\).*/\1/p') + cert_expiry_check "$h" "$cend" || true + local cfg + cfg=$(printf '%s' "$probe_detail" | sed -n 's/.*cfg=\([^ ]*\).*/\1/p') + if [ -n "$cfg" ]; then + info "version" "$h config/api hint: $cfg" + ver="$cfg" + fi + else + info "https" "$h:$p no HTTP response · ${probe_detail:-}" + fi + ;; + 80|8080) + if probe_detail=$(probe_http "$h" "$p"); then + any_proto_ok=1 + ok "http" "$h:$p up · $probe_detail" + else + info "http" "$h:$p no HTTP response" + fi + ;; + 22) + if probe_detail=$(probe_ssh_banner "$h" "$p"); then + any_proto_ok=1 + ok "ssh" "$h:$p banner · $probe_detail" + ver=$(printf '%s' "$probe_detail" | sed 's/banner=//') + else + info "ssh" "$h:$p no SSH banner" + fi + ;; + *) + if tcp_open "$h" "$p"; then + info "tcp" "$h:$p open · protocol unknown" + fi + ;; + esac + done + + # Version summary + if [ -n "$server_hdr" ]; then + info "server-header" "$h · $server_hdr" + pkg=$(guess_package "$server_hdr") + ver_soft=$(extract_version "$server_hdr") + if [ -n "$pkg" ] && [ -n "$ver_soft" ]; then + info "version" "$h software $pkg $ver_soft (from Server header)" + check_cves "$pkg" "$ver_soft" "$h" || true + elif [ -n "$server_hdr" ]; then + warn "version" "$h Server header present but version unknown · $server_hdr" + fi + elif [ -n "$ver" ]; then + info "version" "$h · $ver" + else + if [ "$any_proto_ok" = "1" ]; then + warn "version" "$h reachable but software version unknown" + fi + fi + + # Expected service must answer protocol (not just ping/port) + # Note: ${array[*]:-x} is NOT valid default syntax (bash treats : as slice). + local ports_txt="${open_ports[*]}" + ports_txt=${ports_txt:-none} + if [ "$expect" = "1" ]; then + if [ "$any_proto_ok" = "1" ]; then + ok "reachability" "$h catalogued service OK (protocol confirmed)" + else + err "reachability" "$h catalogued but not reachable via ${proto}/protocol" \ + "dns=$dns ping=$ping_ok open_ports=${ports_txt} (ICMP/TCP alone not enough; protocol failed)" + fi + else + if [ "$any_proto_ok" = "1" ]; then + info "reachability" "$h optional host responds" + else + info "reachability" "$h optional · no protocol response (ok)" + fi + fi +} + +# --- main --- +# Scope (remote only): +# ./taler-monitoring.sh surface → full ecosystem catalog +# ./taler-monitoring.sh -d hacktivism.ch surface → only that domain +if [ "${SURFACE_SCOPE:-}" = "ecosystem" ]; then + load_catalog + expand_ecosystem_scope +elif [ "${SURFACE_SCOPE:-}" = "domain" ] || [ "${TALER_DOMAIN_FROM_CLI:-0}" = "1" ]; then + # domain mode: do not load whole ecosystem first + CATALOG="${SURFACE_CATALOG:-$ROOT/surface-catalog.conf}" + expand_domain_scope "${TALER_DOMAIN:-hacktivism.ch}" +elif [ "${SURFACE_SCOPE:-auto}" = "auto" ]; then + # no -d → ecosystem; with -d → domain (TALER_DOMAIN_FROM_CLI) + if [ "${TALER_DOMAIN_FROM_CLI:-0}" = "1" ]; then + CATALOG="${SURFACE_CATALOG:-$ROOT/surface-catalog.conf}" + expand_domain_scope "$TALER_DOMAIN" + else + load_catalog + expand_ecosystem_scope + fi +else + load_catalog + expand_ecosystem_scope +fi + +info "inventory" "${#ORDERED_HOSTS[@]} hosts to probe (remote only · no SSH · no local podman on targets)" +info "flags" "SURFACE_CVE=${SURFACE_CVE:-1} PORT_TIMEOUT=${PORT_TIMEOUT:-2} HTTP_TIMEOUT=${HTTP_TIMEOUT:-12}" + +for h in "${ORDERED_HOSTS[@]}"; do + [ -n "$h" ] || continue + scan_host "$h" || true +done + +# mon_disk on the *runner* only (optional) — not remote servers' disks +if [ "${SURFACE_CHECK_RUNNER_DISK:-0}" = "1" ]; then + set_group disk + if declare -F mon_disk_check_host >/dev/null 2>&1; then + mon_disk_check_host "runner" || true + else + warn "disk" "mon_disk_check_host not available" + fi +fi + +# summary returns non-zero when FAIL_N>0; do not trip set -e mid-script +summary || true +if [ "${FAIL_N:-0}" -eq 0 ]; then + exit 0 +fi +exit 1 diff --git a/scripts/taler-monitoring/check_versions.sh b/scripts/taler-monitoring/check_versions.sh index 3580732..8fc7c66 100755 --- a/scripts/taler-monitoring/check_versions.sh +++ b/scripts/taler-monitoring/check_versions.sh @@ -166,16 +166,16 @@ fi # --------------------------------------------------------------------------- # 2) INSIDE — containers can reach deb.taler.net (pasta / install path) -# koopa: KOOPA_SSH · stage-lfp: INSIDE_SSH (stagepaysan, low-priv) +# host-podman: INSIDE_PODMAN=1 on GOA host · ssh: KOOPA_SSH / INSIDE_SSH from laptop # --------------------------------------------------------------------------- -set_group inside -section "versions · inside · containers → deb.taler.net" - +_VERS_LOCAL=0 _VERS_SSH_HOST="" _VERS_BANK="taler-hacktivism-bank" _VERS_EX="taler-hacktivism-exchange-ansible" _VERS_MER="taler-hacktivism" -if [ "${INSIDE_PROFILE:-}" = "stage-lfp" ] \ +if [ "${INSIDE_PODMAN:-0}" = "1" ] || [ "${INSIDE_MODE:-}" = "local-podman" ]; then + _VERS_LOCAL=1 +elif [ "${INSIDE_PROFILE:-}" = "stage-lfp" ] \ || [ "${EXPECT_CURRENCY:-}" = "TESTPAYSAN" ] \ || [ -n "${INSIDE_SSH:-}" ]; then _VERS_SSH_HOST="${INSIDE_SSH:-francpaysan-stage-user}" @@ -183,24 +183,44 @@ if [ "${INSIDE_PROFILE:-}" = "stage-lfp" ] \ _VERS_EX="${INSIDE_EXCHANGE_CTR:-stage-lfp-exchange-ansible}" _VERS_MER="${INSIDE_MERCHANT_CTR:-stage-lfp-merchant}" elif [ "${LOCAL_STACK:-0}" = "1" ] && [ "${SKIP_SSH:-0}" != "1" ]; then + # laptop → koopa via SSH (not host-podman) _VERS_SSH_HOST="${KOOPA_SSH}" fi -if [ -z "$_VERS_SSH_HOST" ]; then - info "ssh" "no container SSH for this domain — outside-only package index" +if [ "$_VERS_LOCAL" = "1" ]; then + VERS_ACCESS=host-podman + set_group host +else + VERS_ACCESS=ssh + set_group ssh +fi + +section "versions · inside · containers → deb.taler.net (access=${VERS_ACCESS})" +info "flags" "VERS_ACCESS=${VERS_ACCESS} INSIDE_PODMAN=${INSIDE_PODMAN:-0} INSIDE_MODE=${INSIDE_MODE:-} LOCAL_STACK=${LOCAL_STACK:-0} SKIP_SSH=${SKIP_SSH:-0} KOOPA_SSH=${KOOPA_SSH:-} INSIDE_SSH=${INSIDE_SSH:-}" + +if [ "$_VERS_LOCAL" != "1" ] && [ -z "$_VERS_SSH_HOST" ]; then + info "access" "no host-podman and no SSH target — outside-only package index" info "outside-only" "deb.taler.net public checks completed above" summary exit 0 fi -if [ "$_VERS_SSH_HOST" = "${KOOPA_SSH:-}" ] || [ "${INSIDE_PROFILE:-}" = "koopa" ]; then - if ! koopa_ssh_ok; then - err "ssh" "cannot reach ${KOOPA_SSH} — set SKIP_SSH=1 to skip container checks" +if [ "$_VERS_LOCAL" = "1" ]; then + if ! command -v podman >/dev/null 2>&1; then + err "host" "INSIDE_PODMAN/host-podman but podman missing" info "outside-only" "deb.taler.net public checks completed above" summary exit 1 fi - ok "ssh ${KOOPA_SSH}" + ok "host→container" "podman exec on this host (VERS_ACCESS=host-podman · no SSH)" +elif [ "$_VERS_SSH_HOST" = "${KOOPA_SSH:-}" ] || [ "${INSIDE_PROFILE:-}" = "koopa" ]; then + if ! koopa_ssh_ok; then + err "ssh" "cannot reach ${KOOPA_SSH} — from laptop set KOOPA_SSH=; on host set INSIDE_PODMAN=1" + info "outside-only" "deb.taler.net public checks completed above" + summary + exit 1 + fi + ok "ssh ${KOOPA_SSH}" "remote then podman (VERS_ACCESS=ssh)" else if ! mon_ssh_ok "$_VERS_SSH_HOST"; then err "ssh" "cannot reach ${_VERS_SSH_HOST} (stagepaysan) — skip container package compare" @@ -208,7 +228,7 @@ else summary exit 1 fi - ok "ssh ${_VERS_SSH_HOST}" "stagepaysan container package compare" + ok "ssh ${_VERS_SSH_HOST}" "stagepaysan (VERS_ACCESS=ssh)" fi INRELEASE_URL="${APT_BASE}/dists/${SUITE}/InRelease" @@ -257,7 +277,9 @@ done REMOTE } >"$tmp/remote-versions.sh" -if [ "$_VERS_SSH_HOST" = "${KOOPA_SSH:-}" ] || [ "${INSIDE_PROFILE:-}" = "koopa" ]; then +if [ "$_VERS_LOCAL" = "1" ]; then + REMOTE_OUT=$(bash "$tmp/remote-versions.sh" 2>/dev/null || true) +elif [ "$_VERS_SSH_HOST" = "${KOOPA_SSH:-}" ] || [ "${INSIDE_PROFILE:-}" = "koopa" ]; then REMOTE_OUT=$(koopa_ssh_bash 60 <"$tmp/remote-versions.sh" || true) else REMOTE_OUT=$(mon_ssh_bash "$_VERS_SSH_HOST" 60 <"$tmp/remote-versions.sh" || true) @@ -501,4 +523,41 @@ done info "suite" "deb.taler.net ${SUITE} (testing_index=${HAVE_TESTING})" info "tally" "match=${ok_n} ahead=${ahead_n} behind=${behind_n}" +# Disk free on the machine(s) we just probed (host-podman or SSH target) +set_group disk +section "versions · disk free space" +if [ "${_VERS_LOCAL:-0}" = "1" ]; then + mon_disk_check_host "host" || true + while read -r _vc; do + [ -n "$_vc" ] || continue + mon_disk_check_podman "$_vc" || true + done < <(podman ps --format '{{.Names}}' 2>/dev/null | grep -iE 'hacktivism|taler-|stage-lfp|lfp-' || true) +elif [ -n "${_VERS_SSH_HOST:-}" ]; then + _vdisk=$(mon_ssh_bash "$_VERS_SSH_HOST" 20 <<'DISK' || true +set +e +echo "###HOST###" +df -Pk / /var /home /tmp 2>/dev/null || df -Pk +echo "###CTRS###" +for c in $(podman ps --format '{{.Names}}' 2>/dev/null); do + echo "###CTR $c###" + podman exec "$c" df -Pk / /var /tmp 2>/dev/null || true +done +DISK +) + mon_disk_check_remote_text "ssh:${_VERS_SSH_HOST}" \ + "$(printf '%s\n' "$_vdisk" | sed -n '/^###HOST###$/,/^###CTRS###$/p' | sed '1d;$d')" || true + _ctr=""; _buf="" + while IFS= read -r _line || [ -n "$_line" ]; do + case "$_line" in + '###CTR '*) + [ -n "$_ctr" ] && mon_disk_check_remote_text "ctr:${_ctr}" "$_buf" || true + _ctr=${_line####CTR }; _ctr=${_ctr%###}; _buf="" + ;; + '###CTRS###'|'###HOST###') ;; + *) [ -n "$_ctr" ] && _buf="${_buf}${_line}"$'\n' ;; + esac + done <<<"$_vdisk" + [ -n "$_ctr" ] && mon_disk_check_remote_text "ctr:${_ctr}" "$_buf" || true +fi + summary diff --git a/scripts/taler-monitoring/host-agent/README.md b/scripts/taler-monitoring/host-agent/README.md new file mode 100644 index 0000000..3acfe59 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/README.md @@ -0,0 +1,183 @@ +# host-agent — reporting generation (GOA + FP + any stack) + +**Global pipeline** lives in **`run-host-report.sh`** (all stacks share it): + +| Behaviour | Default | +|-----------|---------| +| Suite refresh | `update-suite.sh` → `origin/main` | +| Wall clock | `RUN_TIMEOUT=600` | +| Log | line-buffered `run-*.log` (continuous write) | +| HTML | always (first run / empty tree too) + commit URL | +| Timeout notice | end of log + top jump on `/monitoring_err/` | + +Thin wrappers only set stack defaults: + +| Wrapper | Stack | Typical user | +|---------|-------|----------------| +| `run-hacktivism-monitoring.sh` | GOA `hacktivism.ch` | `hernani` @ koopa | +| `run-fp-stage-monitoring.sh` | FP stage | `stagepaysan` | +| `run-fp-prod-monitoring.sh` | FP prod | `francpaysan` | +| `run-aptdeploy.sh` | CLI: ensure + **aptdeploy** (no HTML) | local if hostname=koopa, else **ssh koopa-external** | +| `run-aptdeploy-monitoring.sh` | **aptdeploy** → `/taler-monitoring-aptdeploy*` | `hernani` @ koopa (4h timer, taler only) | +| `run-surface-monitoring.sh` | **surface** → `/taler-monitoring-surface*` | `hernani` @ koopa (hourly, taler only) | + +### apt-src deploy tests (hacktivism `/monitoring`) + +| Container | Mode | +|-----------|------| +| `koopa-taler-deploy-test-apt-src-trixie` | **fresh**: recreate when repo package fingerprint changes | +| `koopa-taler-deploy-test-apt-src-trixie-testing` | **fresh** for `trixie-testing` | +| `koopa-taler-deploy-test-apt-src-trixie-upgrade` | **upgrade**: initial install, then mytops-style `apt install` + `upgrade` | +| `koopa-taler-deploy-test-apt-src-trixie-testing-upgrade` | **upgrade** for testing suite | + +```bash +# from laptop / clementine → triggers on koopa-external +./scripts/taler-monitoring/host-agent/run-aptdeploy.sh +./scripts/taler-monitoring/host-agent/run-aptdeploy.sh upgrade +./scripts/taler-monitoring/host-agent/run-aptdeploy.sh fresh +./scripts/taler-monitoring/host-agent/run-aptdeploy.sh check-only + +# on koopa +./taler-monitoring.sh -d hacktivism.ch aptdeploy +``` + +Checks: package table, `httpd --version`, libtalerutil/`ldd` on failure, `systemctl` for +`taler-merchant.target`, basic local `/config` probe. No nginx/HTTPS in containers. + +### surface + aptdeploy HTML (taler.hacktivism.ch only) + +| URL | Role | Timer | +|-----|------|-------| +| `…/taler-monitoring-surface/` | ecosystem surface scan | 1h | +| `…/taler-monitoring-surface_err/` | errors + jump list | ″ | +| `…/taler-monitoring-aptdeploy/` | apt-src container deploy tests | 4h | +| `…/taler-monitoring-aptdeploy_err/` | errors + jump list | ″ | + +Not on bank/exchange. Staging: +`~/monitoring-sites-staging/taler.hacktivism.ch/taler-monitoring-{surface,aptdeploy}*`. + +```bash +systemctl --user enable --now taler-monitoring-surface.timer +systemctl --user enable --now taler-monitoring-aptdeploy.timer +systemctl --user start taler-monitoring-surface.service +systemctl --user start taler-monitoring-aptdeploy.service +# Caddy: handles only in taler.hacktivism.ch block (root) +``` + +Env examples: **`env/*.env.example`**. FP docs: `francpaysan-admin-log/docs/taler-monitoring-host-agents.md`. + +## GOA (koopa) — host-podman + +Runs **as user `hernani`** (systemd user, linger). **Primary** mode: + +- **`INSIDE_PODMAN=1`** / **`INSIDE_MODE=local-podman`**: access mode **`host-podman`** — + `podman exec` on this host into + `taler-hacktivism-bank` / `taler-hacktivism-exchange-ansible` / `taler-hacktivism` + (IDs `inside.host-*`, not `inside.ssh-*`). + From a **laptop**, leave `INSIDE_PODMAN` off → access mode **`ssh`** via `KOOPA_SSH` + (`inside.ssh-*`). +- Public **`urls` + `versions`** for the stack (`-d hacktivism.ch`). +- Writes console HTML for all three sites under + `~/monitoring-sites-staging/{bank,exchange,taler}.hacktivism.ch/`. + +Caddy (host root) can serve that tree at `/monitoring` — you configure Caddy. + +## Latest software + config isolation + +**Source of truth is always Forgejo** (`git.hacktivism.ch`), not a laptop rsync copy. + +Every host-agent / aptdeploy run: + +1. **`update-suite.sh`**: `git fetch` + **`SUITE_UPDATE_MODE=reset`** (default) → `origin/main` + into `SUITE_DIR` (prefer `~/src/koopa-admin-log`). +2. **Re-exec** host-agent scripts from that tree so the running code matches `COMMIT_URL`. +3. Local overrides only in **`~/.config/taler-monitoring/env`** — never overwritten by git. +4. HTML footer = exact commit: + `https://git.hacktivism.ch/hernani/koopa-admin-log/src/commit/` +5. **`RUN_TIMEOUT=600`**, line-buffered logs, HTML always written. + +If fetch fails → previous tree + WARN. +**To ship features (aptdeploy, surface, …): push to `origin/main`.** Local-only edits vanish on the next reset. + +If pull fails (no network / auth), the previous tree is used and a WARN is logged. + +## Install (on koopa as hernani) + +```bash +# once: local config (optional) +mkdir -p ~/.config/taler-monitoring +cp ~/koopa-admin-log/scripts/taler-monitoring/host-agent/env.example \ + ~/.config/taler-monitoring/env + +# laptop: +./scripts/taler-monitoring/host-agent/install-host-agent.sh --remote koopa-external + +# or on koopa: +~/koopa-admin-log/scripts/taler-monitoring/host-agent/install-host-agent.sh +``` + +Units: + +| Unit | Role | +|------|------| +| `taler-monitoring-hacktivism.path` | fires on software stamp + landing/caddy config changes | +| `taler-monitoring-hacktivism.timer` | every **4h** fallback | +| `taler-monitoring-hacktivism.service` | oneshot → `/monitoring*` (urls inside versions) | +| `taler-monitoring-surface.timer` | every **1h** | +| `taler-monitoring-surface.service` | → `/taler-monitoring-surface*` (taler only) | +| `taler-monitoring-aptdeploy.timer` | every **4h** | +| `taler-monitoring-aptdeploy.service` | → `/taler-monitoring-aptdeploy*` (taler only) | + +## After package / image / landing upgrades + +```bash +~/koopa-admin-log/scripts/taler-monitoring/host-agent/touch-software-stamp.sh +# → path unit starts the service +``` + +## Manual + +```bash +systemctl --user start taler-monitoring-hacktivism.service +systemctl --user status taler-monitoring-hacktivism.service --no-pager +journalctl --user -u taler-monitoring-hacktivism.service -n 50 --no-pager +ls ~/monitoring-sites-staging/bank.hacktivism.ch/monitoring_err/ +``` + +## Relation to firecuda outside-only timer + +| | host-agent (koopa) | firecuda site-gen | +|--|--------------------|-------------------| +| Access | **podman on host** | public HTTPS only | +| Phases | urls **inside** versions | urls versions | +| Trigger | software path + 4h | *could* launchd 4h | +| Primary for GOA | **yes** | **off** (optional only) | + +**firecuda-external is not scheduled** right now (launchd disabled). Scripts under +`site-gen/` can still one-shot a public run if needed; see `site-gen/README.md`. + +No secrets required for default phases. `secrets.env` optional for e2e later. + +## Git commit on HTML pages + +Header of each generated page includes a link like: + +`https://git.hacktivism.ch/hernani/koopa-admin-log/src/commit/` + +That SHA is the tree **after** `update-suite.sh` for that run. + +## Packages on koopa + +See **[../DEPENDENCIES.md](../DEPENDENCIES.md)**. + +**Default host-agent needs no new packages** if curl/python3/podman/git are present (they are). + +**Root once** only for full QR checks in `urls`: + +```bash +zypper in qrencode zbar +``` + +Without them, monitoring still runs; QR group WARNs and skips encode/decode. + +Caddy `/var/www/monitoring-sites` is separate (you configured hernani write access). diff --git a/scripts/taler-monitoring/host-agent/ROOT-APPLY-MONITORING.md b/scripts/taler-monitoring/host-agent/ROOT-APPLY-MONITORING.md new file mode 100644 index 0000000..17dd386 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/ROOT-APPLY-MONITORING.md @@ -0,0 +1,57 @@ +# Root: make monitoring pages live on koopa (GOA) + +## Symptom if not applied + +Public URLs return **merchant/bank JSON** like: + +```json +{ "code": 21, "hint": "There is no endpoint defined for the URL provided..." } +``` + +or a **404 HTML** page from the app reverse_proxy. + +Cause: live `/etc/caddy/Caddyfile` has **no** `handle /monitoring*` (falls through to +API). Prepared config is only under **`/home/hernani/koopa-caddy/Caddyfile`**. + +## One-shot as root (recommended) + +```bash +sudo /home/hernani/koopa-caddy/apply-monitoring-live.sh +``` + +That script: + +1. `rsync` staging HTML → `/var/www/monitoring-sites/` (incl. surface + aptdeploy) +2. Makes tree writable for `hernani` (group `caddy` + ACL) so later runs self-sync +3. Installs prepared Caddyfile → `/etc/caddy/Caddyfile`, validates, reloads Caddy +4. Smoke-curls all monitoring URLs (must **not** be JSON `code:21`) + +## Paths + +| Path | Host | Source timer | +|------|------|----------------| +| `/monitoring/` + `/monitoring_err/` | bank + exchange + taler | `taler-monitoring-hacktivism.timer` (4h) | +| `/taler-monitoring-surface/` + `/_err/` | **taler.hacktivism.ch only** | `taler-monitoring-surface.timer` (1h) | +| `/taler-monitoring-aptdeploy/` + `/_err/` | **taler.hacktivism.ch only** | `taler-monitoring-aptdeploy.timer` (4h) | + +HTML is generated as **hernani** into +`/home/hernani/monitoring-sites-staging/`. + +## Manual equivalent + +```bash +rsync -a --delete /home/hernani/monitoring-sites-staging/ /var/www/monitoring-sites/ +chown -R hernani:caddy /var/www/monitoring-sites && chmod -R g+rwX /var/www/monitoring-sites +sudo /home/hernani/koopa-caddy/apply.sh # or apply-monitoring-live.sh +``` + +Expect **200** (HTML with sticky bar). Not merchant JSON `code:21`. + +## After each monitoring run (if www is not hernani-writable) + +```bash +rsync -a --delete /home/hernani/monitoring-sites-staging/ /var/www/monitoring-sites/ +chown -R caddy:caddy /var/www/monitoring-sites +``` + +Or grant `hernani` write on `/var/www/monitoring-sites` once so the host-agent can rsync itself. diff --git a/scripts/taler-monitoring/host-agent/ensure-apt-deploy-test-containers.sh b/scripts/taler-monitoring/host-agent/ensure-apt-deploy-test-containers.sh new file mode 100755 index 0000000..75fa3e4 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/ensure-apt-deploy-test-containers.sh @@ -0,0 +1,373 @@ +#!/usr/bin/env bash +# ensure-apt-deploy-test-containers.sh +# +# Four rootless podman containers on koopa (no nginx / no HTTPS): +# +# Fresh rebuild when repo fingerprint changes: +# koopa-taler-deploy-test-apt-src-trixie +# koopa-taler-deploy-test-apt-src-trixie-testing +# +# Persistent upgrade path (mytops upgrade.sh style): +# koopa-taler-deploy-test-apt-src-trixie-upgrade +# koopa-taler-deploy-test-apt-src-trixie-testing-upgrade +# +# Run as hernani on koopa. From elsewhere use run-aptdeploy.sh (ssh koopa-external). +# +set -euo pipefail + +PODMAN_BIN="${APT_DEPLOY_PODMAN:-podman}" +IMG="${APT_DEPLOY_BASE_IMAGE:-docker.io/library/debian:trixie}" +STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/taler-apt-deploy" +URI_BASE="${APT_DEPLOY_URI:-https://deb.taler.net/apt/debian}" +mkdir -p "$STATE_DIR" + +# Packages we track for "new version → rebuild fresh containers" +TRACK_PKGS="${APT_DEPLOY_TRACK_PKGS:-taler-merchant libtalermerchant libtalerexchange libdonau}" + +# Canonical four containers (keep). Everything else matching legacy names is removed. +CANONICAL_CONTAINERS=( + koopa-taler-deploy-test-apt-src-trixie + koopa-taler-deploy-test-apt-src-trixie-testing + koopa-taler-deploy-test-apt-src-trixie-upgrade + koopa-taler-deploy-test-apt-src-trixie-testing-upgrade +) + +# Legacy / typo names from earlier iterations — always delete if present +LEGACY_CONTAINERS=( + koopa-taler-build-test-apt-trixie + koopa-taler-build-test-apt-trixie-testing + koopa-taler-deploy-test-apt-trixie + koopa-taler-deploy-test-apt-trixie-testing + koopa-taler-buidl-test-apt-trixie + koopa-taler-buidl-test-apt-trixie-testing +) + +cleanup_legacy() { + local n img + echo "======== cleanup legacy containers / images ========" + for n in "${LEGACY_CONTAINERS[@]}"; do + if "$PODMAN_BIN" container exists "$n" 2>/dev/null; then + echo "remove legacy container $n" + "$PODMAN_BIN" rm -f "$n" 2>/dev/null || true + fi + # old setup images for that name + if "$PODMAN_BIN" image exists "localhost/${n}:setup" 2>/dev/null; then + echo "remove legacy image localhost/${n}:setup" + "$PODMAN_BIN" rmi -f "localhost/${n}:setup" 2>/dev/null || true + fi + done + # dangling setup images not in canonical set + while read -r img; do + [ -n "$img" ] || continue + local base keep=0 + base=${img#localhost/} + base=${base%:setup} + for n in "${CANONICAL_CONTAINERS[@]}"; do + [ "$base" = "$n" ] && keep=1 && break + done + if [ "$keep" -eq 0 ] && [[ "$img" == localhost/koopa-taler-*-test-apt* ]]; then + echo "remove non-canonical image $img" + "$PODMAN_BIN" rmi -f "$img" 2>/dev/null || true + fi + done < <("$PODMAN_BIN" images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null | grep -E 'localhost/koopa-taler-.*-test-apt' || true) +} + +# Prefer single suite tree: if both ~/src/koopa-admin-log and ~/koopa-admin-log are +# full git clones, keep the newer (by HEAD commit time) and replace the older with a symlink. +dedupe_suite_repos() { + local a="$HOME/src/koopa-admin-log" b="$HOME/koopa-admin-log" + local ta tb + [ "${APT_DEPLOY_DEDUPE_REPOS:-1}" = "1" ] || return 0 + if [ -d "$a/.git" ] && [ -d "$b/.git" ] && [ ! -L "$a" ] && [ ! -L "$b" ]; then + ta=$(git -C "$a" log -1 --format=%ct 2>/dev/null || echo 0) + tb=$(git -C "$b" log -1 --format=%ct 2>/dev/null || echo 0) + echo "======== dedupe suite repos (both are git clones) ========" + echo " $a HEAD_ts=$ta" + echo " $b HEAD_ts=$tb" + if [ "$ta" -ge "$tb" ]; then + echo "keep $a · remove $b · symlink $b → $a" + rm -rf "$b" + ln -sfn "$a" "$b" + else + echo "keep $b · remove $a · symlink $a → $b" + rm -rf "$a" + mkdir -p "$(dirname "$a")" + ln -sfn "$b" "$a" + fi + elif [ -d "$a/scripts/taler-monitoring" ] && [ -d "$b/scripts/taler-monitoring" ] \ + && [ ! -d "$a/.git" ] && [ -d "$b/.git" ] && [ ! -L "$a" ]; then + echo "remove non-git tree $a · symlink → $b" + rm -rf "$a" + mkdir -p "$(dirname "$a")" + ln -sfn "$b" "$a" + elif [ -d "$b/scripts/taler-monitoring" ] && [ -d "$a/scripts/taler-monitoring" ] \ + && [ ! -d "$b/.git" ] && [ -d "$a/.git" ] && [ ! -L "$b" ]; then + echo "remove non-git tree $b · symlink → $a" + rm -rf "$b" + ln -sfn "$a" "$b" + fi +} + +cleanup_legacy +dedupe_suite_repos +# --------------------------------------------------------------------------- +# Repo fingerprint: Version: lines from suite Packages index +# --------------------------------------------------------------------------- +repo_pkg_version() { + local suite=$1 pkg=$2 + curl -fsS --max-time 45 \ + "${URI_BASE}/dists/${suite}/main/binary-amd64/Packages" 2>/dev/null \ + | awk -v p="$pkg" ' + $0 == "Package: " p { want=1; next } + /^Package: / { want=0 } + want && /^Version: / { print $2; exit } + ' +} + +repo_fingerprint() { + local suite=$1 pkg v line="" + for pkg in $TRACK_PKGS; do + v=$(repo_pkg_version "$suite" "$pkg" || true) + [ -n "$v" ] || v="?" + line="${line}${pkg}=${v};" + done + printf '%s' "$line" +} + +container_fingerprint() { + local name=$1 + "$PODMAN_BIN" exec "$name" bash -lc ' + for p in '"$TRACK_PKGS"'; do + v=$(dpkg-query -W -f="\${Version}" "$p" 2>/dev/null || echo missing) + printf "%s=%s;" "$p" "$v" + done + ' 2>/dev/null || echo "missing" +} + +# --------------------------------------------------------------------------- +# Bootstrap: install base + taler source + packages (shared) +# --------------------------------------------------------------------------- +install_base_and_merchant() { + local name=$1 suite=$2 srcfile=$3 + "$PODMAN_BIN" exec -u root "$name" bash -lc ' + set -euo pipefail + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y -qq ca-certificates curl wget gnupg systemd systemd-sysv \ + dbus procps iproute2 + ' + "$PODMAN_BIN" exec -u root "$name" bash -lc " + set -euo pipefail + export DEBIAN_FRONTEND=noninteractive + mkdir -p /etc/apt/keyrings + wget -q -O /etc/apt/keyrings/taler-systems.gpg https://taler.net/taler-systems.gpg + cat >/etc/apt/sources.list.d/${srcfile} </dev/null + systemctl enable taler-merchant.target 2>/dev/null + systemctl start taler-merchant.target 2>/dev/null + sleep 2 + systemctl is-active taler-merchant-httpd.service 2>/dev/null || true + ' || true +} + +create_from_scratch() { + local name=$1 suite=$2 srcfile=$3 + echo "======== CREATE (fresh) $name Suites: $suite ========" + "$PODMAN_BIN" pull "$IMG" >/dev/null + "$PODMAN_BIN" rm -f "$name" 2>/dev/null || true + # drop previous setup image so we do not accumulate duplicates + "$PODMAN_BIN" rmi -f "localhost/${name}:setup" 2>/dev/null || true + "$PODMAN_BIN" run -d --name "$name" --hostname "$name" "$IMG" sleep infinity + install_base_and_merchant "$name" "$suite" "$srcfile" + commit_and_systemd "$name" + printf '%s\n' "$(repo_fingerprint "$suite")" >"$STATE_DIR/${name}.fp" + printf '%s\n' "$(container_fingerprint "$name")" >"$STATE_DIR/${name}.installed" + echo "created $name fp=$(cat "$STATE_DIR/${name}.fp")" +} + +# --------------------------------------------------------------------------- +# Upgrade path (mytops-admin-log upgrade.sh merchant packages, no restic/nginx) +# Prints a markdown-ish table of before → after versions. +# --------------------------------------------------------------------------- +upgrade_inside() { + local name=$1 + local before after pkg v0 v1 + echo "======== UPGRADE $name (mytops-style apt install + upgrade) ========" + + before=$("$PODMAN_BIN" exec "$name" bash -lc ' + for p in taler-merchant libtalermerchant libtalerexchange libdonau libgnunet \ + taler-merchant-webui taler-merchant-typst taler-terms-generator; do + v=$(dpkg-query -W -f="\${Version}" "$p" 2>/dev/null || echo "-") + printf "%s %s\n" "$p" "$v" + done + ' 2>/dev/null || true) + + # capture upgrade output + set +e + "$PODMAN_BIN" exec -u root "$name" bash -lc ' + set -euo pipefail + export DEBIAN_FRONTEND=noninteractive + systemctl stop taler-merchant.target 2>/dev/null || true + systemctl stop taler-merchant-donaukeyupdate.service 2>/dev/null || true + apt-get update -qq + apt-get install -y \ + taler-merchant \ + taler-merchant-typst \ + taler-merchant-webui \ + taler-terms-generator \ + || true + if dpkg -l challenger-httpd 2>/dev/null | grep -q "^ii"; then + apt-get remove -y challenger-httpd || true + fi + apt-get upgrade -y + systemctl daemon-reload 2>/dev/null || true + systemctl enable taler-merchant.target 2>/dev/null || true + # try dbconfig if present (may no-op without full config) + command -v taler-merchant-dbconfig >/dev/null && taler-merchant-dbconfig 2>/dev/null || true + systemctl reset-failed "taler-merchant*" 2>/dev/null || true + systemctl start taler-merchant.target 2>/dev/null || true + sleep 2 + ' 2>&1 | tee "$STATE_DIR/${name}.upgrade.log" | tail -30 + set -e + + after=$("$PODMAN_BIN" exec "$name" bash -lc ' + for p in taler-merchant libtalermerchant libtalerexchange libdonau libgnunet \ + taler-merchant-webui taler-merchant-typst taler-terms-generator; do + v=$(dpkg-query -W -f="\${Version}" "$p" 2>/dev/null || echo "-") + printf "%s %s\n" "$p" "$v" + done + ' 2>/dev/null || true) + + echo + echo "| package | before | after | change |" + echo "|---------|--------|-------|--------|" + while read -r pkg v0; do + [ -n "${pkg:-}" ] || continue + v1=$(printf '%s\n' "$after" | awk -v p="$pkg" '$1==p {print $2; exit}') + v1=${v1:--} + if [ "$v0" = "$v1" ]; then + ch="=" + else + ch="UPGRADED" + fi + echo "| $pkg | $v0 | $v1 | $ch |" + done <<<"$before" + + printf '%s\n' "$(container_fingerprint "$name")" >"$STATE_DIR/${name}.installed" +} + +# --------------------------------------------------------------------------- +# Fresh mode: recreate if missing / not running / repo fp changed +# --------------------------------------------------------------------------- +ensure_fresh() { + local name=$1 suite=$2 srcfile=$3 + local want have running + + want=$(repo_fingerprint "$suite") + echo "repo fingerprint [$suite]: $want" + + if ! "$PODMAN_BIN" container exists "$name" 2>/dev/null; then + create_from_scratch "$name" "$suite" "$srcfile" + return + fi + + running=$("$PODMAN_BIN" inspect -f '{{.State.Running}}' "$name" 2>/dev/null || echo false) + have=$(container_fingerprint "$name") + echo "installed in $name: $have" + + if [ "$running" != "true" ] \ + || ! "$PODMAN_BIN" exec "$name" bash -lc 'command -v taler-merchant-httpd' >/dev/null 2>&1 \ + || [ "$want" != "$have" ]; then + echo "rebuild $name (running=$running fp_match=$([ "$want" = "$have" ] && echo yes || echo NO))" + create_from_scratch "$name" "$suite" "$srcfile" + else + echo "OK $name up-to-date (fresh container matches repo)" + fi +} + +# --------------------------------------------------------------------------- +# Upgrade mode: create once, then always upgrade_inside +# --------------------------------------------------------------------------- +ensure_upgrade() { + local name=$1 suite=$2 srcfile=$3 + local running + + if ! "$PODMAN_BIN" container exists "$name" 2>/dev/null; then + create_from_scratch "$name" "$suite" "$srcfile" + echo "(initial install for upgrade-track container $name)" + return + fi + + running=$("$PODMAN_BIN" inspect -f '{{.State.Running}}' "$name" 2>/dev/null || echo false) + if [ "$running" != "true" ]; then + echo "start $name" + "$PODMAN_BIN" start "$name" || { + echo "recreate broken $name" + create_from_scratch "$name" "$suite" "$srcfile" + return + } + sleep 3 + fi + + if ! "$PODMAN_BIN" exec "$name" bash -lc 'command -v taler-merchant-httpd' >/dev/null 2>&1; then + echo "recreate $name (no httpd after start)" + create_from_scratch "$name" "$suite" "$srcfile" + return + fi + + upgrade_inside "$name" +} + +MODE="${1:-all}" # all | fresh | upgrade + +case "$MODE" in + all|fresh) + ensure_fresh koopa-taler-deploy-test-apt-src-trixie trixie taler.sources + ensure_fresh koopa-taler-deploy-test-apt-src-trixie-testing trixie-testing taler-testing.sources + ;;& + all|upgrade) + ensure_upgrade koopa-taler-deploy-test-apt-src-trixie-upgrade trixie taler.sources + ensure_upgrade koopa-taler-deploy-test-apt-src-trixie-testing-upgrade trixie-testing taler-testing.sources + ;; + *) + echo "usage: $0 [all|fresh|upgrade]" >&2 + exit 2 + ;; +esac + +echo "done ensure-apt-deploy-test-containers mode=$MODE" diff --git a/scripts/taler-monitoring/host-agent/env.example b/scripts/taler-monitoring/host-agent/env.example new file mode 100644 index 0000000..15f6bae --- /dev/null +++ b/scripts/taler-monitoring/host-agent/env.example @@ -0,0 +1,26 @@ +# Copy to: ~/.config/taler-monitoring/env +# This file is NOT inside the git tree and is never overwritten by update-suite.sh. +# +# Full stack examples (same reporting generation for all): +# env/hacktivism.env.example +# env/stagepaysan.env.example +# env/francpaysan.env.example +# +# SUITE_GIT_URL=https://git.hacktivism.ch/hernani/koopa-admin-log.git +# SUITE_GIT_REF=main +# SUITE_DIR=$HOME/src/koopa-admin-log +# SUITE_UPDATE_MODE=reset +# +# TALER_DOMAIN=hacktivism.ch +# INSIDE_PODMAN=1 +# INSIDE_MODE=local-podman +# CONTINUE_ON_ERROR=1 +# PHASES for the *default* hacktivism timer only (urls inside versions). +# Dedicated wrappers always override PHASES + HTML paths: +# run-surface-monitoring.sh → /taler-monitoring-surface* +# run-aptdeploy-monitoring.sh → /taler-monitoring-aptdeploy* +# PHASES="urls inside versions" +# RUN_TIMEOUT=600 +# HTML_OUT=$HOME/monitoring-sites-staging +# MON_HOSTS="bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch" +# SOURCE_REPO_WEB=https://git.hacktivism.ch/hernani/koopa-admin-log diff --git a/scripts/taler-monitoring/host-agent/env/francpaysan.env.example b/scripts/taler-monitoring/host-agent/env/francpaysan.env.example new file mode 100644 index 0000000..ab1c534 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/env/francpaysan.env.example @@ -0,0 +1,24 @@ +# Copy to: ~/.config/taler-monitoring/env (user francpaysan) +# Never inside the git tree — update-suite never overwrites this file. +# +# Shared reporting generation (run-host-report.sh / run-fp-prod-monitoring.sh): +# RUN_TIMEOUT, line-buffered log, always HTML, commit link on git.hacktivism.ch + +TALER_DOMAIN=lefrancpaysan.ch +INSIDE_PODMAN=1 +INSIDE_MODE=local-podman +LOCAL_STACK=0 +CONTINUE_ON_ERROR=1 +PHASES="urls inside versions" +RUN_TIMEOUT=600 + +HTML_OUT=$HOME/monitoring-sites-staging +MON_HOSTS="bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch" + +SUITE_DIR=$HOME/src/koopa-admin-log +SUITE_GIT_URL=https://git.hacktivism.ch/hernani/koopa-admin-log.git +SUITE_GIT_REF=main +SUITE_UPDATE_MODE=reset +SOURCE_REPO_WEB=https://git.hacktivism.ch/hernani/koopa-admin-log + +# DEPLOY_WWW_ROOT=/var/www/monitoring-sites diff --git a/scripts/taler-monitoring/host-agent/env/hacktivism.env.example b/scripts/taler-monitoring/host-agent/env/hacktivism.env.example new file mode 100644 index 0000000..6b57280 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/env/hacktivism.env.example @@ -0,0 +1,24 @@ +# Copy to: ~/.config/taler-monitoring/env (user hernani on koopa) +# Shared reporting generation: RUN_TIMEOUT, line-buffered log, always HTML. + +TALER_DOMAIN=hacktivism.ch +INSIDE_PODMAN=1 +INSIDE_MODE=local-podman +LOCAL_STACK=1 +CONTINUE_ON_ERROR=1 +PHASES="urls inside versions" +# aptdeploy HTML: run-aptdeploy-monitoring.sh → /taler-monitoring-aptdeploy* +RUN_TIMEOUT=600 +# APT_DEPLOY_ENSURE=1 +# APT_DEPLOY_SKIP=0 + +HTML_OUT=$HOME/monitoring-sites-staging +MON_HOSTS="bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch" + +SUITE_DIR=$HOME/src/koopa-admin-log +SUITE_GIT_URL=https://git.hacktivism.ch/hernani/koopa-admin-log.git +SUITE_GIT_REF=main +SUITE_UPDATE_MODE=reset +SOURCE_REPO_WEB=https://git.hacktivism.ch/hernani/koopa-admin-log + +DEPLOY_WWW_ROOT=/var/www/monitoring-sites diff --git a/scripts/taler-monitoring/host-agent/env/stagepaysan.env.example b/scripts/taler-monitoring/host-agent/env/stagepaysan.env.example new file mode 100644 index 0000000..0959ebc --- /dev/null +++ b/scripts/taler-monitoring/host-agent/env/stagepaysan.env.example @@ -0,0 +1,27 @@ +# Copy to: ~/.config/taler-monitoring/env (user stagepaysan) +# Never inside the git tree — update-suite never overwrites this file. +# +# Shared reporting generation (run-host-report.sh / run-fp-stage-monitoring.sh): +# RUN_TIMEOUT, line-buffered log, always HTML, commit link on git.hacktivism.ch + +TALER_DOMAIN=stage.lefrancpaysan.ch +INSIDE_PODMAN=1 +INSIDE_MODE=local-podman +INSIDE_PROFILE=stage-lfp +LOCAL_STACK=0 +CONTINUE_ON_ERROR=1 +PHASES="urls inside versions" +RUN_TIMEOUT=600 + +HTML_OUT=$HOME/monitoring-sites-staging +MON_HOSTS="stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch" + +# Git clone of koopa-admin-log (repo root, not …/scripts/taler-monitoring) +SUITE_DIR=$HOME/src/koopa-admin-log +SUITE_GIT_URL=https://git.hacktivism.ch/hernani/koopa-admin-log.git +SUITE_GIT_REF=main +SUITE_UPDATE_MODE=reset +SOURCE_REPO_WEB=https://git.hacktivism.ch/hernani/koopa-admin-log + +# Optional public docroot if this user can write it +# DEPLOY_WWW_ROOT=/var/www/monitoring-sites diff --git a/scripts/taler-monitoring/host-agent/install-host-agent.sh b/scripts/taler-monitoring/host-agent/install-host-agent.sh new file mode 100755 index 0000000..adcce66 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/install-host-agent.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# install-host-agent.sh — install hernani user systemd units on koopa +# +# # on koopa as hernani: +# ~/koopa-admin-log/scripts/taler-monitoring/host-agent/install-host-agent.sh +# # or from laptop: +# ./install-host-agent.sh --remote koopa-external +# +set -euo pipefail +ROOT=$(cd "$(dirname "$0")" && pwd) +REMOTE="${1:-}" +[ "${1:-}" = "--remote" ] && REMOTE="${2:-koopa-external}" + +install_local() { + local mon home_unit + mon=$(cd "$ROOT/.." && pwd) + # Prefer ~/koopa-admin-log on live host + if [ -d "$HOME/koopa-admin-log/scripts/taler-monitoring" ]; then + mon="$HOME/koopa-admin-log/scripts/taler-monitoring" + fi + home_unit="$HOME/.config/systemd/user" + mkdir -p "$home_unit" \ + "$HOME/.local/state/taler-hacktivism-monitoring" \ + "$HOME/monitoring-sites-staging" \ + "$HOME/.config/taler-monitoring" \ + "$HOME/src" + + # Local env (never in git) — create from example if missing + if [ ! -f "$HOME/.config/taler-monitoring/env" ] && [ -f "$ROOT/env.example" ]; then + cp "$ROOT/env.example" "$HOME/.config/taler-monitoring/env" + echo "created ~/.config/taler-monitoring/env" + fi + + # Prefer a real git clone for update-suite.sh + if [ ! -d "$HOME/src/koopa-admin-log/.git" ] && [ ! -d "$HOME/koopa-admin-log/.git" ]; then + if [ -d "$HOME/koopa-admin-log/scripts/taler-monitoring" ] && [ ! -d "$HOME/koopa-admin-log/.git" ]; then + echo "note: $HOME/koopa-admin-log has no .git — update-suite will clone to ~/src/koopa-admin-log" + fi + git clone https://git.hacktivism.ch/hernani/koopa-admin-log.git "$HOME/src/koopa-admin-log" 2>/dev/null \ + || echo "WARN: clone failed (auth/network) — will use existing tree if any" + fi + + # Ensure suite is executable + chmod +x "$mon"/taler-monitoring.sh "$mon"/check_*.sh 2>/dev/null || true + chmod +x "$mon/host-agent"/*.sh 2>/dev/null || true + chmod +x "$mon/site-gen"/*.sh "$mon/site-gen/console_to_html.py" 2>/dev/null || true + + # Units reference %h/koopa-admin-log — prefer git clone or symlink + if [ -d "$HOME/src/koopa-admin-log/.git" ] && [ ! -e "$HOME/koopa-admin-log" ]; then + ln -sfn "$HOME/src/koopa-admin-log" "$HOME/koopa-admin-log" + echo "symlink $HOME/koopa-admin-log → $HOME/src/koopa-admin-log" + elif [ ! -e "$HOME/koopa-admin-log" ]; then + admin_root=$(cd "$mon/../.." && pwd) + ln -sfn "$admin_root" "$HOME/koopa-admin-log" + echo "symlink $HOME/koopa-admin-log → $admin_root" + fi + + install -m 644 "$ROOT/taler-monitoring-hacktivism.service" "$home_unit/" + install -m 644 "$ROOT/taler-monitoring-hacktivism.path" "$home_unit/" + install -m 644 "$ROOT/taler-monitoring-hacktivism.timer" "$home_unit/" + install -m 644 "$ROOT/taler-monitoring-surface.service" "$home_unit/" + install -m 644 "$ROOT/taler-monitoring-surface.timer" "$home_unit/" + install -m 644 "$ROOT/taler-monitoring-aptdeploy.service" "$home_unit/" + install -m 644 "$ROOT/taler-monitoring-aptdeploy.timer" "$home_unit/" + + # Fix ExecStart if install path differs (units use %h/koopa-admin-log) + systemctl --user daemon-reload + systemctl --user enable --now taler-monitoring-hacktivism.path + systemctl --user enable --now taler-monitoring-hacktivism.timer + systemctl --user enable --now taler-monitoring-surface.timer + systemctl --user enable --now taler-monitoring-aptdeploy.timer + # initial stamp so path exists + "$ROOT/touch-software-stamp.sh" + systemctl --user start taler-monitoring-hacktivism.service || true + systemctl --user start taler-monitoring-surface.service || true + systemctl --user start taler-monitoring-aptdeploy.service || true + + echo "--- status ---" + systemctl --user status taler-monitoring-hacktivism.path --no-pager -l | head -15 + systemctl --user status taler-monitoring-hacktivism.timer --no-pager -l | head -12 + systemctl --user status taler-monitoring-surface.timer --no-pager -l | head -12 + systemctl --user status taler-monitoring-aptdeploy.timer --no-pager -l | head -12 + systemctl --user list-timers --all | grep taler-monitoring || true + echo "OK host-agent (hacktivism 4h + surface 1h + aptdeploy 4h)" + echo "manual: systemctl --user start taler-monitoring-hacktivism.service" + echo "manual surface: systemctl --user start taler-monitoring-surface.service" + echo "manual aptdeploy: systemctl --user start taler-monitoring-aptdeploy.service" + echo "after upgrades: $ROOT/touch-software-stamp.sh" +} + +if [ -n "$REMOTE" ] && [ "$REMOTE" != "--remote" ]; then + echo "install on $REMOTE …" + rsync -az --delete \ + --exclude secrets.env \ + --exclude 'android-test/artifacts' \ + "$ROOT/../../" \ + "${REMOTE}:koopa-admin-log/scripts/taler-monitoring/" + # ensure full tree for %h/koopa-admin-log if missing configs etc. + ssh -o BatchMode=yes -o ConnectTimeout=20 "$REMOTE" \ + 'test -d $HOME/koopa-admin-log || git clone https://git.hacktivism.ch/hernani/koopa-admin-log.git $HOME/koopa-admin-log 2>/dev/null || true' + rsync -az "$ROOT/" "${REMOTE}:koopa-admin-log/scripts/taler-monitoring/host-agent/" + rsync -az "$ROOT/../site-gen/" "${REMOTE}:koopa-admin-log/scripts/taler-monitoring/site-gen/" 2>/dev/null || true + ssh -o BatchMode=yes "$REMOTE" \ + 'bash $HOME/koopa-admin-log/scripts/taler-monitoring/host-agent/install-host-agent.sh' +else + install_local +fi diff --git a/scripts/taler-monitoring/host-agent/run-aptdeploy-monitoring.sh b/scripts/taler-monitoring/host-agent/run-aptdeploy-monitoring.sh new file mode 100755 index 0000000..fe81257 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/run-aptdeploy-monitoring.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# run-aptdeploy-monitoring.sh — apt-src deploy tests → public HTML +# Public only on taler.hacktivism.ch: +# https://taler.hacktivism.ch/taler-monitoring-aptdeploy/ +# https://taler.hacktivism.ch/taler-monitoring-aptdeploy_err/ +# +# CLI one-shot without HTML remains: run-aptdeploy.sh [all|fresh|upgrade|check-only] +# +set -uo pipefail +AGENT_DIR=$(cd "$(dirname "$0")" && pwd) + +export AGENT_LABEL="${AGENT_LABEL:-aptdeploy-host-agent}" +export STATE_NAME="${STATE_NAME:-taler-aptdeploy-monitoring}" +export TALER_DOMAIN="${TALER_DOMAIN:-hacktivism.ch}" +# containers are local on koopa +export INSIDE_PODMAN=1 +export INSIDE_MODE=local-podman +export LOCAL_STACK=1 +export SKIP_SSH=0 +export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" +# four containers + checks; allow long wall +export RUN_TIMEOUT="${RUN_TIMEOUT:-1800}" +# dedicated job: always aptdeploy only +export PHASES=aptdeploy +# Only merchant public front — not bank/exchange +export MON_HOSTS="${MON_HOSTS:-taler.hacktivism.ch}" +export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}" +export DEPLOY_WWW_ROOT="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" + +export HTML_OK_DIR="taler-monitoring-aptdeploy" +export HTML_ERR_DIR="taler-monitoring-aptdeploy_err" +export HTML_URL_OK="/taler-monitoring-aptdeploy/" +export HTML_URL_ERR="/taler-monitoring-aptdeploy_err/" +export PAGE_LABEL="taler-monitoring-aptdeploy" + +# Ensure all four apt-src containers (fresh rebuild + upgrade track) +if [ "${APT_DEPLOY_ENSURE:-1}" = "1" ] \ + && [ -x "$AGENT_DIR/ensure-apt-deploy-test-containers.sh" ]; then + bash "$AGENT_DIR/ensure-apt-deploy-test-containers.sh" all \ + || echo "WARN: ensure-apt-deploy-test-containers failed (aptdeploy may ERROR)" >&2 +fi + +exec bash "$AGENT_DIR/run-host-report.sh" "$@" diff --git a/scripts/taler-monitoring/host-agent/run-aptdeploy.sh b/scripts/taler-monitoring/host-agent/run-aptdeploy.sh new file mode 100755 index 0000000..fbf5afd --- /dev/null +++ b/scripts/taler-monitoring/host-agent/run-aptdeploy.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +# run-aptdeploy.sh — ensure containers + aptdeploy phase only. +# +# Keep monitoring current: +# 1) Prefer Forgejo origin/main (update-suite reset) +# 2) If Forgejo is behind (no aptdeploy yet), re-apply suite-overlay saved +# from this agent tree before reset (rsync from laptop fills that tree) +# 3) Run ensure + taler-monitoring.sh aptdeploy +# +# Host: on koopa → local; else ssh APT_DEPLOY_SSH + rsync mon tree first. +# +set -euo pipefail + +AGENT_DIR=$(cd "$(dirname "$0")" && pwd) +BOOT_MON=$(cd "$AGENT_DIR/.." && pwd) +MODE="${1:-all}" +APT_DEPLOY_SSH="${APT_DEPLOY_SSH:-koopa-external}" +OVERLAY_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring/suite-overlay" + +on_koopa() { + local h + h=$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo "") + case "$h" in koopa|koopa.*) return 0 ;; esac + [ "${KOOPA_FORCE_LOCAL:-0}" = "1" ] && return 0 + return 1 +} + +mon_has_aptdeploy() { + local mon=$1 + [ -f "$mon/taler-monitoring.sh" ] && grep -qE 'aptdeploy\|apt-deploy' "$mon/taler-monitoring.sh" +} + +save_overlay_from() { + local src=$1 + mon_has_aptdeploy "$src" || return 0 + mkdir -p "$OVERLAY_DIR" + rsync -a --delete \ + --exclude secrets.env \ + --exclude 'android-test/artifacts' \ + --exclude 'android-test/apks' \ + --exclude 'android-test/out*' \ + --exclude '__pycache__' \ + "$src/" "$OVERLAY_DIR/" + echo "saved suite-overlay from $src → $OVERLAY_DIR" +} + +apply_overlay_to() { + local dest=$1 + [ -f "$OVERLAY_DIR/taler-monitoring.sh" ] || return 1 + mon_has_aptdeploy "$OVERLAY_DIR" || return 1 + mkdir -p "$dest" + rsync -a \ + --exclude secrets.env \ + --exclude 'android-test/artifacts' \ + --exclude 'android-test/apks' \ + --exclude 'android-test/out*' \ + --exclude '__pycache__' \ + "$OVERLAY_DIR/" "$dest/" + echo "applied suite-overlay → $dest (Forgejo lag; push origin/main when possible)" + return 0 +} + +pick_mon_tree() { + # Preserve working mon before reset wipes tracked files + save_overlay_from "$BOOT_MON" + + # shellcheck source=update-suite.sh + source "$AGENT_DIR/update-suite.sh" + + local suite_mon="$SUITE_DIR/scripts/taler-monitoring" + mkdir -p "$suite_mon" + + if mon_has_aptdeploy "$suite_mon"; then + MON_DIR="$suite_mon" + echo "using Forgejo suite mon @ ${COMMIT_SHORT:-?} · $MON_DIR" + elif apply_overlay_to "$suite_mon" && mon_has_aptdeploy "$suite_mon"; then + MON_DIR="$suite_mon" + echo "using suite mon + overlay @ ${COMMIT_SHORT:-?} · $MON_DIR" + elif mon_has_aptdeploy "$BOOT_MON"; then + MON_DIR="$BOOT_MON" + echo "WARN: using boot mon $MON_DIR (overlay apply failed)" >&2 + else + echo "ERROR: no mon tree with aptdeploy (Forgejo + overlay empty)" >&2 + echo " rsync suite from laptop or push to git.hacktivism.ch" >&2 + exit 2 + fi + + AGENT_DIR="$MON_DIR/host-agent" + export MON_DIR AGENT_DIR SUITE_DIR COMMIT COMMIT_SHORT COMMIT_URL + chmod +x "$AGENT_DIR"/*.sh "$MON_DIR"/taler-monitoring.sh "$MON_DIR"/check_*.sh 2>/dev/null || true +} + +run_local() { + export PATH="${HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin:${PATH:-}" + export PYTHONUNBUFFERED=1 + export PROGRESS_OFF="${PROGRESS_OFF:-0}" + + echo "======== aptdeploy local on $(hostname) mode=$MODE ========" + pick_mon_tree + + if [ "$MODE" != "check-only" ]; then + bash "$AGENT_DIR/ensure-apt-deploy-test-containers.sh" "$MODE" + fi + + cd "$MON_DIR" + export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" + export INSIDE_PODMAN="${INSIDE_PODMAN:-1}" + export LOCAL_STACK="${LOCAL_STACK:-1}" + export RUN_TIMEOUT="${RUN_TIMEOUT:-600}" + echo "run: $MON_DIR/taler-monitoring.sh -d hacktivism.ch aptdeploy · suite=${COMMIT_SHORT:-?}" + ./taler-monitoring.sh -d hacktivism.ch aptdeploy +} + +run_remote() { + echo "======== aptdeploy remote via ssh ${APT_DEPLOY_SSH} mode=$MODE ========" + if [ "${APT_DEPLOY_RSYNC:-1}" = "1" ]; then + echo "rsync mon suite → ${APT_DEPLOY_SSH}:src/koopa-admin-log/scripts/taler-monitoring/" + rsync -az \ + --exclude secrets.env \ + --exclude 'android-test/artifacts' \ + --exclude 'android-test/apks' \ + --exclude 'android-test/out*' \ + --exclude '__pycache__' \ + "$BOOT_MON/" \ + "${APT_DEPLOY_SSH}:src/koopa-admin-log/scripts/taler-monitoring/" \ + || echo "WARN: rsync failed" >&2 + fi + ssh -o BatchMode=yes -o ConnectTimeout=25 "$APT_DEPLOY_SSH" bash -s -- "$MODE" <<'EOS' +set -euo pipefail +MODE=${1:-all} +export PATH="${HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin:${PATH:-}" +export KOOPA_FORCE_LOCAL=1 +AGENT="$HOME/src/koopa-admin-log/scripts/taler-monitoring/host-agent" +if [ ! -x "$AGENT/run-aptdeploy.sh" ]; then + AGENT="$HOME/koopa-admin-log/scripts/taler-monitoring/host-agent" +fi +if [ ! -x "$AGENT/run-aptdeploy.sh" ]; then + echo "ERROR: run-aptdeploy.sh missing on remote" >&2 + exit 1 +fi +echo "remote AGENT=$AGENT" +exec bash "$AGENT/run-aptdeploy.sh" "$MODE" +EOS +} + +if on_koopa; then + run_local +else + run_remote +fi diff --git a/scripts/taler-monitoring/host-agent/run-fp-prod-monitoring.sh b/scripts/taler-monitoring/host-agent/run-fp-prod-monitoring.sh new file mode 100755 index 0000000..be876a8 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/run-fp-prod-monitoring.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# FrancPaysan PROD host-agent (user francpaysan on francpaysan-host). +# Shared reporting: run-host-report.sh (timeout, line-buffered log, first-run HTML). +set -uo pipefail +AGENT_DIR=$(cd "$(dirname "$0")" && pwd) + +export AGENT_LABEL="${AGENT_LABEL:-fp-prod-host-agent}" +export STATE_NAME="${STATE_NAME:-taler-monitoring-lfp-prod}" +export TALER_DOMAIN="${TALER_DOMAIN:-lefrancpaysan.ch}" +export INSIDE_PODMAN="${INSIDE_PODMAN:-1}" +export INSIDE_MODE="${INSIDE_MODE:-local-podman}" +export LOCAL_STACK="${LOCAL_STACK:-0}" +export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" +export RUN_TIMEOUT="${RUN_TIMEOUT:-600}" +export PHASES="${PHASES:-urls inside versions}" +export MON_HOSTS="${MON_HOSTS:-bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch}" +export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}" +export DEPLOY_WWW_ROOT="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" + +exec bash "$AGENT_DIR/run-host-report.sh" "$@" diff --git a/scripts/taler-monitoring/host-agent/run-fp-stage-monitoring.sh b/scripts/taler-monitoring/host-agent/run-fp-stage-monitoring.sh new file mode 100755 index 0000000..70518f0 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/run-fp-stage-monitoring.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# FrancPaysan STAGE host-agent (user stagepaysan on francpaysan-host). +# Shared reporting: run-host-report.sh (timeout, line-buffered log, first-run HTML). +set -uo pipefail +AGENT_DIR=$(cd "$(dirname "$0")" && pwd) + +export AGENT_LABEL="${AGENT_LABEL:-fp-stage-host-agent}" +export STATE_NAME="${STATE_NAME:-taler-monitoring-stage-lfp}" +export TALER_DOMAIN="${TALER_DOMAIN:-stage.lefrancpaysan.ch}" +export INSIDE_PODMAN="${INSIDE_PODMAN:-1}" +export INSIDE_MODE="${INSIDE_MODE:-local-podman}" +export INSIDE_PROFILE="${INSIDE_PROFILE:-stage-lfp}" +export LOCAL_STACK="${LOCAL_STACK:-0}" +export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" +export RUN_TIMEOUT="${RUN_TIMEOUT:-600}" +export PHASES="${PHASES:-urls inside versions}" +export MON_HOSTS="${MON_HOSTS:-stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch}" +export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}" +export DEPLOY_WWW_ROOT="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" + +exec bash "$AGENT_DIR/run-host-report.sh" "$@" diff --git a/scripts/taler-monitoring/host-agent/run-hacktivism-monitoring.sh b/scripts/taler-monitoring/host-agent/run-hacktivism-monitoring.sh new file mode 100755 index 0000000..8fd98fd --- /dev/null +++ b/scripts/taler-monitoring/host-agent/run-hacktivism-monitoring.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# GOA / hacktivism host-agent on koopa (hernani). +# Stack defaults only — reporting pipeline is shared: run-host-report.sh +set -uo pipefail +AGENT_DIR=$(cd "$(dirname "$0")" && pwd) + +export AGENT_LABEL="${AGENT_LABEL:-hacktivism-host-agent}" +export STATE_NAME="${STATE_NAME:-taler-hacktivism-monitoring}" +export TALER_DOMAIN="${TALER_DOMAIN:-hacktivism.ch}" +export INSIDE_PODMAN="${INSIDE_PODMAN:-1}" +export INSIDE_MODE="${INSIDE_MODE:-local-podman}" +export LOCAL_STACK="${LOCAL_STACK:-1}" +export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" +export RUN_TIMEOUT="${RUN_TIMEOUT:-600}" +# stack checks only — aptdeploy has its own HTML path + timer: +# run-aptdeploy-monitoring.sh → /taler-monitoring-aptdeploy* (taler.hacktivism.ch) +export PHASES="${PHASES:-urls inside versions}" +export MON_HOSTS="${MON_HOSTS:-bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch}" +export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}" +export DEPLOY_WWW_ROOT="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" +export APT_DEPLOY_ENSURE=0 + +exec bash "$AGENT_DIR/run-host-report.sh" "$@" diff --git a/scripts/taler-monitoring/host-agent/run-host-report.sh b/scripts/taler-monitoring/host-agent/run-host-report.sh new file mode 100755 index 0000000..1be1def --- /dev/null +++ b/scripts/taler-monitoring/host-agent/run-host-report.sh @@ -0,0 +1,358 @@ +#!/usr/bin/env bash +# run-host-report.sh — global reporting generation for all stacks +# (GOA/hacktivism, FP stage, FP prod, …). +# +# Shared behaviour (always): +# 1) update-suite → latest git.hacktivism.ch (local ~/.config never overwritten) +# 2) line-buffered run log (written continuously) +# 3) RUN_TIMEOUT wall clock for taler-monitoring.sh (default 600s) +# 4) always write HTML (first run / empty staging too) + Forgejo commit link +# 5) optional rsync to DEPLOY_WWW_ROOT if writable +# +# Configure via env file (not in git): +# ${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring/env +# or TALER_MONITORING_ENV=/path/to/env +# +# Thin wrappers only set stack defaults then exec this script: +# run-hacktivism-monitoring.sh +# run-fp-stage-monitoring.sh +# run-fp-prod-monitoring.sh +# +set -uo pipefail + +export PATH="${HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin:${PATH:-}" +export PYTHONUNBUFFERED=1 + +AGENT_DIR=$(cd "$(dirname "$0")" && pwd) +AGENT_MON=$(cd "$AGENT_DIR/.." && pwd) +CFG_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring" +ENV_FILE="${TALER_MONITORING_ENV:-$CFG_DIR/env}" +mkdir -p "$CFG_DIR" + +# Thin wrappers (surface, aptdeploy-only, …) export stack knobs before exec. +# Snapshot them so ~/.config/taler-monitoring/env cannot clobber dedicated jobs +# (env often has PHASES="urls inside versions aptdeploy" for the main timer). +_wrap_keys=( + PHASES HTML_OUT HTML_OK_DIR HTML_ERR_DIR HTML_URL_OK HTML_URL_ERR PAGE_LABEL + RUN_TIMEOUT STATE_NAME AGENT_LABEL MON_HOSTS TALER_DOMAIN DEPLOY_WWW_ROOT + INSIDE_PODMAN INSIDE_MODE LOCAL_STACK SKIP_SSH CONTINUE_ON_ERROR AUTH401_CONTINUE + APT_DEPLOY_ENSURE SURFACE_SCOPE TALER_DOMAIN_FROM_CLI STRICT_EXIT SOURCE_SUITE_PATH +) +for _k in "${_wrap_keys[@]}"; do + if [ -n "${!_k+x}" ]; then + printf -v "_WRAP_SET_${_k}" '%s' 1 + printf -v "_WRAP_VAL_${_k}" '%s' "${!_k}" + fi +done +unset _k + +if [ -f "$ENV_FILE" ]; then + # shellcheck disable=SC1090 + set -a + # shellcheck source=/dev/null + source "$ENV_FILE" + set +a +fi +# Tell update-suite.sh not to re-source env (would clobber wrapper PHASES etc.) +export _TALER_MON_ENV_LOADED=1 + +# Restore wrapper overrides (dedicated jobs win over shared env file) +for _k in "${_wrap_keys[@]}"; do + _set_var="_WRAP_SET_${_k}" + _val_var="_WRAP_VAL_${_k}" + if [ "${!_set_var:-}" = "1" ]; then + printf -v "$_k" '%s' "${!_val_var}" + export "$_k" + fi +done +unset _k _set_var _val_var + +# --- global report defaults (env file / wrapper wins if already set) --- +export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" +export AUTH401_CONTINUE="${AUTH401_CONTINUE:-${CONTINUE_ON_ERROR}}" +export RUN_TIMEOUT="${RUN_TIMEOUT:-600}" +export TALER_DOMAIN="${TALER_DOMAIN:-hacktivism.ch}" +PHASES="${PHASES:-urls inside versions}" +HTML_BASE="${HTML_OUT:-$HOME/monitoring-sites-staging}" +MON_HOSTS="${MON_HOSTS:-}" +SUITE_PATH="${SOURCE_SUITE_PATH:-scripts/taler-monitoring}" +AGENT_LABEL="${AGENT_LABEL:-host-report}" +STATE_NAME="${STATE_NAME:-taler-monitoring-${TALER_DOMAIN}}" +STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/${STATE_NAME}" +LOG_DIR="$STATE_DIR/logs" +DEPLOY_WWW="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" +# HTML path layout under each host (monitoring / surface / …) +HTML_OK_DIR="${HTML_OK_DIR:-monitoring}" +HTML_ERR_DIR="${HTML_ERR_DIR:-monitoring_err}" +HTML_URL_OK="${HTML_URL_OK:-/monitoring/}" +HTML_URL_ERR="${HTML_URL_ERR:-/monitoring_err/}" +PAGE_LABEL="${PAGE_LABEL:-monitoring}" + +# Sensible MON_HOSTS from domain if not set +if [ -z "$MON_HOSTS" ]; then + case "$TALER_DOMAIN" in + hacktivism.ch|koopa) + MON_HOSTS="bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch" + ;; + stage.lefrancpaysan.ch|stage.*lefrancpaysan*) + MON_HOSTS="stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch" + ;; + lefrancpaysan.ch) + MON_HOSTS="bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch" + ;; + *) + MON_HOSTS="${TALER_DOMAIN}" + ;; + esac +fi + +mkdir -p "$LOG_DIR" "$HTML_BASE" + +STAMP=$(date +%Y%m%d-%H%M%S) +LOG="$LOG_DIR/run-${STAMP}.log" +: >"$LOG" +if command -v stdbuf >/dev/null 2>&1; then + exec > >(stdbuf -oL -eL tee -a "$LOG") 2>&1 +else + exec > >(tee -a "$LOG") 2>&1 +fi + +echo "======== $(date -Iseconds 2>/dev/null || date) ${AGENT_LABEL} ========" +echo "env_file=$ENV_FILE" +echo "domain=$TALER_DOMAIN · phases=$PHASES · RUN_TIMEOUT=${RUN_TIMEOUT}s" +echo "hosts=$MON_HOSTS" +echo "flags INSIDE_PODMAN=${INSIDE_PODMAN:-} INSIDE_MODE=${INSIDE_MODE:-} LOCAL_STACK=${LOCAL_STACK:-} SKIP_SSH=${SKIP_SSH:-} INSIDE_PROFILE=${INSIDE_PROFILE:-}" + +# --- refresh from Forgejo; re-apply suite-overlay if Forgejo lacks needed phases --- +OVERLAY_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring/suite-overlay" +# preserve capable mon before reset wipes tracked files +if [ -f "$AGENT_MON/taler-monitoring.sh" ]; then + mkdir -p "$OVERLAY_DIR" + rsync -a \ + --exclude secrets.env --exclude 'android-test/artifacts' --exclude 'android-test/apks' \ + --exclude 'android-test/out*' --exclude '__pycache__' \ + "$AGENT_MON/" "$OVERLAY_DIR/" 2>/dev/null || true +fi + +# shellcheck source=update-suite.sh +source "$AGENT_DIR/update-suite.sh" + +SUITE_MON="${SUITE_DIR}/scripts/taler-monitoring" +ROOT="$SUITE_MON" +_need_overlay=0 +# Detect missing features in Forgejo tree (untracked helper files alone are not enough — +# taler-monitoring.sh must accept the phase name). +case " $PHASES " in + *" aptdeploy "*|*" apt-deploy "*|*" apt_src "*) + if ! grep -qE 'aptdeploy\|apt-deploy' "$SUITE_MON/taler-monitoring.sh" 2>/dev/null; then + _need_overlay=1 + fi + ;; +esac +case " $PHASES " in + *" surface "*|*" ecosystem "*) + if ! grep -qE 'surface\|ecosystem' "$SUITE_MON/taler-monitoring.sh" 2>/dev/null \ + || [ ! -f "$SUITE_MON/check_surface.sh" ]; then + _need_overlay=1 + fi + ;; +esac +# Wrapper-preserve + HTML path knobs may live only in local overlay until pushed +if [ -f "$OVERLAY_DIR/host-agent/run-host-report.sh" ] \ + && ! grep -q '_wrap_keys\|_WRAP_SET_' "$SUITE_MON/host-agent/run-host-report.sh" 2>/dev/null \ + && grep -q '_wrap_keys\|_WRAP_SET_' "$OVERLAY_DIR/host-agent/run-host-report.sh" 2>/dev/null; then + _need_overlay=1 +fi +if [ "$_need_overlay" = "1" ] && [ -f "$OVERLAY_DIR/taler-monitoring.sh" ]; then + echo "WARN: Forgejo @ ${COMMIT_SHORT:-?} lagging local suite-overlay (phases=[$PHASES])" >&2 + echo " re-applying suite-overlay → $SUITE_MON" >&2 + mkdir -p "$SUITE_MON" + rsync -a \ + --exclude secrets.env --exclude 'android-test/artifacts' --exclude 'android-test/apks' \ + --exclude 'android-test/out*' --exclude '__pycache__' \ + "$OVERLAY_DIR/" "$SUITE_MON/" + ROOT="$SUITE_MON" +elif [ "$_need_overlay" = "1" ]; then + echo "WARN: Forgejo missing features and no suite-overlay — using agent mon $AGENT_MON" >&2 + ROOT="$AGENT_MON" +else + echo "using suite mon @ ${COMMIT_SHORT:-?} · $ROOT" +fi +# Always restore host-agent + site-gen from pre-reset snapshot when present +# (systemd ExecStart points at koopa-admin-log which may be the reset suite tree). +if [ -d "$OVERLAY_DIR/host-agent" ]; then + mkdir -p "$SUITE_MON/host-agent" 2>/dev/null || true + rsync -a "$OVERLAY_DIR/host-agent/" "$SUITE_MON/host-agent/" 2>/dev/null || true +fi +if [ -d "$OVERLAY_DIR/site-gen" ]; then + mkdir -p "$SUITE_MON/site-gen" + rsync -a "$OVERLAY_DIR/site-gen/" "$SUITE_MON/site-gen/" 2>/dev/null || true +fi +unset _need_overlay + +if [ -f "$ROOT/site-gen/console_to_html.py" ]; then + SITE_GEN="$ROOT/site-gen" +elif [ -f "$AGENT_MON/site-gen/console_to_html.py" ]; then + SITE_GEN="$AGENT_MON/site-gen" + echo "note: using agent site-gen at $SITE_GEN" +elif [ -f "$SUITE_MON/site-gen/console_to_html.py" ]; then + SITE_GEN="$SUITE_MON/site-gen" +else + SITE_GEN="" +fi + +if [ ! -x "$ROOT/taler-monitoring.sh" ]; then + if [ -x "$AGENT_MON/taler-monitoring.sh" ]; then + ROOT="$AGENT_MON" + echo "note: using agent mon tree $ROOT" + else + echo "ERROR: suite missing at $ROOT" >&2 + COMMIT="${COMMIT:-unknown}" + COMMIT_SHORT="${COMMIT_SHORT:-unknown}" + COMMIT_URL="${COMMIT_URL:-}" + fi +fi + +echo "commit=${COMMIT_SHORT:-?} · ${COMMIT_URL:-}" + +ec=1 +if [ -x "${ROOT:-}/taler-monitoring.sh" ]; then + cd "$ROOT" || exit 1 + chmod +x taler-monitoring.sh check_*.sh host-agent/*.sh site-gen/*.sh site-gen/*.py 2>/dev/null || true + + # Overlay newer inside/versions host-podman bits if suite clone is stale + for f in check_inside.sh check_versions.sh taler-monitoring.sh lib.sh; do + if [ -f "$AGENT_MON/$f" ] && [ -f "$ROOT/$f" ]; then + if ! grep -q 'host-podman\|INSIDE_ACCESS\|RUN_TIMEOUT' "$ROOT/$f" 2>/dev/null \ + && grep -q 'host-podman\|INSIDE_ACCESS\|RUN_TIMEOUT' "$AGENT_MON/$f" 2>/dev/null; then + echo "note: overlay $f from agent install" + cp -f "$AGENT_MON/$f" "$ROOT/$f" + chmod +x "$ROOT/$f" 2>/dev/null || true + fi + fi + done + + # shellcheck disable=SC2086 + ./taler-monitoring.sh -d "$TALER_DOMAIN" $PHASES + ec=$? +fi + +# --- always generate HTML (first run / missing pages / every run) --- +write_bootstrap_html() { + local out="$1" host="$2" mode="$3" + mkdir -p "$(dirname "$out")" + local cshort="${COMMIT_SHORT:-unknown}" + local curl="${COMMIT_URL:-}" + local clink="$cshort" + local gen iso + gen=$(date -u +"%Y-%m-%d %H:%M:%SZ" 2>/dev/null || date) + iso=$(date -u +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -Iseconds) + if [ -n "$curl" ]; then + clink="$cshort" + fi + cat >"$out" < + + +monitoring ${mode} · ${host} + + +
+

Bootstrap page (converter missing or first install). See host-agent log.

+
$(tail -n 80 "$LOG" 2>/dev/null | sed 's/&/\&/g;s/
+
+ +EOF + echo "bootstrap html → $out" +} + +htmlify_host() { + local host="$1" + local mon="$HTML_BASE/$host/${HTML_OK_DIR}/index.html" + local mon_err="$HTML_BASE/$host/${HTML_ERR_DIR}/index.html" + mkdir -p "$HTML_BASE/$host/${HTML_OK_DIR}" "$HTML_BASE/$host/${HTML_ERR_DIR}" + + if [ -n "$SITE_GEN" ] && [ -f "$SITE_GEN/console_to_html.py" ]; then + python3 "$SITE_GEN/console_to_html.py" \ + --log "$LOG" \ + --out "$mon_err" \ + --hostname "$host" \ + --mode err \ + --commit "${COMMIT:-}" \ + --commit-url "${COMMIT_URL:-}" \ + --suite-path "$SUITE_PATH" \ + --page-label "$PAGE_LABEL" \ + --path-err "$HTML_URL_ERR" \ + --link-other "$HTML_URL_OK" + if [ "$ec" -eq 0 ]; then + python3 "$SITE_GEN/console_to_html.py" \ + --log "$LOG" \ + --out "$mon" \ + --hostname "$host" \ + --mode ok \ + --commit "${COMMIT:-}" \ + --commit-url "${COMMIT_URL:-}" \ + --suite-path "$SUITE_PATH" \ + --page-label "$PAGE_LABEL" \ + --path-err "$HTML_URL_ERR" \ + --link-other "" + rm -rf "$HTML_BASE/$host/${HTML_ERR_DIR}" + echo "html $host → ${HTML_URL_OK} only (clean · ${COMMIT_SHORT:-?})" + else + python3 "$SITE_GEN/console_to_html.py" \ + --log "$LOG" \ + --out "$mon" \ + --hostname "$host" \ + --mode redirect \ + --commit "${COMMIT:-}" \ + --commit-url "${COMMIT_URL:-}" \ + --suite-path "$SUITE_PATH" \ + --page-label "$PAGE_LABEL" \ + --path-err "$HTML_URL_ERR" + echo "html $host → ${HTML_URL_OK} stub + ${HTML_URL_ERR} (ec=$ec · ${COMMIT_SHORT:-?})" + fi + else + echo "WARN: console_to_html.py missing — bootstrap pages" + write_bootstrap_html "$mon_err" "$host" "err" + if [ "$ec" -eq 0 ]; then + write_bootstrap_html "$mon" "$host" "ok" + rm -rf "$HTML_BASE/$host/${HTML_ERR_DIR}" + else + write_bootstrap_html "$mon" "$host" "redirect" + fi + fi + + if [ ! -f "$mon" ]; then + write_bootstrap_html "$mon" "$host" "err" + fi + + printf '%s\n' "${COMMIT:-unknown}" >"$HTML_BASE/$host/COMMIT" + printf '%s\n' "${COMMIT_URL:-}" >"$HTML_BASE/$host/COMMIT_URL" +} +for host in $MON_HOSTS; do + htmlify_host "$host" +done + +if [ -n "$DEPLOY_WWW" ] && [ -w "$DEPLOY_WWW" ] 2>/dev/null; then + rsync -a --delete "$HTML_BASE/" "$DEPLOY_WWW/" && \ + echo "synced → $DEPLOY_WWW/" +fi + +ls -1t "$LOG_DIR"/run-*.log 2>/dev/null | tail -n +30 | xargs rm -f 2>/dev/null || true + +echo "======== done ec=$ec · log=$LOG · ${COMMIT_URL:-} ========" +if [ "${STRICT_EXIT:-0}" = "1" ]; then + exit "$ec" +fi +exit 0 diff --git a/scripts/taler-monitoring/host-agent/run-surface-monitoring.sh b/scripts/taler-monitoring/host-agent/run-surface-monitoring.sh new file mode 100755 index 0000000..1b7a8b8 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/run-surface-monitoring.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# run-surface-monitoring.sh — hourly remote-only ecosystem surface scan +# Public only on taler.hacktivism.ch: +# https://taler.hacktivism.ch/taler-monitoring-surface/ +# https://taler.hacktivism.ch/taler-monitoring-surface_err/ +# +set -uo pipefail +AGENT_DIR=$(cd "$(dirname "$0")" && pwd) + +export AGENT_LABEL="${AGENT_LABEL:-surface-host-agent}" +export STATE_NAME="${STATE_NAME:-taler-surface-monitoring}" +export TALER_DOMAIN="${TALER_DOMAIN:-hacktivism.ch}" +# surface is remote-only — force flags (ignore shared env) +export INSIDE_PODMAN=0 +export LOCAL_STACK=0 +export SKIP_SSH=1 +export INSIDE_MODE="${INSIDE_MODE:-none}" +export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" +# many hosts; allow up to 45 min wall (phase also has RUN_TIMEOUT) +export RUN_TIMEOUT="${RUN_TIMEOUT:-2400}" +# dedicated job: always surface only (never inherit PHASES from shared env) +export PHASES=surface +# Only merchant public front — not bank/exchange +export MON_HOSTS="${MON_HOSTS:-taler.hacktivism.ch}" +export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}" +export DEPLOY_WWW_ROOT="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" + +export HTML_OK_DIR="taler-monitoring-surface" +export HTML_ERR_DIR="taler-monitoring-surface_err" +export HTML_URL_OK="/taler-monitoring-surface/" +export HTML_URL_ERR="/taler-monitoring-surface_err/" +export PAGE_LABEL="taler-monitoring-surface" + +# ecosystem catalog by default (not domain-only) +export SURFACE_SCOPE="${SURFACE_SCOPE:-ecosystem}" +export TALER_DOMAIN_FROM_CLI=0 + +# Do not run apt-deploy ensure for surface-only jobs +export APT_DEPLOY_ENSURE=0 + +exec bash "$AGENT_DIR/run-host-report.sh" "$@" diff --git a/scripts/taler-monitoring/host-agent/taler-monitoring-aptdeploy.service b/scripts/taler-monitoring/host-agent/taler-monitoring-aptdeploy.service new file mode 100644 index 0000000..3d71eb1 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/taler-monitoring-aptdeploy.service @@ -0,0 +1,18 @@ +[Unit] +Description=Taler monitoring aptdeploy (apt-src container deploy tests) +Documentation=file:%h/koopa-admin-log/scripts/taler-monitoring/host-agent/README.md +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin +Environment=CONTINUE_ON_ERROR=1 +Environment=RUN_TIMEOUT=1800 +WorkingDirectory=%h/koopa-admin-log/scripts/taler-monitoring +ExecStart=%h/koopa-admin-log/scripts/taler-monitoring/host-agent/run-aptdeploy-monitoring.sh +Nice=10 +TimeoutStartSec=45min + +[Install] +WantedBy=default.target diff --git a/scripts/taler-monitoring/host-agent/taler-monitoring-aptdeploy.timer b/scripts/taler-monitoring/host-agent/taler-monitoring-aptdeploy.timer new file mode 100644 index 0000000..602ce8b --- /dev/null +++ b/scripts/taler-monitoring/host-agent/taler-monitoring-aptdeploy.timer @@ -0,0 +1,14 @@ +[Unit] +Description=Periodic taler-monitoring aptdeploy (apt-src deploy tests) +Documentation=file:%h/koopa-admin-log/scripts/taler-monitoring/host-agent/README.md + +[Timer] +OnBootSec=35min +OnUnitActiveSec=4h +AccuracySec=5min +Persistent=true +RandomizedDelaySec=10min +Unit=taler-monitoring-aptdeploy.service + +[Install] +WantedBy=timers.target diff --git a/scripts/taler-monitoring/host-agent/taler-monitoring-hacktivism.path b/scripts/taler-monitoring/host-agent/taler-monitoring-hacktivism.path new file mode 100644 index 0000000..1cbf003 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/taler-monitoring-hacktivism.path @@ -0,0 +1,19 @@ +[Unit] +Description=Watch hacktivism software/config changes → re-run monitoring +Documentation=file:%h/koopa-admin-log/scripts/taler-monitoring/host-agent/README.md + +[Path] +# Explicit stamp after package/image upgrades (touch-software-stamp.sh) +PathChanged=%h/.local/state/taler-hacktivism-monitoring/software.stamp +PathModified=%h/.local/state/taler-hacktivism-monitoring/software.stamp +# Landing + caddy config in admin-log (when checked out under ~) +PathChanged=%h/koopa-admin-log/configs/bank-landing +PathChanged=%h/koopa-admin-log/configs/exchange-landing +PathChanged=%h/koopa-admin-log/configs/merchant-landing +PathChanged=%h/koopa-admin-log/configs/caddy +# Debounce: wait for quiet period before firing +TriggerLimitIntervalSec=120 +TriggerLimitBurst=3 + +[Install] +WantedBy=default.target diff --git a/scripts/taler-monitoring/host-agent/taler-monitoring-hacktivism.service b/scripts/taler-monitoring/host-agent/taler-monitoring-hacktivism.service new file mode 100644 index 0000000..712920c --- /dev/null +++ b/scripts/taler-monitoring/host-agent/taler-monitoring-hacktivism.service @@ -0,0 +1,21 @@ +[Unit] +Description=Taler monitoring GOA/hacktivism (host-podman inside + public urls) +Documentation=file:%h/koopa-admin-log/scripts/taler-monitoring/host-agent/README.md +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +# Primary: host can podman-exec into taler-hacktivism* containers +Environment=INSIDE_PODMAN=1 +Environment=CONTINUE_ON_ERROR=1 +Environment=RUN_TIMEOUT=600 +Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin +WorkingDirectory=%h/koopa-admin-log/scripts/taler-monitoring +ExecStart=%h/koopa-admin-log/scripts/taler-monitoring/host-agent/run-hacktivism-monitoring.sh +Nice=10 +# Suite budget is RUN_TIMEOUT (10m) + git/HTML overhead +TimeoutStartSec=15min + +[Install] +WantedBy=default.target diff --git a/scripts/taler-monitoring/host-agent/taler-monitoring-hacktivism.timer b/scripts/taler-monitoring/host-agent/taler-monitoring-hacktivism.timer new file mode 100644 index 0000000..b65c139 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/taler-monitoring-hacktivism.timer @@ -0,0 +1,14 @@ +[Unit] +Description=Periodic hacktivism monitoring (fallback if no path events) +Documentation=file:%h/koopa-admin-log/scripts/taler-monitoring/host-agent/README.md + +[Timer] +# Every 4h + boot +OnBootSec=10min +OnUnitActiveSec=4h +AccuracySec=5min +Persistent=true +Unit=taler-monitoring-hacktivism.service + +[Install] +WantedBy=timers.target diff --git a/scripts/taler-monitoring/host-agent/taler-monitoring-surface.service b/scripts/taler-monitoring/host-agent/taler-monitoring-surface.service new file mode 100644 index 0000000..6018dee --- /dev/null +++ b/scripts/taler-monitoring/host-agent/taler-monitoring-surface.service @@ -0,0 +1,18 @@ +[Unit] +Description=Taler monitoring surface (remote ecosystem inventory) +Documentation=file:%h/koopa-admin-log/scripts/taler-monitoring/host-agent/README.md +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin +Environment=CONTINUE_ON_ERROR=1 +Environment=RUN_TIMEOUT=2400 +WorkingDirectory=%h/koopa-admin-log/scripts/taler-monitoring +ExecStart=%h/koopa-admin-log/scripts/taler-monitoring/host-agent/run-surface-monitoring.sh +Nice=10 +TimeoutStartSec=50min + +[Install] +WantedBy=default.target diff --git a/scripts/taler-monitoring/host-agent/taler-monitoring-surface.timer b/scripts/taler-monitoring/host-agent/taler-monitoring-surface.timer new file mode 100644 index 0000000..a9ed8c9 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/taler-monitoring-surface.timer @@ -0,0 +1,14 @@ +[Unit] +Description=Hourly taler-monitoring surface scan (ecosystem) +Documentation=file:%h/koopa-admin-log/scripts/taler-monitoring/host-agent/README.md + +[Timer] +OnBootSec=20min +OnUnitActiveSec=1h +AccuracySec=2min +Persistent=true +RandomizedDelaySec=5min +Unit=taler-monitoring-surface.service + +[Install] +WantedBy=timers.target diff --git a/scripts/taler-monitoring/host-agent/touch-software-stamp.sh b/scripts/taler-monitoring/host-agent/touch-software-stamp.sh new file mode 100755 index 0000000..0aff400 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/touch-software-stamp.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Call after upgrading hacktivism packages / redeploying landings / image rebuilds. +# systemd path unit watches this stamp → re-runs monitoring. +set -euo pipefail +STATE="${XDG_STATE_HOME:-$HOME/.local/state}/taler-hacktivism-monitoring" +mkdir -p "$STATE" +STAMP="$STATE/software.stamp" +date -Iseconds >"$STAMP" +# also record container image ids for debugging +if command -v podman >/dev/null 2>&1; then + { + echo "# images $(date -Iseconds)" + podman images --format '{{.Repository}}:{{.Tag}} {{.ID}} {{.Digest}}' 2>/dev/null | grep -i hacktivism || true + podman ps --format '{{.Names}} {{.ImageID}} {{.Status}}' 2>/dev/null | grep -i hacktivism || true + } >"$STATE/software-images.txt" || true +fi +echo "touched $STAMP" diff --git a/scripts/taler-monitoring/host-agent/update-suite.sh b/scripts/taler-monitoring/host-agent/update-suite.sh new file mode 100755 index 0000000..70d348d --- /dev/null +++ b/scripts/taler-monitoring/host-agent/update-suite.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# update-suite.sh — ensure taler-monitoring code is latest from Forgejo. +# +# Local config (NEVER overwritten by git): +# ${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring/env +# +# Env overrides (also settable in that file): +# SUITE_GIT_URL default https://git.hacktivism.ch/hernani/koopa-admin-log.git +# SUITE_GIT_REF default main +# SUITE_DIR default $HOME/src/koopa-admin-log (or $HOME/koopa-admin-log) +# SUITE_UPDATE_MODE ff (default) | reset (reset = hard to origin/REF) +# +set -uo pipefail + +CFG_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring" +ENV_FILE="${TALER_MONITORING_ENV:-$CFG_DIR/env}" +# When sourced from run-host-report.sh the env file is already loaded and +# wrapper PHASES/HTML_* must not be clobbered. Only load here if needed. +if [ -f "$ENV_FILE" ] && [ "${_TALER_MON_ENV_LOADED:-0}" != "1" ]; then + # shellcheck disable=SC1090 + set -a + # shellcheck source=/dev/null + source "$ENV_FILE" + set +a +fi + +SUITE_GIT_URL="${SUITE_GIT_URL:-https://git.hacktivism.ch/hernani/koopa-admin-log.git}" +SUITE_GIT_REF="${SUITE_GIT_REF:-main}" +# Host agents default to reset so "always latest from Forgejo" wins over local +# diverged trees. Override with SUITE_UPDATE_MODE=ff for careful pulls. +SUITE_UPDATE_MODE="${SUITE_UPDATE_MODE:-reset}" + +if [ -z "${SUITE_DIR:-}" ]; then + if [ -d "$HOME/src/koopa-admin-log/.git" ]; then + SUITE_DIR="$HOME/src/koopa-admin-log" + elif [ -d "$HOME/koopa-admin-log/.git" ]; then + SUITE_DIR="$HOME/koopa-admin-log" + else + SUITE_DIR="$HOME/src/koopa-admin-log" + fi +fi + +export SUITE_DIR + +mkdir -p "$(dirname "$SUITE_DIR")" "$CFG_DIR" + +if [ ! -d "$SUITE_DIR/.git" ]; then + echo "clone $SUITE_GIT_URL → $SUITE_DIR" + git clone --branch "$SUITE_GIT_REF" "$SUITE_GIT_URL" "$SUITE_DIR" \ + || git clone "$SUITE_GIT_URL" "$SUITE_DIR" +fi + +cd "$SUITE_DIR" || exit 1 +git remote set-url origin "$SUITE_GIT_URL" 2>/dev/null \ + || git remote add origin "$SUITE_GIT_URL" 2>/dev/null || true + +echo "fetch origin ($SUITE_GIT_REF) …" +if ! git fetch --tags origin 2>&1; then + echo "WARN: git fetch failed — using existing tree at $SUITE_DIR" >&2 + git rev-parse HEAD 2>/dev/null || true + return 0 2>/dev/null || exit 0 +fi + +case "$SUITE_UPDATE_MODE" in + reset) + git checkout -B "$SUITE_GIT_REF" "origin/$SUITE_GIT_REF" 2>/dev/null \ + || git checkout -B "$SUITE_GIT_REF" "FETCH_HEAD" + git reset --hard "origin/$SUITE_GIT_REF" 2>/dev/null \ + || git reset --hard FETCH_HEAD + ;; + *) + git checkout "$SUITE_GIT_REF" 2>/dev/null || git checkout -B "$SUITE_GIT_REF" "origin/$SUITE_GIT_REF" 2>/dev/null || true + if ! git pull --ff-only origin "$SUITE_GIT_REF" 2>&1; then + echo "WARN: ff-only pull failed — try SUITE_UPDATE_MODE=reset or fix local commits" >&2 + fi + ;; +esac + +COMMIT=$(git rev-parse HEAD) +COMMIT_SHORT=$(git rev-parse --short=12 HEAD) +# Forgejo web UI +SOURCE_REPO_WEB="${SOURCE_REPO_WEB:-https://git.hacktivism.ch/hernani/koopa-admin-log}" +COMMIT_URL="${SOURCE_REPO_WEB}/src/commit/${COMMIT}" +export COMMIT COMMIT_SHORT COMMIT_URL SOURCE_REPO_WEB + +echo "suite @ $COMMIT_SHORT $COMMIT_URL" +echo " dir=$SUITE_DIR" +echo " config=$ENV_FILE (not touched by git)" +echo " note: always latest = origin/${SUITE_GIT_REF} on git.hacktivism.ch (push local work there)" + +# Symlink convenience path if missing or not a git clone +if [ -d "$SUITE_DIR/.git" ]; then + if [ ! -e "$HOME/koopa-admin-log" ]; then + ln -sfn "$SUITE_DIR" "$HOME/koopa-admin-log" 2>/dev/null || true + elif [ -L "$HOME/koopa-admin-log" ]; then + ln -sfn "$SUITE_DIR" "$HOME/koopa-admin-log" 2>/dev/null || true + fi +fi + +# Make suite scripts executable after reset +if [ -d "$SUITE_DIR/scripts/taler-monitoring" ]; then + chmod +x \ + "$SUITE_DIR"/scripts/taler-monitoring/taler-monitoring.sh \ + "$SUITE_DIR"/scripts/taler-monitoring/check_*.sh \ + "$SUITE_DIR"/scripts/taler-monitoring/host-agent/*.sh \ + "$SUITE_DIR"/scripts/taler-monitoring/site-gen/*.sh \ + "$SUITE_DIR"/scripts/taler-monitoring/site-gen/*.py \ + 2>/dev/null || true +fi diff --git a/scripts/taler-monitoring/lib.sh b/scripts/taler-monitoring/lib.sh index 841d6e8..8455c6b 100755 --- a/scripts/taler-monitoring/lib.sh +++ b/scripts/taler-monitoring/lib.sh @@ -23,6 +23,9 @@ MERCHANT_INSTANCE=$(printf '%s' "${MERCHANT_INSTANCE}" | tr '[:upper:]' '[:lower : "${TIMEOUT:=12}" : "${E2E_TIMEOUT:=55}" # whole e2e budget; skip rest when exceeded (e2e raises as needed) : "${E2E_PAY_SECS:=22}" # dedicated seconds for pay handle-uri (avoid Alarm clock) +# Whole-run wall clock for taler-monitoring.sh (seconds). 0 = unlimited. +# Host-agent default phases (urls/inside/versions) should finish under this. +: "${RUN_TIMEOUT:=600}" # Local GOA: public paivana paywall (https://paivana.hacktivism.ch · template GOA:4200) : "${PAIVANA_PUBLIC:=https://paivana.hacktivism.ch}" : "${E2E_PAIVANA:=1}" # 0 = skip paivana section in e2e @@ -855,6 +858,113 @@ summary() { [ "$FAIL_N" -eq 0 ] } +# --------------------------------------------------------------------------- +# Disk space on hosts / containers under test +# WARN when tight, ERROR when full / critically low +# DISK_WARN_USED_PCT=85 # free below 15% → WARN +# DISK_ERR_USED_PCT=95 # free below 5% → ERROR +# DISK_ERR_FREE_BYTES=0 # free == 0 always ERROR +# --------------------------------------------------------------------------- +: "${DISK_WARN_USED_PCT:=85}" +: "${DISK_ERR_USED_PCT:=95}" + +# Parse one POSIX `df -P` body line (no header): fs size used avail capacity mount +# size/used/avail in 1K-blocks when using df -Pk +_mon_disk_eval_line() { + local where="$1" fs="$2" size="$3" used="$4" avail="$5" cap="$6" mnt="$7" + local pct label detail + pct=${cap%%%} + # non-numeric capacity → skip + case "$pct" in + ''|*[!0-9]*) return 0 ;; + esac + label="disk ${where} ${mnt}" + detail="fs=$fs size_1k=$size used_1k=$used avail_1k=$avail use=${pct}%" + if [ "$avail" = "0" ] || [ "$pct" -ge 100 ]; then + err "disk" "${where} ${mnt} FULL / overflow" "$detail" + return 1 + fi + if [ "$pct" -ge "${DISK_ERR_USED_PCT}" ]; then + err "disk" "${where} ${mnt} critically low free space (≥${DISK_ERR_USED_PCT}% used)" "$detail" + return 1 + fi + if [ "$pct" -ge "${DISK_WARN_USED_PCT}" ]; then + warn "disk" "${where} ${mnt} free space tight (≥${DISK_WARN_USED_PCT}% used)" "$detail" + return 0 + fi + ok "$label" "$detail" + return 0 +} + +# Report all filesystems from `df -P` / `df -Pk` output (with header). +# $1 = where label (host, container:name, ssh:host) +# $2 = full df text +mon_disk_report_df() { + local where="$1" text="$2" line fs size used avail cap mnt rest ec=0 + [ -n "$text" ] || { + warn "disk" "${where}: no df output" + return 0 + } + while IFS= read -r line || [ -n "$line" ]; do + [ -n "$line" ] || continue + case "$line" in + Filesystem*|Filesystem*) continue ;; + esac + # df -P: Filesystem 1024-blocks Used Available Capacity Mounted on + # shellcheck disable=SC2086 + set -- $line + [ "$#" -ge 6 ] || continue + fs=$1 + size=$2 + used=$3 + avail=$4 + cap=$5 + shift 5 + mnt=$* + # skip special/pseudo if tiny and not root-ish + case "$fs" in + tmpfs|devtmpfs|overlay|shm|nsfs|proc|sysfs|cgroup*) + # still check root-like mounts that fill (overlay / in containers) + case "$mnt" in + /|/var|/var/*|/home|/home/*|/data|/mnt/*) ;; + *) continue ;; + esac + ;; + esac + if ! _mon_disk_eval_line "$where" "$fs" "$size" "$used" "$avail" "$cap" "$mnt"; then + ec=1 + fi + done <<<"$text" + return "$ec" +} + +# Local host: important mounts +mon_disk_check_host() { + local where text + where="${1:-host}" + text="" + text=$(df -Pk / /var /home /tmp 2>/dev/null | awk 'NR==1 || !seen[$1,$6]++' || true) + # fallback whole table + if [ -z "$text" ]; then + text=$(df -Pk 2>/dev/null || true) + fi + mon_disk_report_df "$where" "$text" +} + +# Run df inside podman container +mon_disk_check_podman() { + local cname="$1" text + local bin="${2:-podman}" + text=$("$bin" exec "$cname" df -Pk / /var /tmp 2>/dev/null || "$bin" exec "$cname" df -Pk 2>/dev/null || true) + mon_disk_report_df "ctr:${cname}" "$text" +} + +# df text already collected remotely +mon_disk_check_remote_text() { + local where="$1" text="$2" + mon_disk_report_df "$where" "$text" +} + http_code() { local url="$1"; shift curl -skS --max-redirs 0 -m "${TIMEOUT}" -o /dev/null -w '%{http_code}' "$@" "$url" 2>/dev/null || echo 000 diff --git a/scripts/taler-monitoring/site-gen/README.md b/scripts/taler-monitoring/site-gen/README.md new file mode 100644 index 0000000..82e52d2 --- /dev/null +++ b/scripts/taler-monitoring/site-gen/README.md @@ -0,0 +1,81 @@ +# monitoring site-gen + +Build **console-style HTML** from `taler-monitoring` runs and stage them for +**host Caddy** (`/monitoring`, `/monitoring_err`). + +Same **global reporting defaults** as host-agents (`run-host-report.sh`): + +- **`RUN_TIMEOUT=600`** on each suite run +- **line-buffered** logs (continuous write) +- **always** write HTML (first run / empty logs too) +- Forgejo **commit link** in the page header + +Applies to all nine fronts (GOA + FP stage + FP prod) in the default `SITES` list. + +| Host | Role | +|------|------| +| **firecuda-external** | optional runner (public DNS) | +| **koopa-external** | deploy staging + Caddy (root service) | +| **host-agents** | primary: GOA + FP on-box (`../host-agent/`) | + +No secrets in `settings.conf` — only hosts, paths, site list. + +## firecuda-external (outside-only) — **disabled for now** + +**firecuda-external could** run public-only monitoring (`SKIP_SSH=1`, phases +`urls versions`) on a schedule (launchd every 4h) and write HTML under +`~/var/taler-monitoring-sites-work/html/`. Scripts for that still live here +(`install-firecuda-timer.sh`, `run-on-firecuda.sh`, plist template). + +**Current policy: do not run a real launchd job on firecuda.** +Primary GOA monitoring is **koopa host-agent** (`host-agent/`, user `hernani`, +local `podman exec`). Re-enable firecuda only if you explicitly want a second, +outside-only view. + +```bash +# optional one-shot (no timer): +# ssh firecuda-external '~/taler-monitoring-site-gen/site-gen/run-on-firecuda.sh' + +# if a timer was installed earlier, ensure it is off: +# ssh firecuda-external 'launchctl list | grep taler-monitoring || echo off' +``` + +Disabled plists may remain as `*.plist.disabled-*` under `~/Library/LaunchAgents/`. + +## Quick start (laptop) + +```bash +cd scripts/taler-monitoring/site-gen +cp settings.conf.example settings.conf # optional +chmod +x generate-monitoring-sites.sh deploy-monitoring-sites.sh console_to_html.py + +ONLY_HOSTS="bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch" \ + SKIP_SSH=1 RUNNER_SSH= \ + ./generate-monitoring-sites.sh + +./deploy-monitoring-sites.sh +``` + +Then on koopa **as root**: see **[ROOT-ON-KOOPA.md](./ROOT-ON-KOOPA.md)**. + +## URL rules + +| Path | When | +|------|------| +| `/monitoring_err/` | always: full console log + error list on top (CONTINUE_ON_ERROR run) | +| `/monitoring/` | **clean** full log if last strict run exit 0; else **stub** linking to `/monitoring_err/` | + +Footer links **Forgejo commit** of the admin-log tree used for that run. + +## Layout + +```text +site-gen/ + settings.conf.example + generate-monitoring-sites.sh + deploy-monitoring-sites.sh + console_to_html.py + caddy-monitoring-handles.snippet + ROOT-ON-KOOPA.md + README.md +``` diff --git a/scripts/taler-monitoring/site-gen/ROOT-ON-KOOPA.md b/scripts/taler-monitoring/site-gen/ROOT-ON-KOOPA.md new file mode 100644 index 0000000..604b997 --- /dev/null +++ b/scripts/taler-monitoring/site-gen/ROOT-ON-KOOPA.md @@ -0,0 +1,92 @@ +# Root on koopa (hacktivism monitoring sites) + +Host Caddy runs as **root/systemd** (`caddy` user). Static monitoring HTML is +**not** put into Taler containers — only host paths + Caddy `handle`. + +After `generate-monitoring-sites.sh` + `deploy-monitoring-sites.sh` (as hernani), +files live in: + +```text +/home/hernani/monitoring-sites-staging//monitoring/index.html +/home/hernani/monitoring-sites-staging//monitoring_err/index.html +``` + +## One-shot as root (copy + Caddy) + +```bash +# on koopa (koopa-external), as root: + +# 1) web root +install -d -o caddy -g caddy -m 755 /var/www/monitoring-sites +rsync -a --delete /home/hernani/monitoring-sites-staging/ /var/www/monitoring-sites/ +chown -R caddy:caddy /var/www/monitoring-sites + +# 2) Caddyfile — merge handles from admin-log mirror, then: +# (either edit /etc/caddy/Caddyfile by hand using snippet below, +# or copy full mirror after review) +# +# install -m 644 /home/hernani/koopa-admin-log/configs/caddy/Caddyfile /etc/caddy/Caddyfile +# # or: ~/koopa-caddy/apply.sh after syncing Caddyfile into ~/koopa-caddy/ + +caddy validate --config /etc/caddy/Caddyfile +systemctl reload caddy +systemctl is-active caddy +``` + +## Snippet per hacktivism site (inside each `*.hacktivism.ch { }` block) + +Place **before** the catch-all `reverse_proxy` (same idea as `/intro*`): + +```caddy + # Public taler-monitoring console HTML (static; host only) + handle /monitoring_err* { + root * /var/www/monitoring-sites/{host} + rewrite * /monitoring_err{uri} + # uri is /monitoring_err or /monitoring_err/ → serve directory index + file_server + } + handle /monitoring* { + root * /var/www/monitoring-sites/{host} + file_server + } +``` + +Simpler layout (recommended): path on disk matches URL under host root: + +```text +/var/www/monitoring-sites/bank.hacktivism.ch/monitoring/index.html +/var/www/monitoring-sites/bank.hacktivism.ch/monitoring_err/index.html +``` + +```caddy + handle_path /monitoring_err/* { + root * /var/www/monitoring-sites/{host}/monitoring_err + file_server + } + handle /monitoring_err { + redir /monitoring_err/ 302 + } + handle_path /monitoring/* { + root * /var/www/monitoring-sites/{host}/monitoring + file_server + } + handle /monitoring { + redir /monitoring/ 302 + } +``` + +Canonical snippet is also in `caddy-monitoring-handles.snippet` and applied +in `configs/caddy/Caddyfile` for the three GOA hosts. + +## Check + +```bash +curl -sS -o /dev/null -w '%{http_code}\n' https://bank.hacktivism.ch/monitoring/ +curl -sS -o /dev/null -w '%{http_code}\n' https://bank.hacktivism.ch/monitoring_err/ +curl -sS https://bank.hacktivism.ch/monitoring_err/ | head +``` + +## Not needed as root + +- Running `taler-monitoring` (use **firecuda-external** / laptop) +- Writing into podman Taler containers diff --git a/scripts/taler-monitoring/site-gen/caddy-monitoring-handles.snippet b/scripts/taler-monitoring/site-gen/caddy-monitoring-handles.snippet new file mode 100644 index 0000000..1c09531 --- /dev/null +++ b/scripts/taler-monitoring/site-gen/caddy-monitoring-handles.snippet @@ -0,0 +1,57 @@ +# Host Caddy on koopa (root service). Paste BEFORE catch-all reverse_proxy. +# +# Disk layout: +# /var/www/monitoring-sites/{host}/monitoring/index.html +# /var/www/monitoring-sites/{host}/monitoring_err/index.html +# /var/www/monitoring-sites/taler.hacktivism.ch/taler-monitoring-surface*/index.html +# /var/www/monitoring-sites/taler.hacktivism.ch/taler-monitoring-aptdeploy*/index.html +# +# /monitoring* → bank + exchange + taler.hacktivism.ch +# /taler-monitoring-surface* → taler.hacktivism.ch only +# /taler-monitoring-aptdeploy* → taler.hacktivism.ch only + +# --- only inside taler.hacktivism.ch { ... } --- + handle /taler-monitoring-surface_err { + redir /taler-monitoring-surface_err/ 302 + } + handle /taler-monitoring-surface_err/ { + root * /var/www/monitoring-sites/taler.hacktivism.ch/taler-monitoring-surface_err + file_server + } + handle /taler-monitoring-surface { + redir /taler-monitoring-surface/ 302 + } + handle /taler-monitoring-surface/ { + root * /var/www/monitoring-sites/taler.hacktivism.ch/taler-monitoring-surface + file_server + } + handle /taler-monitoring-aptdeploy_err { + redir /taler-monitoring-aptdeploy_err/ 302 + } + handle /taler-monitoring-aptdeploy_err/ { + root * /var/www/monitoring-sites/taler.hacktivism.ch/taler-monitoring-aptdeploy_err + file_server + } + handle /taler-monitoring-aptdeploy { + redir /taler-monitoring-aptdeploy/ 302 + } + handle /taler-monitoring-aptdeploy/ { + root * /var/www/monitoring-sites/taler.hacktivism.ch/taler-monitoring-aptdeploy + file_server + } + +# --- bank + exchange + taler (each site block) --- + handle /monitoring_err { + redir /monitoring_err/ 302 + } + handle /monitoring_err/ { + root * /var/www/monitoring-sites/{host}/monitoring_err + file_server + } + handle /monitoring { + redir /monitoring/ 302 + } + handle /monitoring/ { + root * /var/www/monitoring-sites/{host}/monitoring + file_server + } diff --git a/scripts/taler-monitoring/site-gen/com.hacktivism.taler-monitoring-sites.plist b/scripts/taler-monitoring/site-gen/com.hacktivism.taler-monitoring-sites.plist new file mode 100644 index 0000000..0b6f68b --- /dev/null +++ b/scripts/taler-monitoring/site-gen/com.hacktivism.taler-monitoring-sites.plist @@ -0,0 +1,34 @@ + + + + + + Label + com.hacktivism.taler-monitoring-sites + ProgramArguments + + /bin/bash + __SITE_GEN__/run-on-firecuda.sh + + StartInterval + 14400 + RunAtLoad + + StandardOutPath + __HOME__/Library/Logs/taler-monitoring-sites/launchd.out.log + StandardErrorPath + __HOME__/Library/Logs/taler-monitoring-sites/launchd.err.log + EnvironmentVariables + + PATH + /opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin + SKIP_SSH + 1 + CONTINUE_ON_ERROR + 1 + + + diff --git a/scripts/taler-monitoring/site-gen/console_to_html.py b/scripts/taler-monitoring/site-gen/console_to_html.py new file mode 100755 index 0000000..14aef2a --- /dev/null +++ b/scripts/taler-monitoring/site-gen/console_to_html.py @@ -0,0 +1,935 @@ +#!/usr/bin/env python3 +"""Convert taler-monitoring console log to a console-feel HTML page.""" +from __future__ import annotations + +import argparse +import html +import re +from datetime import datetime, timezone +from pathlib import Path + +# ANSI strip +ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") +ERR_BULLET_RE = re.compile(r"^\s*[•·*-]\s+(?P.+)$") +TID_RE = re.compile(r"\b(?P[a-z0-9_.-]+-\d+)\b", re.I) + + +def strip_ansi(s: str) -> str: + return ANSI_RE.sub("", s) + + +def classify(line: str) -> str: + u = line.upper() + # section / summary headers before badge matching (they contain the word ERROR) + if "SUMMARY" in u or "--- ERRORS" in u or "ERRORS ·" in u or "ERRORS (" in u: + return "meta" + if "numbered checks" in line.lower() or "totals:" in line.lower(): + return "meta" + if "BLOCKER" in u or "┌ BLOCKER" in u: + return "blocker" + if "ERROR" in u or "┌ ERROR" in u: + return "error" + if "WARN" in u or "┌ WARN" in u: + return "warn" + if "INFO" in u or "┌ INFO" in u: + return "info" + if " OK" in u or "[OK" in u or "┌ OK" in u or u.strip().startswith("OK"): + return "ok" + if u.strip().startswith("-- ") or u.strip().startswith("=="): + return "section" + return "plain" + + +def is_run_timeout_text(text: str) -> bool: + u = text.upper() + return "RUN.TIMEOUT" in u or "RUN_TIMEOUT" in u or "RUN TIMEOUT" in u + + +def is_summary_error_line(ln: str) -> bool: + """Skip summary/header lines that contain ERROR but are not checks.""" + low = ln.lower() + if "failed — see" in low or "failed - see" in low: + return True + if "errors · failed" in low or "errors (failed" in low: + return True + if "--- errors" in low: + return True + if "┌ summary" in low or "[ summary" in low: + return True + if "totals:" in low: + return True + # header-ish RUN_TIMEOUT= without error badge + if is_run_timeout_text(ln) and "ERROR" not in ln.upper() and "run.timeout" not in low: + return True + return False + + +def extract_errors(lines: list[str]) -> list[str]: + errs: list[str] = [] + timeout_detail = "" + in_block = False + for ln in lines: + if "--- ERRORS" in ln or "ERRORS (failed" in ln or "RUN TIMEOUT · extraordinary" in ln: + in_block = True + continue + if in_block: + if ln.strip().startswith("---") or "[ SUMMARY" in ln or "totals:" in ln: + in_block = False + continue + m = ERR_BULLET_RE.match(ln) + if m: + body = m.group("body").strip() + if is_run_timeout_text(body): + timeout_detail = body + else: + errs.append(body) + elif "ERROR" in ln.upper() and ln.strip() and not is_run_timeout_text(ln): + errs.append(ln.strip()) + if re.search(r"\bERROR\b", ln, re.I) and "--- ERRORS" not in ln: + if is_run_timeout_text(ln): + if "run.timeout-01" in ln.lower() or "RUN_TIMEOUT exceeded" in ln: + timeout_detail = re.sub(r"\s+", " ", strip_ansi(ln)).strip() + continue + if is_summary_error_line(ln): + continue + body = re.sub(r"^.*\bERROR\b\s*\]?\s*", "", ln, flags=re.I).strip(" ·") + if body and body not in errs and "failed — see" not in body.lower(): + if re.search( + r"#\d+|www\.|e2e\.|auth401\.|versions\.|inside\.|surface\.|aptdeploy\.", + body, + ): + errs.append(body) + seen: set[str] = set() + out: list[str] = [] + for e in errs: + if e in seen or is_run_timeout_text(e): + continue + seen.add(e) + out.append(e) + if timeout_detail or any( + is_run_timeout_text(l) and ("ERROR" in l.upper() or "run.timeout" in l.lower()) + for l in lines + ): + # only if real timeout ERROR badge, not mere RUN_TIMEOUT= header + if timeout_detail or any( + "run.timeout-01" in l.lower() or "RUN_TIMEOUT exceeded" in l for l in lines + ): + canon = timeout_detail or "run.timeout-01 [run] RUN_TIMEOUT exceeded" + out = [canon] + out + return out + + +def count_status(lines: list[str]) -> tuple[int, int, int | None, int | None]: + """Return (error_count, warn_count, first_error_idx, first_warn_idx).""" + n_err = 0 + n_warn = 0 + first_err: int | None = None + first_warn: int | None = None + for i, ln in enumerate(lines): + kind = classify(ln) + if kind in ("error", "blocker"): + if is_summary_error_line(ln): + continue + n_err += 1 + if first_err is None: + first_err = i + elif kind == "warn": + n_warn += 1 + if first_warn is None: + first_warn = i + return n_err, n_warn, first_err, first_warn + + +def slug_error(i: int, text: str) -> str: + if is_run_timeout_text(text): + return "err-run-timeout" + m = TID_RE.search(text) + if m: + return "err-" + re.sub(r"[^a-z0-9_-]+", "-", m.group("tid").lower()) + return f"err-{i:03d}" + + +def render_line(ln: str, err_slugs: dict[str, str], extra_ids: list[str] | None = None) -> str: + kind = classify(ln) + esc = html.escape(ln) + for tid, slug in err_slugs.items(): + if tid in ln: + esc = esc.replace( + html.escape(tid), + f'{html.escape(tid)}', + ) + id_attr = "" + if extra_ids: + id_attr = f' id="{" ".join(extra_ids)}"' # invalid multi-id; use first only + # HTML allows one id — join carefully + id_attr = f' id="{html.escape(extra_ids[0])}"' + if len(extra_ids) > 1: + # secondary anchors as empty spans prepended + spans = "".join(f'' for x in extra_ids[1:]) + return f'{spans}{esc}\n' + return f'{esc}\n' + + +def status_level(n_err: int, n_warn: int) -> str: + if n_err > 0: + return "red" + if n_warn > 0: + return "yellow" + return "green" + + +def extract_phases(log_text: str) -> str: + """Best-effort phases string from host-agent / taler-monitoring log header.""" + for pat in ( + r"phases=([a-z0-9 _.-]+)", + r"· phases=([a-z0-9 _.-]+)", + r"phases\s{2,}([a-z0-9 _.-]+)", + r"^\s*phases\s+([a-z0-9 _.-]+)\s*$", + ): + m = re.search(pat, log_text, re.I | re.M) + if m: + return re.sub(r"\s+", " ", m.group(1)).strip(" ·") + return "" + + +def monitoring_scope( + page_label: str, + hostname: str, + log_text: str = "", +) -> dict[str, object]: + """ + Human-readable description of what this monitoring page covers. + Returned keys: title, summary, items (list[str]), kind + """ + label = (page_label or "monitoring").lower().replace("_", "-") + phases = extract_phases(log_text) + host = hostname or "?" + + if "surface" in label: + return { + "kind": "surface", + "title": "Remote ecosystem surface inventory", + "summary": "Public catalog hosts · DNS / TCP / HTTPS / TLS · Server version · CVE", + "items": [ + "Outside-in only: no SSH and no podman exec on remote targets", + "Catalog (surface-catalog.conf): taler.net, demo/test, gnunet.org, " + "taler-ops.ch, taler-systems.com, deb.taler.net, ftp.gnu.org, …", + "Per host: DNS resolve, ICMP (info), open ports, protocol probes, " + "TLS cert expiry, Server-header software version, optional CVE check", + f"HTML published only on taler.hacktivism.ch " + f"(this page host: {host})", + "Timer: taler-monitoring-surface.timer (hourly)", + ], + } + + if "aptdeploy" in label or "apt-deploy" in label or "apt_src" in label: + return { + "kind": "aptdeploy", + "title": "apt-src merchant deploy tests (podman on koopa)", + "summary": "4 containers · fresh + upgrade tracks · trixie & trixie-testing", + "items": [ + "koopa-taler-deploy-test-apt-src-trixie (fresh, suite trixie)", + "koopa-taler-deploy-test-apt-src-trixie-testing (fresh, trixie-testing)", + "koopa-taler-deploy-test-apt-src-trixie-upgrade (upgrade track)", + "koopa-taler-deploy-test-apt-src-trixie-testing-upgrade (upgrade track)", + "Checks: packages, taler-merchant-httpd --version / ldd, systemd unit, " + "optional local /config probe — not public nginx/HTTPS", + f"HTML published only on taler.hacktivism.ch (this page host: {host})", + "Timer: taler-monitoring-aptdeploy.timer (4h)", + ], + } + + # Default: stack /monitoring pages (bank, exchange, merchant) + phase_txt = phases or "urls · inside · versions" + role = "stack" + if host.startswith("bank."): + role = "bank" + focus = "Libeufin bank public HTTPS + container inside checks" + elif host.startswith("exchange.") or host.startswith("stage.exchange."): + role = "exchange" + focus = "Exchange public HTTPS + container inside checks" + elif host.startswith("taler.") or "merchant" in host or host.startswith("monnaie."): + role = "merchant" + focus = "Merchant backend / SPA public HTTPS + container inside checks" + else: + focus = "Taler stack public + inside checks" + + items = [ + f"Focus: {focus}", + f"Phases this run: {phase_txt}", + "Typical: public URLs (HTTPS, QR, perf), inside (podman/ssh), package versions", + f"HTML host: {host}", + "Timer: taler-monitoring-hacktivism (path + 4h) · /monitoring on bank, exchange, taler", + ] + if phases: + items.insert(2, f"Log header phases= {phases}") + + return { + "kind": role, + "title": f"Stack monitoring · {host}", + "summary": f"{phase_txt} · public + inside", + "items": items, + } + + +def sticky_bar_html( + *, + generated_at: str, + generated_iso: str, + n_err: int, + n_warn: int, + hostname: str, + page_label: str, + mode: str, + commit_html: str, + suite_path: str, + link_other: str | None, + first_err_href: str | None, + first_warn_href: str | None, + scope: dict[str, object] | None = None, +) -> str: + level = status_level(n_err, n_warn) + if level == "red": + status_txt = "ERRORS" + elif level == "yellow": + status_txt = "WARNINGS" + else: + status_txt = "OK" + + if first_err_href and n_err: + err_stat = ( + f'{n_err} error{"s" if n_err != 1 else ""}' + ) + else: + err_stat = f'{n_err} errors' + + if first_warn_href and n_warn: + warn_stat = ( + f'{n_warn} warning{"s" if n_warn != 1 else ""}' + ) + else: + warn_stat = f'{n_warn} warnings' + + other = "" + if link_other: + other = ( + f'' + f"{html.escape(link_other)}" + ) + + if scope is None: + scope = monitoring_scope(page_label, hostname, "") + scope_title = html.escape(str(scope.get("title") or "Monitoring")) + scope_summary = html.escape(str(scope.get("summary") or "")) + scope_kind = html.escape(str(scope.get("kind") or "monitoring")) + lis = "\n".join( + f"
  • {html.escape(str(it))}
    • " for it in (scope.get("items") or []) + ) + suite_bit = ( + f" · {html.escape(suite_path)}" if suite_path else "" + ) + + # Compact sticky row always visible; "Was geprüft wird" expands inside sticky-bar + return f""" + +""" + + +STICKY_CSS = """ +.sticky-bar { + position: sticky; top: 0; z-index: 100; + border-bottom: 2px solid var(--border); + padding: 10px 14px 8px; + backdrop-filter: blur(8px); + -webkit-backdrop-filter: blur(8px); + box-shadow: 0 2px 12px #0008; +} +.sticky-bar.sticky-green { + background: linear-gradient(180deg, #0d1f14f2, #0a1410ee); + border-bottom-color: #1f6b3a; +} +.sticky-bar.sticky-yellow { + background: linear-gradient(180deg, #2a2210f2, #1a160cee); + border-bottom-color: #a68b2d; +} +.sticky-bar.sticky-red { + background: linear-gradient(180deg, #2a1010f2, #1a0808ee); + border-bottom-color: #a33; +} +.sticky-bar-row { + display: flex; flex-wrap: wrap; align-items: center; gap: 6px 10px; + max-width: 1200px; +} +.sticky-bar-row.secondary { + margin-top: 8px; font-size: 12px; color: var(--dim); +} +.sticky-bar-row.secondary a { color: var(--info); text-decoration: none; } +.sticky-bar-row.secondary a:hover { text-decoration: underline; } +.status-pill { + display: inline-block; font-size: 11px; font-weight: 800; + letter-spacing: 0.06em; padding: 2px 10px; border-radius: 3px; + border: 1px solid; +} +.status-pill.sticky-green { color: var(--ok); border-color: #1f6b3a; background: #0a1a10; } +.status-pill.sticky-yellow { color: var(--warn); border-color: #a68b2d; background: #1a1608; } +.status-pill.sticky-red { color: var(--err); border-color: #a33; background: #1a0a0a; } +.host { font-weight: 600; color: #eee; font-size: 13px; } +.sep { color: var(--dim); } +.stat { + font-weight: 700; font-size: 12px; text-decoration: none; + padding: 1px 8px; border-radius: 3px; border: 1px solid transparent; +} +.stat.err { color: var(--err); border-color: #522; background: #1a0a0a; } +.stat.warn { color: var(--warn); border-color: #664; background: #1a1608; } +.stat.muted { opacity: 0.55; font-weight: 600; } +a.stat:hover { text-decoration: underline; filter: brightness(1.15); } +.generated { color: var(--dim); font-size: 12px; } +.generated time { color: #bbb; } +.generated .age { color: #888; margin-left: 4px; } +.generated .age:not(:empty)::before { content: "("; } +.generated .age:not(:empty)::after { content: ")"; } +.badge { + display: inline-block; padding: 1px 8px; border-radius: 3px; + font-size: 11px; font-weight: 700; margin-right: 4px; + border: 1px solid var(--border); +} +.badge.mode-err { color: var(--err); border-color: #522; background: #1a0a0a; } +.badge.mode-ok { color: var(--ok); border-color: #143; background: #0a1a10; } +.badge.mode-redirect { color: var(--warn); border-color: #664; background: #1a1608; } +/* collapsible block inside sticky-bar */ +.scope-toggle { + margin-left: auto; + display: inline-flex; + align-items: center; + gap: 8px; + max-width: min(42rem, 100%); + cursor: pointer; + font: inherit; + font-size: 12px; + color: #ddd; + background: #0006; + border: 1px solid #444; + border-radius: 4px; + padding: 4px 10px; + text-align: left; +} +.scope-toggle:hover { border-color: #888; background: #111a; color: #fff; } +.scope-toggle:focus-visible { outline: 2px solid var(--info); outline-offset: 2px; } +.scope-toggle[aria-expanded="true"] { + border-color: #777; + background: #141414; +} +.scope-chevron { + display: inline-block; + width: 0; height: 0; + border-left: 5px solid #aaa; + border-top: 4px solid transparent; + border-bottom: 4px solid transparent; + transition: transform 0.12s ease; + flex-shrink: 0; +} +.scope-toggle[aria-expanded="true"] .scope-chevron { + transform: rotate(90deg); + border-left-color: #eee; +} +.scope-toggle-label { + font-weight: 700; + letter-spacing: 0.03em; + text-transform: uppercase; + font-size: 11px; + white-space: nowrap; +} +.scope-toggle-hint { + color: #888; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + min-width: 0; +} +.sticky-scope-panel { + margin-top: 8px; + max-width: 1200px; + border: 1px solid #333; + border-radius: 4px; + background: #080808e6; + padding: 8px 10px 10px; +} +.sticky-scope-panel[hidden] { display: none !important; } +.scope-body { padding: 4px 4px 2px 6px; } +.scope-title { + margin: 4px 0 4px; + font-size: 13px; + font-weight: 600; + color: #eee; +} +.scope-lead { + margin: 0 0 6px; + font-size: 12px; + color: #999; +} +.scope-list { + margin: 0; + padding-left: 1.15em; + color: #b0b0b0; + font-size: 12px; + line-height: 1.5; +} +.scope-list li { margin: 3px 0; } +#first-error, #first-warn, +[id^="err-"][id$="-line"] { + scroll-margin-top: 120px; +} +@media (max-width: 720px) { + .scope-toggle { margin-left: 0; width: 100%; } + .scope-toggle-hint { display: none; } +} +""" + +STICKY_JS = """ + +""" + + +def build_html( + *, + title: str, + hostname: str, + mode: str, + log_text: str, + commit: str, + commit_url: str, + suite_path: str, + generated_at: str, + generated_iso: str, + link_other: str | None, + page_label: str = "monitoring", +) -> str: + raw_lines = [strip_ansi(l).rstrip("\n") for l in log_text.splitlines()] + while raw_lines and not raw_lines[-1].strip(): + raw_lines.pop() + + n_err, n_warn, first_err_i, first_warn_i = count_status(raw_lines) + errors = extract_errors(raw_lines) + + err_slugs: dict[str, str] = {} + for i, e in enumerate(errors, 1): + slug = slug_error(i, e) + err_slugs[e] = slug + m = TID_RE.search(e) + if m: + err_slugs[m.group("tid")] = slug + + tid_to_slug = { + k: v for k, v in err_slugs.items() if re.match(r"^[a-z0-9_.-]+-\d+$", k, re.I) + } + + # Build body with anchors for first error/warn + per-error slugs + body_lines: list[str] = [] + claimed_err: set[str] = set() + for idx, ln in enumerate(raw_lines): + extra: list[str] = [] + if first_err_i is not None and idx == first_err_i: + extra.append("first-error") + if first_warn_i is not None and idx == first_warn_i: + extra.append("first-warn") + # per-error jump targets (all modes) + if classify(ln) in ("error", "blocker") and not is_summary_error_line(ln): + for i, e in enumerate(errors, 1): + slug = slug_error(i, e) + if slug in claimed_err: + continue + tid_m = TID_RE.search(e) + if is_run_timeout_text(e): + if "run.timeout-01" in ln and "ERROR" in ln.upper(): + extra.append(f"{slug}-line") + claimed_err.add(slug) + break + continue + key = tid_m.group(1) if tid_m else e[:40] + if key and key in ln: + extra.append(f"{slug}-line") + claimed_err.add(slug) + break + body_lines.append(render_line(ln, tid_to_slug, extra or None)) + + err_nav = "" + if errors and mode == "err": + items = [] + timeout_items = [] + for i, e in enumerate(errors, 1): + slug = slug_error(i, e) + li = ( + f'
  • ' + f'{html.escape(e)}
    • ' + ) + if is_run_timeout_text(e): + timeout_items.append(li) + else: + items.append(li) + if timeout_items: + err_nav += ( + '
    \n' + "

    EXTRAORDINARY · RUN TIMEOUT

    \n" + "

    Whole-run wall clock exceeded. " + 'Jump to detail →

    \n' + '
      \n' + + "\n".join(timeout_items[:1]) + + "\n
    \n
    \n" + ) + rest = items if timeout_items else (timeout_items + items) + if rest or not timeout_items: + err_nav += ( + '
    \n' + f"

    ERRORS ({len(errors)}) — jump to detail

    \n" + '
      \n' + + "\n".join(rest if rest else items) + + "\n
    \n
    \n" + ) + + commit_short = commit[:12] if commit else "unknown" + commit_html = ( + f'{html.escape(commit_short)}' + if commit_url + else html.escape(commit_short) + ) + + first_err_href = "#first-error" if first_err_i is not None else None + first_warn_href = "#first-warn" if first_warn_i is not None else None + scope = monitoring_scope(page_label, hostname, log_text) + + bar = sticky_bar_html( + generated_at=generated_at, + generated_iso=generated_iso, + n_err=n_err, + n_warn=n_warn, + hostname=hostname, + page_label=page_label, + mode=mode, + commit_html=commit_html, + suite_path=suite_path, + link_other=link_other, + first_err_href=first_err_href, + first_warn_href=first_warn_href, + scope=scope, + ) + + return f""" + + + + + +{html.escape(title)} + + + +{bar} +
    +{err_nav} +
    +{"".join(body_lines)} +
    +
    + Console-style render of taler-monitoring output. + Sticky bar: green = clean · yellow = warnings · red = errors. + Commit pins the exact tree used for this run. +
    +
    +{STICKY_JS} + + +""" + + +def build_redirect_html( + *, + hostname: str, + commit: str, + commit_url: str, + path_err: str = "/monitoring_err/", + page_label: str = "monitoring", + generated_at: str = "", + generated_iso: str = "", + n_err: int = 0, + n_warn: int = 0, + log_text: str = "", +) -> str: + commit_short = commit[:12] if commit else "unknown" + link = path_err if path_err.endswith("/") else path_err + "/" + c = ( + f'{html.escape(commit_short)}' + if commit_url + else html.escape(commit_short) + ) + if not generated_at: + now = datetime.now(timezone.utc) + generated_iso = now.strftime("%Y-%m-%dT%H:%M:%SZ") + generated_at = now.strftime("%Y-%m-%d %H:%M:%SZ") + # Redirect pages exist only when the run failed → treat as red if n_err unknown + if n_err <= 0: + n_err = max(n_err, 1) + scope = monitoring_scope(page_label, hostname, log_text) + bar = sticky_bar_html( + generated_at=generated_at, + generated_iso=generated_iso, + n_err=n_err, + n_warn=n_warn, + hostname=hostname, + page_label=page_label, + mode="redirect", + commit_html=c, + suite_path="", + link_other=link, + first_err_href=link, + first_warn_href=None, + scope=scope, + ) + return f""" + + + + + + +{html.escape(page_label)} · errors · {html.escape(hostname)} + + + +{bar} +
    +

    {html.escape(page_label)} has failures.

    +

    See {html.escape(link)} for the full console log, + error index, and sticky status bar.

    +

    generated {html.escape(generated_at)} · source {c}

    +
    +{STICKY_JS} + + +""" + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--log", required=True, type=Path) + ap.add_argument("--out", required=True, type=Path) + ap.add_argument("--hostname", required=True) + ap.add_argument("--mode", choices=("ok", "err", "redirect"), required=True) + ap.add_argument("--commit", default="") + ap.add_argument("--commit-url", default="") + ap.add_argument("--suite-path", default="scripts/taler-monitoring") + ap.add_argument("--title", default="") + ap.add_argument("--link-other", default="") + ap.add_argument( + "--path-err", + default="/monitoring_err/", + help="URL path for error page (redirect target)", + ) + ap.add_argument( + "--page-label", + default="monitoring", + help="Short name shown in titles/redirect box", + ) + args = ap.parse_args() + + log_text = args.log.read_text(errors="replace") if args.log.is_file() else "" + now = datetime.now(timezone.utc) + generated_iso = now.strftime("%Y-%m-%dT%H:%M:%SZ") + generated_at = now.strftime("%Y-%m-%d %H:%M:%SZ") + title = args.title or f"{args.page_label} {args.mode} · {args.hostname}" + + raw_for_counts = [strip_ansi(l).rstrip("\n") for l in log_text.splitlines()] + n_err, n_warn, _, _ = count_status(raw_for_counts) + + if args.mode == "redirect": + html_out = build_redirect_html( + hostname=args.hostname, + commit=args.commit, + commit_url=args.commit_url, + path_err=args.path_err, + page_label=args.page_label, + generated_at=generated_at, + generated_iso=generated_iso, + n_err=n_err, + n_warn=n_warn, + log_text=log_text, + ) + else: + html_out = build_html( + title=title, + hostname=args.hostname, + mode=args.mode, + log_text=log_text, + commit=args.commit, + commit_url=args.commit_url, + suite_path=args.suite_path, + generated_at=generated_at, + generated_iso=generated_iso, + link_other=args.link_other or None, + page_label=args.page_label, + ) + + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(html_out, encoding="utf-8") + print(f"wrote {args.out} (errors={n_err} warnings={n_warn} level={status_level(n_err, n_warn)})") + + +if __name__ == "__main__": + main() diff --git a/scripts/taler-monitoring/site-gen/deploy-monitoring-sites.sh b/scripts/taler-monitoring/site-gen/deploy-monitoring-sites.sh new file mode 100755 index 0000000..c5718e1 --- /dev/null +++ b/scripts/taler-monitoring/site-gen/deploy-monitoring-sites.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# deploy-monitoring-sites.sh — rsync generated HTML to DEPLOY_WWW_ROOT on koopa. +# Does NOT reload Caddy (needs root — see ROOT-ON-KOOPA.md). +set -uo pipefail + +ROOT=$(cd "$(dirname "$0")" && pwd) +SETTINGS="${SITE_GEN_SETTINGS:-$ROOT/settings.conf}" +[ -f "$SETTINGS" ] || SETTINGS="$ROOT/settings.conf.example" +# shellcheck disable=SC1090 +set -a +source <(grep -v '^\s*#' "$SETTINGS" | grep -v '^\s*$' | sed 's/\r$//') +set +a + +WORK_ROOT="${WORK_ROOT:-/tmp/taler-monitoring-sites-work}" +HTML_DIR="$WORK_ROOT/html" +DEPLOY_SSH="${DEPLOY_SSH:-koopa-external}" +DEPLOY_WWW_ROOT="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" + +if [ ! -d "$HTML_DIR" ]; then + echo "no HTML_DIR $HTML_DIR — run generate-monitoring-sites.sh first" >&2 + exit 1 +fi + +echo "deploy $HTML_DIR → ${DEPLOY_SSH}:${DEPLOY_WWW_ROOT}/" +# Prefer writing to a hernani-owned staging dir if /var/www not writable +STAGE_REMOTE="${DEPLOY_STAGING:-/home/hernani/monitoring-sites-staging}" + +ssh -o BatchMode=yes -o ConnectTimeout=20 "$DEPLOY_SSH" \ + "mkdir -p '$STAGE_REMOTE' '$DEPLOY_WWW_ROOT' 2>/dev/null || mkdir -p '$STAGE_REMOTE'" + +rsync -az --delete "$HTML_DIR/" "${DEPLOY_SSH}:${STAGE_REMOTE}/" +echo "staged on $DEPLOY_SSH:$STAGE_REMOTE" +echo +echo "If $DEPLOY_WWW_ROOT is root-owned, as root on koopa:" +echo " mkdir -p $DEPLOY_WWW_ROOT" +echo " rsync -a --delete $STAGE_REMOTE/ $DEPLOY_WWW_ROOT/" +echo " chown -R caddy:caddy $DEPLOY_WWW_ROOT # or root:caddy — match other www" +echo " # then Caddyfile handles + reload — see ROOT-ON-KOOPA.md" +echo +# try direct rsync to DEPLOY_WWW_ROOT if writable +if ssh -o BatchMode=yes "$DEPLOY_SSH" "test -w '$DEPLOY_WWW_ROOT' 2>/dev/null || test -w \"\$(dirname '$DEPLOY_WWW_ROOT')\""; then + rsync -az --delete "$HTML_DIR/" "${DEPLOY_SSH}:${DEPLOY_WWW_ROOT}/" && \ + echo "also synced directly to $DEPLOY_WWW_ROOT" +fi diff --git a/scripts/taler-monitoring/site-gen/generate-monitoring-sites.sh b/scripts/taler-monitoring/site-gen/generate-monitoring-sites.sh new file mode 100755 index 0000000..f45e5bd --- /dev/null +++ b/scripts/taler-monitoring/site-gen/generate-monitoring-sites.sh @@ -0,0 +1,273 @@ +#!/usr/bin/env bash +# generate-monitoring-sites.sh — run taler-monitoring, build console HTML for +# /monitoring and /monitoring_err (static files for host Caddy later). +# +# Usage: +# ./generate-monitoring-sites.sh # all SITES (or ONLY_HOSTS) +# ONLY_HOSTS="bank.hacktivism.ch taler.hacktivism.ch" ./generate-monitoring-sites.sh +# ./generate-monitoring-sites.sh --dry-run # HTML from empty/missing logs only +# ./generate-monitoring-sites.sh --skip-run # only convert existing logs +# +# Settings: settings.conf (from settings.conf.example). No secrets in settings. +# Secrets for e2e: ../secrets.env (optional; default phases are urls versions). +# +set -uo pipefail + +export PYTHONUNBUFFERED=1 +# Global reporting defaults (same as host-agent run-host-report.sh) +: "${RUN_TIMEOUT:=600}" +export RUN_TIMEOUT + +ROOT=$(cd "$(dirname "$0")" && pwd) +SETTINGS="${SITE_GEN_SETTINGS:-$ROOT/settings.conf}" +EXAMPLE="$ROOT/settings.conf.example" + +if [ ! -f "$SETTINGS" ]; then + if [ -f "$EXAMPLE" ]; then + echo "note: no $SETTINGS — using example defaults (copy to settings.conf to customize)" + SETTINGS="$EXAMPLE" + else + echo "missing settings: $EXAMPLE" >&2 + exit 2 + fi +fi + +# Load KEY=value lines; allow SITES already set in environment to win. +# Multiline SITES in the conf file: use SITES_FILE= or env SITES= instead. +while IFS= read -r _line || [ -n "$_line" ]; do + _line=${_line%%#*} + _line=$(echo "$_line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + [ -z "$_line" ] && continue + case "$_line" in + SITES=*|SITES_FILE=*) continue ;; # handled below / env + *=*) + _k=${_line%%=*} + _v=${_line#*=} + # do not override pre-set env + if [ -z "${!_k+x}" ] 2>/dev/null || [ -z "${!_k}" ]; then + export "$_k=$_v" + fi + ;; + esac +done < "$SETTINGS" + +# SITES from dedicated file or keep env; else parse heredoc-style from example list +if [ -n "${SITES_FILE:-}" ] && [ -f "$SITES_FILE" ]; then + SITES=$(grep -v '^\s*#' "$SITES_FILE" | grep -v '^\s*$') + export SITES +elif [ -z "${SITES:-}" ]; then + # default 9 fronts + SITES="bank.hacktivism.ch|hacktivism.ch +exchange.hacktivism.ch|hacktivism.ch +taler.hacktivism.ch|hacktivism.ch +stage.bank.lefrancpaysan.ch|stage.lefrancpaysan.ch +stage.exchange.lefrancpaysan.ch|stage.lefrancpaysan.ch +stage.monnaie.lefrancpaysan.ch|stage.lefrancpaysan.ch +bank.lefrancpaysan.ch|lefrancpaysan.ch +exchange.lefrancpaysan.ch|lefrancpaysan.ch +monnaie.lefrancpaysan.ch|lefrancpaysan.ch" + export SITES +fi + +SKIP_RUN=0 +DRY=0 +while [ $# -gt 0 ]; do + case "$1" in + --skip-run) SKIP_RUN=1; shift ;; + --dry-run) DRY=1; SKIP_RUN=1; shift ;; + -h|--help) sed -n '1,20p' "$0"; exit 0 ;; + *) echo "unknown arg: $1" >&2; exit 2 ;; + esac +done + +MONITORING_ROOT=$(cd "$ROOT/${MONITORING_ROOT:-..}" && pwd) +WORK_ROOT="${WORK_ROOT:-/tmp/taler-monitoring-sites-work}" +LOG_DIR="$WORK_ROOT/logs" +HTML_DIR="$WORK_ROOT/html" +mkdir -p "$LOG_DIR" "$HTML_DIR" + +# commit of monitoring suite used for this run +if git -C "$MONITORING_ROOT/../.." rev-parse HEAD >/dev/null 2>&1; then + REPO_ROOT=$(cd "$MONITORING_ROOT/../.." && pwd) +elif git -C "$MONITORING_ROOT" rev-parse HEAD >/dev/null 2>&1; then + REPO_ROOT=$(cd "$MONITORING_ROOT" && pwd) +else + REPO_ROOT=$(cd "$ROOT/../../.." && pwd) +fi +COMMIT=$(git -C "$REPO_ROOT" rev-parse HEAD 2>/dev/null || echo unknown) +COMMIT_SHORT=$(git -C "$REPO_ROOT" rev-parse --short=12 HEAD 2>/dev/null || echo unknown) +SOURCE_REPO_WEB="${SOURCE_REPO_WEB:-https://git.hacktivism.ch/hernani/koopa-admin-log}" +TMPL="${SOURCE_COMMIT_URL_TMPL:-\{repo\}/src/commit/\{commit\}}" +COMMIT_URL=$(printf '%s' "$TMPL" | sed "s|{repo}|$SOURCE_REPO_WEB|g; s|{commit}|$COMMIT|g") +SUITE_PATH="${SOURCE_SUITE_PATH:-scripts/taler-monitoring}" + +# Outside-only public checks (no SSH into koopa/stage containers) +PHASES_ERR="${PHASES_ERR:-urls versions}" +PHASES_OK="${PHASES_OK:-urls versions}" +CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" +SKIP_SSH="${SKIP_SSH:-1}" +# Drop SSH phases if someone passes "all" / inside / server / e2e by mistake +filter_phases_outside() { + local out=() p + for p in "$@"; do + case "$p" in + inside|server|e2e|ladder|goa-ladder|auth401) continue ;; + all) out+=(urls versions);; + *) out+=("$p") ;; + esac + done + printf '%s\n' "${out[@]}" +} + +run_mon() { + local domain="$1" phases="$2" logfile="$3" cont="$4" + # shellcheck disable=SC2206 + local ph=( $phases ) + mapfile -t ph < <(filter_phases_outside "${ph[@]}") + [ "${#ph[@]}" -gt 0 ] || ph=(urls versions) + + : >"$logfile" + echo "+ -d $domain · phases=${ph[*]} · CONTINUE=$cont · SKIP_SSH=$SKIP_SSH · RUN_TIMEOUT=$RUN_TIMEOUT" + + if [ -n "${RUNNER_SSH:-}" ]; then + echo "+ via ssh $RUNNER_SSH" + rsync -az --delete \ + --exclude secrets.env \ + --exclude 'android-test/artifacts' \ + --exclude '.git' \ + "$MONITORING_ROOT/" \ + "${RUNNER_SSH}:${RUNNER_REMOTE_WORKDIR}/" \ + || return 1 + # shellcheck disable=SC2029 + if command -v stdbuf >/dev/null 2>&1; then + ssh -o BatchMode=yes -o ConnectTimeout=30 "$RUNNER_SSH" \ + "cd '${RUNNER_REMOTE_WORKDIR}' && chmod +x taler-monitoring.sh check_*.sh 2>/dev/null; \ + export CONTINUE_ON_ERROR='$cont' AUTH401_CONTINUE='$cont' SKIP_SSH='$SKIP_SSH' RUN_TIMEOUT='$RUN_TIMEOUT'; \ + stdbuf -oL -eL ./taler-monitoring.sh -d '$domain' ${ph[*]}" \ + 2>&1 | stdbuf -oL -eL tee -a "$logfile" + else + ssh -o BatchMode=yes -o ConnectTimeout=30 "$RUNNER_SSH" \ + "cd '${RUNNER_REMOTE_WORKDIR}' && chmod +x taler-monitoring.sh check_*.sh 2>/dev/null; \ + export CONTINUE_ON_ERROR='$cont' AUTH401_CONTINUE='$cont' SKIP_SSH='$SKIP_SSH' RUN_TIMEOUT='$RUN_TIMEOUT'; \ + ./taler-monitoring.sh -d '$domain' ${ph[*]}" \ + 2>&1 | tee -a "$logfile" + fi + return "${PIPESTATUS[0]}" + fi + + if command -v stdbuf >/dev/null 2>&1; then + ( cd "$MONITORING_ROOT" && \ + CONTINUE_ON_ERROR="$cont" AUTH401_CONTINUE="$cont" SKIP_SSH="$SKIP_SSH" \ + RUN_TIMEOUT="$RUN_TIMEOUT" \ + stdbuf -oL -eL ./taler-monitoring.sh -d "$domain" "${ph[@]}" ) \ + 2>&1 | stdbuf -oL -eL tee -a "$logfile" + else + ( cd "$MONITORING_ROOT" && \ + CONTINUE_ON_ERROR="$cont" AUTH401_CONTINUE="$cont" SKIP_SSH="$SKIP_SSH" \ + RUN_TIMEOUT="$RUN_TIMEOUT" \ + ./taler-monitoring.sh -d "$domain" "${ph[@]}" ) \ + 2>&1 | tee -a "$logfile" + fi + return "${PIPESTATUS[0]}" +} + +htmlify() { + local host="$1" mode="$2" log="$3" out="$4" other="$5" + mkdir -p "$(dirname "$out")" + if [ ! -f "$ROOT/console_to_html.py" ]; then + echo "WARN: console_to_html.py missing — bootstrap $out" + { + echo "$host $mode" + echo "

    $host · $mode

    commit $COMMIT_SHORT

    " + echo "
    "
+      tail -n 100 "$log" 2>/dev/null | sed 's/&/\&/g;s/"
+    } >"$out"
+    return 0
+  fi
+  python3 "$ROOT/console_to_html.py" \
+    --log "$log" \
+    --out "$out" \
+    --hostname "$host" \
+    --mode "$mode" \
+    --commit "$COMMIT" \
+    --commit-url "$COMMIT_URL" \
+    --suite-path "$SUITE_PATH" \
+    --link-other "$other"
+}
+
+# Parse SITES
+mapfile -t SITE_LINES < <(printf '%s\n' "$SITES" | sed '/^\s*$/d' | sed 's/#.*//')
+ONLY="${ONLY_HOSTS:-}"
+
+for line in "${SITE_LINES[@]}"; do
+  line=$(echo "$line" | tr -d ' \t')
+  [ -z "$line" ] && continue
+  host=${line%%|*}
+  domain=${line#*|}
+  if [ -n "$ONLY" ]; then
+    echo " $ONLY " | grep -q " $host " || continue
+  fi
+
+  echo "======== $host  (domain=$domain)  commit=$COMMIT_SHORT ========"
+  base_html="$HTML_DIR/$host"
+  mkdir -p "$base_html/monitoring" "$base_html/monitoring_err"
+  log_err="$LOG_DIR/${host}.err.log"
+  log_ok="$LOG_DIR/${host}.ok.log"
+  ec_err=1
+  ec_ok=1
+
+  if [ "$SKIP_RUN" != "1" ]; then
+    run_mon "$domain" "$PHASES_ERR" "$log_err" "$CONTINUE_ON_ERROR"
+    ec_err=$?
+    # ok-run without continue (strict)
+    run_mon "$domain" "$PHASES_OK" "$log_ok" "0"
+    ec_ok=$?
+  else
+    [ -f "$log_err" ] || : >"$log_err"
+    [ -f "$log_ok" ] || : >"$log_ok"
+    # infer exit from log if present
+    if grep -q '\[ ERROR \]' "$log_err" 2>/dev/null || grep -q 'ERROR' "$log_err" 2>/dev/null; then
+      ec_err=1
+    else
+      ec_err=0
+    fi
+    ec_ok=$ec_err
+  fi
+
+  # Always write pages (first run / empty logs too)
+  [ -f "$log_err" ] || : >"$log_err"
+  [ -f "$log_ok" ] || : >"$log_ok"
+
+  htmlify "$host" "err" "$log_err" \
+    "$base_html/monitoring_err/index.html" "/monitoring/"
+
+  if [ "$ec_ok" -eq 0 ] && [ "$ec_err" -eq 0 ]; then
+    htmlify "$host" "ok" "$log_ok" \
+      "$base_html/monitoring/index.html" "/monitoring_err/"
+    echo "  → /monitoring (clean) + /monitoring_err"
+  else
+    htmlify "$host" "redirect" "$log_err" \
+      "$base_html/monitoring/index.html" "/monitoring_err/"
+    echo "  → /monitoring (redirect stub) + /monitoring_err (errors)  ec_err=$ec_err ec_ok=$ec_ok"
+  fi
+
+  # first-run safety
+  if [ ! -f "$base_html/monitoring/index.html" ]; then
+    htmlify "$host" "err" "$log_err" \
+      "$base_html/monitoring/index.html" "/monitoring_err/"
+  fi
+
+  printf '%s\n' "$COMMIT" >"$base_html/COMMIT"
+  printf '%s\n' "$COMMIT_URL" >"$base_html/COMMIT_URL"
+  printf 'host=%s domain=%s ec_err=%s ec_ok=%s commit=%s run_timeout=%s\n' \
+    "$host" "$domain" "$ec_err" "$ec_ok" "$COMMIT" "$RUN_TIMEOUT" | tee "$base_html/STATUS.txt"
+done
+
+echo
+echo "HTML under $HTML_DIR"
+echo "commit $COMMIT_SHORT  $COMMIT_URL"
+echo "RUN_TIMEOUT=${RUN_TIMEOUT}s (global; 0=unlimited)"
+echo "Deploy (as hernani, may need sudo for /var/www):"
+echo "  ./deploy-monitoring-sites.sh"
+echo "Root on koopa: see ROOT-ON-KOOPA.md"
diff --git a/scripts/taler-monitoring/site-gen/install-firecuda-timer.sh b/scripts/taler-monitoring/site-gen/install-firecuda-timer.sh
new file mode 100755
index 0000000..9615b96
--- /dev/null
+++ b/scripts/taler-monitoring/site-gen/install-firecuda-timer.sh
@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+# install-firecuda-timer.sh — OPTIONAL: rsync suite to firecuda + launchd every 4h
+#
+# POLICY (2026-07): firecuda launchd is **disabled**. Prefer koopa host-agent.
+# This script remains for documentation / future re-enable only.
+# Do not run unless you intentionally want a second outside-only scheduler.
+#
+#   ./install-firecuda-timer.sh
+#   ./install-firecuda-timer.sh --run-now
+#
+set -euo pipefail
+
+ROOT=$(cd "$(dirname "$0")" && pwd)
+MON_ROOT=$(cd "$ROOT/.." && pwd)
+FIRECUDA="${FIRECUDA_SSH:-firecuda-external}"
+REMOTE_BASE="${FIRECUDA_INSTALL_DIR:-taler-monitoring-site-gen}"
+RUN_NOW=0
+[ "${1:-}" = "--run-now" ] && RUN_NOW=1
+
+echo "install → $FIRECUDA:~/$REMOTE_BASE (outside-only, every 4h)"
+
+ssh -o BatchMode=yes -o ConnectTimeout=20 "$FIRECUDA" \
+  "mkdir -p \"\$HOME/$REMOTE_BASE\" \"\$HOME/Library/Logs/taler-monitoring-sites\" \"\$HOME/var/taler-monitoring-sites-work\" \"\$HOME/Library/LaunchAgents\""
+
+# Sync full taler-monitoring suite (no secrets.env)
+rsync -az --delete \
+  --exclude secrets.env \
+  --exclude 'android-test/artifacts' \
+  --exclude '.git' \
+  --exclude 'site-gen/settings.conf' \
+  "$MON_ROOT/" \
+  "${FIRECUDA}:$REMOTE_BASE/"
+
+# settings on firecuda: outside only, run locally on firecuda, deploy to koopa
+ssh -o BatchMode=yes "$FIRECUDA" "cat > \"\$HOME/$REMOTE_BASE/site-gen/settings.conf\" <<'EOF'
+# auto-installed by install-firecuda-timer.sh — no secrets
+RUNNER_SSH=
+SKIP_SSH=1
+CONTINUE_ON_ERROR=1
+PHASES_ERR=urls versions
+PHASES_OK=urls versions
+DEPLOY_SSH=koopa-external
+DEPLOY_STAGING=/home/hernani/monitoring-sites-staging
+WORK_ROOT=
+MONITORING_ROOT=..
+SOURCE_REPO_WEB=https://git.hacktivism.ch/hernani/koopa-admin-log
+SOURCE_COMMIT_URL_TMPL={repo}/src/commit/{commit}
+SOURCE_SUITE_PATH=scripts/taler-monitoring
+EOF
+# WORK_ROOT expanded on remote
+sed -i.bak \"s|^WORK_ROOT=.*|WORK_ROOT=\$HOME/var/taler-monitoring-sites-work|\" \"\$HOME/$REMOTE_BASE/site-gen/settings.conf\" 2>/dev/null || \
+  sed -i '' \"s|^WORK_ROOT=.*|WORK_ROOT=\$HOME/var/taler-monitoring-sites-work|\" \"\$HOME/$REMOTE_BASE/site-gen/settings.conf\"
+"
+
+# launchd plist with absolute paths
+ssh -o BatchMode=yes "$FIRECUDA" bash -s < "\$PLIST_DST"
+chmod +x "\$SITE_GEN"/*.sh "\$SITE_GEN"/console_to_html.py 2>/dev/null || true
+chmod +x "\$SITE_GEN/run-on-firecuda.sh"
+launchctl bootout "gui/\$(id -u)/com.hacktivism.taler-monitoring-sites" 2>/dev/null || true
+launchctl unload "\$PLIST_DST" 2>/dev/null || true
+launchctl bootstrap "gui/\$(id -u)" "\$PLIST_DST" 2>/dev/null || launchctl load "\$PLIST_DST"
+launchctl enable "gui/\$(id -u)/com.hacktivism.taler-monitoring-sites" 2>/dev/null || true
+launchctl print "gui/\$(id -u)/com.hacktivism.taler-monitoring-sites" 2>/dev/null | head -20 || launchctl list | grep taler-monitoring || true
+echo "OK: launchd com.hacktivism.taler-monitoring-sites (StartInterval=14400)"
+REMOTE
+
+if [ "$RUN_NOW" = "1" ]; then
+  echo "run once now…"
+  ssh -o BatchMode=yes "$FIRECUDA" "\"\$HOME/$REMOTE_BASE/site-gen/run-on-firecuda.sh\"" || true
+fi
+
+echo "done. logs: $FIRECUDA:~/Library/Logs/taler-monitoring-sites/"
+echo "manual: ssh $FIRECUDA '~/taler-monitoring-site-gen/site-gen/run-on-firecuda.sh'"
diff --git a/scripts/taler-monitoring/site-gen/pull-firecuda-to-koopa.sh b/scripts/taler-monitoring/site-gen/pull-firecuda-to-koopa.sh
new file mode 100755
index 0000000..b1b10ec
--- /dev/null
+++ b/scripts/taler-monitoring/site-gen/pull-firecuda-to-koopa.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# pull-firecuda-to-koopa.sh — copy HTML from firecuda work dir → koopa staging
+# Run on laptop (hernani) where both SSH aliases work.
+set -euo pipefail
+FIRECUDA="${FIRECUDA_SSH:-firecuda-external}"
+KOOPA="${DEPLOY_SSH:-koopa-external}"
+REMOTE_HTML="${FIRECUDA_HTML:-var/taler-monitoring-sites-work/html/}"
+STAGE="${DEPLOY_STAGING:-/home/hernani/monitoring-sites-staging}"
+
+echo "firecuda:$REMOTE_HTML → $KOOPA:$STAGE"
+ssh -o BatchMode=yes -o ConnectTimeout=20 "$KOOPA" "mkdir -p '$STAGE'"
+rsync -az --delete \
+  "${FIRECUDA}:${REMOTE_HTML}" \
+  "${KOOPA}:${STAGE}/"
+echo "OK staged. Root: rsync to /var/www/monitoring-sites (see ROOT-ON-KOOPA.md)"
diff --git a/scripts/taler-monitoring/site-gen/run-on-firecuda.sh b/scripts/taler-monitoring/site-gen/run-on-firecuda.sh
new file mode 100755
index 0000000..6f77de6
--- /dev/null
+++ b/scripts/taler-monitoring/site-gen/run-on-firecuda.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# run-on-firecuda.sh — entrypoint for launchd/cron on firecuda-external.
+# Outside-only monitoring (SKIP_SSH=1), all 9 sites, then stage HTML to koopa.
+#
+# Installed path on firecuda (default):
+#   ~/taler-monitoring-site-gen/site-gen/run-on-firecuda.sh
+#
+set -uo pipefail
+
+export PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:${PATH:-}"
+
+ROOT=$(cd "$(dirname "$0")" && pwd)
+LOG_DIR="${FIRECUDA_LOG_DIR:-$HOME/Library/Logs/taler-monitoring-sites}"
+mkdir -p "$LOG_DIR"
+STAMP=$(date +%Y%m%d-%H%M%S)
+LOG="$LOG_DIR/run-${STAMP}.log"
+exec >>"$LOG" 2>&1
+
+echo "======== $(date -Iseconds 2>/dev/null || date) firecuda monitoring sites ========"
+
+# Always: public DNS only, no SSH checks into stacks
+export SKIP_SSH=1
+export CONTINUE_ON_ERROR=1
+export AUTH401_CONTINUE=1
+export RUNNER_SSH=
+export PHASES_ERR="${PHASES_ERR:-urls versions}"
+export PHASES_OK="${PHASES_OK:-urls versions}"
+export WORK_ROOT="${WORK_ROOT:-$HOME/var/taler-monitoring-sites-work}"
+export MONITORING_ROOT="${MONITORING_ROOT:-$ROOT/..}"
+export DEPLOY_SSH="${DEPLOY_SSH:-koopa-external}"
+export DEPLOY_STAGING="${DEPLOY_STAGING:-/home/hernani/monitoring-sites-staging}"
+
+# Prefer full 9-site list from settings or defaults inside generate script
+unset ONLY_HOSTS
+
+cd "$ROOT" || exit 1
+chmod +x generate-monitoring-sites.sh deploy-monitoring-sites.sh console_to_html.py 2>/dev/null || true
+
+./generate-monitoring-sites.sh
+ec=$?
+
+# HTML stays on firecuda under WORK_ROOT/html.
+# Optional push to koopa if this host can SSH there (often only from laptop):
+if [ -n "${DEPLOY_SSH:-}" ] && [ "${DEPLOY_FROM_FIRECUDA:-0}" = "1" ]; then
+  if [ -x ./deploy-monitoring-sites.sh ]; then
+    ./deploy-monitoring-sites.sh || echo "WARN: deploy staging failed"
+  fi
+else
+  echo "note: HTML at ${WORK_ROOT}/html — pull from laptop:"
+  echo "  rsync -az firecuda-external:var/taler-monitoring-sites-work/html/ \\"
+  echo "    koopa-external:monitoring-sites-staging/"
+  echo "  # or: ./pull-firecuda-to-koopa.sh"
+fi
+
+# keep last 20 logs
+ls -1t "$LOG_DIR"/run-*.log 2>/dev/null | tail -n +21 | xargs rm -f 2>/dev/null || true
+
+echo "======== done ec=$ec ========"
+exit "$ec"
diff --git a/scripts/taler-monitoring/site-gen/settings.conf.example b/scripts/taler-monitoring/site-gen/settings.conf.example
new file mode 100644
index 0000000..3d14fc2
--- /dev/null
+++ b/scripts/taler-monitoring/site-gen/settings.conf.example
@@ -0,0 +1,55 @@
+# monitoring site-gen — non-secret settings (safe as example)
+# Copy:  cp settings.conf.example settings.conf
+# Never put passwords/tokens here.
+
+# On firecuda the suite runs *locally* (RUNNER_SSH empty). From a laptop you
+# can set RUNNER_SSH=firecuda-external to drive a one-off remote run instead.
+RUNNER_SSH=
+RUNNER_SSH_FALLBACKS=
+RUNNER_REMOTE_WORKDIR=/tmp/taler-monitoring-site-gen
+MONITORING_ROOT=..
+
+# Outside-only: never SSH into bank/exchange/merchant containers
+SKIP_SSH=1
+
+# Deploy staging on koopa (host Caddy later — root copies to /var/www)
+DEPLOY_SSH=koopa-external
+DEPLOY_SSH_FALLBACKS=koopa
+# Static files for Caddy file_server (created per hostname)
+DEPLOY_WWW_ROOT=/var/www/monitoring-sites
+DEPLOY_SUDO=sudo
+CADDY_CONFIG=/etc/caddy/Caddyfile
+CADDY_MIRROR_REL=configs/caddy/Caddyfile
+# Working tree on deploy host for apply (optional)
+KOOPA_CADDY_DIR=/home/hernani/koopa-caddy
+
+# Source link for HTML footer (Forgejo)
+SOURCE_REPO_WEB=https://git.hacktivism.ch/hernani/koopa-admin-log
+SOURCE_COMMIT_URL_TMPL={repo}/src/commit/{commit}
+SOURCE_SUITE_PATH=scripts/taler-monitoring
+
+CONTINUE_ON_ERROR=1
+PHASES_ERR=urls versions
+PHASES_OK=urls versions
+SKIP_E2E=1
+
+# Local work dir for logs + HTML
+WORK_ROOT=/tmp/taler-monitoring-sites-work
+
+# Sites: hostname|domain_profile  (9 fronts)
+# Start with hacktivism; stage + prod LFP follow
+SITES="
+bank.hacktivism.ch|hacktivism.ch
+exchange.hacktivism.ch|hacktivism.ch
+taler.hacktivism.ch|hacktivism.ch
+stage.bank.lefrancpaysan.ch|stage.lefrancpaysan.ch
+stage.exchange.lefrancpaysan.ch|stage.lefrancpaysan.ch
+stage.monnaie.lefrancpaysan.ch|stage.lefrancpaysan.ch
+bank.lefrancpaysan.ch|lefrancpaysan.ch
+exchange.lefrancpaysan.ch|lefrancpaysan.ch
+monnaie.lefrancpaysan.ch|lefrancpaysan.ch
+"
+
+# Only process this subset (space-separated hostnames). Empty = all SITES.
+# Example for first deploy: ONLY_HOSTS=bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch
+ONLY_HOSTS=
diff --git a/scripts/taler-monitoring/surface-catalog.conf b/scripts/taler-monitoring/surface-catalog.conf
new file mode 100644
index 0000000..16d5755
--- /dev/null
+++ b/scripts/taler-monitoring/surface-catalog.conf
@@ -0,0 +1,52 @@
+# surface-catalog.conf — known public Taler / GNUnet / Taler Systems hosts
+# Format (whitespace-separated; # comments):
+#   host   ports   expect_proto   label
+# expect_proto: https | http | ssh | tcp | any
+# ports: comma-separated (empty = defaults for proto)
+#
+# This list is expanded at runtime (DNS discovery, cert SANs, -d domain profile).
+# Phase `surface` is NOT in default phases — run explicitly:
+#   ./taler-monitoring.sh surface
+#   ./taler-monitoring.sh -d hacktivism.ch surface
+
+# --- taler.net ecosystem ---
+www.taler.net                 443     https   taler-www
+docs.taler.net                443     https   taler-docs
+git.taler.net                 443,22  https   taler-git
+bugs.taler.net                443     https   taler-bugs
+lists.taler.net               443     https   taler-lists
+tutorials.taler.net           443     https   taler-tutorials
+bank.demo.taler.net           443     https   demo-bank
+exchange.demo.taler.net       443     https   demo-exchange
+backend.demo.taler.net        443     https   demo-merchant
+shop.demo.taler.net           443     https   demo-shop
+bank.test.taler.net           443     https   test-bank
+exchange.test.taler.net       443     https   test-exchange
+backend.test.taler.net        443     https   test-merchant
+
+# --- taler-ops / TOPS ---
+www.taler-ops.ch              443     https   tops-www
+taler-ops.ch                  443     https   tops-apex
+bank.taler-ops.ch             443     https   tops-bank
+exchange.taler-ops.ch         443     https   tops-exchange
+my.taler-ops.ch               443     https   tops-merchant
+stage.taler-ops.ch            443     https   tops-stage-www
+stage.my.taler-ops.ch         443     https   tops-stage-merchant
+exchange.stage.taler-ops.ch   443     https   tops-stage-exchange
+nexus.stage.taler-ops.ch      443     https   tops-stage-nexus
+shops.taler-ops.ch            443     https   tops-shops
+mattermost.taler.net          443     https   mattermost
+
+# --- gnunet.org ---
+www.gnunet.org                443,80  https   gnunet-www
+gnunet.org                    443,80  https   gnunet-apex
+git.gnunet.org                443,22  https   gnunet-git
+bugs.gnunet.org               443     https   gnunet-bugs
+
+# --- taler-systems.com ---
+www.taler-systems.com         443     https   tsa-www
+taler-systems.com             443     https   tsa-apex
+
+# --- common infra hostnames (remote only; may 404/redirect) ---
+deb.taler.net                 443     https   deb-taler
+ftp.gnu.org                   443     https   gnu-ftp
diff --git a/scripts/taler-monitoring/taler-monitoring.sh b/scripts/taler-monitoring/taler-monitoring.sh
index 8b20e72..557c7a0 100755
--- a/scripts/taler-monitoring/taler-monitoring.sh
+++ b/scripts/taler-monitoring/taler-monitoring.sh
@@ -12,6 +12,12 @@
 set -euo pipefail
 ROOT=$(cd "$(dirname "$0")" && pwd)
 
+# Line-buffered stdout/stderr so redirected logs (tee/host-agent) flush each line.
+if [ "${_TALER_MON_STDBUF:-0}" != "1" ] && command -v stdbuf >/dev/null 2>&1; then
+  export _TALER_MON_STDBUF=1
+  exec stdbuf -oL -eL bash "$0" "$@"
+fi
+
 usage() {
   cat <<'EOF'
 taler-monitoring — bank / exchange / merchant checks
@@ -29,8 +35,14 @@ Phases:
   e2e      withdraw + pay (small amounts; remote aborts on login/KYC)
   ladder   withdraw/pay amount ladder (GOA ceiling or stage TESTPAYSAN max_wire)
   auth401  merchant Basic-auth / case matrix (HTTP 401 paths; may create throwaway instance)
+  aptdeploy  koopa podman apt-src smoke: taler-merchant in
+             koopa-taler-deploy-test-apt-src-trixie{,-testing}
+  surface  REMOTE-ONLY public inventory (NOT in default/all/full):
+             ecosystem hosts (taler.net, gnunet.org, taler-systems.com, mattermost, …)
+             or -d DOMAIN → that domain’s surface; port/protocol/TLS/CVE (OSV)
   all      urls + inside + versions + sanity + e2e  (SSH phases only on koopa)
   full     all + server + ladder + auth401  (maximum; long-running, needs secrets)
+           NOTE: surface is never included in all/full — pass it explicitly
 
 Options:
   -d, --domain DOMAIN   load profile from domains.conf (bank/exchange/merchant)
@@ -61,6 +73,10 @@ Env (same meaning):
   TALER_DOMAINS_CONF  SKIP_SSH=1
   NO_COLOR=1 / CLICOLOR=0   disable green/yellow/red tags (default: coloured)
   SKIP_SSH=1    NO_COLOR=1
+  RUN_TIMEOUT=600   whole-run wall clock seconds (default 600; 0=unlimited)
+                    long phases (ladder/full/e2e) need a higher value or 0
+  DISK_WARN_USED_PCT=85  disk free: WARN when used ≥ this %
+  DISK_ERR_USED_PCT=95   disk free: ERROR when used ≥ this % (or avail=0 / 100%)
   PERF_WARN_MS  PERF_FAIL_MS   (urls latency; default 8000 / 20000)
   QR_CHECK=0    skip QR form + qrencode/zbarimg (urls phase)
   QR_ECC=M      qrencode ECC level (default M)
@@ -73,11 +89,23 @@ Env (same meaning):
   AUTH401_*                     see check_auth401.sh / secrets.env.example
   AUTH401_CONTINUE=1 / CONTINUE_ON_ERROR=1
                         auth401: run all groups, collect every ERROR (no mid-run halt)
+  APT_DEPLOY_SKIP=1     skip aptdeploy phase
+  APT_DEPLOY_CONTAINERS="name:suite …"  default trixie + trixie-testing deploy-test pods
+  SURFACE_CVE=0         disable CVE queries in surface phase
+  SURFACE_CVE_LEVEL=warn|error  bare Server-header versions default warn
+                        (Debian package versions default error)
+  SURFACE_CATALOG=path  override surface-catalog.conf
 
 Full load (GOA / hacktivism):
   ./taler-monitoring.sh -d hacktivism.ch full
   # or explicit:
   ./taler-monitoring.sh -d hacktivism.ch urls inside versions sanity server e2e ladder auth401
+  # hacktivism host-agent also runs aptdeploy (apt-src merchant containers on koopa)
+
+Remote surface / ecosystem (explicit only — never default):
+  ./taler-monitoring.sh surface
+  ./taler-monitoring.sh -d hacktivism.ch surface
+  ./taler-monitoring.sh -d lefrancpaysan.ch surface
 
 SPA pin after selfbuild:
   EXPECT_WEBUI_VERSION=1.6.11 EXPECT_WEBUI_OVERLAY=selfbuild-v1.6.11 \\
@@ -111,6 +139,7 @@ while [ $# -gt 0 ]; do
       [ $# -ge 2 ] || { echo "missing arg for $1" >&2; exit 2; }
       apply_taler_domain "$2"
       DOMAIN_SET=1
+      TALER_DOMAIN_FROM_CLI=1
       shift 2
       ;;
     --bank)
@@ -130,12 +159,13 @@ while [ $# -gt 0 ]; do
       CURRENCY_OVERRIDE="$2"; shift 2
       ;;
     --no-probe) NO_PROBE=1; shift ;;
-    urls|inside|versions|sanity|server|e2e|ladder|goa-ladder|auth401|all|full) PHASES+=("$1"); shift ;;
+    urls|inside|versions|sanity|server|e2e|ladder|goa-ladder|auth401|aptdeploy|apt-deploy|apt_src|surface|ecosystem|all|full) PHASES+=("$1"); shift ;;
     *)
       # bare domain shorthand: ./taler-monitoring.sh taler.net
       if [[ "$1" == *.* && "$1" != *://* && "$1" != -* ]]; then
         apply_taler_domain "$1"
         DOMAIN_SET=1
+        TALER_DOMAIN_FROM_CLI=1
         shift
       else
         echo "Unknown: $1" >&2; usage >&2; exit 2
@@ -170,6 +200,8 @@ fi
 # Export so check_*.sh (re-source lib) see the same targets via env
 export TALER_DOMAIN BANK_PUBLIC EXCHANGE_PUBLIC MERCHANT_PUBLIC
 export EXPECT_CURRENCY SKIP_SSH LOCAL_STACK TALER_DOMAIN_PROBE CHECK_LANDING
+export TALER_DOMAIN_FROM_CLI="${TALER_DOMAIN_FROM_CLI:-0}"
+export DOMAIN_SET="${DOMAIN_SET:-0}"
 export WITHDRAW_AMT PAY_AMT CREDIT_AMT MERCHANT_INSTANCE
 export E2E_FAKE_INCOMING E2E_REMOTE E2E_VARIABLE E2E_ATM_MAX
 export E2E_WITHDRAW_VALUES E2E_PAY_VALUES E2E_USE_TEMPLATES E2E_TEMPLATE_MAP
@@ -288,33 +320,158 @@ if [ "${PROGRESS_TOTAL:-0}" = "0" ] || [ -z "${PROGRESS_TOTAL:-}" ]; then
       e2e)      _pt=$((_pt + 240)) ;;
       ladder|goa-ladder) _pt=$((_pt + 120)) ;;
       auth401)  _pt=$((_pt + 70)) ;;
+      aptdeploy) _pt=$((_pt + 20)) ;;
+      surface|ecosystem) _pt=$((_pt + 80)) ;;
     esac
   done
   set_progress_total "$_pt"
   unset _pt
 fi
 
+# Whole-run wall clock (default 10 min). Override: RUN_TIMEOUT=0 (unlimited).
+: "${RUN_TIMEOUT:=600}"
+export RUN_TIMEOUT
+MON_T0=$(date +%s)
+export MON_T0
+RUN_TIMED_OUT=0
+RUN_TIMEOUT_AT_PHASE=""
+RUN_SKIPPED_PHASES=()
+
+mon_seconds_left() {
+  if [ "${RUN_TIMEOUT:-0}" -eq 0 ]; then
+    printf '%s' "999999"
+    return 0
+  fi
+  local now left
+  now=$(date +%s)
+  left=$((RUN_TIMEOUT - (now - MON_T0)))
+  [ "$left" -lt 0 ] && left=0
+  printf '%s' "$left"
+}
+
+# Run one phase script under remaining RUN_TIMEOUT budget.
+# Exit 124 (timeout utility) → mark RUN_TIMED_OUT.
+run_phase() {
+  local phase="$1" script="$2" left rc
+  left=$(mon_seconds_left)
+  if [ "${RUN_TIMEOUT:-0}" -gt 0 ] && [ "$left" -le 0 ]; then
+    RUN_TIMED_OUT=1
+    RUN_TIMEOUT_AT_PHASE="${RUN_TIMEOUT_AT_PHASE:-$phase}"
+    return 1
+  fi
+  if [ "${RUN_TIMEOUT:-0}" -eq 0 ]; then
+    "$script"
+    return $?
+  fi
+  # Keep at least 2s so timeout(1) can start the child.
+  [ "$left" -lt 2 ] && left=2
+  set +e
+  with_timeout "$left" "$script"
+  rc=$?
+  set -e
+  if [ "$rc" -eq 124 ]; then
+    RUN_TIMED_OUT=1
+    RUN_TIMEOUT_AT_PHASE="$phase"
+    return 1
+  fi
+  return "$rc"
+}
+
+# How inside/versions will reach containers (host-podman vs ssh)
+_INSIDE_ACCESS_HINT=ssh
+if [ "${INSIDE_PODMAN:-0}" = "1" ] || [ "${INSIDE_MODE:-}" = "local-podman" ]; then
+  _INSIDE_ACCESS_HINT=host-podman
+elif [ "${SKIP_SSH:-0}" = "1" ]; then
+  _INSIDE_ACCESS_HINT=skipped
+elif [ "${LOCAL_STACK:-0}" = "1" ] && command -v podman >/dev/null 2>&1 \
+  && podman ps --format '{{.Names}}' 2>/dev/null | grep -qE 'taler-hacktivism'; then
+  _INSIDE_ACCESS_HINT=host-podman
+fi
+
 printf 'target domain=%s\n' "${TALER_DOMAIN}"
 printf '  bank     %s\n' "$BANK_PUBLIC"
 printf '  exchange %s\n' "$EXCHANGE_PUBLIC"
 printf '  merchant %s\n' "$MERCHANT_PUBLIC"
-printf '  currency expect=%s  skip_ssh=%s\n' "${EXPECT_CURRENCY:-any}" "$SKIP_SSH"
+printf '  currency expect=%s\n' "${EXPECT_CURRENCY:-any}"
 printf '  phases   %s\n' "${PHASES[*]}"
+printf '  flags    LOCAL_STACK=%s SKIP_SSH=%s INSIDE_PODMAN=%s INSIDE_MODE=%s\n' \
+  "${LOCAL_STACK:-}" "${SKIP_SSH:-0}" "${INSIDE_PODMAN:-0}" "${INSIDE_MODE:-}"
+printf '  flags    KOOPA_SSH=%s INSIDE_SSH=%s INSIDE_PROFILE=%s\n' \
+  "${KOOPA_SSH:-}" "${INSIDE_SSH:-}" "${INSIDE_PROFILE:-}"
+printf '  access   inside/versions → %s' "$_INSIDE_ACCESS_HINT"
+case "$_INSIDE_ACCESS_HINT" in
+  host-podman) printf ' (podman exec on this host; IDs inside.host-*)\n' ;;
+  ssh)         printf ' (SSH then podman; IDs inside.ssh-*)\n' ;;
+  skipped)     printf ' (SKIP_SSH=1)\n' ;;
+  *)           printf '\n' ;;
+esac
+if [ "${RUN_TIMEOUT:-0}" -eq 0 ]; then
+  printf '  run_timeout unlimited (RUN_TIMEOUT=0)\n'
+else
+  printf '  run_timeout %ss wall clock (RUN_TIMEOUT=; 0=unlimited)\n' "$RUN_TIMEOUT"
+fi
 printf '  progress total≈%s (set PROGRESS_TOTAL= to override; PROGRESS_OFF=1 to hide)\n' "${PROGRESS_TOTAL:-0}"
+unset _INSIDE_ACCESS_HINT
 
 chmod +x "$ROOT"/check_*.sh 2>/dev/null || true
 
 ec=0
 for p in "${PHASES[@]}"; do
+  if [ "$RUN_TIMED_OUT" = "1" ]; then
+    RUN_SKIPPED_PHASES+=("$p")
+    continue
+  fi
+  left=$(mon_seconds_left)
+  if [ "${RUN_TIMEOUT:-0}" -gt 0 ] && [ "$left" -le 0 ]; then
+    RUN_TIMED_OUT=1
+    RUN_TIMEOUT_AT_PHASE="${RUN_TIMEOUT_AT_PHASE:-$p}"
+    RUN_SKIPPED_PHASES+=("$p")
+    continue
+  fi
   case "$p" in
-    urls)     "$ROOT/check_urls.sh"     || ec=1 ;;
-    inside)   "$ROOT/check_inside.sh"   || ec=1 ;;
-    versions) "$ROOT/check_versions.sh" || ec=1 ;;
-    sanity)   "$ROOT/check_sanity.sh"   || ec=1 ;;
-    server)   "$ROOT/check_server.sh"   || ec=1 ;;
-    e2e)      "$ROOT/check_e2e.sh"      || ec=1 ;;
-    ladder|goa-ladder) "$ROOT/check_goa_ladder.sh" || ec=1 ;;
-    auth401)  "$ROOT/check_auth401.sh"  || ec=1 ;;
+    urls)     run_phase urls     "$ROOT/check_urls.sh"     || ec=1 ;;
+    inside)   run_phase inside   "$ROOT/check_inside.sh"   || ec=1 ;;
+    versions) run_phase versions "$ROOT/check_versions.sh" || ec=1 ;;
+    sanity)   run_phase sanity   "$ROOT/check_sanity.sh"   || ec=1 ;;
+    server)   run_phase server   "$ROOT/check_server.sh"   || ec=1 ;;
+    e2e)      run_phase e2e      "$ROOT/check_e2e.sh"      || ec=1 ;;
+    ladder|goa-ladder) run_phase ladder "$ROOT/check_goa_ladder.sh" || ec=1 ;;
+    auth401)  run_phase auth401  "$ROOT/check_auth401.sh"  || ec=1 ;;
+    aptdeploy|apt-deploy|apt_src) run_phase aptdeploy "$ROOT/check_apt_deploy.sh" || ec=1 ;;
+    surface|ecosystem) run_phase surface "$ROOT/check_surface.sh" || ec=1 ;;
   esac
 done
+
+# Extraordinary run-budget failure: always report at end; HTML links top → here.
+if [ "$RUN_TIMED_OUT" = "1" ]; then
+  ec=1
+  elapsed=$(( $(date +%s) - MON_T0 ))
+  skipped="${RUN_SKIPPED_PHASES[*]:-}"
+  # Stable id for jump links (console HTML + err-top banner)
+  printf '\n'
+  printf '╔══════════════════════════════════════════════════════════╗\n'
+  printf '║ RUN TIMEOUT · extraordinary (see jump target below)     ║\n'
+  printf '╚══════════════════════════════════════════════════════════╝\n'
+  printf '┌ ERROR  ┐ #run.timeout-01  RUN_TIMEOUT exceeded · budget %ss · elapsed %ss\n' \
+    "$RUN_TIMEOUT" "$elapsed"
+  printf '  detail: wall-clock limit hit'
+  if [ -n "${RUN_TIMEOUT_AT_PHASE:-}" ]; then
+    printf ' during/after phase "%s"' "$RUN_TIMEOUT_AT_PHASE"
+  fi
+  if [ -n "$skipped" ]; then
+    printf ' · skipped: %s' "$skipped"
+  fi
+  printf '\n'
+  printf '  id=run.timeout-01\n'
+  printf '  hint: raise RUN_TIMEOUT= (seconds) or set RUN_TIMEOUT=0 for unlimited\n'
+  printf -- '┌ RUN TIMEOUT · extraordinary ┐\n'
+  printf -- '  • run.timeout-01 [run] RUN_TIMEOUT=%ss exceeded (elapsed %ss)%s%s\n' \
+    "$RUN_TIMEOUT" "$elapsed" \
+    "${RUN_TIMEOUT_AT_PHASE:+ · phase $RUN_TIMEOUT_AT_PHASE}" \
+    "${skipped:+ · skipped $skipped}"
+  printf -- '--- ERRORS (failed checks) ---\n'
+  printf -- '  • run.timeout-01 [run] RUN_TIMEOUT=%ss exceeded (elapsed %ss)\n' \
+    "$RUN_TIMEOUT" "$elapsed"
+fi
+
 exit "$ec"