From 6436ab0dd90d986948a5b6637dda0f4af5a9a90e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sat, 18 Jul 2026 19:57:21 +0200 Subject: [PATCH] monitoring: host-agent timers and HTML report pipeline --- scripts/taler-monitoring/host-agent/README.md | 183 +++++++++ .../host-agent/ROOT-APPLY-MONITORING.md | 57 +++ .../ensure-apt-deploy-test-containers.sh | 373 ++++++++++++++++++ .../taler-monitoring/host-agent/env.example | 26 ++ .../host-agent/env/francpaysan.env.example | 24 ++ .../host-agent/env/hacktivism.env.example | 24 ++ .../host-agent/env/stagepaysan.env.example | 27 ++ .../host-agent/install-host-agent.sh | 107 +++++ .../host-agent/run-aptdeploy-monitoring.sh | 43 ++ .../host-agent/run-aptdeploy.sh | 150 +++++++ .../host-agent/run-fp-prod-monitoring.sh | 20 + .../host-agent/run-fp-stage-monitoring.sh | 21 + .../host-agent/run-hacktivism-monitoring.sh | 23 ++ .../host-agent/run-host-report.sh | 358 +++++++++++++++++ .../host-agent/run-surface-monitoring.sh | 41 ++ .../taler-monitoring-aptdeploy.service | 18 + .../taler-monitoring-aptdeploy.timer | 14 + .../taler-monitoring-hacktivism.path | 19 + .../taler-monitoring-hacktivism.service | 21 + .../taler-monitoring-hacktivism.timer | 14 + .../taler-monitoring-surface.service | 18 + .../host-agent/taler-monitoring-surface.timer | 14 + .../host-agent/touch-software-stamp.sh | 17 + .../host-agent/update-suite.sh | 109 +++++ 24 files changed, 1721 insertions(+) create mode 100644 scripts/taler-monitoring/host-agent/README.md create mode 100644 scripts/taler-monitoring/host-agent/ROOT-APPLY-MONITORING.md create mode 100755 scripts/taler-monitoring/host-agent/ensure-apt-deploy-test-containers.sh create mode 100644 scripts/taler-monitoring/host-agent/env.example create mode 100644 scripts/taler-monitoring/host-agent/env/francpaysan.env.example create mode 100644 scripts/taler-monitoring/host-agent/env/hacktivism.env.example create mode 100644 scripts/taler-monitoring/host-agent/env/stagepaysan.env.example create mode 100755 scripts/taler-monitoring/host-agent/install-host-agent.sh create mode 100755 scripts/taler-monitoring/host-agent/run-aptdeploy-monitoring.sh create mode 100755 scripts/taler-monitoring/host-agent/run-aptdeploy.sh create mode 100755 scripts/taler-monitoring/host-agent/run-fp-prod-monitoring.sh create mode 100755 scripts/taler-monitoring/host-agent/run-fp-stage-monitoring.sh create mode 100755 scripts/taler-monitoring/host-agent/run-hacktivism-monitoring.sh create mode 100755 scripts/taler-monitoring/host-agent/run-host-report.sh create mode 100755 scripts/taler-monitoring/host-agent/run-surface-monitoring.sh create mode 100644 scripts/taler-monitoring/host-agent/taler-monitoring-aptdeploy.service create mode 100644 scripts/taler-monitoring/host-agent/taler-monitoring-aptdeploy.timer create mode 100644 scripts/taler-monitoring/host-agent/taler-monitoring-hacktivism.path create mode 100644 scripts/taler-monitoring/host-agent/taler-monitoring-hacktivism.service create mode 100644 scripts/taler-monitoring/host-agent/taler-monitoring-hacktivism.timer create mode 100644 scripts/taler-monitoring/host-agent/taler-monitoring-surface.service create mode 100644 scripts/taler-monitoring/host-agent/taler-monitoring-surface.timer create mode 100755 scripts/taler-monitoring/host-agent/touch-software-stamp.sh create mode 100755 scripts/taler-monitoring/host-agent/update-suite.sh diff --git a/scripts/taler-monitoring/host-agent/README.md b/scripts/taler-monitoring/host-agent/README.md new file mode 100644 index 0000000..3acfe59 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/README.md @@ -0,0 +1,183 @@ +# host-agent — reporting generation (GOA + FP + any stack) + +**Global pipeline** lives in **`run-host-report.sh`** (all stacks share it): + +| Behaviour | Default | +|-----------|---------| +| Suite refresh | `update-suite.sh` → `origin/main` | +| Wall clock | `RUN_TIMEOUT=600` | +| Log | line-buffered `run-*.log` (continuous write) | +| HTML | always (first run / empty tree too) + commit URL | +| Timeout notice | end of log + top jump on `/monitoring_err/` | + +Thin wrappers only set stack defaults: + +| Wrapper | Stack | Typical user | +|---------|-------|----------------| +| `run-hacktivism-monitoring.sh` | GOA `hacktivism.ch` | `hernani` @ koopa | +| `run-fp-stage-monitoring.sh` | FP stage | `stagepaysan` | +| `run-fp-prod-monitoring.sh` | FP prod | `francpaysan` | +| `run-aptdeploy.sh` | CLI: ensure + **aptdeploy** (no HTML) | local if hostname=koopa, else **ssh koopa-external** | +| `run-aptdeploy-monitoring.sh` | **aptdeploy** → `/taler-monitoring-aptdeploy*` | `hernani` @ koopa (4h timer, taler only) | +| `run-surface-monitoring.sh` | **surface** → `/taler-monitoring-surface*` | `hernani` @ koopa (hourly, taler only) | + +### apt-src deploy tests (hacktivism `/monitoring`) + +| Container | Mode | +|-----------|------| +| `koopa-taler-deploy-test-apt-src-trixie` | **fresh**: recreate when repo package fingerprint changes | +| `koopa-taler-deploy-test-apt-src-trixie-testing` | **fresh** for `trixie-testing` | +| `koopa-taler-deploy-test-apt-src-trixie-upgrade` | **upgrade**: initial install, then mytops-style `apt install` + `upgrade` | +| `koopa-taler-deploy-test-apt-src-trixie-testing-upgrade` | **upgrade** for testing suite | + +```bash +# from laptop / clementine → triggers on koopa-external +./scripts/taler-monitoring/host-agent/run-aptdeploy.sh +./scripts/taler-monitoring/host-agent/run-aptdeploy.sh upgrade +./scripts/taler-monitoring/host-agent/run-aptdeploy.sh fresh +./scripts/taler-monitoring/host-agent/run-aptdeploy.sh check-only + +# on koopa +./taler-monitoring.sh -d hacktivism.ch aptdeploy +``` + +Checks: package table, `httpd --version`, libtalerutil/`ldd` on failure, `systemctl` for +`taler-merchant.target`, basic local `/config` probe. No nginx/HTTPS in containers. + +### surface + aptdeploy HTML (taler.hacktivism.ch only) + +| URL | Role | Timer | +|-----|------|-------| +| `…/taler-monitoring-surface/` | ecosystem surface scan | 1h | +| `…/taler-monitoring-surface_err/` | errors + jump list | ″ | +| `…/taler-monitoring-aptdeploy/` | apt-src container deploy tests | 4h | +| `…/taler-monitoring-aptdeploy_err/` | errors + jump list | ″ | + +Not on bank/exchange. Staging: +`~/monitoring-sites-staging/taler.hacktivism.ch/taler-monitoring-{surface,aptdeploy}*`. + +```bash +systemctl --user enable --now taler-monitoring-surface.timer +systemctl --user enable --now taler-monitoring-aptdeploy.timer +systemctl --user start taler-monitoring-surface.service +systemctl --user start taler-monitoring-aptdeploy.service +# Caddy: handles only in taler.hacktivism.ch block (root) +``` + +Env examples: **`env/*.env.example`**. FP docs: `francpaysan-admin-log/docs/taler-monitoring-host-agents.md`. + +## GOA (koopa) — host-podman + +Runs **as user `hernani`** (systemd user, linger). **Primary** mode: + +- **`INSIDE_PODMAN=1`** / **`INSIDE_MODE=local-podman`**: access mode **`host-podman`** — + `podman exec` on this host into + `taler-hacktivism-bank` / `taler-hacktivism-exchange-ansible` / `taler-hacktivism` + (IDs `inside.host-*`, not `inside.ssh-*`). + From a **laptop**, leave `INSIDE_PODMAN` off → access mode **`ssh`** via `KOOPA_SSH` + (`inside.ssh-*`). +- Public **`urls` + `versions`** for the stack (`-d hacktivism.ch`). +- Writes console HTML for all three sites under + `~/monitoring-sites-staging/{bank,exchange,taler}.hacktivism.ch/`. + +Caddy (host root) can serve that tree at `/monitoring` — you configure Caddy. + +## Latest software + config isolation + +**Source of truth is always Forgejo** (`git.hacktivism.ch`), not a laptop rsync copy. + +Every host-agent / aptdeploy run: + +1. **`update-suite.sh`**: `git fetch` + **`SUITE_UPDATE_MODE=reset`** (default) → `origin/main` + into `SUITE_DIR` (prefer `~/src/koopa-admin-log`). +2. **Re-exec** host-agent scripts from that tree so the running code matches `COMMIT_URL`. +3. Local overrides only in **`~/.config/taler-monitoring/env`** — never overwritten by git. +4. HTML footer = exact commit: + `https://git.hacktivism.ch/hernani/koopa-admin-log/src/commit/` +5. **`RUN_TIMEOUT=600`**, line-buffered logs, HTML always written. + +If fetch fails → previous tree + WARN. +**To ship features (aptdeploy, surface, …): push to `origin/main`.** Local-only edits vanish on the next reset. + +If pull fails (no network / auth), the previous tree is used and a WARN is logged. + +## Install (on koopa as hernani) + +```bash +# once: local config (optional) +mkdir -p ~/.config/taler-monitoring +cp ~/koopa-admin-log/scripts/taler-monitoring/host-agent/env.example \ + ~/.config/taler-monitoring/env + +# laptop: +./scripts/taler-monitoring/host-agent/install-host-agent.sh --remote koopa-external + +# or on koopa: +~/koopa-admin-log/scripts/taler-monitoring/host-agent/install-host-agent.sh +``` + +Units: + +| Unit | Role | +|------|------| +| `taler-monitoring-hacktivism.path` | fires on software stamp + landing/caddy config changes | +| `taler-monitoring-hacktivism.timer` | every **4h** fallback | +| `taler-monitoring-hacktivism.service` | oneshot → `/monitoring*` (urls inside versions) | +| `taler-monitoring-surface.timer` | every **1h** | +| `taler-monitoring-surface.service` | → `/taler-monitoring-surface*` (taler only) | +| `taler-monitoring-aptdeploy.timer` | every **4h** | +| `taler-monitoring-aptdeploy.service` | → `/taler-monitoring-aptdeploy*` (taler only) | + +## After package / image / landing upgrades + +```bash +~/koopa-admin-log/scripts/taler-monitoring/host-agent/touch-software-stamp.sh +# → path unit starts the service +``` + +## Manual + +```bash +systemctl --user start taler-monitoring-hacktivism.service +systemctl --user status taler-monitoring-hacktivism.service --no-pager +journalctl --user -u taler-monitoring-hacktivism.service -n 50 --no-pager +ls ~/monitoring-sites-staging/bank.hacktivism.ch/monitoring_err/ +``` + +## Relation to firecuda outside-only timer + +| | host-agent (koopa) | firecuda site-gen | +|--|--------------------|-------------------| +| Access | **podman on host** | public HTTPS only | +| Phases | urls **inside** versions | urls versions | +| Trigger | software path + 4h | *could* launchd 4h | +| Primary for GOA | **yes** | **off** (optional only) | + +**firecuda-external is not scheduled** right now (launchd disabled). Scripts under +`site-gen/` can still one-shot a public run if needed; see `site-gen/README.md`. + +No secrets required for default phases. `secrets.env` optional for e2e later. + +## Git commit on HTML pages + +Header of each generated page includes a link like: + +`https://git.hacktivism.ch/hernani/koopa-admin-log/src/commit/` + +That SHA is the tree **after** `update-suite.sh` for that run. + +## Packages on koopa + +See **[../DEPENDENCIES.md](../DEPENDENCIES.md)**. + +**Default host-agent needs no new packages** if curl/python3/podman/git are present (they are). + +**Root once** only for full QR checks in `urls`: + +```bash +zypper in qrencode zbar +``` + +Without them, monitoring still runs; QR group WARNs and skips encode/decode. + +Caddy `/var/www/monitoring-sites` is separate (you configured hernani write access). diff --git a/scripts/taler-monitoring/host-agent/ROOT-APPLY-MONITORING.md b/scripts/taler-monitoring/host-agent/ROOT-APPLY-MONITORING.md new file mode 100644 index 0000000..17dd386 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/ROOT-APPLY-MONITORING.md @@ -0,0 +1,57 @@ +# Root: make monitoring pages live on koopa (GOA) + +## Symptom if not applied + +Public URLs return **merchant/bank JSON** like: + +```json +{ "code": 21, "hint": "There is no endpoint defined for the URL provided..." } +``` + +or a **404 HTML** page from the app reverse_proxy. + +Cause: live `/etc/caddy/Caddyfile` has **no** `handle /monitoring*` (falls through to +API). Prepared config is only under **`/home/hernani/koopa-caddy/Caddyfile`**. + +## One-shot as root (recommended) + +```bash +sudo /home/hernani/koopa-caddy/apply-monitoring-live.sh +``` + +That script: + +1. `rsync` staging HTML → `/var/www/monitoring-sites/` (incl. surface + aptdeploy) +2. Makes tree writable for `hernani` (group `caddy` + ACL) so later runs self-sync +3. Installs prepared Caddyfile → `/etc/caddy/Caddyfile`, validates, reloads Caddy +4. Smoke-curls all monitoring URLs (must **not** be JSON `code:21`) + +## Paths + +| Path | Host | Source timer | +|------|------|----------------| +| `/monitoring/` + `/monitoring_err/` | bank + exchange + taler | `taler-monitoring-hacktivism.timer` (4h) | +| `/taler-monitoring-surface/` + `/_err/` | **taler.hacktivism.ch only** | `taler-monitoring-surface.timer` (1h) | +| `/taler-monitoring-aptdeploy/` + `/_err/` | **taler.hacktivism.ch only** | `taler-monitoring-aptdeploy.timer` (4h) | + +HTML is generated as **hernani** into +`/home/hernani/monitoring-sites-staging/`. + +## Manual equivalent + +```bash +rsync -a --delete /home/hernani/monitoring-sites-staging/ /var/www/monitoring-sites/ +chown -R hernani:caddy /var/www/monitoring-sites && chmod -R g+rwX /var/www/monitoring-sites +sudo /home/hernani/koopa-caddy/apply.sh # or apply-monitoring-live.sh +``` + +Expect **200** (HTML with sticky bar). Not merchant JSON `code:21`. + +## After each monitoring run (if www is not hernani-writable) + +```bash +rsync -a --delete /home/hernani/monitoring-sites-staging/ /var/www/monitoring-sites/ +chown -R caddy:caddy /var/www/monitoring-sites +``` + +Or grant `hernani` write on `/var/www/monitoring-sites` once so the host-agent can rsync itself. diff --git a/scripts/taler-monitoring/host-agent/ensure-apt-deploy-test-containers.sh b/scripts/taler-monitoring/host-agent/ensure-apt-deploy-test-containers.sh new file mode 100755 index 0000000..75fa3e4 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/ensure-apt-deploy-test-containers.sh @@ -0,0 +1,373 @@ +#!/usr/bin/env bash +# ensure-apt-deploy-test-containers.sh +# +# Four rootless podman containers on koopa (no nginx / no HTTPS): +# +# Fresh rebuild when repo fingerprint changes: +# koopa-taler-deploy-test-apt-src-trixie +# koopa-taler-deploy-test-apt-src-trixie-testing +# +# Persistent upgrade path (mytops upgrade.sh style): +# koopa-taler-deploy-test-apt-src-trixie-upgrade +# koopa-taler-deploy-test-apt-src-trixie-testing-upgrade +# +# Run as hernani on koopa. From elsewhere use run-aptdeploy.sh (ssh koopa-external). +# +set -euo pipefail + +PODMAN_BIN="${APT_DEPLOY_PODMAN:-podman}" +IMG="${APT_DEPLOY_BASE_IMAGE:-docker.io/library/debian:trixie}" +STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/taler-apt-deploy" +URI_BASE="${APT_DEPLOY_URI:-https://deb.taler.net/apt/debian}" +mkdir -p "$STATE_DIR" + +# Packages we track for "new version → rebuild fresh containers" +TRACK_PKGS="${APT_DEPLOY_TRACK_PKGS:-taler-merchant libtalermerchant libtalerexchange libdonau}" + +# Canonical four containers (keep). Everything else matching legacy names is removed. +CANONICAL_CONTAINERS=( + koopa-taler-deploy-test-apt-src-trixie + koopa-taler-deploy-test-apt-src-trixie-testing + koopa-taler-deploy-test-apt-src-trixie-upgrade + koopa-taler-deploy-test-apt-src-trixie-testing-upgrade +) + +# Legacy / typo names from earlier iterations — always delete if present +LEGACY_CONTAINERS=( + koopa-taler-build-test-apt-trixie + koopa-taler-build-test-apt-trixie-testing + koopa-taler-deploy-test-apt-trixie + koopa-taler-deploy-test-apt-trixie-testing + koopa-taler-buidl-test-apt-trixie + koopa-taler-buidl-test-apt-trixie-testing +) + +cleanup_legacy() { + local n img + echo "======== cleanup legacy containers / images ========" + for n in "${LEGACY_CONTAINERS[@]}"; do + if "$PODMAN_BIN" container exists "$n" 2>/dev/null; then + echo "remove legacy container $n" + "$PODMAN_BIN" rm -f "$n" 2>/dev/null || true + fi + # old setup images for that name + if "$PODMAN_BIN" image exists "localhost/${n}:setup" 2>/dev/null; then + echo "remove legacy image localhost/${n}:setup" + "$PODMAN_BIN" rmi -f "localhost/${n}:setup" 2>/dev/null || true + fi + done + # dangling setup images not in canonical set + while read -r img; do + [ -n "$img" ] || continue + local base keep=0 + base=${img#localhost/} + base=${base%:setup} + for n in "${CANONICAL_CONTAINERS[@]}"; do + [ "$base" = "$n" ] && keep=1 && break + done + if [ "$keep" -eq 0 ] && [[ "$img" == localhost/koopa-taler-*-test-apt* ]]; then + echo "remove non-canonical image $img" + "$PODMAN_BIN" rmi -f "$img" 2>/dev/null || true + fi + done < <("$PODMAN_BIN" images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null | grep -E 'localhost/koopa-taler-.*-test-apt' || true) +} + +# Prefer single suite tree: if both ~/src/koopa-admin-log and ~/koopa-admin-log are +# full git clones, keep the newer (by HEAD commit time) and replace the older with a symlink. +dedupe_suite_repos() { + local a="$HOME/src/koopa-admin-log" b="$HOME/koopa-admin-log" + local ta tb + [ "${APT_DEPLOY_DEDUPE_REPOS:-1}" = "1" ] || return 0 + if [ -d "$a/.git" ] && [ -d "$b/.git" ] && [ ! -L "$a" ] && [ ! -L "$b" ]; then + ta=$(git -C "$a" log -1 --format=%ct 2>/dev/null || echo 0) + tb=$(git -C "$b" log -1 --format=%ct 2>/dev/null || echo 0) + echo "======== dedupe suite repos (both are git clones) ========" + echo " $a HEAD_ts=$ta" + echo " $b HEAD_ts=$tb" + if [ "$ta" -ge "$tb" ]; then + echo "keep $a · remove $b · symlink $b → $a" + rm -rf "$b" + ln -sfn "$a" "$b" + else + echo "keep $b · remove $a · symlink $a → $b" + rm -rf "$a" + mkdir -p "$(dirname "$a")" + ln -sfn "$b" "$a" + fi + elif [ -d "$a/scripts/taler-monitoring" ] && [ -d "$b/scripts/taler-monitoring" ] \ + && [ ! -d "$a/.git" ] && [ -d "$b/.git" ] && [ ! -L "$a" ]; then + echo "remove non-git tree $a · symlink → $b" + rm -rf "$a" + mkdir -p "$(dirname "$a")" + ln -sfn "$b" "$a" + elif [ -d "$b/scripts/taler-monitoring" ] && [ -d "$a/scripts/taler-monitoring" ] \ + && [ ! -d "$b/.git" ] && [ -d "$a/.git" ] && [ ! -L "$b" ]; then + echo "remove non-git tree $b · symlink → $a" + rm -rf "$b" + ln -sfn "$a" "$b" + fi +} + +cleanup_legacy +dedupe_suite_repos +# --------------------------------------------------------------------------- +# Repo fingerprint: Version: lines from suite Packages index +# --------------------------------------------------------------------------- +repo_pkg_version() { + local suite=$1 pkg=$2 + curl -fsS --max-time 45 \ + "${URI_BASE}/dists/${suite}/main/binary-amd64/Packages" 2>/dev/null \ + | awk -v p="$pkg" ' + $0 == "Package: " p { want=1; next } + /^Package: / { want=0 } + want && /^Version: / { print $2; exit } + ' +} + +repo_fingerprint() { + local suite=$1 pkg v line="" + for pkg in $TRACK_PKGS; do + v=$(repo_pkg_version "$suite" "$pkg" || true) + [ -n "$v" ] || v="?" + line="${line}${pkg}=${v};" + done + printf '%s' "$line" +} + +container_fingerprint() { + local name=$1 + "$PODMAN_BIN" exec "$name" bash -lc ' + for p in '"$TRACK_PKGS"'; do + v=$(dpkg-query -W -f="\${Version}" "$p" 2>/dev/null || echo missing) + printf "%s=%s;" "$p" "$v" + done + ' 2>/dev/null || echo "missing" +} + +# --------------------------------------------------------------------------- +# Bootstrap: install base + taler source + packages (shared) +# --------------------------------------------------------------------------- +install_base_and_merchant() { + local name=$1 suite=$2 srcfile=$3 + "$PODMAN_BIN" exec -u root "$name" bash -lc ' + set -euo pipefail + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y -qq ca-certificates curl wget gnupg systemd systemd-sysv \ + dbus procps iproute2 + ' + "$PODMAN_BIN" exec -u root "$name" bash -lc " + set -euo pipefail + export DEBIAN_FRONTEND=noninteractive + mkdir -p /etc/apt/keyrings + wget -q -O /etc/apt/keyrings/taler-systems.gpg https://taler.net/taler-systems.gpg + cat >/etc/apt/sources.list.d/${srcfile} </dev/null + systemctl enable taler-merchant.target 2>/dev/null + systemctl start taler-merchant.target 2>/dev/null + sleep 2 + systemctl is-active taler-merchant-httpd.service 2>/dev/null || true + ' || true +} + +create_from_scratch() { + local name=$1 suite=$2 srcfile=$3 + echo "======== CREATE (fresh) $name Suites: $suite ========" + "$PODMAN_BIN" pull "$IMG" >/dev/null + "$PODMAN_BIN" rm -f "$name" 2>/dev/null || true + # drop previous setup image so we do not accumulate duplicates + "$PODMAN_BIN" rmi -f "localhost/${name}:setup" 2>/dev/null || true + "$PODMAN_BIN" run -d --name "$name" --hostname "$name" "$IMG" sleep infinity + install_base_and_merchant "$name" "$suite" "$srcfile" + commit_and_systemd "$name" + printf '%s\n' "$(repo_fingerprint "$suite")" >"$STATE_DIR/${name}.fp" + printf '%s\n' "$(container_fingerprint "$name")" >"$STATE_DIR/${name}.installed" + echo "created $name fp=$(cat "$STATE_DIR/${name}.fp")" +} + +# --------------------------------------------------------------------------- +# Upgrade path (mytops-admin-log upgrade.sh merchant packages, no restic/nginx) +# Prints a markdown-ish table of before → after versions. +# --------------------------------------------------------------------------- +upgrade_inside() { + local name=$1 + local before after pkg v0 v1 + echo "======== UPGRADE $name (mytops-style apt install + upgrade) ========" + + before=$("$PODMAN_BIN" exec "$name" bash -lc ' + for p in taler-merchant libtalermerchant libtalerexchange libdonau libgnunet \ + taler-merchant-webui taler-merchant-typst taler-terms-generator; do + v=$(dpkg-query -W -f="\${Version}" "$p" 2>/dev/null || echo "-") + printf "%s %s\n" "$p" "$v" + done + ' 2>/dev/null || true) + + # capture upgrade output + set +e + "$PODMAN_BIN" exec -u root "$name" bash -lc ' + set -euo pipefail + export DEBIAN_FRONTEND=noninteractive + systemctl stop taler-merchant.target 2>/dev/null || true + systemctl stop taler-merchant-donaukeyupdate.service 2>/dev/null || true + apt-get update -qq + apt-get install -y \ + taler-merchant \ + taler-merchant-typst \ + taler-merchant-webui \ + taler-terms-generator \ + || true + if dpkg -l challenger-httpd 2>/dev/null | grep -q "^ii"; then + apt-get remove -y challenger-httpd || true + fi + apt-get upgrade -y + systemctl daemon-reload 2>/dev/null || true + systemctl enable taler-merchant.target 2>/dev/null || true + # try dbconfig if present (may no-op without full config) + command -v taler-merchant-dbconfig >/dev/null && taler-merchant-dbconfig 2>/dev/null || true + systemctl reset-failed "taler-merchant*" 2>/dev/null || true + systemctl start taler-merchant.target 2>/dev/null || true + sleep 2 + ' 2>&1 | tee "$STATE_DIR/${name}.upgrade.log" | tail -30 + set -e + + after=$("$PODMAN_BIN" exec "$name" bash -lc ' + for p in taler-merchant libtalermerchant libtalerexchange libdonau libgnunet \ + taler-merchant-webui taler-merchant-typst taler-terms-generator; do + v=$(dpkg-query -W -f="\${Version}" "$p" 2>/dev/null || echo "-") + printf "%s %s\n" "$p" "$v" + done + ' 2>/dev/null || true) + + echo + echo "| package | before | after | change |" + echo "|---------|--------|-------|--------|" + while read -r pkg v0; do + [ -n "${pkg:-}" ] || continue + v1=$(printf '%s\n' "$after" | awk -v p="$pkg" '$1==p {print $2; exit}') + v1=${v1:--} + if [ "$v0" = "$v1" ]; then + ch="=" + else + ch="UPGRADED" + fi + echo "| $pkg | $v0 | $v1 | $ch |" + done <<<"$before" + + printf '%s\n' "$(container_fingerprint "$name")" >"$STATE_DIR/${name}.installed" +} + +# --------------------------------------------------------------------------- +# Fresh mode: recreate if missing / not running / repo fp changed +# --------------------------------------------------------------------------- +ensure_fresh() { + local name=$1 suite=$2 srcfile=$3 + local want have running + + want=$(repo_fingerprint "$suite") + echo "repo fingerprint [$suite]: $want" + + if ! "$PODMAN_BIN" container exists "$name" 2>/dev/null; then + create_from_scratch "$name" "$suite" "$srcfile" + return + fi + + running=$("$PODMAN_BIN" inspect -f '{{.State.Running}}' "$name" 2>/dev/null || echo false) + have=$(container_fingerprint "$name") + echo "installed in $name: $have" + + if [ "$running" != "true" ] \ + || ! "$PODMAN_BIN" exec "$name" bash -lc 'command -v taler-merchant-httpd' >/dev/null 2>&1 \ + || [ "$want" != "$have" ]; then + echo "rebuild $name (running=$running fp_match=$([ "$want" = "$have" ] && echo yes || echo NO))" + create_from_scratch "$name" "$suite" "$srcfile" + else + echo "OK $name up-to-date (fresh container matches repo)" + fi +} + +# --------------------------------------------------------------------------- +# Upgrade mode: create once, then always upgrade_inside +# --------------------------------------------------------------------------- +ensure_upgrade() { + local name=$1 suite=$2 srcfile=$3 + local running + + if ! "$PODMAN_BIN" container exists "$name" 2>/dev/null; then + create_from_scratch "$name" "$suite" "$srcfile" + echo "(initial install for upgrade-track container $name)" + return + fi + + running=$("$PODMAN_BIN" inspect -f '{{.State.Running}}' "$name" 2>/dev/null || echo false) + if [ "$running" != "true" ]; then + echo "start $name" + "$PODMAN_BIN" start "$name" || { + echo "recreate broken $name" + create_from_scratch "$name" "$suite" "$srcfile" + return + } + sleep 3 + fi + + if ! "$PODMAN_BIN" exec "$name" bash -lc 'command -v taler-merchant-httpd' >/dev/null 2>&1; then + echo "recreate $name (no httpd after start)" + create_from_scratch "$name" "$suite" "$srcfile" + return + fi + + upgrade_inside "$name" +} + +MODE="${1:-all}" # all | fresh | upgrade + +case "$MODE" in + all|fresh) + ensure_fresh koopa-taler-deploy-test-apt-src-trixie trixie taler.sources + ensure_fresh koopa-taler-deploy-test-apt-src-trixie-testing trixie-testing taler-testing.sources + ;;& + all|upgrade) + ensure_upgrade koopa-taler-deploy-test-apt-src-trixie-upgrade trixie taler.sources + ensure_upgrade koopa-taler-deploy-test-apt-src-trixie-testing-upgrade trixie-testing taler-testing.sources + ;; + *) + echo "usage: $0 [all|fresh|upgrade]" >&2 + exit 2 + ;; +esac + +echo "done ensure-apt-deploy-test-containers mode=$MODE" diff --git a/scripts/taler-monitoring/host-agent/env.example b/scripts/taler-monitoring/host-agent/env.example new file mode 100644 index 0000000..15f6bae --- /dev/null +++ b/scripts/taler-monitoring/host-agent/env.example @@ -0,0 +1,26 @@ +# Copy to: ~/.config/taler-monitoring/env +# This file is NOT inside the git tree and is never overwritten by update-suite.sh. +# +# Full stack examples (same reporting generation for all): +# env/hacktivism.env.example +# env/stagepaysan.env.example +# env/francpaysan.env.example +# +# SUITE_GIT_URL=https://git.hacktivism.ch/hernani/koopa-admin-log.git +# SUITE_GIT_REF=main +# SUITE_DIR=$HOME/src/koopa-admin-log +# SUITE_UPDATE_MODE=reset +# +# TALER_DOMAIN=hacktivism.ch +# INSIDE_PODMAN=1 +# INSIDE_MODE=local-podman +# CONTINUE_ON_ERROR=1 +# PHASES for the *default* hacktivism timer only (urls inside versions). +# Dedicated wrappers always override PHASES + HTML paths: +# run-surface-monitoring.sh → /taler-monitoring-surface* +# run-aptdeploy-monitoring.sh → /taler-monitoring-aptdeploy* +# PHASES="urls inside versions" +# RUN_TIMEOUT=600 +# HTML_OUT=$HOME/monitoring-sites-staging +# MON_HOSTS="bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch" +# SOURCE_REPO_WEB=https://git.hacktivism.ch/hernani/koopa-admin-log diff --git a/scripts/taler-monitoring/host-agent/env/francpaysan.env.example b/scripts/taler-monitoring/host-agent/env/francpaysan.env.example new file mode 100644 index 0000000..ab1c534 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/env/francpaysan.env.example @@ -0,0 +1,24 @@ +# Copy to: ~/.config/taler-monitoring/env (user francpaysan) +# Never inside the git tree — update-suite never overwrites this file. +# +# Shared reporting generation (run-host-report.sh / run-fp-prod-monitoring.sh): +# RUN_TIMEOUT, line-buffered log, always HTML, commit link on git.hacktivism.ch + +TALER_DOMAIN=lefrancpaysan.ch +INSIDE_PODMAN=1 +INSIDE_MODE=local-podman +LOCAL_STACK=0 +CONTINUE_ON_ERROR=1 +PHASES="urls inside versions" +RUN_TIMEOUT=600 + +HTML_OUT=$HOME/monitoring-sites-staging +MON_HOSTS="bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch" + +SUITE_DIR=$HOME/src/koopa-admin-log +SUITE_GIT_URL=https://git.hacktivism.ch/hernani/koopa-admin-log.git +SUITE_GIT_REF=main +SUITE_UPDATE_MODE=reset +SOURCE_REPO_WEB=https://git.hacktivism.ch/hernani/koopa-admin-log + +# DEPLOY_WWW_ROOT=/var/www/monitoring-sites diff --git a/scripts/taler-monitoring/host-agent/env/hacktivism.env.example b/scripts/taler-monitoring/host-agent/env/hacktivism.env.example new file mode 100644 index 0000000..6b57280 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/env/hacktivism.env.example @@ -0,0 +1,24 @@ +# Copy to: ~/.config/taler-monitoring/env (user hernani on koopa) +# Shared reporting generation: RUN_TIMEOUT, line-buffered log, always HTML. + +TALER_DOMAIN=hacktivism.ch +INSIDE_PODMAN=1 +INSIDE_MODE=local-podman +LOCAL_STACK=1 +CONTINUE_ON_ERROR=1 +PHASES="urls inside versions" +# aptdeploy HTML: run-aptdeploy-monitoring.sh → /taler-monitoring-aptdeploy* +RUN_TIMEOUT=600 +# APT_DEPLOY_ENSURE=1 +# APT_DEPLOY_SKIP=0 + +HTML_OUT=$HOME/monitoring-sites-staging +MON_HOSTS="bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch" + +SUITE_DIR=$HOME/src/koopa-admin-log +SUITE_GIT_URL=https://git.hacktivism.ch/hernani/koopa-admin-log.git +SUITE_GIT_REF=main +SUITE_UPDATE_MODE=reset +SOURCE_REPO_WEB=https://git.hacktivism.ch/hernani/koopa-admin-log + +DEPLOY_WWW_ROOT=/var/www/monitoring-sites diff --git a/scripts/taler-monitoring/host-agent/env/stagepaysan.env.example b/scripts/taler-monitoring/host-agent/env/stagepaysan.env.example new file mode 100644 index 0000000..0959ebc --- /dev/null +++ b/scripts/taler-monitoring/host-agent/env/stagepaysan.env.example @@ -0,0 +1,27 @@ +# Copy to: ~/.config/taler-monitoring/env (user stagepaysan) +# Never inside the git tree — update-suite never overwrites this file. +# +# Shared reporting generation (run-host-report.sh / run-fp-stage-monitoring.sh): +# RUN_TIMEOUT, line-buffered log, always HTML, commit link on git.hacktivism.ch + +TALER_DOMAIN=stage.lefrancpaysan.ch +INSIDE_PODMAN=1 +INSIDE_MODE=local-podman +INSIDE_PROFILE=stage-lfp +LOCAL_STACK=0 +CONTINUE_ON_ERROR=1 +PHASES="urls inside versions" +RUN_TIMEOUT=600 + +HTML_OUT=$HOME/monitoring-sites-staging +MON_HOSTS="stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch" + +# Git clone of koopa-admin-log (repo root, not …/scripts/taler-monitoring) +SUITE_DIR=$HOME/src/koopa-admin-log +SUITE_GIT_URL=https://git.hacktivism.ch/hernani/koopa-admin-log.git +SUITE_GIT_REF=main +SUITE_UPDATE_MODE=reset +SOURCE_REPO_WEB=https://git.hacktivism.ch/hernani/koopa-admin-log + +# Optional public docroot if this user can write it +# DEPLOY_WWW_ROOT=/var/www/monitoring-sites diff --git a/scripts/taler-monitoring/host-agent/install-host-agent.sh b/scripts/taler-monitoring/host-agent/install-host-agent.sh new file mode 100755 index 0000000..adcce66 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/install-host-agent.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# install-host-agent.sh — install hernani user systemd units on koopa +# +# # on koopa as hernani: +# ~/koopa-admin-log/scripts/taler-monitoring/host-agent/install-host-agent.sh +# # or from laptop: +# ./install-host-agent.sh --remote koopa-external +# +set -euo pipefail +ROOT=$(cd "$(dirname "$0")" && pwd) +REMOTE="${1:-}" +[ "${1:-}" = "--remote" ] && REMOTE="${2:-koopa-external}" + +install_local() { + local mon home_unit + mon=$(cd "$ROOT/.." && pwd) + # Prefer ~/koopa-admin-log on live host + if [ -d "$HOME/koopa-admin-log/scripts/taler-monitoring" ]; then + mon="$HOME/koopa-admin-log/scripts/taler-monitoring" + fi + home_unit="$HOME/.config/systemd/user" + mkdir -p "$home_unit" \ + "$HOME/.local/state/taler-hacktivism-monitoring" \ + "$HOME/monitoring-sites-staging" \ + "$HOME/.config/taler-monitoring" \ + "$HOME/src" + + # Local env (never in git) — create from example if missing + if [ ! -f "$HOME/.config/taler-monitoring/env" ] && [ -f "$ROOT/env.example" ]; then + cp "$ROOT/env.example" "$HOME/.config/taler-monitoring/env" + echo "created ~/.config/taler-monitoring/env" + fi + + # Prefer a real git clone for update-suite.sh + if [ ! -d "$HOME/src/koopa-admin-log/.git" ] && [ ! -d "$HOME/koopa-admin-log/.git" ]; then + if [ -d "$HOME/koopa-admin-log/scripts/taler-monitoring" ] && [ ! -d "$HOME/koopa-admin-log/.git" ]; then + echo "note: $HOME/koopa-admin-log has no .git — update-suite will clone to ~/src/koopa-admin-log" + fi + git clone https://git.hacktivism.ch/hernani/koopa-admin-log.git "$HOME/src/koopa-admin-log" 2>/dev/null \ + || echo "WARN: clone failed (auth/network) — will use existing tree if any" + fi + + # Ensure suite is executable + chmod +x "$mon"/taler-monitoring.sh "$mon"/check_*.sh 2>/dev/null || true + chmod +x "$mon/host-agent"/*.sh 2>/dev/null || true + chmod +x "$mon/site-gen"/*.sh "$mon/site-gen/console_to_html.py" 2>/dev/null || true + + # Units reference %h/koopa-admin-log — prefer git clone or symlink + if [ -d "$HOME/src/koopa-admin-log/.git" ] && [ ! -e "$HOME/koopa-admin-log" ]; then + ln -sfn "$HOME/src/koopa-admin-log" "$HOME/koopa-admin-log" + echo "symlink $HOME/koopa-admin-log → $HOME/src/koopa-admin-log" + elif [ ! -e "$HOME/koopa-admin-log" ]; then + admin_root=$(cd "$mon/../.." && pwd) + ln -sfn "$admin_root" "$HOME/koopa-admin-log" + echo "symlink $HOME/koopa-admin-log → $admin_root" + fi + + install -m 644 "$ROOT/taler-monitoring-hacktivism.service" "$home_unit/" + install -m 644 "$ROOT/taler-monitoring-hacktivism.path" "$home_unit/" + install -m 644 "$ROOT/taler-monitoring-hacktivism.timer" "$home_unit/" + install -m 644 "$ROOT/taler-monitoring-surface.service" "$home_unit/" + install -m 644 "$ROOT/taler-monitoring-surface.timer" "$home_unit/" + install -m 644 "$ROOT/taler-monitoring-aptdeploy.service" "$home_unit/" + install -m 644 "$ROOT/taler-monitoring-aptdeploy.timer" "$home_unit/" + + # Fix ExecStart if install path differs (units use %h/koopa-admin-log) + systemctl --user daemon-reload + systemctl --user enable --now taler-monitoring-hacktivism.path + systemctl --user enable --now taler-monitoring-hacktivism.timer + systemctl --user enable --now taler-monitoring-surface.timer + systemctl --user enable --now taler-monitoring-aptdeploy.timer + # initial stamp so path exists + "$ROOT/touch-software-stamp.sh" + systemctl --user start taler-monitoring-hacktivism.service || true + systemctl --user start taler-monitoring-surface.service || true + systemctl --user start taler-monitoring-aptdeploy.service || true + + echo "--- status ---" + systemctl --user status taler-monitoring-hacktivism.path --no-pager -l | head -15 + systemctl --user status taler-monitoring-hacktivism.timer --no-pager -l | head -12 + systemctl --user status taler-monitoring-surface.timer --no-pager -l | head -12 + systemctl --user status taler-monitoring-aptdeploy.timer --no-pager -l | head -12 + systemctl --user list-timers --all | grep taler-monitoring || true + echo "OK host-agent (hacktivism 4h + surface 1h + aptdeploy 4h)" + echo "manual: systemctl --user start taler-monitoring-hacktivism.service" + echo "manual surface: systemctl --user start taler-monitoring-surface.service" + echo "manual aptdeploy: systemctl --user start taler-monitoring-aptdeploy.service" + echo "after upgrades: $ROOT/touch-software-stamp.sh" +} + +if [ -n "$REMOTE" ] && [ "$REMOTE" != "--remote" ]; then + echo "install on $REMOTE …" + rsync -az --delete \ + --exclude secrets.env \ + --exclude 'android-test/artifacts' \ + "$ROOT/../../" \ + "${REMOTE}:koopa-admin-log/scripts/taler-monitoring/" + # ensure full tree for %h/koopa-admin-log if missing configs etc. + ssh -o BatchMode=yes -o ConnectTimeout=20 "$REMOTE" \ + 'test -d $HOME/koopa-admin-log || git clone https://git.hacktivism.ch/hernani/koopa-admin-log.git $HOME/koopa-admin-log 2>/dev/null || true' + rsync -az "$ROOT/" "${REMOTE}:koopa-admin-log/scripts/taler-monitoring/host-agent/" + rsync -az "$ROOT/../site-gen/" "${REMOTE}:koopa-admin-log/scripts/taler-monitoring/site-gen/" 2>/dev/null || true + ssh -o BatchMode=yes "$REMOTE" \ + 'bash $HOME/koopa-admin-log/scripts/taler-monitoring/host-agent/install-host-agent.sh' +else + install_local +fi diff --git a/scripts/taler-monitoring/host-agent/run-aptdeploy-monitoring.sh b/scripts/taler-monitoring/host-agent/run-aptdeploy-monitoring.sh new file mode 100755 index 0000000..fe81257 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/run-aptdeploy-monitoring.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# run-aptdeploy-monitoring.sh — apt-src deploy tests → public HTML +# Public only on taler.hacktivism.ch: +# https://taler.hacktivism.ch/taler-monitoring-aptdeploy/ +# https://taler.hacktivism.ch/taler-monitoring-aptdeploy_err/ +# +# CLI one-shot without HTML remains: run-aptdeploy.sh [all|fresh|upgrade|check-only] +# +set -uo pipefail +AGENT_DIR=$(cd "$(dirname "$0")" && pwd) + +export AGENT_LABEL="${AGENT_LABEL:-aptdeploy-host-agent}" +export STATE_NAME="${STATE_NAME:-taler-aptdeploy-monitoring}" +export TALER_DOMAIN="${TALER_DOMAIN:-hacktivism.ch}" +# containers are local on koopa +export INSIDE_PODMAN=1 +export INSIDE_MODE=local-podman +export LOCAL_STACK=1 +export SKIP_SSH=0 +export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" +# four containers + checks; allow long wall +export RUN_TIMEOUT="${RUN_TIMEOUT:-1800}" +# dedicated job: always aptdeploy only +export PHASES=aptdeploy +# Only merchant public front — not bank/exchange +export MON_HOSTS="${MON_HOSTS:-taler.hacktivism.ch}" +export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}" +export DEPLOY_WWW_ROOT="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" + +export HTML_OK_DIR="taler-monitoring-aptdeploy" +export HTML_ERR_DIR="taler-monitoring-aptdeploy_err" +export HTML_URL_OK="/taler-monitoring-aptdeploy/" +export HTML_URL_ERR="/taler-monitoring-aptdeploy_err/" +export PAGE_LABEL="taler-monitoring-aptdeploy" + +# Ensure all four apt-src containers (fresh rebuild + upgrade track) +if [ "${APT_DEPLOY_ENSURE:-1}" = "1" ] \ + && [ -x "$AGENT_DIR/ensure-apt-deploy-test-containers.sh" ]; then + bash "$AGENT_DIR/ensure-apt-deploy-test-containers.sh" all \ + || echo "WARN: ensure-apt-deploy-test-containers failed (aptdeploy may ERROR)" >&2 +fi + +exec bash "$AGENT_DIR/run-host-report.sh" "$@" diff --git a/scripts/taler-monitoring/host-agent/run-aptdeploy.sh b/scripts/taler-monitoring/host-agent/run-aptdeploy.sh new file mode 100755 index 0000000..fbf5afd --- /dev/null +++ b/scripts/taler-monitoring/host-agent/run-aptdeploy.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +# run-aptdeploy.sh — ensure containers + aptdeploy phase only. +# +# Keep monitoring current: +# 1) Prefer Forgejo origin/main (update-suite reset) +# 2) If Forgejo is behind (no aptdeploy yet), re-apply suite-overlay saved +# from this agent tree before reset (rsync from laptop fills that tree) +# 3) Run ensure + taler-monitoring.sh aptdeploy +# +# Host: on koopa → local; else ssh APT_DEPLOY_SSH + rsync mon tree first. +# +set -euo pipefail + +AGENT_DIR=$(cd "$(dirname "$0")" && pwd) +BOOT_MON=$(cd "$AGENT_DIR/.." && pwd) +MODE="${1:-all}" +APT_DEPLOY_SSH="${APT_DEPLOY_SSH:-koopa-external}" +OVERLAY_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring/suite-overlay" + +on_koopa() { + local h + h=$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo "") + case "$h" in koopa|koopa.*) return 0 ;; esac + [ "${KOOPA_FORCE_LOCAL:-0}" = "1" ] && return 0 + return 1 +} + +mon_has_aptdeploy() { + local mon=$1 + [ -f "$mon/taler-monitoring.sh" ] && grep -qE 'aptdeploy\|apt-deploy' "$mon/taler-monitoring.sh" +} + +save_overlay_from() { + local src=$1 + mon_has_aptdeploy "$src" || return 0 + mkdir -p "$OVERLAY_DIR" + rsync -a --delete \ + --exclude secrets.env \ + --exclude 'android-test/artifacts' \ + --exclude 'android-test/apks' \ + --exclude 'android-test/out*' \ + --exclude '__pycache__' \ + "$src/" "$OVERLAY_DIR/" + echo "saved suite-overlay from $src → $OVERLAY_DIR" +} + +apply_overlay_to() { + local dest=$1 + [ -f "$OVERLAY_DIR/taler-monitoring.sh" ] || return 1 + mon_has_aptdeploy "$OVERLAY_DIR" || return 1 + mkdir -p "$dest" + rsync -a \ + --exclude secrets.env \ + --exclude 'android-test/artifacts' \ + --exclude 'android-test/apks' \ + --exclude 'android-test/out*' \ + --exclude '__pycache__' \ + "$OVERLAY_DIR/" "$dest/" + echo "applied suite-overlay → $dest (Forgejo lag; push origin/main when possible)" + return 0 +} + +pick_mon_tree() { + # Preserve working mon before reset wipes tracked files + save_overlay_from "$BOOT_MON" + + # shellcheck source=update-suite.sh + source "$AGENT_DIR/update-suite.sh" + + local suite_mon="$SUITE_DIR/scripts/taler-monitoring" + mkdir -p "$suite_mon" + + if mon_has_aptdeploy "$suite_mon"; then + MON_DIR="$suite_mon" + echo "using Forgejo suite mon @ ${COMMIT_SHORT:-?} · $MON_DIR" + elif apply_overlay_to "$suite_mon" && mon_has_aptdeploy "$suite_mon"; then + MON_DIR="$suite_mon" + echo "using suite mon + overlay @ ${COMMIT_SHORT:-?} · $MON_DIR" + elif mon_has_aptdeploy "$BOOT_MON"; then + MON_DIR="$BOOT_MON" + echo "WARN: using boot mon $MON_DIR (overlay apply failed)" >&2 + else + echo "ERROR: no mon tree with aptdeploy (Forgejo + overlay empty)" >&2 + echo " rsync suite from laptop or push to git.hacktivism.ch" >&2 + exit 2 + fi + + AGENT_DIR="$MON_DIR/host-agent" + export MON_DIR AGENT_DIR SUITE_DIR COMMIT COMMIT_SHORT COMMIT_URL + chmod +x "$AGENT_DIR"/*.sh "$MON_DIR"/taler-monitoring.sh "$MON_DIR"/check_*.sh 2>/dev/null || true +} + +run_local() { + export PATH="${HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin:${PATH:-}" + export PYTHONUNBUFFERED=1 + export PROGRESS_OFF="${PROGRESS_OFF:-0}" + + echo "======== aptdeploy local on $(hostname) mode=$MODE ========" + pick_mon_tree + + if [ "$MODE" != "check-only" ]; then + bash "$AGENT_DIR/ensure-apt-deploy-test-containers.sh" "$MODE" + fi + + cd "$MON_DIR" + export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" + export INSIDE_PODMAN="${INSIDE_PODMAN:-1}" + export LOCAL_STACK="${LOCAL_STACK:-1}" + export RUN_TIMEOUT="${RUN_TIMEOUT:-600}" + echo "run: $MON_DIR/taler-monitoring.sh -d hacktivism.ch aptdeploy · suite=${COMMIT_SHORT:-?}" + ./taler-monitoring.sh -d hacktivism.ch aptdeploy +} + +run_remote() { + echo "======== aptdeploy remote via ssh ${APT_DEPLOY_SSH} mode=$MODE ========" + if [ "${APT_DEPLOY_RSYNC:-1}" = "1" ]; then + echo "rsync mon suite → ${APT_DEPLOY_SSH}:src/koopa-admin-log/scripts/taler-monitoring/" + rsync -az \ + --exclude secrets.env \ + --exclude 'android-test/artifacts' \ + --exclude 'android-test/apks' \ + --exclude 'android-test/out*' \ + --exclude '__pycache__' \ + "$BOOT_MON/" \ + "${APT_DEPLOY_SSH}:src/koopa-admin-log/scripts/taler-monitoring/" \ + || echo "WARN: rsync failed" >&2 + fi + ssh -o BatchMode=yes -o ConnectTimeout=25 "$APT_DEPLOY_SSH" bash -s -- "$MODE" <<'EOS' +set -euo pipefail +MODE=${1:-all} +export PATH="${HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin:${PATH:-}" +export KOOPA_FORCE_LOCAL=1 +AGENT="$HOME/src/koopa-admin-log/scripts/taler-monitoring/host-agent" +if [ ! -x "$AGENT/run-aptdeploy.sh" ]; then + AGENT="$HOME/koopa-admin-log/scripts/taler-monitoring/host-agent" +fi +if [ ! -x "$AGENT/run-aptdeploy.sh" ]; then + echo "ERROR: run-aptdeploy.sh missing on remote" >&2 + exit 1 +fi +echo "remote AGENT=$AGENT" +exec bash "$AGENT/run-aptdeploy.sh" "$MODE" +EOS +} + +if on_koopa; then + run_local +else + run_remote +fi diff --git a/scripts/taler-monitoring/host-agent/run-fp-prod-monitoring.sh b/scripts/taler-monitoring/host-agent/run-fp-prod-monitoring.sh new file mode 100755 index 0000000..be876a8 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/run-fp-prod-monitoring.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# FrancPaysan PROD host-agent (user francpaysan on francpaysan-host). +# Shared reporting: run-host-report.sh (timeout, line-buffered log, first-run HTML). +set -uo pipefail +AGENT_DIR=$(cd "$(dirname "$0")" && pwd) + +export AGENT_LABEL="${AGENT_LABEL:-fp-prod-host-agent}" +export STATE_NAME="${STATE_NAME:-taler-monitoring-lfp-prod}" +export TALER_DOMAIN="${TALER_DOMAIN:-lefrancpaysan.ch}" +export INSIDE_PODMAN="${INSIDE_PODMAN:-1}" +export INSIDE_MODE="${INSIDE_MODE:-local-podman}" +export LOCAL_STACK="${LOCAL_STACK:-0}" +export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" +export RUN_TIMEOUT="${RUN_TIMEOUT:-600}" +export PHASES="${PHASES:-urls inside versions}" +export MON_HOSTS="${MON_HOSTS:-bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch}" +export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}" +export DEPLOY_WWW_ROOT="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" + +exec bash "$AGENT_DIR/run-host-report.sh" "$@" diff --git a/scripts/taler-monitoring/host-agent/run-fp-stage-monitoring.sh b/scripts/taler-monitoring/host-agent/run-fp-stage-monitoring.sh new file mode 100755 index 0000000..70518f0 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/run-fp-stage-monitoring.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# FrancPaysan STAGE host-agent (user stagepaysan on francpaysan-host). +# Shared reporting: run-host-report.sh (timeout, line-buffered log, first-run HTML). +set -uo pipefail +AGENT_DIR=$(cd "$(dirname "$0")" && pwd) + +export AGENT_LABEL="${AGENT_LABEL:-fp-stage-host-agent}" +export STATE_NAME="${STATE_NAME:-taler-monitoring-stage-lfp}" +export TALER_DOMAIN="${TALER_DOMAIN:-stage.lefrancpaysan.ch}" +export INSIDE_PODMAN="${INSIDE_PODMAN:-1}" +export INSIDE_MODE="${INSIDE_MODE:-local-podman}" +export INSIDE_PROFILE="${INSIDE_PROFILE:-stage-lfp}" +export LOCAL_STACK="${LOCAL_STACK:-0}" +export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" +export RUN_TIMEOUT="${RUN_TIMEOUT:-600}" +export PHASES="${PHASES:-urls inside versions}" +export MON_HOSTS="${MON_HOSTS:-stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch}" +export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}" +export DEPLOY_WWW_ROOT="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" + +exec bash "$AGENT_DIR/run-host-report.sh" "$@" diff --git a/scripts/taler-monitoring/host-agent/run-hacktivism-monitoring.sh b/scripts/taler-monitoring/host-agent/run-hacktivism-monitoring.sh new file mode 100755 index 0000000..8fd98fd --- /dev/null +++ b/scripts/taler-monitoring/host-agent/run-hacktivism-monitoring.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# GOA / hacktivism host-agent on koopa (hernani). +# Stack defaults only — reporting pipeline is shared: run-host-report.sh +set -uo pipefail +AGENT_DIR=$(cd "$(dirname "$0")" && pwd) + +export AGENT_LABEL="${AGENT_LABEL:-hacktivism-host-agent}" +export STATE_NAME="${STATE_NAME:-taler-hacktivism-monitoring}" +export TALER_DOMAIN="${TALER_DOMAIN:-hacktivism.ch}" +export INSIDE_PODMAN="${INSIDE_PODMAN:-1}" +export INSIDE_MODE="${INSIDE_MODE:-local-podman}" +export LOCAL_STACK="${LOCAL_STACK:-1}" +export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" +export RUN_TIMEOUT="${RUN_TIMEOUT:-600}" +# stack checks only — aptdeploy has its own HTML path + timer: +# run-aptdeploy-monitoring.sh → /taler-monitoring-aptdeploy* (taler.hacktivism.ch) +export PHASES="${PHASES:-urls inside versions}" +export MON_HOSTS="${MON_HOSTS:-bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch}" +export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}" +export DEPLOY_WWW_ROOT="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" +export APT_DEPLOY_ENSURE=0 + +exec bash "$AGENT_DIR/run-host-report.sh" "$@" diff --git a/scripts/taler-monitoring/host-agent/run-host-report.sh b/scripts/taler-monitoring/host-agent/run-host-report.sh new file mode 100755 index 0000000..1be1def --- /dev/null +++ b/scripts/taler-monitoring/host-agent/run-host-report.sh @@ -0,0 +1,358 @@ +#!/usr/bin/env bash +# run-host-report.sh — global reporting generation for all stacks +# (GOA/hacktivism, FP stage, FP prod, …). +# +# Shared behaviour (always): +# 1) update-suite → latest git.hacktivism.ch (local ~/.config never overwritten) +# 2) line-buffered run log (written continuously) +# 3) RUN_TIMEOUT wall clock for taler-monitoring.sh (default 600s) +# 4) always write HTML (first run / empty staging too) + Forgejo commit link +# 5) optional rsync to DEPLOY_WWW_ROOT if writable +# +# Configure via env file (not in git): +# ${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring/env +# or TALER_MONITORING_ENV=/path/to/env +# +# Thin wrappers only set stack defaults then exec this script: +# run-hacktivism-monitoring.sh +# run-fp-stage-monitoring.sh +# run-fp-prod-monitoring.sh +# +set -uo pipefail + +export PATH="${HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin:${PATH:-}" +export PYTHONUNBUFFERED=1 + +AGENT_DIR=$(cd "$(dirname "$0")" && pwd) +AGENT_MON=$(cd "$AGENT_DIR/.." && pwd) +CFG_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring" +ENV_FILE="${TALER_MONITORING_ENV:-$CFG_DIR/env}" +mkdir -p "$CFG_DIR" + +# Thin wrappers (surface, aptdeploy-only, …) export stack knobs before exec. +# Snapshot them so ~/.config/taler-monitoring/env cannot clobber dedicated jobs +# (env often has PHASES="urls inside versions aptdeploy" for the main timer). +_wrap_keys=( + PHASES HTML_OUT HTML_OK_DIR HTML_ERR_DIR HTML_URL_OK HTML_URL_ERR PAGE_LABEL + RUN_TIMEOUT STATE_NAME AGENT_LABEL MON_HOSTS TALER_DOMAIN DEPLOY_WWW_ROOT + INSIDE_PODMAN INSIDE_MODE LOCAL_STACK SKIP_SSH CONTINUE_ON_ERROR AUTH401_CONTINUE + APT_DEPLOY_ENSURE SURFACE_SCOPE TALER_DOMAIN_FROM_CLI STRICT_EXIT SOURCE_SUITE_PATH +) +for _k in "${_wrap_keys[@]}"; do + if [ -n "${!_k+x}" ]; then + printf -v "_WRAP_SET_${_k}" '%s' 1 + printf -v "_WRAP_VAL_${_k}" '%s' "${!_k}" + fi +done +unset _k + +if [ -f "$ENV_FILE" ]; then + # shellcheck disable=SC1090 + set -a + # shellcheck source=/dev/null + source "$ENV_FILE" + set +a +fi +# Tell update-suite.sh not to re-source env (would clobber wrapper PHASES etc.) +export _TALER_MON_ENV_LOADED=1 + +# Restore wrapper overrides (dedicated jobs win over shared env file) +for _k in "${_wrap_keys[@]}"; do + _set_var="_WRAP_SET_${_k}" + _val_var="_WRAP_VAL_${_k}" + if [ "${!_set_var:-}" = "1" ]; then + printf -v "$_k" '%s' "${!_val_var}" + export "$_k" + fi +done +unset _k _set_var _val_var + +# --- global report defaults (env file / wrapper wins if already set) --- +export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" +export AUTH401_CONTINUE="${AUTH401_CONTINUE:-${CONTINUE_ON_ERROR}}" +export RUN_TIMEOUT="${RUN_TIMEOUT:-600}" +export TALER_DOMAIN="${TALER_DOMAIN:-hacktivism.ch}" +PHASES="${PHASES:-urls inside versions}" +HTML_BASE="${HTML_OUT:-$HOME/monitoring-sites-staging}" +MON_HOSTS="${MON_HOSTS:-}" +SUITE_PATH="${SOURCE_SUITE_PATH:-scripts/taler-monitoring}" +AGENT_LABEL="${AGENT_LABEL:-host-report}" +STATE_NAME="${STATE_NAME:-taler-monitoring-${TALER_DOMAIN}}" +STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/${STATE_NAME}" +LOG_DIR="$STATE_DIR/logs" +DEPLOY_WWW="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" +# HTML path layout under each host (monitoring / surface / …) +HTML_OK_DIR="${HTML_OK_DIR:-monitoring}" +HTML_ERR_DIR="${HTML_ERR_DIR:-monitoring_err}" +HTML_URL_OK="${HTML_URL_OK:-/monitoring/}" +HTML_URL_ERR="${HTML_URL_ERR:-/monitoring_err/}" +PAGE_LABEL="${PAGE_LABEL:-monitoring}" + +# Sensible MON_HOSTS from domain if not set +if [ -z "$MON_HOSTS" ]; then + case "$TALER_DOMAIN" in + hacktivism.ch|koopa) + MON_HOSTS="bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch" + ;; + stage.lefrancpaysan.ch|stage.*lefrancpaysan*) + MON_HOSTS="stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch" + ;; + lefrancpaysan.ch) + MON_HOSTS="bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch" + ;; + *) + MON_HOSTS="${TALER_DOMAIN}" + ;; + esac +fi + +mkdir -p "$LOG_DIR" "$HTML_BASE" + +STAMP=$(date +%Y%m%d-%H%M%S) +LOG="$LOG_DIR/run-${STAMP}.log" +: >"$LOG" +if command -v stdbuf >/dev/null 2>&1; then + exec > >(stdbuf -oL -eL tee -a "$LOG") 2>&1 +else + exec > >(tee -a "$LOG") 2>&1 +fi + +echo "======== $(date -Iseconds 2>/dev/null || date) ${AGENT_LABEL} ========" +echo "env_file=$ENV_FILE" +echo "domain=$TALER_DOMAIN · phases=$PHASES · RUN_TIMEOUT=${RUN_TIMEOUT}s" +echo "hosts=$MON_HOSTS" +echo "flags INSIDE_PODMAN=${INSIDE_PODMAN:-} INSIDE_MODE=${INSIDE_MODE:-} LOCAL_STACK=${LOCAL_STACK:-} SKIP_SSH=${SKIP_SSH:-} INSIDE_PROFILE=${INSIDE_PROFILE:-}" + +# --- refresh from Forgejo; re-apply suite-overlay if Forgejo lacks needed phases --- +OVERLAY_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring/suite-overlay" +# preserve capable mon before reset wipes tracked files +if [ -f "$AGENT_MON/taler-monitoring.sh" ]; then + mkdir -p "$OVERLAY_DIR" + rsync -a \ + --exclude secrets.env --exclude 'android-test/artifacts' --exclude 'android-test/apks' \ + --exclude 'android-test/out*' --exclude '__pycache__' \ + "$AGENT_MON/" "$OVERLAY_DIR/" 2>/dev/null || true +fi + +# shellcheck source=update-suite.sh +source "$AGENT_DIR/update-suite.sh" + +SUITE_MON="${SUITE_DIR}/scripts/taler-monitoring" +ROOT="$SUITE_MON" +_need_overlay=0 +# Detect missing features in Forgejo tree (untracked helper files alone are not enough — +# taler-monitoring.sh must accept the phase name). +case " $PHASES " in + *" aptdeploy "*|*" apt-deploy "*|*" apt_src "*) + if ! grep -qE 'aptdeploy\|apt-deploy' "$SUITE_MON/taler-monitoring.sh" 2>/dev/null; then + _need_overlay=1 + fi + ;; +esac +case " $PHASES " in + *" surface "*|*" ecosystem "*) + if ! grep -qE 'surface\|ecosystem' "$SUITE_MON/taler-monitoring.sh" 2>/dev/null \ + || [ ! -f "$SUITE_MON/check_surface.sh" ]; then + _need_overlay=1 + fi + ;; +esac +# Wrapper-preserve + HTML path knobs may live only in local overlay until pushed +if [ -f "$OVERLAY_DIR/host-agent/run-host-report.sh" ] \ + && ! grep -q '_wrap_keys\|_WRAP_SET_' "$SUITE_MON/host-agent/run-host-report.sh" 2>/dev/null \ + && grep -q '_wrap_keys\|_WRAP_SET_' "$OVERLAY_DIR/host-agent/run-host-report.sh" 2>/dev/null; then + _need_overlay=1 +fi +if [ "$_need_overlay" = "1" ] && [ -f "$OVERLAY_DIR/taler-monitoring.sh" ]; then + echo "WARN: Forgejo @ ${COMMIT_SHORT:-?} lagging local suite-overlay (phases=[$PHASES])" >&2 + echo " re-applying suite-overlay → $SUITE_MON" >&2 + mkdir -p "$SUITE_MON" + rsync -a \ + --exclude secrets.env --exclude 'android-test/artifacts' --exclude 'android-test/apks' \ + --exclude 'android-test/out*' --exclude '__pycache__' \ + "$OVERLAY_DIR/" "$SUITE_MON/" + ROOT="$SUITE_MON" +elif [ "$_need_overlay" = "1" ]; then + echo "WARN: Forgejo missing features and no suite-overlay — using agent mon $AGENT_MON" >&2 + ROOT="$AGENT_MON" +else + echo "using suite mon @ ${COMMIT_SHORT:-?} · $ROOT" +fi +# Always restore host-agent + site-gen from pre-reset snapshot when present +# (systemd ExecStart points at koopa-admin-log which may be the reset suite tree). +if [ -d "$OVERLAY_DIR/host-agent" ]; then + mkdir -p "$SUITE_MON/host-agent" 2>/dev/null || true + rsync -a "$OVERLAY_DIR/host-agent/" "$SUITE_MON/host-agent/" 2>/dev/null || true +fi +if [ -d "$OVERLAY_DIR/site-gen" ]; then + mkdir -p "$SUITE_MON/site-gen" + rsync -a "$OVERLAY_DIR/site-gen/" "$SUITE_MON/site-gen/" 2>/dev/null || true +fi +unset _need_overlay + +if [ -f "$ROOT/site-gen/console_to_html.py" ]; then + SITE_GEN="$ROOT/site-gen" +elif [ -f "$AGENT_MON/site-gen/console_to_html.py" ]; then + SITE_GEN="$AGENT_MON/site-gen" + echo "note: using agent site-gen at $SITE_GEN" +elif [ -f "$SUITE_MON/site-gen/console_to_html.py" ]; then + SITE_GEN="$SUITE_MON/site-gen" +else + SITE_GEN="" +fi + +if [ ! -x "$ROOT/taler-monitoring.sh" ]; then + if [ -x "$AGENT_MON/taler-monitoring.sh" ]; then + ROOT="$AGENT_MON" + echo "note: using agent mon tree $ROOT" + else + echo "ERROR: suite missing at $ROOT" >&2 + COMMIT="${COMMIT:-unknown}" + COMMIT_SHORT="${COMMIT_SHORT:-unknown}" + COMMIT_URL="${COMMIT_URL:-}" + fi +fi + +echo "commit=${COMMIT_SHORT:-?} · ${COMMIT_URL:-}" + +ec=1 +if [ -x "${ROOT:-}/taler-monitoring.sh" ]; then + cd "$ROOT" || exit 1 + chmod +x taler-monitoring.sh check_*.sh host-agent/*.sh site-gen/*.sh site-gen/*.py 2>/dev/null || true + + # Overlay newer inside/versions host-podman bits if suite clone is stale + for f in check_inside.sh check_versions.sh taler-monitoring.sh lib.sh; do + if [ -f "$AGENT_MON/$f" ] && [ -f "$ROOT/$f" ]; then + if ! grep -q 'host-podman\|INSIDE_ACCESS\|RUN_TIMEOUT' "$ROOT/$f" 2>/dev/null \ + && grep -q 'host-podman\|INSIDE_ACCESS\|RUN_TIMEOUT' "$AGENT_MON/$f" 2>/dev/null; then + echo "note: overlay $f from agent install" + cp -f "$AGENT_MON/$f" "$ROOT/$f" + chmod +x "$ROOT/$f" 2>/dev/null || true + fi + fi + done + + # shellcheck disable=SC2086 + ./taler-monitoring.sh -d "$TALER_DOMAIN" $PHASES + ec=$? +fi + +# --- always generate HTML (first run / missing pages / every run) --- +write_bootstrap_html() { + local out="$1" host="$2" mode="$3" + mkdir -p "$(dirname "$out")" + local cshort="${COMMIT_SHORT:-unknown}" + local curl="${COMMIT_URL:-}" + local clink="$cshort" + local gen iso + gen=$(date -u +"%Y-%m-%d %H:%M:%SZ" 2>/dev/null || date) + iso=$(date -u +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -Iseconds) + if [ -n "$curl" ]; then + clink="$cshort" + fi + cat >"$out" < + + +monitoring ${mode} · ${host} + + +
+

Bootstrap page (converter missing or first install). See host-agent log.

+
$(tail -n 80 "$LOG" 2>/dev/null | sed 's/&/\&/g;s/
+
+ +EOF + echo "bootstrap html → $out" +} + +htmlify_host() { + local host="$1" + local mon="$HTML_BASE/$host/${HTML_OK_DIR}/index.html" + local mon_err="$HTML_BASE/$host/${HTML_ERR_DIR}/index.html" + mkdir -p "$HTML_BASE/$host/${HTML_OK_DIR}" "$HTML_BASE/$host/${HTML_ERR_DIR}" + + if [ -n "$SITE_GEN" ] && [ -f "$SITE_GEN/console_to_html.py" ]; then + python3 "$SITE_GEN/console_to_html.py" \ + --log "$LOG" \ + --out "$mon_err" \ + --hostname "$host" \ + --mode err \ + --commit "${COMMIT:-}" \ + --commit-url "${COMMIT_URL:-}" \ + --suite-path "$SUITE_PATH" \ + --page-label "$PAGE_LABEL" \ + --path-err "$HTML_URL_ERR" \ + --link-other "$HTML_URL_OK" + if [ "$ec" -eq 0 ]; then + python3 "$SITE_GEN/console_to_html.py" \ + --log "$LOG" \ + --out "$mon" \ + --hostname "$host" \ + --mode ok \ + --commit "${COMMIT:-}" \ + --commit-url "${COMMIT_URL:-}" \ + --suite-path "$SUITE_PATH" \ + --page-label "$PAGE_LABEL" \ + --path-err "$HTML_URL_ERR" \ + --link-other "" + rm -rf "$HTML_BASE/$host/${HTML_ERR_DIR}" + echo "html $host → ${HTML_URL_OK} only (clean · ${COMMIT_SHORT:-?})" + else + python3 "$SITE_GEN/console_to_html.py" \ + --log "$LOG" \ + --out "$mon" \ + --hostname "$host" \ + --mode redirect \ + --commit "${COMMIT:-}" \ + --commit-url "${COMMIT_URL:-}" \ + --suite-path "$SUITE_PATH" \ + --page-label "$PAGE_LABEL" \ + --path-err "$HTML_URL_ERR" + echo "html $host → ${HTML_URL_OK} stub + ${HTML_URL_ERR} (ec=$ec · ${COMMIT_SHORT:-?})" + fi + else + echo "WARN: console_to_html.py missing — bootstrap pages" + write_bootstrap_html "$mon_err" "$host" "err" + if [ "$ec" -eq 0 ]; then + write_bootstrap_html "$mon" "$host" "ok" + rm -rf "$HTML_BASE/$host/${HTML_ERR_DIR}" + else + write_bootstrap_html "$mon" "$host" "redirect" + fi + fi + + if [ ! -f "$mon" ]; then + write_bootstrap_html "$mon" "$host" "err" + fi + + printf '%s\n' "${COMMIT:-unknown}" >"$HTML_BASE/$host/COMMIT" + printf '%s\n' "${COMMIT_URL:-}" >"$HTML_BASE/$host/COMMIT_URL" +} +for host in $MON_HOSTS; do + htmlify_host "$host" +done + +if [ -n "$DEPLOY_WWW" ] && [ -w "$DEPLOY_WWW" ] 2>/dev/null; then + rsync -a --delete "$HTML_BASE/" "$DEPLOY_WWW/" && \ + echo "synced → $DEPLOY_WWW/" +fi + +ls -1t "$LOG_DIR"/run-*.log 2>/dev/null | tail -n +30 | xargs rm -f 2>/dev/null || true + +echo "======== done ec=$ec · log=$LOG · ${COMMIT_URL:-} ========" +if [ "${STRICT_EXIT:-0}" = "1" ]; then + exit "$ec" +fi +exit 0 diff --git a/scripts/taler-monitoring/host-agent/run-surface-monitoring.sh b/scripts/taler-monitoring/host-agent/run-surface-monitoring.sh new file mode 100755 index 0000000..1b7a8b8 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/run-surface-monitoring.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# run-surface-monitoring.sh — hourly remote-only ecosystem surface scan +# Public only on taler.hacktivism.ch: +# https://taler.hacktivism.ch/taler-monitoring-surface/ +# https://taler.hacktivism.ch/taler-monitoring-surface_err/ +# +set -uo pipefail +AGENT_DIR=$(cd "$(dirname "$0")" && pwd) + +export AGENT_LABEL="${AGENT_LABEL:-surface-host-agent}" +export STATE_NAME="${STATE_NAME:-taler-surface-monitoring}" +export TALER_DOMAIN="${TALER_DOMAIN:-hacktivism.ch}" +# surface is remote-only — force flags (ignore shared env) +export INSIDE_PODMAN=0 +export LOCAL_STACK=0 +export SKIP_SSH=1 +export INSIDE_MODE="${INSIDE_MODE:-none}" +export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" +# many hosts; allow up to 45 min wall (phase also has RUN_TIMEOUT) +export RUN_TIMEOUT="${RUN_TIMEOUT:-2400}" +# dedicated job: always surface only (never inherit PHASES from shared env) +export PHASES=surface +# Only merchant public front — not bank/exchange +export MON_HOSTS="${MON_HOSTS:-taler.hacktivism.ch}" +export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}" +export DEPLOY_WWW_ROOT="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" + +export HTML_OK_DIR="taler-monitoring-surface" +export HTML_ERR_DIR="taler-monitoring-surface_err" +export HTML_URL_OK="/taler-monitoring-surface/" +export HTML_URL_ERR="/taler-monitoring-surface_err/" +export PAGE_LABEL="taler-monitoring-surface" + +# ecosystem catalog by default (not domain-only) +export SURFACE_SCOPE="${SURFACE_SCOPE:-ecosystem}" +export TALER_DOMAIN_FROM_CLI=0 + +# Do not run apt-deploy ensure for surface-only jobs +export APT_DEPLOY_ENSURE=0 + +exec bash "$AGENT_DIR/run-host-report.sh" "$@" diff --git a/scripts/taler-monitoring/host-agent/taler-monitoring-aptdeploy.service b/scripts/taler-monitoring/host-agent/taler-monitoring-aptdeploy.service new file mode 100644 index 0000000..3d71eb1 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/taler-monitoring-aptdeploy.service @@ -0,0 +1,18 @@ +[Unit] +Description=Taler monitoring aptdeploy (apt-src container deploy tests) +Documentation=file:%h/koopa-admin-log/scripts/taler-monitoring/host-agent/README.md +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin +Environment=CONTINUE_ON_ERROR=1 +Environment=RUN_TIMEOUT=1800 +WorkingDirectory=%h/koopa-admin-log/scripts/taler-monitoring +ExecStart=%h/koopa-admin-log/scripts/taler-monitoring/host-agent/run-aptdeploy-monitoring.sh +Nice=10 +TimeoutStartSec=45min + +[Install] +WantedBy=default.target diff --git a/scripts/taler-monitoring/host-agent/taler-monitoring-aptdeploy.timer b/scripts/taler-monitoring/host-agent/taler-monitoring-aptdeploy.timer new file mode 100644 index 0000000..602ce8b --- /dev/null +++ b/scripts/taler-monitoring/host-agent/taler-monitoring-aptdeploy.timer @@ -0,0 +1,14 @@ +[Unit] +Description=Periodic taler-monitoring aptdeploy (apt-src deploy tests) +Documentation=file:%h/koopa-admin-log/scripts/taler-monitoring/host-agent/README.md + +[Timer] +OnBootSec=35min +OnUnitActiveSec=4h +AccuracySec=5min +Persistent=true +RandomizedDelaySec=10min +Unit=taler-monitoring-aptdeploy.service + +[Install] +WantedBy=timers.target diff --git a/scripts/taler-monitoring/host-agent/taler-monitoring-hacktivism.path b/scripts/taler-monitoring/host-agent/taler-monitoring-hacktivism.path new file mode 100644 index 0000000..1cbf003 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/taler-monitoring-hacktivism.path @@ -0,0 +1,19 @@ +[Unit] +Description=Watch hacktivism software/config changes → re-run monitoring +Documentation=file:%h/koopa-admin-log/scripts/taler-monitoring/host-agent/README.md + +[Path] +# Explicit stamp after package/image upgrades (touch-software-stamp.sh) +PathChanged=%h/.local/state/taler-hacktivism-monitoring/software.stamp +PathModified=%h/.local/state/taler-hacktivism-monitoring/software.stamp +# Landing + caddy config in admin-log (when checked out under ~) +PathChanged=%h/koopa-admin-log/configs/bank-landing +PathChanged=%h/koopa-admin-log/configs/exchange-landing +PathChanged=%h/koopa-admin-log/configs/merchant-landing +PathChanged=%h/koopa-admin-log/configs/caddy +# Debounce: wait for quiet period before firing +TriggerLimitIntervalSec=120 +TriggerLimitBurst=3 + +[Install] +WantedBy=default.target diff --git a/scripts/taler-monitoring/host-agent/taler-monitoring-hacktivism.service b/scripts/taler-monitoring/host-agent/taler-monitoring-hacktivism.service new file mode 100644 index 0000000..712920c --- /dev/null +++ b/scripts/taler-monitoring/host-agent/taler-monitoring-hacktivism.service @@ -0,0 +1,21 @@ +[Unit] +Description=Taler monitoring GOA/hacktivism (host-podman inside + public urls) +Documentation=file:%h/koopa-admin-log/scripts/taler-monitoring/host-agent/README.md +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +# Primary: host can podman-exec into taler-hacktivism* containers +Environment=INSIDE_PODMAN=1 +Environment=CONTINUE_ON_ERROR=1 +Environment=RUN_TIMEOUT=600 +Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin +WorkingDirectory=%h/koopa-admin-log/scripts/taler-monitoring +ExecStart=%h/koopa-admin-log/scripts/taler-monitoring/host-agent/run-hacktivism-monitoring.sh +Nice=10 +# Suite budget is RUN_TIMEOUT (10m) + git/HTML overhead +TimeoutStartSec=15min + +[Install] +WantedBy=default.target diff --git a/scripts/taler-monitoring/host-agent/taler-monitoring-hacktivism.timer b/scripts/taler-monitoring/host-agent/taler-monitoring-hacktivism.timer new file mode 100644 index 0000000..b65c139 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/taler-monitoring-hacktivism.timer @@ -0,0 +1,14 @@ +[Unit] +Description=Periodic hacktivism monitoring (fallback if no path events) +Documentation=file:%h/koopa-admin-log/scripts/taler-monitoring/host-agent/README.md + +[Timer] +# Every 4h + boot +OnBootSec=10min +OnUnitActiveSec=4h +AccuracySec=5min +Persistent=true +Unit=taler-monitoring-hacktivism.service + +[Install] +WantedBy=timers.target diff --git a/scripts/taler-monitoring/host-agent/taler-monitoring-surface.service b/scripts/taler-monitoring/host-agent/taler-monitoring-surface.service new file mode 100644 index 0000000..6018dee --- /dev/null +++ b/scripts/taler-monitoring/host-agent/taler-monitoring-surface.service @@ -0,0 +1,18 @@ +[Unit] +Description=Taler monitoring surface (remote ecosystem inventory) +Documentation=file:%h/koopa-admin-log/scripts/taler-monitoring/host-agent/README.md +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin +Environment=CONTINUE_ON_ERROR=1 +Environment=RUN_TIMEOUT=2400 +WorkingDirectory=%h/koopa-admin-log/scripts/taler-monitoring +ExecStart=%h/koopa-admin-log/scripts/taler-monitoring/host-agent/run-surface-monitoring.sh +Nice=10 +TimeoutStartSec=50min + +[Install] +WantedBy=default.target diff --git a/scripts/taler-monitoring/host-agent/taler-monitoring-surface.timer b/scripts/taler-monitoring/host-agent/taler-monitoring-surface.timer new file mode 100644 index 0000000..a9ed8c9 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/taler-monitoring-surface.timer @@ -0,0 +1,14 @@ +[Unit] +Description=Hourly taler-monitoring surface scan (ecosystem) +Documentation=file:%h/koopa-admin-log/scripts/taler-monitoring/host-agent/README.md + +[Timer] +OnBootSec=20min +OnUnitActiveSec=1h +AccuracySec=2min +Persistent=true +RandomizedDelaySec=5min +Unit=taler-monitoring-surface.service + +[Install] +WantedBy=timers.target diff --git a/scripts/taler-monitoring/host-agent/touch-software-stamp.sh b/scripts/taler-monitoring/host-agent/touch-software-stamp.sh new file mode 100755 index 0000000..0aff400 --- /dev/null +++ b/scripts/taler-monitoring/host-agent/touch-software-stamp.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Call after upgrading hacktivism packages / redeploying landings / image rebuilds. +# systemd path unit watches this stamp → re-runs monitoring. +set -euo pipefail +STATE="${XDG_STATE_HOME:-$HOME/.local/state}/taler-hacktivism-monitoring" +mkdir -p "$STATE" +STAMP="$STATE/software.stamp" +date -Iseconds >"$STAMP" +# also record container image ids for debugging +if command -v podman >/dev/null 2>&1; then + { + echo "# images $(date -Iseconds)" + podman images --format '{{.Repository}}:{{.Tag}} {{.ID}} {{.Digest}}' 2>/dev/null | grep -i hacktivism || true + podman ps --format '{{.Names}} {{.ImageID}} {{.Status}}' 2>/dev/null | grep -i hacktivism || true + } >"$STATE/software-images.txt" || true +fi +echo "touched $STAMP" diff --git a/scripts/taler-monitoring/host-agent/update-suite.sh b/scripts/taler-monitoring/host-agent/update-suite.sh new file mode 100755 index 0000000..70d348d --- /dev/null +++ b/scripts/taler-monitoring/host-agent/update-suite.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# update-suite.sh — ensure taler-monitoring code is latest from Forgejo. +# +# Local config (NEVER overwritten by git): +# ${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring/env +# +# Env overrides (also settable in that file): +# SUITE_GIT_URL default https://git.hacktivism.ch/hernani/koopa-admin-log.git +# SUITE_GIT_REF default main +# SUITE_DIR default $HOME/src/koopa-admin-log (or $HOME/koopa-admin-log) +# SUITE_UPDATE_MODE ff (default) | reset (reset = hard to origin/REF) +# +set -uo pipefail + +CFG_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring" +ENV_FILE="${TALER_MONITORING_ENV:-$CFG_DIR/env}" +# When sourced from run-host-report.sh the env file is already loaded and +# wrapper PHASES/HTML_* must not be clobbered. Only load here if needed. +if [ -f "$ENV_FILE" ] && [ "${_TALER_MON_ENV_LOADED:-0}" != "1" ]; then + # shellcheck disable=SC1090 + set -a + # shellcheck source=/dev/null + source "$ENV_FILE" + set +a +fi + +SUITE_GIT_URL="${SUITE_GIT_URL:-https://git.hacktivism.ch/hernani/koopa-admin-log.git}" +SUITE_GIT_REF="${SUITE_GIT_REF:-main}" +# Host agents default to reset so "always latest from Forgejo" wins over local +# diverged trees. Override with SUITE_UPDATE_MODE=ff for careful pulls. +SUITE_UPDATE_MODE="${SUITE_UPDATE_MODE:-reset}" + +if [ -z "${SUITE_DIR:-}" ]; then + if [ -d "$HOME/src/koopa-admin-log/.git" ]; then + SUITE_DIR="$HOME/src/koopa-admin-log" + elif [ -d "$HOME/koopa-admin-log/.git" ]; then + SUITE_DIR="$HOME/koopa-admin-log" + else + SUITE_DIR="$HOME/src/koopa-admin-log" + fi +fi + +export SUITE_DIR + +mkdir -p "$(dirname "$SUITE_DIR")" "$CFG_DIR" + +if [ ! -d "$SUITE_DIR/.git" ]; then + echo "clone $SUITE_GIT_URL → $SUITE_DIR" + git clone --branch "$SUITE_GIT_REF" "$SUITE_GIT_URL" "$SUITE_DIR" \ + || git clone "$SUITE_GIT_URL" "$SUITE_DIR" +fi + +cd "$SUITE_DIR" || exit 1 +git remote set-url origin "$SUITE_GIT_URL" 2>/dev/null \ + || git remote add origin "$SUITE_GIT_URL" 2>/dev/null || true + +echo "fetch origin ($SUITE_GIT_REF) …" +if ! git fetch --tags origin 2>&1; then + echo "WARN: git fetch failed — using existing tree at $SUITE_DIR" >&2 + git rev-parse HEAD 2>/dev/null || true + return 0 2>/dev/null || exit 0 +fi + +case "$SUITE_UPDATE_MODE" in + reset) + git checkout -B "$SUITE_GIT_REF" "origin/$SUITE_GIT_REF" 2>/dev/null \ + || git checkout -B "$SUITE_GIT_REF" "FETCH_HEAD" + git reset --hard "origin/$SUITE_GIT_REF" 2>/dev/null \ + || git reset --hard FETCH_HEAD + ;; + *) + git checkout "$SUITE_GIT_REF" 2>/dev/null || git checkout -B "$SUITE_GIT_REF" "origin/$SUITE_GIT_REF" 2>/dev/null || true + if ! git pull --ff-only origin "$SUITE_GIT_REF" 2>&1; then + echo "WARN: ff-only pull failed — try SUITE_UPDATE_MODE=reset or fix local commits" >&2 + fi + ;; +esac + +COMMIT=$(git rev-parse HEAD) +COMMIT_SHORT=$(git rev-parse --short=12 HEAD) +# Forgejo web UI +SOURCE_REPO_WEB="${SOURCE_REPO_WEB:-https://git.hacktivism.ch/hernani/koopa-admin-log}" +COMMIT_URL="${SOURCE_REPO_WEB}/src/commit/${COMMIT}" +export COMMIT COMMIT_SHORT COMMIT_URL SOURCE_REPO_WEB + +echo "suite @ $COMMIT_SHORT $COMMIT_URL" +echo " dir=$SUITE_DIR" +echo " config=$ENV_FILE (not touched by git)" +echo " note: always latest = origin/${SUITE_GIT_REF} on git.hacktivism.ch (push local work there)" + +# Symlink convenience path if missing or not a git clone +if [ -d "$SUITE_DIR/.git" ]; then + if [ ! -e "$HOME/koopa-admin-log" ]; then + ln -sfn "$SUITE_DIR" "$HOME/koopa-admin-log" 2>/dev/null || true + elif [ -L "$HOME/koopa-admin-log" ]; then + ln -sfn "$SUITE_DIR" "$HOME/koopa-admin-log" 2>/dev/null || true + fi +fi + +# Make suite scripts executable after reset +if [ -d "$SUITE_DIR/scripts/taler-monitoring" ]; then + chmod +x \ + "$SUITE_DIR"/scripts/taler-monitoring/taler-monitoring.sh \ + "$SUITE_DIR"/scripts/taler-monitoring/check_*.sh \ + "$SUITE_DIR"/scripts/taler-monitoring/host-agent/*.sh \ + "$SUITE_DIR"/scripts/taler-monitoring/site-gen/*.sh \ + "$SUITE_DIR"/scripts/taler-monitoring/site-gen/*.py \ + 2>/dev/null || true +fi