diff --git a/ports-whitelist.conf b/ports-whitelist.conf new file mode 100644 index 0000000..8bea64c --- /dev/null +++ b/ports-whitelist.conf @@ -0,0 +1,48 @@ +# ports-whitelist.conf — operator override for *intentionally* open public ports +# +# Monitoring is outside-in only (TCP connect from the internet). Anything odd that +# is still open and not listed here stays suspicious — many weird ports ⇒ look closer. +# This file does not invent internal topology; it only silences known-good public opens. +# +# Format: host_or_pattern port[,port…] [# note] +# Patterns: exact FQDN | *.example.com | * (any host) +# Baseline already allows 80/443 on all hosts, and mail ports on mail-catalog hosts. +# Override path: TALER_PORTS_WHITELIST=/path/to/file +# +# Whitelisted ports must NOT raise orange/red on the stack card. + +# --- hacktivism public edge (observed / intentional WAN services) --- +# 22 ssh · 8080 Tor OR · 9200 Forgejo git-SSH (public; not treated as Elasticsearch here) +*.hacktivism.ch 22,8080,9200 +hacktivism.ch 22,8080,9200 + +# Common intentional SSH on public edges: +*.taler.net 22 +*.taler-ops.ch 22 +*.lefrancpaysan.ch 22 +*.taler-systems.com 22 +*.gnunet.org 22 +*.anastasis.lu 22 +*.infomaniak.ch 22 +anastasis.lu 22,25,143,465,587,993 +taler-systems.com 22 +gnunet.org 22 + +# Shared mail/edge ports often co-hosted on public FQDNs (soft noise otherwise): +*.taler.net 25,143,465,587,993,8080 +*.test.taler.net 25,143,465,587,993,8080 +firefly.gnunet.org 22,25,143,465,587,993,9418 +pixel.taler-systems.com 22,25,143,465,587,993 +mail.anastasis.lu 22,25,143,465,587,993 + +# Firefly git daemon (admin-log firefly/14-git-daemon.txt · ListenStream=9418) +git.taler.net 22,9418 +git.gnunet.org 22,9418 + +# VeciGate WAN edge (vecigate-admin-log · public A 212.51.151.254) +# 80/443 baseline; 8080 Tor OR; 9200 Forgejo git-SSH; 2323x host SSH DNAT; +# 1789/1790 Nym; tortuna companion 23230 + 52323 (nat README) +212.51.151.254 22,8080,9200,1789,1790,23230,23235,23236,23237,23238,52323 + +# Optional — add more as you confirm intentional opens: +# lefrancpaysan.ch 21 diff --git a/services-catalog.conf b/services-catalog.conf new file mode 100644 index 0000000..27455f6 --- /dev/null +++ b/services-catalog.conf @@ -0,0 +1,33 @@ +# services-catalog.conf — non-Taler company / firefly / pixel hosts (monitord) +# +# Format (whitespace-separated; # comments): +# id host label zone +# +# zone hints: pixel | firefly | tops | public +# Override: TALER_SERVICES_CONF=/path/to/file + +# --- Pixel / TSA / Anastasis --- +taler-systems.com taler-systems.com company pixel +www.taler-systems.com www.taler-systems.com company pixel +erp.stage.taler-systems.com erp.stage.taler-systems.com ERPnext pixel +anastasis.lu anastasis.lu anastasis pixel +www.anastasis.lu www.anastasis.lu anastasis pixel +pixel.taler-systems.com pixel.taler-systems.com pixel pixel + +# TOPS company web (CHF Taler stack is in domains.conf as taler-ops.ch) +www.taler-ops.ch www.taler-ops.ch company tops + +# --- Firefly / GNUnet (mail plane hosts are in mail-catalog.conf) --- +gnunet.org gnunet.org gnunet firefly +www.gnunet.org www.gnunet.org gnunet firefly +bugs.gnunet.org bugs.gnunet.org MantisBT firefly +taler.net-apex taler.net taler-net firefly +git.taler.net git.taler.net Git firefly +git.gnunet.org git.gnunet.org Git firefly +mattermost.taler.net mattermost.taler.net Mattermost firefly +buildbot.gnunet.org buildbot.gnunet.org Buildbot firefly +buildbot.taler.net buildbot.taler.net Buildbot firefly + +# --- VeciGate edge (+ tortuna companion via WAN DNAT) --- +# Facts: PATH[vecigate-admin-log] docs/port-map-*.md · ip/firewall/nat/README.md +vecigate-wan 212.51.151.254 VeciGate vecigate diff --git a/site-gen/README.md b/site-gen/README.md index 6d95bbf..9657c8b 100644 --- a/site-gen/README.md +++ b/site-gen/README.md @@ -1,14 +1,97 @@ # site-gen — HTML converter + Caddy notes +**Product:** **LLM** = **Lightweight Monitoring** (Taler-stack watcher). CLI/tree still `taler-monitoring` / `taler-monitord`. + **Required for host-agent:** `console_to_html.py` (log → sticky HTML). | File | Role | |------|------| | `console_to_html.py` | Convert suite console log → public mon HTML | +| `stack_status_lib.py` | Domain-stack traffic lights + aggregation | +| `taler-monitord` | Daemon: probe stacks → `stack-status.json` (UI detached) | +| `stack_status_html.py` | Render status cards from JSON only | | `caddy-monitoring-handles.snippet` | Caddy `handle` / bare redir notes | | `ROOT-ON-KOOPA.md` | Publish staging → `/var/www` (paths from env) | | `settings.conf.example` | Optional settings for **legacy** multi-host generator | +## Stack status (taler-monitord) + +Like `llm-controld`: the daemon collects; the page only reads JSON. + +```bash +./site-gen/taler-monitord --once +./site-gen/stack_status_html.py --out /tmp/taler-stack-status.html +# optional: ./site-gen/taler-monitord --interval 20 +# local GOA (hacktivism.ch / taler.hacktivism.ch) is included by default; +# skip with --no-include-local or TALER_MONITORD_INCLUDE_LOCAL=0 +``` + +Lights: green ok · blue present · yellow soft · orange hard/recent · red acute · black long · white gone. +Box colour = weakest member; **muted** if partial; **full** if all members red/black. + +**Total score** (header): `N/M green` across stack+service+mail cards. Ratio thresholds (env override): +1.0 green · ≥0.85 blue · ≥0.60 yellow · ≥0.35 orange · ≥0.01 red · 0 white (gone) / all-black → black. + +Stacks with currency `-` in `domains.conf` are **`static_pending`** (e.g. lefrancpaysan.ch: static site, no Taler prod yet). monitord probes the apex as `site` and treats missing bank/exchange/merchant as **blue `no_prod_yet`**, not orange/red. Stage TESTPAYSAN remains a normal Taler stack. + +**Bank backend:** exchange `/keys` `payto://` methods decide whether `bank.*` is expected. +`payto://x-taler-bank/…` → probe bank (libeufin). +`payto://iban/…` only (EBICS / TOPS CHF) → bank member **blue**, host **`IBAN`**, reason `ebics_iban` (not red). +Mixed (e.g. regio) still probes bank. Override in `domains.conf`: bank column `-` / `ebics` / `none`. + +**Network / SSH:** each stack gets a coarse **zone** (GOA/local, FP stage, TOPS, pixel, firefly, public demo). +SSH = **TCP:22 banner** (service present), not Magikoopa jump login. +Zone-mapped SSH targets come **only** from `~/.config/taler-monitoring/env` (`TALER_MONITORD_SSH_TOPS` / `_FP` / `_GOA` / `_TOPS_PROD` via **taler-monitoring-env**); hostnames are **hidden** in the UI when set. Without env, probes use public stack FQDNs. Disable: `TALER_MONITORD_SSH=0`. + +**Outside CVEs:** `Server` headers (nginx/apache/caddy + version) → OSV. +**CRITICAL → red**, **HIGH → orange**. UI: CVE tags **link to osv.dev**; card click opens detail (severity / version / summary). Logos for nginx/apache/caddy. Disable: `TALER_MONITORD_CVE=0`. + +**Why Degraded:** each card shows a one-line aggregation (`4× OK, 3× Soft → Degraded`); click opens a dialog with per-member reasons. + +**Company / services:** `services-catalog.conf` (ERPnext, MantisBT, company apexes, …). Disable: `TALER_MONITORD_SERVICES=0`. + +**Mail:** `mail-catalog.conf` → MX / SPF / DMARC / A+AAAA / rDNS + SMTP/IMAP TCP ports (587 soft). +**Deliverability** (regular, not every cycle): SMTP `MAIL FROM`/`RCPT TO` smoke to `postmaster@domain` (no DATA). +Cached under monitord state (`mail-deliverability.json`); default interval **6h**, stale after **30h** → yellow. +Send/server failures → orange/red (degraded). Env: `TALER_MONITORD_DELIVER=0`, `_INTERVAL_S`, `_STALE_S`, `_FROM`, `_RCPT`. +Disable plane: `TALER_MONITORD_MAIL=0`; ports only: `TALER_MONITORD_MAIL_PORTS=0`. + +**TLS certs + orphans:** cert days-left + SANs. Orphans = **DNS-missing** SANs not covered by monitored hosts; wildcards/siblings skipped. +HTTP orphans opt-in: `TALER_MONITORD_ORPHAN_HTTP=1`. Disable: `TALER_MONITORD_CERT=0` / `TALER_MONITORD_ORPHAN=0`. + +**Port scan (outside-in only):** TCP connect to notable ports on publicly named hosts. No internal firewall maps required. +Baseline allows 80/443 (+ mail ports on mail-catalog hosts). Anything else open → **suspicious** until listed in `ports-whitelist.conf`. +Many odd open ports ⇒ worth investigating. Whitelist only silences *known intentional public* opens (e.g. hacktivism **8080** Tor OR, **9200** git-SSH) so they do not turn cards orange. +Only **hard** findings (DB/RDP/SMB/…) raise card colour; soft stays informational. Disable: `TALER_MONITORD_PORTSCAN=0`. + +**Software versions:** per card from Taler `/config` (`build_version`), `Server:` (nginx/apache/caddy), SMTP banner (Postfix). +Compared to **published** packages on `deb.taler.net` (`TALER_MONITORD_APT_SUITE`, default trixie): current / behind / ahead; chips show `installed → published` when behind. +**Apps / SPAs:** page section with published WebUI + wallet-cli (+ GNU FTP wallet tarball when listed). Disable: `TALER_MONITORD_PUBLISHED=0`. + +**Publish:** Cyon **`hacktivism`**. Defaults (both): +`…/public_html/stacks/index.html` → **https://hacktivism.ch/stacks/** +`…/public_html/monitoring.hacktivism.ch/index.html` (addon when DNS ready) +Override: `TALER_MONITORD_PUBLISH_PATH` or `TALER_MONITORD_PUBLISH_PATHS=a,b`. Off: `TALER_MONITORD_PUBLISH=0`. + +**Result index (optional):** bottom of the stacks page lists every probe grouped by traffic-light colour. +`TALER_MONITORD_RESULT_INDEX=1` (or `colors`) · with concrete command: `=cmd` / `full`, or `=1` plus `TALER_MONITORD_RESULT_INDEX_CMD=1`. Off by default (`=0`). + +**Git soft probe:** Git-labelled services run infrequent `git ls-remote` over HTTPS + SSH (cached ~6h). Soft yellow on fail. Disable: `TALER_MONITORD_GIT=0`. + +**VeciGate:** `services-catalog.conf` zone `vecigate` (WAN IP) + intentional DNAT ports in `ports-whitelist.conf` (incl. tortuna companion). + +**Payability:** `TALER_MONITORD_PAY=1` shows CHF:0 / zero-pay status on matching stacks. Live wallet run only with `TALER_MONITORD_PAY_RUN=1` (interval `TALER_MONITORD_PAY_INTERVAL_S`, default 4h). Domains: `TALER_MONITORD_PAY_DOMAINS`. + +**Ansible smoke (opt-in phase):** `./taler-monitoring.sh ansible-smoke` → `check_ansible_smoke.sh` against `TALER_DEPLOYMENT` (`regional-currency-ng/site.yml`). Podman tracks: `ANSIBLE_SMOKE_PODMAN=1` on koopa (else reuse `aptdeploy`). + +**Local VCS mirror:** `~/git/www-hacktivism-wiki` rsyncs Cyon → git every **4h** +(`ch.hacktivism.www-wiki-sync`). Tracks `stacks/` (+ optional addon). No push by default. + +**Custom /intro:** only **GOA** (hacktivism) and **FrancPaysan / TESTPAYSAN stage**. +**Landing stats:** on intro stacks, `…/intro/stats.json` → compact tag. +**Currency / coins:** from exchange `/config` `currency_specification`. +**Endpoints:** suite-like paths; soft ≤ yellow. **`/terms` + `/privacy`:** medium → orange **only TOPS prod**. + ## Optional / moved Legacy multi-host generate/deploy and firecuda launchd tooling live under diff --git a/site-gen/stack_status_html.py b/site-gen/stack_status_html.py new file mode 100755 index 0000000..49518e5 --- /dev/null +++ b/site-gen/stack_status_html.py @@ -0,0 +1,1050 @@ +#!/usr/bin/env python3 +"""Render detached status HTML from stack-status.json (no probing).""" +from __future__ import annotations + +import argparse +import html +import json +import os +import sys +import time +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import stack_status_lib as L # noqa: E402 + +# Human labels instead of colour names (llm-control style) +_STATUS_LABEL = { + "green": "OK", + "blue": "Present", + "yellow": "Soft", + "orange": "Degraded", + "red": "Down", + "black": "Long down", + "white": "Gone", +} + +def _logo_svg(pkg: str, *, fill: str, text: str, size: str = "16") -> str: + return ( + f'' + ) + + +_LOGO_SVG = { + "nginx": lambda: _logo_svg("nginx", fill="#009639", text="ngx", size="18"), + "apache": lambda: _logo_svg("apache", fill="#d22128", text="httpd", size="14"), + "caddy": lambda: _logo_svg("caddy", fill="#1f9ceb", text="caddy", size="14"), + "postfix": lambda: _logo_svg("postfix", fill="#f0a000", text="pfx", size="16"), + "mta": lambda: _logo_svg("mta", fill="#c08a00", text="mta", size="16"), + "taler": lambda: _logo_svg("taler", fill="#5a32a3", text="T", size="22"), + "taler-exchange": lambda: _logo_svg("taler-exchange", fill="#5a32a3", text="ex", size="16"), + "taler-merchant": lambda: _logo_svg("taler-merchant", fill="#7b4cc7", text="mer", size="14"), + "taler-bank": lambda: _logo_svg("taler-bank", fill="#3d2a80", text="bnk", size="14"), + "webui": lambda: _logo_svg("webui", fill="#0d9488", text="ui", size="16"), + "wallet": lambda: _logo_svg("wallet", fill="#2563eb", text="wal", size="14"), +} + +CSS = """ +:root { + --bg: #0f1419; --fg: #e7ecf1; --muted: #8b98a5; --card: #1a222c; --line: #243040; + --dot-green: #3ddc84; --dot-blue: #61afef; --dot-yellow: #e6c07b; + --dot-orange: #ff9f43; --dot-red: #ff5c5c; --dot-black: #2a2a2a; --ring-black: #9aa0a6; +} +* { box-sizing: border-box; } +body { + margin: 0; font-family: ui-sans-serif, system-ui, -apple-system, Segoe UI, sans-serif; + background: var(--bg); color: var(--fg); line-height: 1.3; +} +header { + padding: .7rem 1rem; border-bottom: 1px solid var(--line); + display: flex; flex-wrap: wrap; gap: .4rem 1rem; align-items: center; +} +header h1 { font-size: 1rem; margin: 0; font-weight: 650; } +header .meta { color: var(--muted); font-size: .8rem; } +header .score { + display: inline-flex; align-items: center; gap: .4rem; + font-size: .85rem; font-weight: 650; + border: 1px solid #314050; border-radius: 999px; + padding: .2rem .65rem; background: rgba(0,0,0,.2); +} +header .score .dot { width: .65rem; height: .65rem; } +.dot.white { background: #f5f7fa; box-shadow: 0 0 0 1px #9aa0a6; } +.score.light-green { border-color: rgba(61,220,132,.55); } +.score.light-blue { border-color: rgba(97,175,239,.55); } +.score.light-yellow { border-color: rgba(230,192,123,.55); } +.score.light-orange { border-color: rgba(255,159,67,.55); } +.score.light-red { border-color: rgba(255,92,92,.55); } +.score.light-black { border-color: rgba(154,160,166,.45); } +.score.light-white { border-color: rgba(245,247,250,.45); color: #c5ccd4; } +h3.sec { + margin: 1rem 1rem .35rem; font-size: .75rem; color: var(--muted); + text-transform: uppercase; letter-spacing: .04em; font-weight: 600; +} +.grid { + display: grid; gap: .7rem; padding: .55rem 1rem .85rem; + grid-template-columns: repeat(auto-fill, minmax(260px, 1fr)); +} +.card { + background: var(--card); border-radius: 10px; padding: .7rem .8rem; + border: 2px solid #314050; min-width: 0; cursor: pointer; +} +.card:focus-visible { outline: 2px solid var(--dot-blue); outline-offset: 2px; } +.card h2 { margin: 0; font-size: .9rem; font-weight: 650; overflow-wrap: anywhere; } +.card .sub { + color: var(--muted); font-size: .72rem; margin: .15rem 0 .45rem; + display: flex; flex-wrap: wrap; gap: .25rem .55rem; align-items: center; +} +.card .sub .bad { color: var(--dot-orange); } +.card .sub .crit { color: var(--dot-red); } +.card .sub .ok { color: var(--dot-green); } +.card .why { + font-size: .68rem; color: var(--muted); margin: 0 0 .35rem; + white-space: nowrap; overflow: hidden; text-overflow: ellipsis; +} +.rows { display: flex; flex-direction: column; gap: .15rem; } +.row { + display: flex; align-items: center; gap: .4rem; + font-size: .78rem; min-width: 0; +} +.row .role { + color: var(--muted); width: 3.6rem; flex: 0 0 auto; + font-size: .65rem; text-transform: uppercase; +} +.row .host { + flex: 1 1 auto; min-width: 0; + white-space: nowrap; overflow: hidden; text-overflow: ellipsis; +} +.dot { + width: .55rem; height: .55rem; border-radius: 50%; flex: 0 0 auto; + box-shadow: 0 0 0 1px rgba(255,255,255,.1); +} +.dot.green { background: var(--dot-green); } +.dot.blue { background: var(--dot-blue); } +.dot.yellow { background: var(--dot-yellow); } +.dot.orange { background: var(--dot-orange); } +.dot.red { background: var(--dot-red); } +.dot.black { background: var(--dot-black); box-shadow: 0 0 0 1px var(--ring-black); } +.tags { + display: flex; flex-wrap: wrap; gap: .3rem; margin-top: .5rem; + padding-top: .45rem; border-top: 1px solid var(--line); + align-items: center; +} +.tag { + font-size: .62rem; color: var(--muted); + border: 1px solid #314050; border-radius: 999px; + padding: .1rem .45rem; background: rgba(0,0,0,.15); + text-decoration: none; +} +a.tag:hover { filter: brightness(1.15); } +.tag.warn { color: var(--dot-orange); border-color: rgba(255,159,67,.45); } +.tag.crit { color: var(--dot-red); border-color: rgba(255,92,92,.45); } +.logo { width: 1.05rem; height: 1.05rem; vertical-align: middle; flex: 0 0 auto; } +.swrow { + display: flex; flex-wrap: wrap; gap: .35rem; margin-top: .45rem; +} +.sw { + display: inline-flex; align-items: center; gap: .3rem; + font-size: .62rem; border-radius: 8px; padding: .15rem .4rem .15rem .25rem; + border: 1px solid #314050; background: rgba(0,0,0,.18); color: var(--muted); + max-width: 100%; +} +.sw .ver { color: var(--fg); font-variant-numeric: tabular-nums; white-space: nowrap; } +.sw.light-green { border-color: rgba(61,220,132,.55); } +.sw.light-blue { border-color: rgba(97,175,239,.5); } +.sw.light-yellow { border-color: rgba(230,192,123,.55); color: var(--dot-yellow); } +.sw.light-orange { border-color: rgba(255,159,67,.6); color: var(--dot-orange); } +.sw.light-red { border-color: rgba(255,92,92,.65); color: var(--dot-red); } +.card.ok-green { border-color: rgba(61,220,132,.5); } +.card.ok-blue { border-color: rgba(97,175,239,.5); } +.card.muted-green { border-color: rgba(61,220,132,.3); } +.card.muted-blue { border-color: rgba(97,175,239,.3); } +.card.muted-yellow { border-color: rgba(230,192,123,.4); } +.card.muted-orange { border-color: rgba(255,159,67,.45); } +.card.muted-red { border-color: rgba(255,92,92,.4); } +.card.muted-black { border-color: rgba(154,160,166,.35); } +.card.full-red { border-color: var(--dot-red); background: #3a1414; } +.card.full-black { border-color: var(--ring-black); background: #0a0a0a; } +.card.full-orange { border-color: var(--dot-orange); background: #3a2210; } +.stale { color: var(--dot-orange); } +footer { padding: .7rem 1rem 1.2rem; color: var(--muted); font-size: .7rem; } +.result-index { + margin: .4rem 1rem 1.2rem; padding: .75rem .9rem; + border: 1px solid var(--line); border-radius: 10px; background: #121820; +} +.result-index h3 { + margin: 0 0 .55rem; font-size: .78rem; color: var(--muted); + text-transform: uppercase; letter-spacing: .04em; font-weight: 600; +} +.result-index .ri-group { margin: 0 0 .65rem; } +.result-index .ri-group:last-child { margin-bottom: 0; } +.result-index .ri-hd { + display: flex; align-items: center; gap: .4rem; + font-size: .78rem; font-weight: 600; margin: 0 0 .3rem; +} +.result-index ul { + margin: 0; padding-left: 1.05rem; font-size: .72rem; color: var(--muted); +} +.result-index li { margin: .18rem 0; } +.result-index .ri-stack { color: var(--fg); font-weight: 500; } +.result-index .ri-cmd { + display: inline-block; margin-left: .35rem; padding: .05rem .35rem; + border-radius: 4px; border: 1px solid #314050; background: rgba(0,0,0,.25); + color: var(--dot-blue); font-family: ui-monospace, Menlo, Consolas, monospace; + font-size: .68rem; +} +dialog { + border: 1px solid var(--line); border-radius: 12px; background: #121820; color: var(--fg); + padding: 0; max-width: min(520px, 94vw); width: 100%; +} +dialog::backdrop { background: rgba(0,0,0,.55); } +.dlg-hd { + display: flex; justify-content: space-between; gap: .5rem; align-items: start; + padding: .85rem 1rem; border-bottom: 1px solid var(--line); +} +.dlg-hd h2 { margin: 0; font-size: 1rem; } +.dlg-hd button { + background: transparent; border: 1px solid #314050; color: var(--fg); + border-radius: 8px; padding: .2rem .55rem; cursor: pointer; +} +.dlg-bd { padding: .85rem 1rem 1.1rem; font-size: .82rem; } +.dlg-bd .agg { margin: 0 0 .7rem; color: var(--muted); } +.dlg-bd ul { margin: 0; padding-left: 1.1rem; } +.dlg-bd li { margin: .25rem 0; } +.dlg-bd .cve { + margin-top: .8rem; padding-top: .7rem; border-top: 1px solid var(--line); +} +.dlg-bd a { color: var(--dot-blue); } +""" + + +def _esc(s: object) -> str: + return html.escape(str(s if s is not None else ""), quote=True) + + +def _label(light: object) -> str: + return _STATUS_LABEL.get(str(light or ""), str(light or "?")) + + +def _short_host(host: str) -> str: + h = host or "" + if h.upper() == "IBAN": + return "IBAN" + return h + + +def _osv_url(cve_id: str) -> str: + cid = (cve_id or "").strip() + if not cid: + return "https://osv.dev/" + return f"https://osv.dev/vulnerability/{cid}" + + +def _probe_command_for_endpoint(host: str, ep: dict) -> str: + """Concrete outside-in probe (HTTP GET) for one endpoint row.""" + if ep.get("skipped"): + path = ep.get("path") or ep.get("id") or "?" + return f"(skip) GET https://{host}{path}" + url = str(ep.get("url") or "").strip() + if not url: + path = ep.get("path") or "" + url = f"https://{host}{path}" if host and path else path or "?" + return f"GET {url}" + + +def _probe_command_for_member(m: dict) -> str: + role = str(m.get("role") or "member") + host = str(m.get("host") or "") + reason = str(m.get("reason") or "") + if role == "ssh" or reason.startswith("SSH-") or reason in ("ssh_probe", "timeout", "no_host"): + target = host or "?" + return f"TCP {target}:22 banner" + if host.upper() == "IBAN" or reason in ("ebics_iban", "ebics_no_bank"): + return f"(no HTTP) {role} · {reason or 'ebics'}" + if host: + return f"GET https://{host}/config" + return f"probe {role}" + + +def collect_result_index_rows(doc: dict) -> list[dict]: + """Flatten probe/test rows across stacks/services/mail for the bottom index.""" + rows: list[dict] = [] + for section, key in ( + ("stack", "stacks"), + ("service", "services"), + ("mail", "mail"), + ): + for st in doc.get(key) or []: + if not isinstance(st, dict): + continue + sid = str(st.get("id") or "?") + for m in st.get("members") or []: + if not isinstance(m, dict): + continue + role = str(m.get("role") or "") + if role == "cve": + continue + host = str(m.get("host") or "") + endpoints = [e for e in (m.get("endpoints") or []) if isinstance(e, dict)] + if endpoints: + for ep in endpoints: + light = str(ep.get("light") or m.get("light") or "black") + eid = str(ep.get("id") or ep.get("path") or "ep") + reason = str(ep.get("reason") or "") + label = f"{role}/{eid}" + rows.append( + { + "section": section, + "stack": sid, + "role": role, + "host": host, + "id": eid, + "label": label, + "light": light, + "reason": reason, + "command": _probe_command_for_endpoint(host, ep), + } + ) + else: + light = str(m.get("light") or "black") + rows.append( + { + "section": section, + "stack": sid, + "role": role, + "host": host, + "id": role or "member", + "label": role or "member", + "light": light, + "reason": str(m.get("reason") or ""), + "command": _probe_command_for_member(m), + } + ) + net = st.get("network") if isinstance(st.get("network"), dict) else {} + ssh = net.get("ssh") if isinstance(net.get("ssh"), dict) else {} + if ssh.get("expected") and ssh.get("ok") is not None: + shown = str(ssh.get("host") or "") + # Prefer real candidates when not hidden; else keep blank host in cmd + hosts = list(ssh.get("candidates") or []) + cmd_host = shown or (hosts[0] if hosts else "") + rows.append( + { + "section": section, + "stack": sid, + "role": "ssh", + "host": shown, + "id": "ssh", + "label": "ssh", + "light": str(ssh.get("light") or ("green" if ssh.get("ok") else "orange")), + "reason": str(ssh.get("reason") or ""), + "command": f"TCP {cmd_host or '∅'}:22 banner", + } + ) + return rows + + +def _render_result_index(doc: dict, *, mode: str | None = None) -> str: + """Optional bottom section: all tests grouped by light colour (+ optional command).""" + mode = (mode or getattr(L, "RESULT_INDEX_MODE", "off") or "off").lower() + if mode in ("0", "off", "false", "no", ""): + return "" + show_cmd = mode in ("cmd", "full", "commands", "2") + rows = collect_result_index_rows(doc) + if not rows: + return "" + order = ("green", "blue", "yellow", "orange", "red", "black", "white") + by: dict[str, list[dict]] = {} + for r in rows: + by.setdefault(str(r.get("light") or "black"), []).append(r) + groups = [] + for light in order: + items = by.get(light) or [] + if not items: + continue + lis = [] + for r in items: + tip = r.get("reason") or _label(light) + cmd = "" + if show_cmd and r.get("command"): + cmd = f' {_esc(r["command"])}' + host = r.get("host") or "" + host_bit = f" @ {_esc(host)}" if host else "" + lis.append( + "
  • " + f'{_esc(r.get("stack"))} · ' + f'{_esc(r.get("label"))}{host_bit}' + f' ({_esc(_label(light))})' + f"{cmd}" + "
    • " + ) + groups.append( + f'
    ' + f'
    ' + f'{_esc(_label(light))} · {len(items)}
    ' + f'
    ' + ) + title = "all probes by colour" + if show_cmd: + title += " · with command" + return ( + f'
    ' + f"

    {_esc(title)} ({len(rows)})

    " + f'{"".join(groups)}' + f"
    " + ) + + +def _agg_text(st: dict) -> str: + bd = st.get("breakdown") if isinstance(st.get("breakdown"), dict) else {} + by = bd.get("by_light") if isinstance(bd.get("by_light"), dict) else {} + if not by: + # fallback from members + by = {} + for m in st.get("members") or []: + if (m.get("reason") or "") in getattr(L, "_AGG_SKIP_REASONS", ()): + continue + Lgt = str(m.get("light") or "black") + by[Lgt] = by.get(Lgt, 0) + 1 + if not by: + return "" + order = ("green", "blue", "yellow", "orange", "red", "black") + parts = [] + for k in order: + n = int(by.get(k) or 0) + if n: + parts.append(f"{n}× {_label(k)}") + worst = st.get("light") or bd.get("worst") or "" + if not parts: + return "" + return f"{', '.join(parts)} → {_label(worst)}" + + +def _software_items(st: dict) -> list[dict]: + raw = st.get("software") if isinstance(st.get("software"), list) else [] + if raw: + return [x for x in raw if isinstance(x, dict) and x.get("package")] + # Fallback: vulns.servers only + vulns = st.get("vulns") if isinstance(st.get("vulns"), dict) else {} + out = [] + for srv in vulns.get("servers") or []: + p = str((srv or {}).get("package") or "") + if not p: + continue + out.append( + { + "package": p, + "version": (srv or {}).get("version") or "", + "host": (srv or {}).get("host") or "", + "light": "green", + } + ) + return out + + +def _render_software_row(st: dict) -> str: + items = _software_items(st) + if not items: + return "" + # Collapse identical package+version (multi-host) into one chip. + merged: dict[str, dict] = {} + for it in items: + pkg = str(it.get("package") or "") + ver = str(it.get("version") or "") + key = f"{pkg}|{ver}" + if key not in merged: + merged[key] = { + "package": pkg, + "version": ver, + "label": it.get("label") or pkg, + "light": it.get("light") or "green", + "hosts": [], + "source": it.get("source") or "", + "currency_status": it.get("currency_status") or "", + "behind": it.get("behind") or "", + "published_upstream": it.get("published_upstream") or "", + "delta": it.get("delta"), + } + h = str(it.get("host") or "") + if h and h not in merged[key]["hosts"]: + merged[key]["hosts"].append(h) + # keep worst light + Lgt = str(it.get("light") or "green") + cur = str(merged[key]["light"] or "green") + if L.SEVERITY.get(Lgt, 0) > L.SEVERITY.get(cur, 0): + merged[key]["light"] = Lgt + if it.get("currency_status") == "behind": + merged[key]["currency_status"] = "behind" + merged[key]["behind"] = it.get("behind") or merged[key].get("behind") + merged[key]["published_upstream"] = ( + it.get("published_upstream") or merged[key].get("published_upstream") + ) + if it.get("delta") is not None: + merged[key]["delta"] = it.get("delta") + chips = [] + for it in list(merged.values())[:8]: + pkg = str(it.get("package") or "") + ver = str(it.get("version") or "") + light = str(it.get("light") or "green") + hosts = it.get("hosts") or [] + label = str(it.get("label") or pkg) + logo_fn = _LOGO_SVG.get(pkg) + logo = logo_fn() if callable(logo_fn) else _logo_svg(pkg, fill="#445566", text=(pkg[:3] or "?"), size="14") + cur = str(it.get("currency_status") or "") + behind = str(it.get("behind") or it.get("published_upstream") or "") + delta = it.get("delta") + ver_txt = ver or "?" + if cur == "behind" and behind: + lag = f" −{delta}" if isinstance(delta, int) and delta > 0 else "" + ver_txt = f"{ver_txt} → {behind}{lag}" + elif cur == "current" and ver: + ver_txt = f"{ver_txt} ✓" + tip = " · ".join( + x + for x in ( + label, + ver, + f"pub {behind}" if behind else "", + cur, + ", ".join(hosts[:4]), + it.get("source") or "", + ) + if x + ) + chips.append( + f'' + f"{logo}" + f'{_esc(ver_txt)}' + f"" + ) + return f'
    {"".join(chips)}
    ' + + +def _card_dialog_payload(st: dict) -> dict: + """JSON-serializable detail for the open dialog.""" + members = [] + for m in st.get("members") or []: + reason = m.get("reason") or "" + es = m.get("endpoints_summary") if isinstance(m.get("endpoints_summary"), dict) else {} + # If endpoints pulled the light up, surface worst endpoint id/reason. + if es and (m.get("light") or "") not in ("green", "blue", None, ""): + bad = [] + for ep in m.get("endpoints") or []: + if ep.get("skipped"): + continue + el = ep.get("light") or "green" + if el in ("yellow", "orange", "red", "black"): + bad.append(f"{ep.get('id') or ep.get('path')}:{ep.get('reason') or el}") + if bad and reason in ("ok", "", None): + reason = ", ".join(bad[:4]) + elif bad: + reason = f"{reason} · " + ", ".join(bad[:3]) + members.append( + { + "role": m.get("role"), + "host": m.get("host"), + "light": m.get("light"), + "label": _label(m.get("light")), + "reason": reason, + "code": m.get("code"), + } + ) + hits = [] + vulns = st.get("vulns") if isinstance(st.get("vulns"), dict) else {} + for h in vulns.get("hits") or []: + hits.append( + { + "id": h.get("id"), + "severity": h.get("severity"), + "package": h.get("package"), + "version": h.get("version"), + "host": h.get("host"), + "summary": h.get("summary") or "", + "url": _osv_url(str(h.get("id") or "")), + } + ) + tls = st.get("tls") if isinstance(st.get("tls"), dict) else {} + ports = st.get("ports") if isinstance(st.get("ports"), dict) else {} + software = [] + for it in _software_items(st)[:12]: + software.append( + { + "package": it.get("package"), + "label": it.get("label") or it.get("package"), + "version": it.get("version") or "", + "host": it.get("host") or "", + "light": it.get("light") or "green", + "source": it.get("source") or "", + "currency_status": it.get("currency_status") or "", + "published": it.get("published") or "", + "behind": it.get("behind") or "", + "delta": it.get("delta"), + } + ) + return { + "id": st.get("id"), + "light": st.get("light"), + "label": _label(st.get("light")), + "agg": _agg_text(st), + "members": members, + "hits": hits, + "tls": tls, + "ports": ports.get("suspicious") or [], + "software": software, + "deliverability": st.get("deliverability") + if isinstance(st.get("deliverability"), dict) + else {}, + "kind": st.get("kind"), + "label_soft": st.get("service_label") or "", + } + + +def _render_card(st: dict, *, section: str = "stack") -> str: + light = st.get("light") or "black" + intensity = st.get("intensity") or "muted" + klass = f"card {intensity}-{light}" + cur = st.get("currency") or "" + ci = st.get("currency_info") if isinstance(st.get("currency_info"), dict) else {} + symbol = ci.get("symbol") or "" + backend = st.get("bank_backend") or "" + service_label = st.get("service_label") or "" + + bits = [x for x in (service_label, cur, symbol if symbol != cur else "", _label(light)) if x] + if backend == "ebics": + bits.insert(-1 if bits else 0, "IBAN") + elif backend == "x-taler-bank": + bits.insert(-1 if bits else 0, "x-taler-bank") + + net = st.get("network") if isinstance(st.get("network"), dict) else {} + ssh = net.get("ssh") if isinstance(net.get("ssh"), dict) else {} + sub_extra = [] + if net.get("zone"): + sub_extra.append(_esc(str(net.get("zone")))) + if ssh.get("expected"): + if ssh.get("ok") is True: + sub_extra.append('ssh') + elif ssh.get("ok") is False: + sub_extra.append('ssh!') + vulns = st.get("vulns") if isinstance(st.get("vulns"), dict) else {} + hits = vulns.get("hits") or [] + if hits: + top = hits[0] + cls = "crit" if (vulns.get("worst") or "") == "CRITICAL" else "bad" + sub_extra.append( + f'cve' + ) + + why = _agg_text(st) + why_html = f'
    {_esc(why)}
    ' if why else "" + + rows = [] + for m in st.get("members") or []: + role = m.get("role") or "" + if role == "cve": + continue + host = m.get("host") or "" + if not host and (m.get("reason") or "") in ("ebics_iban", "ebics_no_bank"): + host = "IBAN" + ml = m.get("light") or "black" + tip = m.get("reason") or _label(ml) + rows.append( + "
    " + f'' + f'{_esc(role)}' + f'{_esc(_short_host(host))}' + "
    " + ) + + tags = [] + if st.get("commercial_prod"): + tags.append('prod') + if backend: + tags.append(f'{_esc(backend)}') + if net.get("zone"): + tags.append(f'{_esc(net.get("zone"))}') + if ssh.get("expected"): + if ssh.get("ok") is True: + tags.append('ssh') + elif ssh.get("ok") is False: + tags.append('ssh') + tls = st.get("tls") if isinstance(st.get("tls"), dict) else {} + if tls.get("days_left") is not None: + try: + days = int(tls["days_left"]) + except (TypeError, ValueError): + days = None + if days is not None: + if days < 14: + tags.append(f'cert {days}d') + elif days < 30: + tags.append(f'cert {days}d') + else: + tags.append(f'cert') + if st.get("orphan"): + tags.append('orphan?') + deliver = st.get("deliverability") if isinstance(st.get("deliverability"), dict) else {} + if deliver: + dl = str(deliver.get("light") or "yellow") + reason = str(deliver.get("reason") or "deliver") + age = deliver.get("age_s") + age_s = "" + try: + if age is not None: + a = float(age) + age_s = f"{int(a // 3600)}h" if a >= 3600 else f"{int(a // 60)}m" + except (TypeError, ValueError): + age_s = "" + tip = " · ".join(x for x in (reason, deliver.get("rcpt") or "", age_s) if x) + cls = "tag" + if dl in ("orange", "red"): + cls = "tag warn" if dl == "orange" else "tag crit" + elif dl == "yellow" or deliver.get("stale"): + cls = "tag warn" + tags.append(f'deliver') + ports = st.get("ports") if isinstance(st.get("ports"), dict) else {} + n_sus = int(ports.get("n_suspicious") or 0) + if n_sus: + tip = ", ".join( + f"{s.get('label') or s.get('port')}" + + (f"@{s.get('host')}" if s.get("host") else "") + for s in (ports.get("suspicious") or [])[:8] + ) + tags.append( + f'ports:{n_sus}' + ) + for h in hits[:3]: + cid = str(h.get("id") or "cve") + cls = "tag crit" if (h.get("severity") or "") == "CRITICAL" else "tag warn" + tip = " · ".join( + x + for x in ( + h.get("severity") or "", + f"{h.get('package')}/{h.get('version')}" if h.get("package") else "", + (h.get("summary") or "")[:80], + ) + if x + ) + tags.append( + f'{_esc(cid)}' + ) + stats = st.get("stats") if isinstance(st.get("stats"), dict) else {} + if stats.get("present"): + tags.append('stats') + denoms = (ci.get("denoms") or [])[:6] + if denoms: + tags.append( + f'denoms' + ) + + payload = _esc(json.dumps(_card_dialog_payload(st), ensure_ascii=False)) + title = st.get("id") or "?" + sw_html = _render_software_row(st) + return ( + f'
    ' + f"

    {_esc(title)}

    " + f'
    {_esc(" · ".join(bits))}' + f'{(" · " + " ".join(sub_extra)) if sub_extra else ""}
    ' + f"{why_html}" + f'
    {"".join(rows)}
    ' + f"{sw_html}" + f'
    {"".join(tags)}
    ' + "
    " + ) + + +JS = """ +function openStack(el) { + let raw = el.getAttribute('data-detail') || '{}'; + let d; + try { d = JSON.parse(raw); } catch (e) { return; } + const dlg = document.getElementById('stack-dlg'); + const title = document.getElementById('dlg-title'); + const body = document.getElementById('dlg-body'); + title.textContent = (d.id || '?') + ' · ' + (d.label || ''); + let html = ''; + if (d.agg) html += '

    ' + escapeHtml(d.agg) + '

    '; + html += ''; + if (d.tls && (d.tls.not_after || d.tls.days_left != null)) { + html += '

    TLS · days_left=' + escapeHtml(d.tls.days_left) + + (d.tls.not_after ? ' · until ' + escapeHtml(d.tls.not_after) : '') + + '

    '; + if (d.tls.sans && d.tls.sans.length) { + html += '

    SANs: ' + escapeHtml(d.tls.sans.join(', ')) + '

    '; + } + } + if (d.hits && d.hits.length) { + html += '
    CVEs
    '; + } + if (d.software && d.software.length) { + html += '
    Software
    '; + } + if (d.deliverability && (d.deliverability.reason || d.deliverability.light)) { + const deliv = d.deliverability; + html += '
    Deliverability
    '; + } + if (d.ports && d.ports.length) { + html += '
    Suspicious ports
    '; + } + body.innerHTML = html; + if (typeof dlg.showModal === 'function') dlg.showModal(); + else dlg.setAttribute('open', ''); +} +function escapeHtml(s) { + return String(s == null ? '' : s) + .replace(/&/g,'&').replace(//g,'>') + .replace(/"/g,'"'); +} +""" + + +def render_html(doc: dict) -> str: + stacks = doc.get("stacks") or [] + services = doc.get("services") or [] + mail = doc.get("mail") or [] + orphans = doc.get("orphans") or [] + ports_doc = doc.get("ports") if isinstance(doc.get("ports"), dict) else {} + published = doc.get("published") if isinstance(doc.get("published"), dict) else {} + ts = doc.get("ts") + age = None + try: + age = time.time() - float(ts) + except (TypeError, ValueError): + pass + stale = age is not None and age > 3 * float(doc.get("acute_down_s") or L.ACUTE_DOWN_S) + if age is None: + updated = "?" + elif age < 120: + updated = f"{int(age)}s" + elif age < 3600: + updated = f"{int(age // 60)}m" + else: + updated = f"{age / 3600:.1f}h" + try: + updated += " · " + time.strftime("%H:%M", time.localtime(float(ts))) + except (TypeError, ValueError, OSError): + pass + + stack_cards = [_render_card(st, section="stack") for st in stacks] + # Firefly-plane (git / Mattermost / buildbot / gnunet) near mail; VeciGate edge; rest = company + def _zone(st: dict) -> str: + return str((st.get("network") or {}).get("zone") or st.get("zone") or "").lower() + + firefly_services = [st for st in services if _zone(st) == "firefly"] + vecigate_services = [st for st in services if _zone(st) == "vecigate"] + other_services = [ + st for st in services if _zone(st) not in ("firefly", "vecigate") + ] + service_cards = [_render_card(st, section="service") for st in other_services] + firefly_cards = [_render_card(st, section="service") for st in firefly_services] + vecigate_cards = [_render_card(st, section="service") for st in vecigate_services] + mail_cards = [_render_card(st, section="mail") for st in mail] + + score = doc.get("score") if isinstance(doc.get("score"), dict) else None + if not score: + score = L.compute_total_score(stacks, services, mail) + s_light = str(score.get("light") or "white") + s_green = int(score.get("green") or 0) + s_total = int(score.get("total") or 0) + s_pct = int(score.get("pct") or 0) + score_html = ( + f'
    ' + f'' + f'{s_green}/{s_total} green' + f'({s_pct}%) · {_esc(_label(s_light))}' + f"
    " + ) + + orphan_html = "" + if orphans: + items = "".join( + f'
  • {_esc(o.get("host"))} · {_esc(o.get("reason") or "orphan")}
    • ' + for o in orphans[:40] + ) + orphan_html = f'

    orphans

    ' + + ports_html = "" + sus_all = ports_doc.get("suspicious") or [] + if ports_doc.get("enabled") and sus_all: + n_hard = int(ports_doc.get("n_hard") or 0) + items = "".join( + f'
  • ' + f'{_esc(s.get("label") or s.get("port"))}' + f'{(" @ " + _esc(s.get("host"))) if s.get("host") else ""}' + f' · {_esc(s.get("severity") or "soft")}
    • ' + for s in sus_all[:60] + ) + ports_html = ( + f'

    suspicious ports ({len(sus_all)}' + f'{f", {n_hard} hard" if n_hard else ""}) · whitelist: ports-whitelist.conf

    ' + f'' + ) + + sections = [] + sections.append("

    taler stacks

    ") + empty = '

    No stacks.

    ' + sections.append(f'
    {"".join(stack_cards) if stack_cards else empty}
    ') + if service_cards: + sections.append('

    company / services

    ') + sections.append(f'
    {"".join(service_cards)}
    ') + if firefly_cards or mail_cards: + sections.append('

    firefly plane · git / Mattermost / buildbot / mail

    ') + if firefly_cards: + sections.append(f'
    {"".join(firefly_cards)}
    ') + if mail_cards: + sections.append(f'
    {"".join(mail_cards)}
    ') + if vecigate_cards: + sections.append('

    VeciGate edge · WAN / tortuna companion ports

    ') + sections.append(f'
    {"".join(vecigate_cards)}
    ') + apps = published.get("apps") or [] + apps_html = "" + if apps: + chips = [] + for app in apps[:12]: + if not isinstance(app, dict): + continue + logo_key = str(app.get("logo") or app.get("kind") or "webui") + logo_fn = _LOGO_SVG.get(logo_key) or _LOGO_SVG.get("webui") + logo = logo_fn() if callable(logo_fn) else _logo_svg(logo_key, fill="#0d9488", text="app", size="14") + pub = str(app.get("published_upstream") or app.get("published") or "?") + light = str(app.get("light") or "green") + tip = " · ".join( + x + for x in ( + app.get("label") or app.get("package"), + app.get("package"), + app.get("published"), + published.get("suite") or "", + app.get("source") or "deb.taler.net", + ) + if x + ) + name = str(app.get("label") or app.get("package") or "app") + chips.append( + f'' + f"{logo}" + f'{_esc(name)} {_esc(pub)}' + f"" + ) + suite = published.get("suite") or "" + apps_html = ( + f'

    published apps / SPAs' + f'{(" · " + _esc(suite)) if suite else ""} · deb.taler.net

    ' + f'
    {"".join(chips)}
    ' + ) + + if apps_html: + sections.append(apps_html) + if ports_html: + sections.append(ports_html) + if orphan_html: + sections.append(orphan_html) + + result_index_html = _render_result_index(doc) + + return f""" + + + + + +LLM Lightweight Monitoring · stacks + + + +
    +

    LLM · stacks

    + {score_html} +
    updated {_esc(updated)} · Lightweight Monitoring
    +
    +{''.join(sections)} +{result_index_html} + + +
    +

    detail

    + +
    +
    +     + + + +""" + + +def main() -> int: + ap = argparse.ArgumentParser(description="Render stack status HTML from JSON") + ap.add_argument("--status", type=Path, default=None) + ap.add_argument("--out", type=Path, required=True) + args = ap.parse_args() + if L.load_monitoring_env(): + L.refresh_runtime_flags_from_env() + doc = L.read_status(args.status) + if not doc: + doc = {"schema": L.SCHEMA, "ts": time.time(), "stacks": [], "legend": L.LEGEND} + html_text = render_html(doc) + args.out.parent.mkdir(parents=True, exist_ok=True) + tmp = args.out.with_name(f".{args.out.name}.tmp.{os.getpid()}") + try: + tmp.write_text(html_text, encoding="utf-8") + os.replace(tmp, args.out) + finally: + if tmp.exists(): + try: + tmp.unlink() + except OSError: + pass + print(args.out) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/site-gen/stack_status_lib.py b/site-gen/stack_status_lib.py new file mode 100644 index 0000000..045de76 --- /dev/null +++ b/site-gen/stack_status_lib.py @@ -0,0 +1,4410 @@ +#!/usr/bin/env python3 +"""Stack traffic lights for taler-monitoring (daemon → JSON; UI reads only).""" +from __future__ import annotations + +import json +import os +import re +import socket +import ssl +import subprocess +import time +import urllib.error +import urllib.request +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from typing import Any, Callable + +SCHEMA = 8 +LIGHTS = ("green", "blue", "yellow", "orange", "red", "black") +SEVERITY = {name: i for i, name in enumerate(LIGHTS)} +# Soft endpoint fails (intro/terms/…) cap at yellow unless hard=True. +ENDPOINT_SOFT_MAX = "yellow" + + +def _env_truthy(name: str, default: str = "1") -> bool: + return os.environ.get(name, default) not in ("0", "false", "no", "") + + +def refresh_runtime_flags_from_env() -> None: + """Re-read module toggles after load_monitoring_env() (import-time values go stale).""" + global PUBLISHED_ENABLED, PUBLISHED_SUITE, PUBLISHED_APT_BASE + global ACUTE_DOWN_S, PROBE_TIMEOUT_S, STATS_STALE_S, SSH_PROBE_TIMEOUT_S, TLS_VERIFY + global SSH_PROBES_ENABLED, CVE_PROBES_ENABLED, SERVICES_ENABLED, MAIL_ENABLED + global CERT_PROBES_ENABLED, ORPHAN_PROBES_ENABLED, ORPHAN_HTTP_ENABLED + global MAIL_PORTS_ENABLED, MAIL_PORT_TIMEOUT_S + global DELIVER_ENABLED, DELIVER_INTERVAL_S, DELIVER_STALE_S, DELIVER_TIMEOUT_S + global PORT_SCAN_ENABLED, PORT_SCAN_TIMEOUT_S, PORT_SCAN_WORKERS + global CERT_WARN_DAYS, CERT_CRIT_DAYS + global SCORE_RATIO_GREEN, SCORE_RATIO_BLUE, SCORE_RATIO_YELLOW + global SCORE_RATIO_ORANGE, SCORE_RATIO_RED + global RESULT_INDEX_MODE + global GIT_PROBE_ENABLED, GIT_PROBE_INTERVAL_S, GIT_PROBE_STALE_S, GIT_PROBE_TIMEOUT_S + global PAY_ENABLED, PAY_RUN, PAY_INTERVAL_S, PAY_STALE_S, PAY_TIMEOUT_S, PAY_DOMAINS + PUBLISHED_ENABLED = _env_truthy("TALER_MONITORD_PUBLISHED") + PUBLISHED_SUITE = os.environ.get("TALER_MONITORD_APT_SUITE", "trixie") + PUBLISHED_APT_BASE = os.environ.get( + "TALER_MONITORD_APT_BASE", "https://deb.taler.net/apt/debian" + ).rstrip("/") + ACUTE_DOWN_S = float(os.environ.get("TALER_MONITORD_ACUTE_DOWN_S", "90")) + PROBE_TIMEOUT_S = float(os.environ.get("TALER_MONITORD_PROBE_TIMEOUT_S", "5")) + STATS_STALE_S = float(os.environ.get("TALER_MONITORD_STATS_STALE_S", str(6 * 3600))) + SSH_PROBE_TIMEOUT_S = float(os.environ.get("TALER_MONITORD_SSH_TIMEOUT_S", "4")) + TLS_VERIFY = _env_truthy("TALER_MONITORD_TLS_VERIFY", "0") + SSH_PROBES_ENABLED = _env_truthy("TALER_MONITORD_SSH") + CVE_PROBES_ENABLED = _env_truthy("TALER_MONITORD_CVE") + SERVICES_ENABLED = _env_truthy("TALER_MONITORD_SERVICES") + MAIL_ENABLED = _env_truthy("TALER_MONITORD_MAIL") + CERT_PROBES_ENABLED = _env_truthy("TALER_MONITORD_CERT") + ORPHAN_PROBES_ENABLED = _env_truthy("TALER_MONITORD_ORPHAN") + ORPHAN_HTTP_ENABLED = _env_truthy("TALER_MONITORD_ORPHAN_HTTP", "0") + MAIL_PORTS_ENABLED = _env_truthy("TALER_MONITORD_MAIL_PORTS") + MAIL_PORT_TIMEOUT_S = float(os.environ.get("TALER_MONITORD_MAIL_PORT_TIMEOUT_S", "4")) + DELIVER_ENABLED = _env_truthy("TALER_MONITORD_DELIVER") + DELIVER_INTERVAL_S = float( + os.environ.get("TALER_MONITORD_DELIVER_INTERVAL_S", str(6 * 3600)) + ) + DELIVER_STALE_S = float( + os.environ.get("TALER_MONITORD_DELIVER_STALE_S", str(30 * 3600)) + ) + DELIVER_TIMEOUT_S = float(os.environ.get("TALER_MONITORD_DELIVER_TIMEOUT_S", "12")) + PORT_SCAN_ENABLED = _env_truthy("TALER_MONITORD_PORTSCAN") + PORT_SCAN_TIMEOUT_S = float(os.environ.get("TALER_MONITORD_PORTSCAN_TIMEOUT_S", "0.8")) + PORT_SCAN_WORKERS = int(os.environ.get("TALER_MONITORD_PORTSCAN_WORKERS", "24")) + CERT_WARN_DAYS = int(os.environ.get("TALER_MONITORD_CERT_WARN_DAYS", "30")) + CERT_CRIT_DAYS = int(os.environ.get("TALER_MONITORD_CERT_CRIT_DAYS", "14")) + SCORE_RATIO_GREEN = float(os.environ.get("TALER_MONITORD_SCORE_GREEN", "1.0")) + SCORE_RATIO_BLUE = float(os.environ.get("TALER_MONITORD_SCORE_BLUE", "0.85")) + SCORE_RATIO_YELLOW = float(os.environ.get("TALER_MONITORD_SCORE_YELLOW", "0.60")) + SCORE_RATIO_ORANGE = float(os.environ.get("TALER_MONITORD_SCORE_ORANGE", "0.35")) + SCORE_RATIO_RED = float(os.environ.get("TALER_MONITORD_SCORE_RED", "0.01")) + # Optional bottom index: 0/off | 1/colors | cmd/full (+ concrete probe command) + raw_idx = (os.environ.get("TALER_MONITORD_RESULT_INDEX") or "0").strip().lower() + if raw_idx in ("0", "false", "no", "off", ""): + RESULT_INDEX_MODE = "off" + elif raw_idx in ("cmd", "full", "commands", "2"): + RESULT_INDEX_MODE = "cmd" + else: + RESULT_INDEX_MODE = "colors" + if RESULT_INDEX_MODE == "colors" and _env_truthy( + "TALER_MONITORD_RESULT_INDEX_CMD", "0" + ): + RESULT_INDEX_MODE = "cmd" + GIT_PROBE_ENABLED = _env_truthy("TALER_MONITORD_GIT") + GIT_PROBE_INTERVAL_S = float( + os.environ.get("TALER_MONITORD_GIT_INTERVAL_S", str(6 * 3600)) + ) + GIT_PROBE_STALE_S = float( + os.environ.get("TALER_MONITORD_GIT_STALE_S", str(30 * 3600)) + ) + GIT_PROBE_TIMEOUT_S = float(os.environ.get("TALER_MONITORD_GIT_TIMEOUT_S", "12")) + # Payability (wallet-cli zero-pay) — opt-in; not every monitord tick + PAY_ENABLED = _env_truthy("TALER_MONITORD_PAY", "0") + PAY_RUN = _env_truthy("TALER_MONITORD_PAY_RUN", "0") + PAY_INTERVAL_S = float(os.environ.get("TALER_MONITORD_PAY_INTERVAL_S", str(4 * 3600))) + PAY_STALE_S = float(os.environ.get("TALER_MONITORD_PAY_STALE_S", str(30 * 3600))) + PAY_TIMEOUT_S = float(os.environ.get("TALER_MONITORD_PAY_TIMEOUT_S", "180")) + raw_pay = (os.environ.get("TALER_MONITORD_PAY_DOMAINS") or "").strip() + PAY_DOMAINS = [ + p.strip() + for p in re.split(r"[\s,]+", raw_pay) + if p.strip() + ] or ["stage.taler-ops.ch"] + + +# Placeholders; filled by refresh_runtime_flags_from_env() below. +PUBLISHED_ENABLED = True +PUBLISHED_SUITE = "trixie" +PUBLISHED_APT_BASE = "https://deb.taler.net/apt/debian" +ACUTE_DOWN_S = 90.0 +PROBE_TIMEOUT_S = 5.0 +STATS_STALE_S = float(6 * 3600) +SSH_PROBE_TIMEOUT_S = 4.0 +TLS_VERIFY = False +SSH_PROBES_ENABLED = True +CVE_PROBES_ENABLED = True +SERVICES_ENABLED = True +MAIL_ENABLED = True +CERT_PROBES_ENABLED = True +ORPHAN_PROBES_ENABLED = True +ORPHAN_HTTP_ENABLED = False +MAIL_PORTS_ENABLED = True +MAIL_PORT_TIMEOUT_S = 4.0 +DELIVER_ENABLED = True +DELIVER_INTERVAL_S = float(6 * 3600) +DELIVER_STALE_S = float(30 * 3600) +DELIVER_TIMEOUT_S = 12.0 +# Outside-in notable port scan (public TCP only). Suspicious = open ∩ ¬whitelist ∩ ¬baseline. +# Whitelist = operator-known intentional public opens — not an inventory of private topology. +PORT_SCAN_ENABLED = True +PORT_SCAN_TIMEOUT_S = 0.8 +PORT_SCAN_WORKERS = 24 +CERT_WARN_DAYS = 30 +CERT_CRIT_DAYS = 14 +SCORE_RATIO_GREEN = 1.0 +SCORE_RATIO_BLUE = 0.85 +SCORE_RATIO_YELLOW = 0.60 +SCORE_RATIO_ORANGE = 0.35 +SCORE_RATIO_RED = 0.01 +RESULT_INDEX_MODE = "off" +GIT_PROBE_ENABLED = True +GIT_PROBE_INTERVAL_S = float(6 * 3600) +GIT_PROBE_STALE_S = float(30 * 3600) +GIT_PROBE_TIMEOUT_S = 12.0 +PAY_ENABLED = False +PAY_RUN = False +PAY_INTERVAL_S = float(4 * 3600) +PAY_STALE_S = float(30 * 3600) +PAY_TIMEOUT_S = 180.0 +PAY_DOMAINS = ["stage.taler-ops.ch"] +refresh_runtime_flags_from_env() +# Notable TCP ports (not a full nmap range). Override: TALER_MONITORD_PORTSCAN_PORTS=22,80,443,… +_DEFAULT_SCAN_PORTS = ( + 21, + 22, + 23, + 25, + 80, + 110, + 135, + 139, + 143, + 443, + 445, + 465, + 587, + 993, + 995, + 1433, + 1521, + 1789, + 1790, + 2049, + 3306, + 3389, + 5432, + 5900, + 6379, + 8080, + 8443, + 9200, + 9418, + 11211, + 23230, + 23235, + 23236, + 23237, + 23238, + 27017, + 52323, +) +# Baseline always-ok for public HTTPS endpoints (still overridable by denying via empty whitelist only — +# whitelist *adds* allowance; baseline is the default allow for web). +_BASELINE_WEB_PORTS = frozenset({80, 443}) +_BASELINE_MAIL_PORTS = frozenset({25, 110, 143, 465, 587, 993, 995}) +# Port → short label for UI +_PORT_LABELS = { + 21: "ftp", + 22: "ssh", + 23: "telnet", + 25: "smtp", + 80: "http", + 110: "pop3", + 135: "msrpc", + 139: "netbios", + 143: "imap", + 443: "https", + 445: "smb", + 465: "smtps", + 587: "submission", + 993: "imaps", + 995: "pop3s", + 1433: "mssql", + 1521: "oracle", + 2049: "nfs", + 3306: "mysql", + 3389: "rdp", + 5432: "postgres", + 5900: "vnc", + 6379: "redis", + 8080: "http-alt", + 8443: "https-alt", + 1789: "nym-sphinx", + 1790: "nym-verloc", + 9200: "git-ssh", + 9418: "git-daemon", + 11211: "memcached", + 23230: "ssh-dnat", + 23235: "ssh-dnat", + 23236: "ssh-dnat", + 23237: "ssh-dnat", + 23238: "ssh-dnat", + 27017: "mongodb", + 52323: "tortuna", +} +# Member reasons that are informational only (do not weaken the stack box). +_AGG_SKIP_REASONS = frozenset({"ebics_iban", "ebics_no_bank"}) + +# Suite-like public endpoints per machine (paths relative to https://host). +# need_intro: custom /intro stack only (GOA + FrancPaysan/TESTPAYSAN stage). +# severity: soft (≤yellow) | medium (missing → orange) | hard (full classify). +ROLE_ENDPOINTS: dict[str, list[dict[str, Any]]] = { + "bank": [ + {"id": "integration", "path": "/taler-integration/config", "expect": (200,), "severity": "hard"}, + {"id": "config", "path": "/config", "expect": (200,), "severity": "hard"}, + {"id": "webui", "path": "/webui/", "expect": (200, 301, 302), "severity": "soft"}, + {"id": "intro", "path": "/intro/", "expect": (200, 301, 302), "severity": "soft", "need_intro": True}, + {"id": "stats", "path": "/intro/stats.json", "expect": (200,), "severity": "soft", "need_intro": True}, + # Legal docs: severity overridden at probe time (medium only for commercial prod). + {"id": "terms", "path": "/terms", "expect": (200,), "severity": "soft", "need_intro": True, "legal": True}, + {"id": "privacy", "path": "/privacy", "expect": (200,), "severity": "soft", "need_intro": True, "legal": True}, + ], + "exchange": [ + {"id": "config", "path": "/config", "expect": (200,), "severity": "hard"}, + {"id": "keys", "path": "/keys", "expect": (200,), "severity": "hard"}, + {"id": "terms", "path": "/terms", "expect": (200,), "severity": "soft", "legal": True}, + {"id": "privacy", "path": "/privacy", "expect": (200,), "severity": "soft", "legal": True}, + {"id": "intro", "path": "/intro/", "expect": (200, 301, 302), "severity": "soft", "need_intro": True}, + ], + "merchant": [ + {"id": "config", "path": "/config", "expect": (200,), "severity": "hard"}, + {"id": "webui", "path": "/webui/", "expect": (200, 301, 302), "severity": "soft"}, + {"id": "intro", "path": "/intro/", "expect": (200, 301, 302), "severity": "soft", "need_intro": True}, + {"id": "stats", "path": "/intro/stats.json", "expect": (200,), "severity": "soft", "need_intro": True}, + {"id": "terms", "path": "/terms", "expect": (200,), "severity": "soft", "legal": True}, + {"id": "privacy", "path": "/privacy", "expect": (200,), "severity": "soft", "legal": True}, + ], + "site": [ + {"id": "apex", "path": "/", "expect": (200, 301, 302), "severity": "hard"}, + ], +} + + +def stack_has_custom_intro(st: dict[str, Any]) -> bool: + """True only for custom /intro landings: GOA, FrancPaysan, TESTPAYSAN stage. + + Prefer domains.conf landing=1; also recognize by currency / hostname so + demo/test/regio/TOPS never get /intro probes by accident. + """ + cur = str(st.get("currency") or "").upper() + if cur in ("GOA", "TESTPAYSAN"): + return True + blob = " ".join( + [ + str(st.get("id") or ""), + " ".join(str(a) for a in (st.get("aliases") or [])), + str(st.get("bank") or ""), + str(st.get("exchange") or ""), + str(st.get("merchant") or ""), + ] + ).lower() + if "hacktivism" in blob or "lefrancpaysan" in blob or "francpaysan" in blob: + # static_pending FP prod has no Taler /intro APIs yet + if (st.get("kind") or "") == "static_pending" or not cur: + return False + return True + # Explicit landing=1 only if it is one of the known custom stacks above + return False + + +def stack_is_commercial_prod(st: dict[str, Any]) -> bool: + """Commercial production (legal docs must be solid) — not stage/demo/test/GOA. + + Today: TOPS CHF prod (taler-ops.ch / my.taler-ops.ch). Stage and sandboxes stay soft. + """ + if st.get("local"): + return False + if (st.get("kind") or "") == "static_pending": + return False + sid = str(st.get("id") or "").lower() + blob = " ".join( + [ + sid, + " ".join(str(a) for a in (st.get("aliases") or [])), + str(st.get("merchant") or ""), + str(st.get("exchange") or ""), + ] + ).lower() + if "stage." in sid or sid.startswith("stage.") or ".stage." in sid: + return False + if "stage." in blob and "taler-ops" in blob: + return False + # TOPS production CHF + if "taler-ops.ch" in blob and "stage" not in sid: + cur = str(st.get("currency") or "").upper() + if cur in ("", "CHF"): + return True + return False + + +def legal_doc_severity(st: dict[str, Any]) -> str: + """terms/privacy: medium (orange) only for commercial production.""" + return "medium" if stack_is_commercial_prod(st) else "soft" + + +def _stack_blob(st: dict[str, Any]) -> str: + return " ".join( + [ + str(st.get("id") or ""), + " ".join(str(a) for a in (st.get("aliases") or [])), + str(st.get("bank") or ""), + str(st.get("exchange") or ""), + str(st.get("merchant") or ""), + str(st.get("currency") or ""), + ] + ).lower() + + +def classify_network_zone(st: dict[str, Any]) -> dict[str, str]: + """Map a stack to a coarse network area (not a full topology).""" + blob = _stack_blob(st) + cur = str(st.get("currency") or "").upper() + if st.get("local") or "hacktivism" in blob or cur == "GOA": + return {"zone": "goa-local", "label": "GOA / local (koopa)"} + if cur == "TESTPAYSAN" or "stage.lefrancpaysan" in blob: + return {"zone": "fp-stage", "label": "FrancPaysan stage"} + if (st.get("kind") or "") == "static_pending" or ( + "lefrancpaysan" in blob and not cur + ): + return {"zone": "fp-edge", "label": "FrancPaysan edge (static)"} + if stack_is_commercial_prod(st): + return {"zone": "tops-prod", "label": "TOPS prod CHF"} + if "stage.taler-ops" in blob or "stage.my.taler-ops" in blob: + return {"zone": "tops-stage", "label": "TOPS stage"} + if cur in ("KUDOS", "TESTKUDOS", "FLODOS") or "demo.taler.net" in blob or "test.taler.net" in blob: + return {"zone": "public-demo", "label": "public demo/test"} + if "taler-systems" in blob or "anastasis" in blob or "pixel" in blob: + return {"zone": "pixel", "label": "pixel / TSA / Anastasis"} + if "gnunet.org" in blob or (st.get("kind") or "") == "mail": + zone = str(st.get("zone") or "") + if zone == "mail" or (st.get("kind") or "") == "mail": + return {"zone": "mail", "label": "mail plane"} + return {"zone": "firefly", "label": "firefly / gnunet"} + return {"zone": "public", "label": "public HTTPS"} + + +def _env_ssh_hosts(*keys: str) -> list[str]: + out: list[str] = [] + for k in keys: + raw = (os.environ.get(k) or "").strip() + if not raw: + continue + for part in re.split(r"[\s,]+", raw): + if part and part not in out: + out.append(part) + return out + + +def monitoring_env_path() -> Path: + """Path to taler-monitoring-env install (~/.config/taler-monitoring/env).""" + raw = (os.environ.get("TALER_MONITORING_ENV") or "").strip() + if raw: + return Path(raw) + xdg = os.environ.get("XDG_CONFIG_HOME") or str(Path.home() / ".config") + return Path(xdg) / "taler-monitoring" / "env" + + +def _expand_env_value(val: str) -> str: + """Expand $VAR / ${VAR} using current environ (shell-like, for env profiles).""" + + def repl(m: re.Match[str]) -> str: + name = m.group(1) or m.group(2) + return os.environ.get(name, "") + + return re.sub( + r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}|\$([A-Za-z_][A-Za-z0-9_]*)", + repl, + val, + ) + + +def load_monitoring_env(*, path: Path | None = None, override: bool = True) -> Path | None: + """Load KEY=value from taler-monitoring-env profile into os.environ (like lib.sh). + + Skips comments/blanks. Expands $HOME / $VAR. Returns path if loaded, else None. + """ + p = path or monitoring_env_path() + if not p.is_file(): + return None + try: + text = p.read_text(encoding="utf-8") + except OSError: + return None + for line in text.splitlines(): + s = line.strip() + if not s or s.startswith("#"): + continue + if s.startswith("export "): + s = s[7:].lstrip() + if "=" not in s: + continue + key, _, val = s.partition("=") + key = key.strip() + if not key or not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", key): + continue + if not override and key in os.environ: + continue + val = val.strip() + if len(val) >= 2 and val[0] == val[-1] and val[0] in ("'", '"'): + val = val[1:-1] + os.environ[key] = _expand_env_value(val) + return p + + +def ssh_targets_for_stack(st: dict[str, Any]) -> dict[str, Any]: + """Hostnames to probe for an SSH daemon (port 22 banner) — not jump-host login. + + Zone-mapped SSH targets come only from taler-monitoring-env + (`TALER_MONITORD_SSH_TOPS` / `_FP` / `_GOA` / `_TOPS_PROD`). No suite hardcodes. + When env supplies hosts, hide them in UI/JSON display fields. + """ + zone = classify_network_zone(st)["zone"] + # Prefer apex / merchant hostname that is publicly meaningful + candidates: list[str] = [] + for key in ("id", "merchant", "exchange", "bank"): + h = _host_only(str(st.get(key) or "")) + if h and h.upper() != "IBAN" and h not in candidates: + candidates.append(h) + env_map = { + "goa-local": "TALER_MONITORD_SSH_GOA", + "fp-stage": "TALER_MONITORD_SSH_FP", + "tops-stage": "TALER_MONITORD_SSH_TOPS", + "tops-prod": "TALER_MONITORD_SSH_TOPS_PROD", + } + env_hosts = _env_ssh_hosts(env_map[zone]) if zone in env_map else [] + if env_hosts: + hosts = env_hosts + hide_host = True + else: + hosts = candidates[:2] + hide_host = False + # Public demos: still check if SSH is exposed (security signal) + expected = True + if zone in ("fp-edge",): + expected = bool(hosts) + return { + "expected": expected and bool(hosts), + "hosts": hosts, + "via": "tcp22-banner", + "hide_host": hide_host, + } + + +def probe_ssh_host(host: str, *, timeout_s: float = SSH_PROBE_TIMEOUT_S) -> dict[str, Any]: + """Check whether an SSH service answers (banner), not whether our keys work. + + SSH-2.0-… banner ⇒ present (ok/green). Connection refused / timeout ⇒ absent. + """ + host = _host_only(host) + if not host: + return {"ok": False, "light": "blue", "reason": "no_host", "host": "", "elapsed": 0.0} + t0 = time.monotonic() + try: + with socket.create_connection((host, 22), timeout=max(1.0, float(timeout_s))) as sock: + sock.settimeout(max(1.0, float(timeout_s))) + banner = sock.recv(256).decode("utf-8", "replace").strip().splitlines() + line = banner[0] if banner else "" + elapsed = time.monotonic() - t0 + if line.upper().startswith("SSH-"): + return { + "ok": True, + "light": "green", + "reason": line[:80], + "host": host, + "elapsed": round(elapsed, 3), + "banner": line[:120], + } + return { + "ok": False, + "light": "orange", + "reason": f"no_ssh_banner:{line[:60]}", + "host": host, + "elapsed": round(elapsed, 3), + } + except (socket.timeout, TimeoutError): + return { + "ok": False, + "light": "orange", + "reason": "timeout", + "host": host, + "elapsed": round(time.monotonic() - t0, 3), + } + except OSError as e: + # refused / unreachable = no public SSH (often OK for hardened edge) + return { + "ok": False, + "light": "blue", + "reason": type(e).__name__, + "host": host, + "elapsed": round(time.monotonic() - t0, 3), + } + + +def build_network_status( + st: dict[str, Any], + *, + members: list[dict[str, Any]] | None = None, + ssh_cache: dict[str, dict[str, Any]] | None = None, + ssh_probe_fn: Callable[[str], dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Zone + public HTTPS summary + optional SSH reachability for the bundle.""" + zone = classify_network_zone(st) + members = members or [] + # Public path: worst non-informational member light + pub_lights = [ + m.get("light") or "black" + for m in members + if (m.get("reason") or "") not in _AGG_SKIP_REASONS + ] + if not pub_lights: + pub_lights = [m.get("light") or "black" for m in members] or ["blue"] + pub_worst = LIGHTS[max(SEVERITY.get(L, 5) for L in pub_lights)] + targets = ssh_targets_for_stack(st) + hide_host = bool(targets.get("hide_host")) + ssh_block: dict[str, Any] = { + "expected": bool(targets.get("expected")), + "via": targets.get("via") or "", + "candidates": [] if hide_host else list(targets.get("hosts") or []), + "ok": None, + "host": "", + "light": "blue", + "reason": "not_expected", + "elapsed": 0.0, + } + if not SSH_PROBES_ENABLED: + ssh_block["reason"] = "ssh_probes_disabled" + ssh_block["light"] = "blue" + elif targets.get("expected") and targets.get("hosts"): + cache = ssh_cache if ssh_cache is not None else {} + probe = ssh_probe_fn or probe_ssh_host + best: dict[str, Any] | None = None + for h in targets["hosts"]: + if h not in cache: + cache[h] = probe(h) + got = cache[h] + if got.get("ok"): + best = got + break + if best is None: + best = got + if best: + # Keep real probe host out of UI for mapped zones (e.g. betel ≠ stage FQDN). + shown_host = "" if targets.get("hide_host") else (best.get("host") or "") + reason = best.get("reason") or "" + if targets.get("hide_host") and reason and not str(reason).upper().startswith("SSH-"): + # Drop accidental hostname leakage in soft errors; keep SSH banner text. + if shown_host and shown_host in str(reason): + reason = "ssh_probe" + ssh_block.update( + { + "ok": bool(best.get("ok")), + "host": shown_host, + "light": best.get("light") or ("green" if best.get("ok") else "orange"), + "reason": reason, + "elapsed": best.get("elapsed") or 0.0, + } + ) + return { + "zone": zone["zone"], + "label": zone["label"], + "public_light": pub_worst, + "ssh": ssh_block, + } + +LEGEND = { + "green": "ok", + "blue": "present", + "yellow": "degraded", + "orange": "recent_or_hard_degraded", + "red": "down_acute", + "black": "down_long", +} + + +def monitord_state_dir() -> Path: + base = os.environ.get("TALER_MONITORD_STATE") or os.environ.get("XDG_RUNTIME_DIR") + if base: + d = Path(base) / "taler-monitord" + else: + d = Path.home() / ".local" / "state" / "taler-monitord" + d.mkdir(parents=True, exist_ok=True) + return d + + +def monitord_data_dir() -> Path: + raw = os.environ.get("TALER_MONITORD_DATA") + if raw: + d = Path(raw) + else: + d = Path.home() / ".local" / "share" / "taler-monitord" + d.mkdir(parents=True, exist_ok=True) + return d + + +def status_json_paths() -> list[Path]: + return [monitord_state_dir() / "stack-status.json", monitord_data_dir() / "stack-status.json"] + + +def default_domains_conf() -> Path: + env = os.environ.get("TALER_DOMAINS_CONF") or "" + if env and Path(env).is_file(): + return Path(env) + here = Path(__file__).resolve().parent + for p in (here.parent / "domains.conf", Path("domains.conf")): + if p.is_file(): + return p + return here.parent / "domains.conf" + + +def _repo_conf(name: str, env_key: str) -> Path: + env = os.environ.get(env_key) or "" + if env and Path(env).is_file(): + return Path(env) + here = Path(__file__).resolve().parent + for p in (here.parent / name, Path(name)): + if p.is_file(): + return p + return here.parent / name + + +def default_services_conf() -> Path: + return _repo_conf("services-catalog.conf", "TALER_SERVICES_CONF") + + +def default_mail_catalog() -> Path: + return _repo_conf("mail-catalog.conf", "MAIL_CATALOG") + + +def default_ports_whitelist_conf() -> Path: + return _repo_conf("ports-whitelist.conf", "TALER_PORTS_WHITELIST") + + +def _host_only(s: str) -> str: + s = (s or "").strip() + s = s.removeprefix("https://").removeprefix("http://") + return s.split("/")[0].split(":")[0] + + +# domains.conf bank column: no public libeufin / x-taler-bank host expected +_BANK_SKIP_TOKENS = frozenset({"-", ".", "none", "n/a", "na", "ebics", "iban", "skip", "off"}) + + +def bank_host_configured(bank: str) -> bool: + """False when domains.conf marks bank as absent (EBICS / IBAN wire).""" + b = (bank or "").strip().lower() + return bool(b) and b not in _BANK_SKIP_TOKENS + + +def payto_wire_methods(blob: bytes | str | None) -> set[str]: + """Extract payto://METHOD/… wire methods from exchange /keys (or similar) body.""" + if blob is None: + return set() + if isinstance(blob, bytes): + text = blob.decode("utf-8", "replace") + else: + text = str(blob) + return {m.lower() for m in re.findall(r"payto://([a-z0-9-]+)/", text, flags=re.I)} + + +def classify_bank_backend(methods: set[str], *, bank_configured: bool = True) -> str: + """Return x-taler-bank | ebics | unknown. + + EBICS/IBAN stacks (e.g. TOPS CHF) have no public bank.* — only payto://iban. + Demo/stage libeufin stacks advertise payto://x-taler-bank/…. + Mixed (regio: both) still expects bank.* because x-taler-bank is present. + """ + if not bank_configured: + return "ebics" + m = {x.lower() for x in (methods or set())} + if "x-taler-bank" in m: + return "x-taler-bank" + if "iban" in m: + return "ebics" + return "unknown" + + +def _looks_locale(tok: str) -> bool: + t = tok.lower().replace("_", "-") + return bool(t in ("fr-ch", "de-ch", "en-ch", "ch-fr", "ch-de", "ch-en") or ( + len(t) == 5 and t[2] == "-" and t[:2].isalpha() and t[3:].isalpha() + )) + + +def _looks_lang(tok: str) -> bool: + return tok.lower() in ("en", "fr", "de", "eng", "fra", "deu", "ger", "german", "deutsch", "french", "english") + + +def parse_domains_conf(path: Path | None = None) -> list[dict[str, Any]]: + """Return unique stacks keyed by (bank, exchange, merchant).""" + path = path or default_domains_conf() + by_endpoints: dict[tuple[str, str, str], dict[str, Any]] = {} + if not path.is_file(): + return [] + + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.split("#", 1)[0].strip() + if not line: + continue + parts = line.split() + if len(parts) < 5: + continue + name, bank, exchange, merchant, currency = parts[0], parts[1], parts[2], parts[3], parts[4] + local_stack = parts[5] if len(parts) > 5 else "0" + landing = parts[6] if len(parts) > 6 else "" + rest = parts[7:] + + # Legacy: 7th field was canonical (contains a dot, not 0/1) + if landing and landing not in ("0", "1") and "." in landing: + rest = [landing] + rest + landing = "" + + canon = name + if rest: + if _looks_lang(rest[0]): + if len(rest) >= 2 and _looks_locale(rest[1]): + canon = rest[2] if len(rest) >= 3 else name + elif len(rest) >= 2 and "." in rest[1]: + canon = rest[1] + elif "." in rest[0]: + canon = rest[0] + + bank_configured = bank_host_configured(bank) + bank_h = _host_only(bank) if bank_configured else "" + ex_h, mer_h = _host_only(exchange), _host_only(merchant) + # Keep skip-token in key so EBICS rows do not collide with empty bank hosts. + key = (bank_h or (bank.strip().lower() if bank.strip() else "-"), ex_h, mer_h) + cur = currency if currency not in ("-", "") else "" + # No currency ⇒ static site / Taler prod not provisioned yet (e.g. lefrancpaysan.ch). + kind = "taler" if cur else "static_pending" + row = { + "id": canon, + "bank": bank_h, + "bank_configured": bank_configured, + "exchange": ex_h, + "merchant": mer_h, + "currency": cur, + "kind": kind, + "local": local_stack in ("1", "true", "yes"), + "landing": landing in ("1", "true", "yes"), + "aliases": [name], + } + if key not in by_endpoints: + by_endpoints[key] = row + else: + prev = by_endpoints[key] + prev["aliases"].append(name) + # Prefer non-service hostname as id when alias list grows + if prev["id"] in (bank_h, ex_h, mer_h) and canon not in (bank_h, ex_h, mer_h): + prev["id"] = canon + elif canon == name and name not in (bank_h, ex_h, mer_h) and len(name) <= len(prev["id"]): + # keep shorter apex-like id when equal quality + if prev["id"] in prev["aliases"] and name.count(".") <= prev["id"].count("."): + prev["id"] = canon + if cur and not prev["currency"]: + prev["currency"] = cur + prev["kind"] = "taler" + + return sorted(by_endpoints.values(), key=lambda s: s["id"]) + + +def member_probe_url(role: str, host: str) -> str: + host = _host_only(host) + if role == "bank": + # Prefer integration config (wallet-facing); fallback handled in probe + return f"https://{host}/taler-integration/config" + return f"https://{host}/config" + + +def stats_probe_url(host: str) -> str: + return f"https://{_host_only(host)}/intro/stats.json" + + +def _amount_value(amount: str) -> str: + """Strip CUR: prefix from Taler amount string.""" + s = (amount or "").strip() + if ":" in s: + return s.split(":", 1)[1] + return s + + +def _amount_sort_key(val: str) -> tuple: + try: + return (0, float(val)) + except (TypeError, ValueError): + return (1, str(val)) + + +def extract_denom_values(blob: bytes | str | None, currency: str = "") -> list[str]: + """Unique denomination values from exchange /keys body (best-effort).""" + if blob is None: + return [] + text = blob.decode("utf-8", "replace") if isinstance(blob, bytes) else str(blob) + vals: set[str] = set() + # "value": "CUR:N" or amount fields + for m in re.finditer(r'"value"\s*:\s*"([^"]+)"', text): + raw = m.group(1) + if currency and raw.startswith(currency + ":"): + vals.add(_amount_value(raw)) + elif ":" in raw and re.match(r"^[A-Z][A-Z0-9_]+:", raw): + vals.add(_amount_value(raw)) + # Prefer positive coin values; drop zero-only noise + cleaned = [v for v in vals if v not in ("", "0", "0.0", "0.00")] + return sorted(cleaned, key=_amount_sort_key)[:32] + + +def summarize_currency_spec( + cs: dict[str, Any] | None, + *, + keys_body: bytes | str | None = None, + fallback_currency: str = "", +) -> dict[str, Any] | None: + """Currency code, coin symbols (alt_unit_names), denomination list for UI.""" + cs = cs if isinstance(cs, dict) else {} + currency = str(cs.get("currency") or fallback_currency or "").strip() + alt_raw = cs.get("alt_unit_names") if isinstance(cs.get("alt_unit_names"), dict) else {} + alt = {str(k): str(v) for k, v in alt_raw.items() if str(v).strip()} + if not currency and not alt: + return None + symbol = alt.get("0") or currency + # Subunits: "-2" → Rp./ct — sorted by exponent + def _exp_key(k: str) -> int: + try: + return int(k) + except ValueError: + return 0 + + subunits: list[dict[str, str]] = [] + for k in sorted((x for x in alt if x != "0"), key=_exp_key, reverse=True): + subunits.append({"exp": k, "symbol": alt[k]}) + + denoms: list[str] = [] + for a in cs.get("common_amounts") or []: + v = _amount_value(str(a)) + if v and v not in denoms: + denoms.append(v) + if not denoms and keys_body is not None: + denoms = extract_denom_values(keys_body, currency) + + unit_bits = [symbol] + [s["symbol"] for s in subunits] + # unique preserve order + seen: set[str] = set() + units: list[str] = [] + for u in unit_bits: + if u and u not in seen: + seen.add(u) + units.append(u) + + return { + "currency": currency, + "name": str(cs.get("name") or ""), + "symbol": symbol, + "alt_unit_names": alt, + "subunits": subunits, + "units_label": " / ".join(units) if units else currency, + "denoms": denoms, + "denoms_label": " · ".join(denoms[:16]), + } + + +def _json_body(raw: dict[str, Any] | None) -> dict[str, Any] | None: + if not raw: + return None + body = raw.get("body") + if body is None: + return None + if isinstance(body, bytes): + text = body.decode("utf-8", "replace") + else: + text = str(body) + text = text.strip() + if not text: + return None + try: + data = json.loads(text) + except json.JSONDecodeError: + return None + return data if isinstance(data, dict) else None + + +def summarize_landing_stats( + data: dict[str, Any], + *, + now: float | None = None, + stale_s: float = STATS_STALE_S, + role: str = "", + host: str = "", + url: str = "", +) -> dict[str, Any]: + """Compact, UI-friendly summary of public …/intro/stats.json (no full dump).""" + now = time.time() if now is None else now + gu = data.get("generated_at_unix") + age: int | None = None + if gu is not None and str(gu).strip() != "": + try: + src_u = int(float(gu)) + if src_u > 10_000_000_000: + src_u //= 1000 + age = int(now - src_u) + except (TypeError, ValueError): + age = None + + lines: list[str] = [] + ba = data.get("bank_accounts") if isinstance(data.get("bank_accounts"), dict) else {} + wd = data.get("withdraws") if isinstance(data.get("withdraws"), dict) else {} + if ba or wd: + n_acc = ba.get("total") if ba.get("total") is not None else ba.get("users") + if n_acc is not None: + lines.append(f"accounts {n_acc}") + if wd.get("count") is not None: + lines.append(f"withdraws {wd.get('count')}") + amt = wd.get("total_amount_alt") or wd.get("total_amount") + if amt: + lines.append(str(amt)) + if "reserves" in data: + lines.append(f"reserves {data.get('reserves')}") + if data.get("wire_in_count") is not None: + lines.append(f"wire_in {data.get('wire_in_count')}") + coins = data.get("known_coins") + if coins is None: + coins = data.get("coins_live") + if coins is not None: + lines.append(f"coins {coins}") + if data.get("instances") is not None: + lines.append(f"instances {data.get('instances')}") + if data.get("orders") is not None: + lines.append(f"orders {data.get('orders')}") + bal = data.get("balance_explorer_alt") or data.get("balance_explorer") + if bal and not any(str(bal) in x for x in lines): + lines.append(f"explorer {bal}") + + return { + "present": True, + "ok": data.get("ok") is not False, + "age_s": age, + "stale": bool(age is not None and age > stale_s), + "generated_at_human": data.get("generated_at_human") or data.get("generated_at") or "", + "source": str(data.get("source") or "")[:72], + "currency": data.get("currency") or "", + "lines": lines[:6], + "role": role, + "host": host, + "url": url, + } + + +def fetch_stack_stats( + st: dict[str, Any], + *, + bank_backend: str, + probe_fn: Callable[[str], dict[str, Any]], + now: float | None = None, + stale_s: float = STATS_STALE_S, +) -> dict[str, Any] | None: + """Try public /intro/stats.json (custom intro stacks only). Soft-miss → None.""" + now = time.time() if now is None else now + kind = st.get("kind") or "taler" + if kind == "static_pending": + return None + if not stack_has_custom_intro(st): + return None + + candidates: list[tuple[str, str]] = [] + if bank_backend == "x-taler-bank" and bank_host_configured(st.get("bank") or ""): + candidates.append(("bank", _host_only(st["bank"]))) + if st.get("merchant"): + candidates.append(("merchant", _host_only(st["merchant"]))) + if st.get("exchange"): + candidates.append(("exchange", _host_only(st["exchange"]))) + # de-dupe hosts while keeping order + seen: set[str] = set() + ordered: list[tuple[str, str]] = [] + for role, host in candidates: + if not host or host in seen: + continue + seen.add(host) + ordered.append((role, host)) + + for role, host in ordered: + url = stats_probe_url(host) + raw = probe_fn(url) + if not raw.get("ok"): + continue + data = _json_body(raw) + if not data: + continue + return summarize_landing_stats( + data, now=now, stale_s=stale_s, role=role, host=host, url=url + ) + return None + + +def _ssl_context() -> ssl.SSLContext | None: + if TLS_VERIFY: + return None + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + return ctx + + +def _header_server(headers: Any) -> str: + if headers is None: + return "" + try: + return str(headers.get("Server") or headers.get("server") or "").strip() + except Exception: + return "" + + +def probe_http(url: str, *, timeout: float = PROBE_TIMEOUT_S) -> dict[str, Any]: + """Probe one URL. Returns ok/code/elapsed/error/server (no light yet).""" + t0 = time.monotonic() + req = urllib.request.Request(url, method="GET", headers={"User-Agent": "taler-monitord/1"}) + try: + with urllib.request.urlopen(req, timeout=timeout, context=_ssl_context()) as resp: + body = resp.read(65536) + code = getattr(resp, "status", None) or resp.getcode() + elapsed = time.monotonic() - t0 + return { + "ok": 200 <= int(code) < 400, + "code": int(code), + "elapsed": elapsed, + "error": None, + "bytes": len(body), + "body": body, + "server": _header_server(getattr(resp, "headers", None)), + "url": url, + } + except urllib.error.HTTPError as e: + elapsed = time.monotonic() - t0 + try: + err_body = e.read(65536) or b"" + except Exception: + err_body = b"" + return { + "ok": False, + "code": int(e.code), + "elapsed": elapsed, + "error": f"http_{e.code}", + "bytes": len(err_body), + "body": err_body, + "server": _header_server(getattr(e, "headers", None)), + "url": url, + } + except (urllib.error.URLError, TimeoutError, socket.timeout, OSError) as e: + elapsed = time.monotonic() - t0 + err = type(e).__name__ + reason = getattr(e, "reason", None) + if reason is not None: + err = f"{err}:{reason}" + return { + "ok": False, + "code": 0, + "elapsed": elapsed, + "error": err, + "bytes": 0, + "body": b"", + "server": "", + "url": url, + } + + +def guess_http_package(server: str) -> str: + s = (server or "").lower() + if "nginx" in s: + return "nginx" + if "apache" in s or "httpd" in s: + return "apache" + if "caddy" in s: + return "caddy" + return "" + + +def extract_http_version(server: str) -> str: + m = re.search(r"/([0-9]+(?:\.[0-9]+)*)", server or "") + return m.group(1) if m else "" + + +def _taler_pkg_from_name(name: str) -> str: + n = (name or "").lower() + if "exchange" in n: + return "taler-exchange" + if "merchant" in n: + return "taler-merchant" + if "corebank" in n or n.endswith("bank") or "libeufin" in n: + return "taler-bank" + if "bank-integration" in n or "integration" in n: + return "taler-bank" + if n.startswith("taler"): + return "taler" + return "" + + +def software_from_taler_config(body: bytes | str | dict | None, *, host: str = "", role: str = "") -> list[dict[str, Any]]: + """Extract Taler component name + build_version from /config JSON.""" + if isinstance(body, dict): + data = body + elif body is None: + data = None + else: + data = _json_body({"body": body}) + if not isinstance(data, dict): + return [] + name = str(data.get("name") or "") + pkg = _taler_pkg_from_name(name) + if not pkg and role in ("exchange", "merchant", "bank"): + pkg = {"exchange": "taler-exchange", "merchant": "taler-merchant", "bank": "taler-bank"}[role] + if not pkg: + return [] + build = str(data.get("build_version") or "").strip() + api = str(data.get("version") or "").strip() + ver = build or api + return [ + { + "package": pkg, + "label": name or pkg, + "version": ver, + "api_version": api, + "build_version": build, + "host": _host_only(host), + "source": "taler-config", + "role": role, + } + ] + + +def software_from_server_header(server: str, *, host: str = "") -> list[dict[str, Any]]: + pkg = guess_http_package(server) + if not pkg: + return [] + ver = extract_http_version(server) + return [ + { + "package": pkg, + "label": pkg, + "version": ver, + "host": _host_only(host), + "source": "server-header", + "server": (server or "")[:80], + } + ] + + +def smtp_banner_software(host: str, *, timeout_s: float = 4.0) -> list[dict[str, Any]]: + """Read SMTP 220 banner; detect Postfix (+ version if present).""" + h = _host_only(host) + if not h: + return [] + try: + with socket.create_connection((h, 25), timeout=max(1.0, float(timeout_s))) as sock: + sock.settimeout(max(1.0, float(timeout_s))) + banner = sock.recv(512).decode("utf-8", "replace").strip().splitlines() + line = banner[0] if banner else "" + try: + sock.sendall(b"QUIT\r\n") + except OSError: + pass + except OSError: + return [] + low = line.lower() + if "postfix" not in low: + if "esmtp" in low or line.startswith("220"): + return [ + { + "package": "mta", + "label": "SMTP", + "version": "", + "host": h, + "source": "smtp-banner", + "banner": line[:100], + } + ] + return [] + ver = "" + m = re.search(r"postfix[^\d]*([0-9]+(?:\.[0-9]+)*)", line, flags=re.I) + if m: + ver = m.group(1) + return [ + { + "package": "postfix", + "label": "Postfix", + "version": ver, + "host": h, + "source": "smtp-banner", + "banner": line[:100], + } + ] + + +def _sev_to_light(sev: str) -> str: + s = (sev or "").upper() + if s == "CRITICAL": + return "red" + if s == "HIGH": + return "orange" + if s == "MEDIUM": + return "yellow" + if s == "LOW": + return "blue" + return "green" + + +_PUBLISHED_CACHE: dict[str, Any] = {"ts": 0.0, "pkgs": {}, "apps": [], "suite": "", "error": None} + +# inventory package → deb.taler.net package name(s) +_SOFT_TO_DEB = { + "taler-exchange": ("taler-exchange",), + "taler-merchant": ("taler-merchant",), + "taler-bank": ("libeufin-bank",), + "taler": ("taler-exchange",), +} + +# Published apps / SPAs we surface even without a live probe +_APP_CATALOG = ( + {"package": "taler-merchant-webui", "label": "Merchant WebUI", "kind": "webui", "logo": "webui"}, + {"package": "libeufin-bank-webui", "label": "Bank WebUI", "kind": "webui", "logo": "webui"}, + {"package": "taler-exchange-kyc-webui", "label": "KYC WebUI", "kind": "webui", "logo": "webui"}, + {"package": "taler-exchange-aml-webui", "label": "AML WebUI", "kind": "webui", "logo": "webui"}, + {"package": "taler-wallet-cli", "label": "Wallet CLI", "kind": "wallet", "logo": "wallet"}, +) + + +def _deb_ord_ch(c: str) -> int: + if c == "~": + return -1 + if c.isdigit(): + return 0 + if c.isalpha(): + return ord(c) + return ord(c) + 256 + + +def _deb_split(v: str) -> tuple[int, str, str]: + epoch = 0 + if ":" in v: + e, v = v.split(":", 1) + try: + epoch = int(e) + except ValueError: + epoch = 0 + if "-" in v: + upstream, deb = v.rsplit("-", 1) + else: + upstream, deb = v, "" + return epoch, upstream, deb + + +def _deb_cmp_part(a: str, b: str) -> int: + ia = ib = 0 + while ia < len(a) or ib < len(b): + while True: + ca = a[ia] if ia < len(a) and not a[ia].isdigit() else "" + cb = b[ib] if ib < len(b) and not b[ib].isdigit() else "" + if ca == "" and cb == "": + break + if ca == "" and cb: + return -1 if _deb_ord_ch(cb) > 0 else (1 if cb == "~" else -1) + if cb == "" and ca: + return 1 if _deb_ord_ch(ca) > 0 else (-1 if ca == "~" else 1) + oa, ob = _deb_ord_ch(ca), _deb_ord_ch(cb) + ia += 1 + ib += 1 + if oa != ob: + return (oa > ob) - (oa < ob) + sa = sb = "" + while ia < len(a) and a[ia].isdigit(): + sa += a[ia] + ia += 1 + while ib < len(b) and b[ib].isdigit(): + sb += b[ib] + ib += 1 + na = int(sa) if sa else 0 + nb = int(sb) if sb else 0 + if na != nb: + return (na > nb) - (na < nb) + return 0 + + +def deb_version_cmp(a: str, b: str) -> int: + """Debian-policy-ish version compare (subset used by check_versions.sh).""" + if a == b: + return 0 + ea, ua, da = _deb_split(a or "") + eb, ub, db = _deb_split(b or "") + if ea != eb: + return (ea > eb) - (ea < eb) + c = _deb_cmp_part(ua, ub) + if c: + return c + return _deb_cmp_part(da, db) + + +def normalize_build_version(v: str) -> str: + """Map Taler build_version toward deb upstream form for comparison. + + Examples: ``X.Y.Z-dev.1`` → ``X.Y.Z~dev1``; plain ``X.Y.Z`` stays. + """ + s = (v or "").strip() + if not s: + return "" + # API versions like 39:0:5 are not package versions + if re.fullmatch(r"\d+:\d+:\d+", s): + return "" + s = re.sub(r"-dev\.(\d+)", r"~dev\1", s, flags=re.I) + s = re.sub(r"-dev(\d+)", r"~dev\1", s, flags=re.I) + s = s.replace("-dev", "~dev") + return s + + +def parse_apt_packages_index(text: str) -> dict[str, str]: + """Return package → highest Version from a Packages file body.""" + pkgs: dict[str, str] = {} + cur = ver = None + for line in (text or "").splitlines(): + if line.startswith("Package: "): + if cur and ver and (cur not in pkgs or deb_version_cmp(ver, pkgs[cur]) > 0): + pkgs[cur] = ver + cur = line[9:].strip() + ver = None + elif line.startswith("Version: ") and cur: + ver = line[9:].strip() + elif line == "" and cur: + if ver and (cur not in pkgs or deb_version_cmp(ver, pkgs[cur]) > 0): + pkgs[cur] = ver + cur = ver = None + if cur and ver and (cur not in pkgs or deb_version_cmp(ver, pkgs[cur]) > 0): + pkgs[cur] = ver + return pkgs + + +def fetch_published_taler_packages( + *, + suite: str | None = None, + apt_base: str | None = None, + cache_s: float = 3600.0, +) -> dict[str, Any]: + """Fetch/cached deb.taler.net Packages for published Taler versions.""" + if not PUBLISHED_ENABLED: + return {"enabled": False, "suite": suite or PUBLISHED_SUITE, "packages": {}, "apps": []} + suite = suite or PUBLISHED_SUITE + apt_base = (apt_base or PUBLISHED_APT_BASE).rstrip("/") + now = time.time() + if ( + _PUBLISHED_CACHE.get("pkgs") + and _PUBLISHED_CACHE.get("suite") == suite + and now - float(_PUBLISHED_CACHE.get("ts") or 0) < cache_s + ): + return { + "enabled": True, + "suite": suite, + "packages": dict(_PUBLISHED_CACHE["pkgs"]), + "apps": list(_PUBLISHED_CACHE.get("apps") or []), + "error": _PUBLISHED_CACHE.get("error"), + "cached": True, + } + url = f"{apt_base}/dists/{suite}/main/binary-amd64/Packages" + # Full Packages index is >>64KiB (probe_http truncates); fetch with a larger cap. + pkgs: dict[str, str] = {} + err = None + try: + req = urllib.request.Request(url, method="GET", headers={"User-Agent": "taler-monitord/1"}) + with urllib.request.urlopen(req, timeout=20, context=_ssl_context()) as resp: + body = resp.read(8_000_000) + code = getattr(resp, "status", None) or resp.getcode() + if 200 <= int(code) < 400 and body: + text = body.decode("utf-8", "replace") + pkgs = parse_apt_packages_index(text) + else: + err = f"http_{code}" + except Exception as e: + err = f"{type(e).__name__}:{e}" + # fallback tiny probe (may be incomplete) + raw = probe_http(url, timeout=max(8.0, PROBE_TIMEOUT_S)) + if raw.get("ok") and raw.get("body"): + body = raw["body"] + text = body.decode("utf-8", "replace") if isinstance(body, bytes) else str(body) + pkgs = parse_apt_packages_index(text) + apps = [] + for app in _APP_CATALOG: + pub = pkgs.get(app["package"]) or "" + apps.append( + { + **app, + "published": pub, + "published_upstream": _deb_split(pub)[1] if pub else "", + "light": "green" if pub else "yellow", + "status": "published" if pub else "missing", + } + ) + # Wallet tarball from GNU FTP (best-effort latest numbered) + try: + ftp = probe_http("https://ftp.gnu.org/gnu/taler/", timeout=8.0) + if ftp.get("ok") and ftp.get("body"): + html = ftp["body"].decode("utf-8", "replace") if isinstance(ftp["body"], bytes) else str(ftp["body"]) + vers = re.findall(r"taler-wallet-([0-9]+\.[0-9]+(?:\.[0-9]+)?)\.tar\.gz", html) + if vers: + latest = sorted(vers, key=lambda v: _ver_tuple(v))[-1] + apps.append( + { + "package": "taler-wallet", + "label": "Wallet (GNU FTP)", + "kind": "wallet", + "logo": "wallet", + "published": latest, + "published_upstream": latest, + "light": "green", + "status": "published", + "source": "ftp.gnu.org", + } + ) + except Exception: + pass + _PUBLISHED_CACHE.update({"ts": now, "pkgs": pkgs, "apps": apps, "suite": suite, "error": err}) + return { + "enabled": True, + "suite": suite, + "packages": pkgs, + "apps": apps, + "error": err, + "cached": False, + "url": url, + } + + +def soft_to_deb_names(item: dict[str, Any]) -> list[str]: + pkg = str(item.get("package") or "") + label = str(item.get("label") or "").lower() + if pkg == "taler-bank" and "integration" in label: + return [] # API version only, not a deb package build + return list(_SOFT_TO_DEB.get(pkg) or ()) + + +def version_currency( + installed: str, + published_deb: str, +) -> dict[str, Any]: + """Compare installed build_version to published deb version; estimate lag.""" + inst_n = normalize_build_version(installed) + pub_up = _deb_split(published_deb)[1] if published_deb else "" + if not inst_n or not pub_up: + return { + "status": "unknown", + "light": "blue", + "published": published_deb, + "published_upstream": pub_up, + "behind": None, + "delta": None, + } + # Compare upstream-like strings via deb rules + cmp = deb_version_cmp(inst_n, pub_up) + if cmp == 0: + return { + "status": "current", + "light": "green", + "published": published_deb, + "published_upstream": pub_up, + "behind": None, + "delta": 0, + } + if cmp > 0: + return { + "status": "ahead", + "light": "blue", + "published": published_deb, + "published_upstream": pub_up, + "behind": None, + "delta": None, + } + # behind — try numeric distance on dotted prefix + ti, tp = _ver_tuple(inst_n.replace("~", ".")), _ver_tuple(pub_up.replace("~", ".")) + delta = None + if ti and tp and len(ti) >= 2 and len(tp) >= 2: + # rough component-wise lag on last differing field + n = max(len(ti), len(tp)) + ti = ti + (0,) * (n - len(ti)) + tp = tp + (0,) * (n - len(tp)) + for i in range(n): + if tp[i] > ti[i]: + delta = int(tp[i] - ti[i]) + break + return { + "status": "behind", + "light": "orange" if (delta is None or delta > 0) else "yellow", + "published": published_deb, + "published_upstream": pub_up, + "behind": pub_up, + "delta": delta, + } + + +def apply_published_currency( + items: list[dict[str, Any]], + published: dict[str, Any] | None, +) -> list[dict[str, Any]]: + """Attach published/current/behind fields; raise light when behind.""" + pkgs = (published or {}).get("packages") or {} + out = [] + for it in items: + row = dict(it) + deb_names = soft_to_deb_names(row) + best = None + for dn in deb_names: + pub = pkgs.get(dn) or "" + if not pub: + continue + cur = version_currency(str(row.get("version") or ""), pub) + cur["deb_package"] = dn + if best is None or ( + cur.get("status") == "behind" + and best.get("status") != "behind" + ): + best = cur + elif cur.get("status") == "behind" and best.get("status") == "behind": + # keep higher published upstream + if deb_version_cmp(str(cur.get("published") or ""), str(best.get("published") or "")) > 0: + best = cur + if best: + row.update( + { + "published": best.get("published"), + "published_upstream": best.get("published_upstream"), + "currency_status": best.get("status"), + "behind": best.get("behind"), + "delta": best.get("delta"), + "deb_package": best.get("deb_package"), + } + ) + # CVE colour wins if worse; else use currency light + if best.get("status") == "behind": + if SEVERITY.get(str(best.get("light")), 0) > SEVERITY.get(str(row.get("light") or "green"), 0): + row["light"] = best.get("light") + elif best.get("status") == "current" and row.get("light") in ("green", "blue", None): + row["light"] = "green" + else: + row.setdefault("currency_status", "n/a") + out.append(row) + return out + + +def colorize_software( + items: list[dict[str, Any]], + vulns: dict[str, Any] | None = None, + published: dict[str, Any] | None = None, +) -> list[dict[str, Any]]: + """Dedupe + CVE colour + published currency (current/behind/ahead).""" + hits = (vulns or {}).get("hits") or [] + by_pkg_ver: dict[str, str] = {} + for h in hits: + pkg = str(h.get("package") or "") + ver = str(h.get("version") or "") + sev = str(h.get("severity") or "") + key = f"{pkg}|{ver}" + light = _sev_to_light(sev) + if key not in by_pkg_ver or SEVERITY.get(light, 0) > SEVERITY.get(by_pkg_ver[key], 0): + by_pkg_ver[key] = light + out: list[dict[str, Any]] = [] + seen: set[str] = set() + for it in items: + pkg = str(it.get("package") or "") + ver = str(it.get("version") or "") + host = str(it.get("host") or "") + key = f"{pkg}|{ver}|{host}|{it.get('source')}" + if not pkg or key in seen: + continue + seen.add(key) + light = by_pkg_ver.get(f"{pkg}|{ver}") or ("green" if ver else "blue") + out.append({**it, "light": light}) + out = apply_published_currency(out, published) + order = { + "taler-exchange": 0, + "taler-merchant": 1, + "taler-bank": 2, + "taler": 3, + "nginx": 10, + "apache": 11, + "caddy": 12, + "postfix": 20, + "mta": 21, + } + out.sort(key=lambda x: (order.get(str(x.get("package")), 50), str(x.get("package")), str(x.get("host")))) + return out + + +def collect_software_from_probes( + *, + probe_cache: dict[str, dict[str, Any]], + members: list[dict[str, Any]], + vulns: dict[str, Any] | None = None, + mail_hosts: list[str] | None = None, + smtp: bool = False, + published: dict[str, Any] | None = None, +) -> list[dict[str, Any]]: + """Assemble software inventory for one card from HTTP/Taler/SMTP signals.""" + items: list[dict[str, Any]] = [] + for url, raw in (probe_cache or {}).items(): + host = _host_only(url) + srv = str((raw or {}).get("server") or "") + items.extend(software_from_server_header(srv, host=host)) + # Taler /config or /keys-adjacent config bodies + path = url.split(host, 1)[-1] if host and host in url else "" + if path.endswith("/config") or "/taler-integration/config" in path: + role = "" + for m in members or []: + if _host_only(str(m.get("host") or "")) == host: + role = str(m.get("role") or "") + break + items.extend(software_from_taler_config((raw or {}).get("body"), host=host, role=role)) + if smtp: + for h in mail_hosts or []: + items.extend(smtp_banner_software(h)) + return colorize_software(items, vulns, published=published) + + +def _ver_tuple(s: str) -> tuple[int, ...]: + nums = [int(x) for x in re.findall(r"\d+", s or "")[:5]] + return tuple(nums) if nums else () + + +def _vt_cmp(a: tuple[int, ...], b: tuple[int, ...]) -> int: + n = max(len(a), len(b)) + a = a + (0,) * (n - len(a)) + b = b + (0,) * (n - len(b)) + return (a > b) - (a < b) + + +def _in_semver_events(ver: str, events: list) -> bool: + vt = _ver_tuple(ver) + if not vt: + return False + intro = None + for e in events or []: + if "introduced" in e: + intro = e.get("introduced") + elif "fixed" in e or "last_affected" in e: + fixed = e.get("fixed") + last = e.get("last_affected") + lo = _ver_tuple("0" if intro in (None, "0") else str(intro)) + if _vt_cmp(vt, lo) < 0: + intro = None + continue + if fixed is not None: + if _vt_cmp(vt, _ver_tuple(str(fixed))) < 0: + return True + elif last is not None: + if _vt_cmp(vt, _ver_tuple(str(last))) <= 0: + return True + intro = None + return False + + +_OSV_SEED = { + "nginx": [ + "CVE-2025-23419", + "CVE-2025-53859", + "CVE-2024-7347", + "CVE-2024-34161", + "CVE-2024-32760", + "CVE-2024-31079", + "CVE-2024-24989", + "CVE-2024-24990", + ], + "apache": [ + "CVE-2024-38474", + "CVE-2024-38476", + "CVE-2024-38477", + "CVE-2023-31122", + "CVE-2023-43622", + ], + "caddy": ["CVE-2022-29718", "CVE-2023-50463"], +} + + +def _osv_get(vid: str) -> dict[str, Any]: + try: + with urllib.request.urlopen(f"https://api.osv.dev/v1/vulns/{vid}", timeout=12) as r: + return json.loads(r.read().decode("utf-8", "replace")) + except Exception: + return {} + + +def _severity_rank(sev: str) -> int: + s = (sev or "").upper() + if s == "CRITICAL": + return 3 + if s == "HIGH": + return 2 + if s == "MEDIUM": + return 1 + return 0 + + +def osv_hits_for_server(pkg: str, ver: str, *, cache: dict[str, list] | None = None) -> list[dict[str, Any]]: + """Actionable OSV hits for a bare upstream Server header version (no Debian rev). + + Same policy as check_surface.sh: use SEMVER extracted_events on known CVE seeds + — avoids Debian ecosystem false positives on bare versions. + """ + if not pkg or not ver: + return [] + cache = cache if cache is not None else {} + key = f"{pkg}|{ver}" + if key in cache: + return list(cache[key]) + hits: list[dict[str, Any]] = [] + for vid in _OSV_SEED.get(pkg, []): + doc = _osv_get(vid) + if not doc: + continue + ok_hit = False + for a in doc.get("affected") or []: + for rg in a.get("ranges") or []: + db = rg.get("database_specific") or {} + extracted = db.get("extracted_events") or [] + if extracted and _in_semver_events(ver, extracted): + blob = json.dumps(a).lower() + if pkg == "nginx" and "nginx" not in blob and "f5" not in blob: + continue + ok_hit = True + break + if rg.get("type") == "SEMVER" and _in_semver_events(ver, rg.get("events") or []): + ok_hit = True + break + if ok_hit: + break + if not ok_hit: + continue + sev = "" + for s in doc.get("severity") or []: + if isinstance(s, dict) and s.get("type") in ("CVSS_V3", "CVSS_V4", "CVSS_V2"): + # score-based + score = s.get("score") + try: + sc = float(str(score).split("/")[0]) if score else 0.0 + except ValueError: + sc = 0.0 + if sc >= 9.0: + sev = "CRITICAL" + elif sc >= 7.0: + sev = "HIGH" + elif sc >= 4.0: + sev = "MEDIUM" + break + dbsev = ((doc.get("database_specific") or {}).get("severity") or "").upper() + if dbsev in ("CRITICAL", "HIGH", "MEDIUM", "LOW") and _severity_rank(dbsev) > _severity_rank(sev): + sev = dbsev + if not sev: + sev = "HIGH" # known actionable seed hit without score → treat serious + hits.append( + { + "id": vid, + "severity": sev, + "package": pkg, + "version": ver, + "summary": str(doc.get("summary") or "")[:160], + } + ) + # keep critical/high only for box colour; still return medium for display + cache[key] = hits + return list(hits) + + +def assess_external_vulns( + *, + probe_cache: dict[str, dict[str, Any]], + members: list[dict[str, Any]], + osv_cache: dict[str, list] | None = None, +) -> dict[str, Any]: + """Scan outside-visible Server headers from probe cache; critical/high → box colour.""" + if not CVE_PROBES_ENABLED: + return {"enabled": False, "hits": [], "worst": "", "light": None, "servers": []} + seen_srv: dict[str, str] = {} + # Prefer apex/config responses already in cache + for url, raw in (probe_cache or {}).items(): + srv = str((raw or {}).get("server") or "").strip() + if not srv: + continue + host = _host_only(url) + if host and host not in seen_srv: + seen_srv[host] = srv + # Also ensure we looked at each member host apex once + for m in members or []: + host = _host_only(str(m.get("host") or "")) + if not host or host.upper() == "IBAN" or host in seen_srv: + continue + url = f"https://{host}/" + if url not in probe_cache: + probe_cache[url] = probe_http(url) + srv = str((probe_cache[url] or {}).get("server") or "").strip() + if srv: + seen_srv[host] = srv + + servers_out = [] + all_hits: list[dict[str, Any]] = [] + osv_cache = osv_cache if osv_cache is not None else {} + for host, srv in sorted(seen_srv.items()): + pkg = guess_http_package(srv) + ver = extract_http_version(srv) + entry = {"host": host, "server": srv, "package": pkg, "version": ver, "hits": []} + if pkg and ver: + hits = osv_hits_for_server(pkg, ver, cache=osv_cache) + for h in hits: + h2 = {**h, "host": host, "server": srv} + entry["hits"].append(h2) + all_hits.append(h2) + servers_out.append(entry) + + worst = "" + light = None + for h in all_hits: + sev = h.get("severity") or "" + if _severity_rank(sev) > _severity_rank(worst): + worst = sev + if worst == "CRITICAL": + light = "red" + elif worst == "HIGH": + light = "orange" + elif worst == "MEDIUM": + light = "yellow" + return { + "enabled": True, + "hits": all_hits[:20], + "worst": worst, + "light": light, + "servers": servers_out, + } + + +def apply_vuln_to_stack( + light: str, + intensity: str, + members: list[dict[str, Any]], + vulns: dict[str, Any], +) -> tuple[str, str, list[dict[str, Any]]]: + """Raise stack colour for critical/high outside-visible CVEs; add quiet member row.""" + vlight = vulns.get("light") + hits = vulns.get("hits") or [] + if not vlight or not hits: + return light, intensity, members + # synthetic member so UI/aggregation stay consistent + top = hits[0] + reason = f"{top.get('id')} {top.get('severity')} ({top.get('package')}/{top.get('version')})" + members = list(members) + [ + { + "role": "cve", + "host": top.get("host") or "edge", + "ok": False, + "light": vlight, + "reason": reason, + "down_since": None, + "code": 0, + "elapsed": 0.0, + "error": None, + "url": None, + "endpoints": [], + "endpoints_summary": {"n": 0, "ok": 0, "soft": 0, "bad": 0, "skipped": 0, "light": "green", "dots": []}, + } + ] + # re-aggregate including CVE member + light2, intensity2 = aggregate_stack(members) + return light2, intensity2, members + + +def classify_probe( + result: dict[str, Any], + *, + prev: dict[str, Any] | None = None, + now: float | None = None, + acute_s: float = ACUTE_DOWN_S, + soft_s: float = 2.5, +) -> dict[str, Any]: + """Map probe result + previous down_since → light/reason.""" + now = time.time() if now is None else now + prev = prev or {} + ok = bool(result.get("ok")) + code = int(result.get("code") or 0) + elapsed = float(result.get("elapsed") or 0) + error = result.get("error") + down_since = prev.get("down_since") + + if ok: + down_since = None + if elapsed >= soft_s: + light, reason = "yellow", "slow" + else: + light, reason = "green", "ok" + else: + # Reachable but bad status → degraded rather than offline + if code >= 400: + down_since = None + if code in (401, 403): + light, reason = "blue", "auth_present" + elif code >= 500: + light, reason = "orange", f"http_{code}" + else: + light, reason = "yellow", f"http_{code}" + else: + prev_ok = prev.get("ok") + try: + inherited = float(prev["down_since"]) if prev.get("down_since") is not None else None + except (TypeError, ValueError): + inherited = None + if prev_ok or inherited is None: + down_since = now + else: + down_since = inherited + age = now - float(down_since) + err_l = str(error or "").lower() + # Timeouts / reset often mean DoS or broken path — orange if still "fresh" + if age < acute_s and ("timeout" in err_l or "timedout" in err_l or "reset" in err_l): + light, reason = "orange", "hard_degraded" + elif age < acute_s: + light, reason = "red", "down_acute" + else: + light, reason = "black", "down_long" + + return { + "ok": ok, + "light": light, + "reason": reason, + "down_since": down_since, + "code": code, + "elapsed": round(elapsed, 3), + "error": error, + "url": result.get("url"), + "ts": now, + } + + +def _cap_light(light: str, max_light: str) -> str: + if SEVERITY.get(light, 5) > SEVERITY.get(max_light, 5): + return max_light + return light + + +def summarize_endpoints(endpoints: list[dict[str, Any]]) -> dict[str, Any]: + """Compact rollup for UI: counts + worst light (dots = probed only, not skips).""" + if not endpoints: + return {"n": 0, "ok": 0, "soft": 0, "bad": 0, "skipped": 0, "light": "green", "dots": []} + dots: list[str] = [] + ok = soft = bad = skipped = 0 + worst = "green" + for ep in endpoints: + if ep.get("skipped"): + skipped += 1 + continue + L = ep.get("light") or "black" + dots.append(L) + if SEVERITY.get(L, 5) > SEVERITY.get(worst, 0): + worst = L + if SEVERITY.get(L, 5) <= SEVERITY["blue"]: + ok += 1 + elif SEVERITY.get(L, 5) <= SEVERITY["yellow"]: + soft += 1 + else: + bad += 1 + return { + "n": len(endpoints), + "ok": ok, + "soft": soft, + "bad": bad, + "skipped": skipped, + "light": worst if dots else "green", + "dots": dots, + } + + +def classify_endpoint_probe( + raw: dict[str, Any], + *, + expect: tuple[int, ...], + severity: str = "soft", + hard: bool | None = None, + prev: dict[str, Any] | None = None, + now: float | None = None, + acute_s: float = ACUTE_DOWN_S, +) -> dict[str, Any]: + """Map HTTP probe to light. + + soft → fail capped at yellow + medium → missing/wrong is at least orange (mittelschwer), not red for 4xx + hard → full classify_probe severity + """ + if hard is not None and not severity: + severity = "hard" if hard else "soft" + sev = (severity or "soft").lower() + code = int(raw.get("code") or 0) + if code in expect: + c = classify_probe(raw, prev=prev, now=now, acute_s=acute_s) + return {**c, "expected": True, "severity": sev} + fake = {**raw, "ok": False} + if code >= 400: + fake["error"] = f"http_{code}" + elif not fake.get("error"): + fake["error"] = "unreachable" + c = classify_probe(fake, prev=prev, now=now, acute_s=acute_s) + reason = c.get("reason") or (f"http_{code}" if code else "unreachable") + if sev == "soft": + c = { + **c, + "light": _cap_light(c.get("light") or "yellow", ENDPOINT_SOFT_MAX), + "reason": reason, + } + elif sev == "medium": + # At least orange; keep 5xx/unreachable from going black via acute age — still medium. + light = c.get("light") or "orange" + if SEVERITY.get(light, 0) < SEVERITY["orange"]: + light = "orange" + elif SEVERITY.get(light, 0) > SEVERITY["orange"]: + light = "orange" + c = {**c, "light": light, "reason": reason, "down_since": None} + # hard: keep classify_probe result + return {**c, "expected": False, "severity": sev} + + +def probe_role_endpoints( + role: str, + host: str, + *, + has_intro: bool = False, + landing: bool | None = None, + legal_severity: str = "soft", + probe_fn: Callable[[str], dict[str, Any]], + cache: dict[str, dict[str, Any]] | None = None, + prev_by_path: dict[str, dict[str, Any]] | None = None, + now: float | None = None, + acute_s: float = ACUTE_DOWN_S, +) -> list[dict[str, Any]]: + """Probe suite-like public endpoints for one machine; return compact rows.""" + host = _host_only(host) + if not host or host.upper() == "IBAN": + return [] + # Compat: older callers passed landing= + if landing is not None: + has_intro = bool(landing) + catalog = ROLE_ENDPOINTS.get(role) or [] + cache = cache if cache is not None else {} + prev_by_path = prev_by_path or {} + now = time.time() if now is None else now + out: list[dict[str, Any]] = [] + for spec in catalog: + if spec.get("need_intro") and not has_intro: + out.append( + { + "id": spec["id"], + "path": spec["path"], + "skipped": True, + "light": "blue", + "reason": "no_intro", + "code": 0, + "elapsed": 0.0, + } + ) + continue + url = f"https://{host}{spec['path']}" + if url in cache: + raw = cache[url] + else: + raw = probe_fn(url) + cache[url] = raw + sev = str(spec.get("severity") or ("hard" if spec.get("hard") else "soft")) + if spec.get("legal"): + sev = legal_severity or sev + classified = classify_endpoint_probe( + raw, + expect=tuple(spec.get("expect") or (200,)), + severity=sev, + prev=prev_by_path.get(spec["path"]), + now=now, + acute_s=acute_s, + ) + out.append( + { + "id": spec["id"], + "path": spec["path"], + "url": url, + "skipped": False, + "light": classified.get("light") or "black", + "reason": classified.get("reason") or "", + "code": classified.get("code") or 0, + "elapsed": classified.get("elapsed") or 0.0, + "ok": bool(classified.get("ok")) and bool(classified.get("expected")), + "severity": sev, + # medium/hard pull machine colour past soft yellow + "hard": sev in ("hard", "medium"), + } + ) + return out + + +def merge_member_endpoint_light(member: dict[str, Any], endpoints: list[dict[str, Any]]) -> dict[str, Any]: + """Attach endpoints + summary; raise member light to worst *hard* endpoint issue.""" + summary = summarize_endpoints(endpoints) + member = {**member, "endpoints": endpoints, "endpoints_summary": summary} + # Informational members stay as-is + if (member.get("reason") or "") in _AGG_SKIP_REASONS: + return member + base = member.get("light") or "black" + # Only hard endpoint failures pull the machine light past yellow via soft cap already applied + worst = base + for ep in endpoints: + if ep.get("skipped"): + continue + L = ep.get("light") or "black" + if ep.get("hard") is False and SEVERITY.get(L, 0) <= SEVERITY["yellow"]: + # soft noise shouldn't turn machine red if primary is green — still show yellow if soft warn + if SEVERITY.get(L, 0) > SEVERITY.get(worst, 0) and SEVERITY.get(L, 0) <= SEVERITY["yellow"]: + worst = L + continue + if SEVERITY.get(L, 0) > SEVERITY.get(worst, 0): + worst = L + member["light"] = worst + return member + + +def score_light_from_ratio(ratio: float, *, all_black: bool = False) -> str: + """Map green/total ratio to overall traffic light (incl. white = gone).""" + if ratio <= 0: + return "black" if all_black else "white" + if ratio >= SCORE_RATIO_GREEN: + return "green" + if ratio >= SCORE_RATIO_BLUE: + return "blue" + if ratio >= SCORE_RATIO_YELLOW: + return "yellow" + if ratio >= SCORE_RATIO_ORANGE: + return "orange" + if ratio >= SCORE_RATIO_RED: + return "red" + return "white" + + +def compute_total_score( + stacks: list[dict[str, Any]], + services: list[dict[str, Any]] | None = None, + mail: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Page-level score: how many boxes are green out of all scored boxes. + + One point = one stack / service / mail card. Strict green only (blue ≠ green). + """ + boxes: list[dict[str, Any]] = [] + for group in (stacks or [], services or [], mail or []): + boxes.extend(group) + total = len(boxes) + by_light: dict[str, int] = {} + for b in boxes: + Lgt = str(b.get("light") or "black") + by_light[Lgt] = by_light.get(Lgt, 0) + 1 + green = int(by_light.get("green") or 0) + ratio = (green / total) if total else 0.0 + all_black = total > 0 and int(by_light.get("black") or 0) == total + light = "white" if total == 0 else score_light_from_ratio(ratio, all_black=all_black) + return { + "green": green, + "total": total, + "ratio": round(ratio, 4), + "pct": int(round(ratio * 100)) if total else 0, + "light": light, + "by_light": by_light, + "thresholds": { + "green": SCORE_RATIO_GREEN, + "blue": SCORE_RATIO_BLUE, + "yellow": SCORE_RATIO_YELLOW, + "orange": SCORE_RATIO_ORANGE, + "red": SCORE_RATIO_RED, + }, + } + + +def aggregate_breakdown(members: list[dict[str, Any]]) -> dict[str, Any]: + """Counts + parts so UI can explain e.g. 4× OK, 3× Soft → Degraded.""" + counted = [m for m in members if (m.get("reason") or "") not in _AGG_SKIP_REASONS] + if not counted: + counted = list(members) + by_light: dict[str, int] = {} + parts: list[dict[str, Any]] = [] + for m in counted: + Lgt = str(m.get("light") or "black") + by_light[Lgt] = by_light.get(Lgt, 0) + 1 + parts.append( + { + "role": m.get("role"), + "host": m.get("host"), + "light": Lgt, + "reason": m.get("reason"), + } + ) + worst = "black" + if by_light: + worst = LIGHTS[max(SEVERITY.get(k, 5) for k in by_light)] + return { + "by_light": by_light, + "worst": worst, + "counted": len(counted), + "parts": parts, + } + + +def aggregate_stack(members: list[dict[str, Any]]) -> tuple[str, str]: + """Return (light, intensity) for a stack box. + + Logical color = weakest *expected* member. Intensity muted unless every + counted member is at worst severity and that severity is red/black (full). + Informational rows (e.g. ebics_no_bank) are shown but ignored for the box. + """ + counted = [m for m in members if (m.get("reason") or "") not in _AGG_SKIP_REASONS] + if not counted: + counted = list(members) + if not counted: + return "black", "full" + lights = [m.get("light") or "black" for m in counted] + worst_rank = max(SEVERITY.get(L, 5) for L in lights) + worst = LIGHTS[worst_rank] + n = len(counted) + k_worst = sum(1 for L in lights if SEVERITY.get(L, 5) == worst_rank) + # green/blue = healthy or expected-present (e.g. static_pending / no_prod_yet) + if worst_rank <= SEVERITY["blue"]: + return worst, "ok" + if worst_rank >= SEVERITY["red"] and k_worst == n: + return worst, "full" + return worst, "muted" + + +def _dig_lines(rrtype: str, name: str) -> list[str]: + """Best-effort dig +short; empty if dig missing/fails.""" + try: + r = subprocess.run( + ["dig", "+short", "+time=2", "+tries=1", rrtype, name], + capture_output=True, + timeout=6, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + return [] + out = (r.stdout or b"").decode("utf-8", "replace") + return [ln.strip().rstrip(".") for ln in out.splitlines() if ln.strip()] + + +def dns_a_aaaa(host: str) -> dict[str, list[str]]: + h = _host_only(host) + a = [x for x in _dig_lines("A", h) if re.match(r"^\d+\.\d+\.\d+\.\d+$", x)] + aaaa = [x for x in _dig_lines("AAAA", h) if ":" in x] + return {"a": a, "aaaa": aaaa} + + +def dns_mx(domain: str) -> list[str]: + lines = _dig_lines("MX", domain) + hosts: list[str] = [] + for ln in lines: + parts = ln.split() + if len(parts) >= 2: + hosts.append(parts[-1].lower().rstrip(".")) + elif parts: + hosts.append(parts[0].lower().rstrip(".")) + return hosts + + +def dns_txt(name: str) -> str: + return " ".join(_dig_lines("TXT", name)).replace('"', "") + + +def dns_ptr(ip: str) -> list[str]: + """Reverse DNS via dig -x.""" + try: + r = subprocess.run( + ["dig", "+short", "+time=2", "+tries=1", "-x", ip], + capture_output=True, + timeout=6, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + return [] + out = (r.stdout or b"").decode("utf-8", "replace") + return [ln.strip().rstrip(".") for ln in out.splitlines() if ln.strip()] + + +def probe_tls_cert(host: str, *, timeout: float = PROBE_TIMEOUT_S) -> dict[str, Any]: + """Fetch peer cert notAfter + SANs (outside-in).""" + h = _host_only(host) + if not h or h.upper() == "IBAN": + return {"ok": False, "host": h, "error": "no_host"} + ctx = ssl.create_default_context() + if not TLS_VERIFY: + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + try: + with socket.create_connection((h, 443), timeout=timeout) as sock: + with ctx.wrap_socket(sock, server_hostname=h) as ssock: + cert = ssock.getpeercert() + except OSError as e: + return {"ok": False, "host": h, "error": type(e).__name__} + if not cert: + # CERT_NONE often yields empty dict — retry binary DER via getpeercert(True) + try: + with socket.create_connection((h, 443), timeout=timeout) as sock: + with ctx.wrap_socket(sock, server_hostname=h) as ssock: + der = ssock.getpeercert(binary_form=True) + if not der: + return {"ok": False, "host": h, "error": "no_cert"} + # Parse notAfter via openssl if available + return _parse_der_cert(h, der) + except OSError as e: + return {"ok": False, "host": h, "error": type(e).__name__} + return _cert_from_dict(h, cert) + + +def _cert_from_dict(host: str, cert: dict[str, Any]) -> dict[str, Any]: + not_after = cert.get("notAfter") or "" + days_left = None + try: + # OpenSSL ASN.1: 'Jun 15 12:00:00 2027 GMT' + exp = time.strptime(str(not_after), "%b %d %H:%M:%S %Y %Z") + days_left = int((time.mktime(exp) - time.time()) / 86400) + except (TypeError, ValueError, OverflowError, OSError): + days_left = None + sans: list[str] = [] + for typ, val in cert.get("subjectAltName") or []: + if typ.lower() == "dns" and val and val not in sans: + sans.append(str(val).lower()) + light = "green" + reason = "cert_ok" + if days_left is None: + light, reason = "yellow", "cert_parse" + elif days_left < 0: + light, reason = "red", "cert_expired" + elif days_left < CERT_CRIT_DAYS: + light, reason = "orange", "cert_expiring" + elif days_left < CERT_WARN_DAYS: + light, reason = "yellow", "cert_warn" + return { + "ok": days_left is not None and days_left >= 0, + "host": host, + "not_after": not_after, + "days_left": days_left, + "sans": sans[:32], + "light": light, + "reason": reason, + "error": None, + } + + +def _parse_der_cert(host: str, der: bytes) -> dict[str, Any]: + """Fallback: openssl x509 -noout -dates -ext subjectAltName.""" + try: + r = subprocess.run( + ["openssl", "x509", "-inform", "DER", "-noout", "-enddate", "-ext", "subjectAltName"], + input=der, + capture_output=True, + timeout=6, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + return {"ok": False, "host": host, "error": "openssl_fail"} + text = (r.stdout or b"").decode("utf-8", "replace") + not_after = "" + for ln in text.splitlines(): + if ln.startswith("notAfter="): + not_after = ln.split("=", 1)[1].strip() + sans: list[str] = [] + for m in re.finditer(r"DNS:([A-Za-z0-9_.*-]+)", text): + v = m.group(1).lower() + if v not in sans: + sans.append(v) + fake = {"notAfter": not_after, "subjectAltName": [("DNS", s) for s in sans]} + out = _cert_from_dict(host, fake) + if out.get("days_left") is None and not_after: + out["error"] = "cert_parse" + return out + + +def parse_services_conf(path: Path | None = None) -> list[dict[str, Any]]: + """Company/service hosts (non-Taler): id host label zone.""" + path = path or default_services_conf() + rows: list[dict[str, Any]] = [] + if not path.is_file(): + return rows + seen: set[str] = set() + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.split("#", 1)[0].strip() + if not line: + continue + parts = line.split() + if len(parts) < 2: + continue + sid, host = parts[0], _host_only(parts[1]) + label = parts[2] if len(parts) > 2 else "service" + zone = parts[3] if len(parts) > 3 else "public" + if not host or host in seen: + continue + seen.add(host) + rows.append( + { + "id": sid, + "host": host, + "service_label": label, + "zone": zone, + "kind": "service", + } + ) + return rows + + +def parse_mail_catalog(path: Path | None = None) -> list[dict[str, Any]]: + path = path or default_mail_catalog() + rows: list[dict[str, Any]] = [] + if not path.is_file(): + return rows + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.split("#", 1)[0].strip() + if not line: + continue + parts = line.split() + if len(parts) < 2: + continue + domain = parts[0] + expect_mx = [x.strip().lower() for x in parts[1].split(",") if x.strip()] + mail_hosts = ( + [x.strip().lower() for x in parts[2].split(",") if x.strip()] + if len(parts) > 2 + else list(expect_mx) + ) + smtp_ports = [ + p.strip() + for p in (parts[3] if len(parts) > 3 else "25,465,587").split(",") + if p.strip() + ] + imap_ports = [ + p.strip() + for p in (parts[4] if len(parts) > 4 else "143,993").split(",") + if p.strip() + ] + label = parts[5] if len(parts) > 5 else domain + rows.append( + { + "domain": domain, + "expect_mx": expect_mx, + "mail_hosts": mail_hosts, + "smtp_ports": smtp_ports, + "imap_ports": imap_ports, + "label": label, + } + ) + return rows + + +def tcp_port_open(host: str, port: int | str, *, timeout_s: float = MAIL_PORT_TIMEOUT_S) -> bool: + try: + with socket.create_connection((_host_only(host), int(port)), timeout=max(1.0, float(timeout_s))): + return True + except (OSError, ValueError, TypeError): + return False + + +def git_probe_state_path() -> Path: + return monitord_state_dir() / "git-probes.json" + + +def deliverability_state_path() -> Path: + return monitord_state_dir() / "mail-deliverability.json" + + +def load_deliverability_state() -> dict[str, Any]: + path = deliverability_state_path() + if not path.is_file(): + return {"schema": 1, "domains": {}} + try: + doc = json.loads(path.read_text(encoding="utf-8")) + if isinstance(doc, dict): + doc.setdefault("domains", {}) + return doc + except (OSError, json.JSONDecodeError): + pass + return {"schema": 1, "domains": {}} + + +def save_deliverability_state(doc: dict[str, Any]) -> None: + path = deliverability_state_path() + atomic_write_json = globals().get("atomic_write_json") + # atomic_write_json is defined later; fall back to simple write if importing early + try: + from typing import cast + + _ = cast(Any, None) + except Exception: + pass + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(f".{path.name}.tmp.{os.getpid()}") + text = json.dumps(doc, ensure_ascii=False, indent=2) + "\n" + try: + tmp.write_text(text, encoding="utf-8") + os.replace(tmp, path) + finally: + if tmp.exists(): + try: + tmp.unlink() + except OSError: + pass + + +def _smtp_read_reply(sock: socket.socket, timeout_s: float) -> tuple[int, str]: + sock.settimeout(timeout_s) + data = b"" + while True: + chunk = sock.recv(4096) + if not chunk: + break + data += chunk + text = data.decode("utf-8", "replace") + lines = text.splitlines() + if not lines: + continue + # multi-line: last line is "NNN " (space) not "NNN-" + last = lines[-1] + if len(last) >= 4 and last[0:3].isdigit() and last[3] == " ": + return int(last[0:3]), text.strip() + if len(data) > 65536: + break + text = data.decode("utf-8", "replace").strip() + code = int(text[:3]) if len(text) >= 3 and text[:3].isdigit() else 0 + return code, text + + +def smtp_deliverability_probe( + mx_host: str, + *, + domain: str, + mail_from: str | None = None, + rcpt_to: str | None = None, + timeout_s: float = DELIVER_TIMEOUT_S, +) -> dict[str, Any]: + """Outside-in SMTP deliverability smoke: banner → EHLO → MAIL FROM → RCPT TO → RSET/QUIT. + + Does not send DATA/body. Acceptance of RCPT for postmaster@domain (or configured + recipient) counts as deliverable. Soft-fail/greylist may still return 4xx. + """ + h = _host_only(mx_host) + mail_from = mail_from or os.environ.get("TALER_MONITORD_DELIVER_FROM", "monitor@taler-monitoring.invalid") + rcpt_to = rcpt_to or os.environ.get("TALER_MONITORD_DELIVER_RCPT") or f"postmaster@{domain}" + t0 = time.monotonic() + steps: list[str] = [] + try: + with socket.create_connection((h, 25), timeout=max(2.0, float(timeout_s))) as sock: + code, text = _smtp_read_reply(sock, timeout_s) + steps.append(f"banner:{code}") + if code != 220: + return { + "ok": False, + "light": "orange", + "reason": f"bad_banner:{code}", + "detail": text[:160], + "mx": h, + "rcpt": rcpt_to, + "elapsed": round(time.monotonic() - t0, 3), + "steps": steps, + } + sock.sendall(b"EHLO taler-monitoring.invalid\r\n") + code, text = _smtp_read_reply(sock, timeout_s) + steps.append(f"ehlo:{code}") + if code != 250: + return { + "ok": False, + "light": "orange", + "reason": f"ehlo_fail:{code}", + "detail": text[:160], + "mx": h, + "rcpt": rcpt_to, + "elapsed": round(time.monotonic() - t0, 3), + "steps": steps, + } + sock.sendall(f"MAIL FROM:<{mail_from}>\r\n".encode()) + code, text = _smtp_read_reply(sock, timeout_s) + steps.append(f"mail:{code}") + if code not in (250, 251): + return { + "ok": False, + "light": "orange", + "reason": f"mail_from_reject:{code}", + "detail": text[:160], + "mx": h, + "rcpt": rcpt_to, + "elapsed": round(time.monotonic() - t0, 3), + "steps": steps, + } + sock.sendall(f"RCPT TO:<{rcpt_to}>\r\n".encode()) + code, text = _smtp_read_reply(sock, timeout_s) + steps.append(f"rcpt:{code}") + try: + sock.sendall(b"RSET\r\n") + _smtp_read_reply(sock, min(3.0, timeout_s)) + sock.sendall(b"QUIT\r\n") + except OSError: + pass + elapsed = round(time.monotonic() - t0, 3) + if code in (250, 251): + return { + "ok": True, + "light": "green", + "reason": "rcpt_accepted", + "detail": text[:160], + "mx": h, + "rcpt": rcpt_to, + "elapsed": elapsed, + "steps": steps, + } + if 400 <= code < 500: + # greylist / temp fail — soft degrade + return { + "ok": False, + "light": "yellow", + "reason": f"rcpt_tempfail:{code}", + "detail": text[:160], + "mx": h, + "rcpt": rcpt_to, + "elapsed": elapsed, + "steps": steps, + } + return { + "ok": False, + "light": "orange", + "reason": f"rcpt_reject:{code}", + "detail": text[:160], + "mx": h, + "rcpt": rcpt_to, + "elapsed": elapsed, + "steps": steps, + } + except (socket.timeout, TimeoutError): + return { + "ok": False, + "light": "orange", + "reason": "smtp_timeout", + "detail": "", + "mx": h, + "rcpt": rcpt_to, + "elapsed": round(time.monotonic() - t0, 3), + "steps": steps, + } + except OSError as e: + return { + "ok": False, + "light": "red" if "refused" in str(e).lower() else "orange", + "reason": f"smtp_connect:{type(e).__name__}", + "detail": str(e)[:120], + "mx": h, + "rcpt": rcpt_to, + "elapsed": round(time.monotonic() - t0, 3), + "steps": steps, + } + + +def ensure_deliverability( + domain: str, + *, + mx_hosts: list[str], + now: float | None = None, + force: bool = False, + state: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Return deliverability status for domain; refresh on interval if due. + + Missing/stale results → yellow (no recent test). Send/server failures → orange/red. + """ + now = time.time() if now is None else now + if not DELIVER_ENABLED: + return { + "ok": True, + "light": "blue", + "reason": "deliver_disabled", + "ts": None, + "age_s": None, + "stale": False, + } + st = state if state is not None else load_deliverability_state() + domains = st.setdefault("domains", {}) + prev = domains.get(domain) if isinstance(domains.get(domain), dict) else {} + prev_ts = prev.get("ts") + try: + age = None if prev_ts is None else now - float(prev_ts) + except (TypeError, ValueError): + age = None + due = force or prev_ts is None or age is None or age >= DELIVER_INTERVAL_S + if due and mx_hosts: + mx = _host_only(mx_hosts[0]) + result = smtp_deliverability_probe(mx, domain=domain) + result["ts"] = now + result["domain"] = domain + domains[domain] = result + st["ts"] = now + try: + save_deliverability_state(st) + except OSError: + pass + prev = result + age = 0.0 + elif not prev: + # never ran and no MX to probe + return { + "ok": False, + "light": "yellow", + "reason": "deliver_never_run" if mx_hosts else "deliver_no_mx", + "ts": None, + "age_s": None, + "stale": True, + "mx": "", + "rcpt": f"postmaster@{domain}", + } + # Classify freshness + try: + age_s = None if prev.get("ts") is None else now - float(prev["ts"]) + except (TypeError, ValueError): + age_s = None + stale = age_s is None or age_s > DELIVER_STALE_S + light = str(prev.get("light") or "yellow") + reason = str(prev.get("reason") or "deliver_unknown") + ok = bool(prev.get("ok")) + if stale and ok: + light, reason, ok = "yellow", "deliver_stale", False + elif stale and not ok: + # keep failure colour but mark stale + if light in ("green", "blue"): + light = "yellow" + reason = f"deliver_stale:{reason}" + return { + "ok": ok and not stale, + "light": light, + "reason": reason, + "ts": prev.get("ts"), + "age_s": None if age_s is None else round(age_s, 1), + "stale": stale, + "mx": prev.get("mx") or "", + "rcpt": prev.get("rcpt") or "", + "detail": prev.get("detail") or "", + "elapsed": prev.get("elapsed"), + "steps": prev.get("steps") or [], + } + + +def assess_mail_domain( + row: dict[str, Any], + *, + port_done: set[str] | None = None, + deliver_state: dict[str, Any] | None = None, + now: float | None = None, +) -> dict[str, Any]: + """Outside-in MX/SPF/DMARC/A/AAAA/rDNS (+ ports + deliverability) for one catalog domain.""" + domain = row["domain"] + members: list[dict[str, Any]] = [] + port_done = port_done if port_done is not None else set() + now = time.time() if now is None else now + mx = dns_mx(domain) + expect = row.get("expect_mx") or [] + mx_ok = bool(mx) + mx_match = True + if expect and mx: + mx_match = any( + any(m == w or m.endswith("." + w) or w in m for m in mx) for w in expect + ) + elif expect and not mx: + mx_match = False + members.append( + { + "role": "mx", + "host": ",".join(mx[:3]) or domain, + "ok": mx_ok and mx_match, + "light": "green" if (mx_ok and mx_match) else ("orange" if mx_ok else "red"), + "reason": "mx_ok" if (mx_ok and mx_match) else ("mx_mismatch" if mx_ok else "mx_missing"), + "down_since": None, + "code": 0, + "elapsed": 0.0, + "error": None, + "url": None, + "endpoints": [], + "endpoints_summary": summarize_endpoints([]), + } + ) + spf = dns_txt(domain) + has_spf = "v=spf1" in spf.lower() + members.append( + { + "role": "spf", + "host": domain, + "ok": has_spf, + "light": "green" if has_spf else "yellow", + "reason": "spf_ok" if has_spf else "spf_missing", + "down_since": None, + "code": 0, + "elapsed": 0.0, + "error": None, + "url": None, + "endpoints": [], + "endpoints_summary": summarize_endpoints([]), + } + ) + dmarc = dns_txt(f"_dmarc.{domain}") + has_dmarc = "v=dmarc1" in dmarc.lower() + members.append( + { + "role": "dmarc", + "host": f"_dmarc.{domain}", + "ok": has_dmarc, + "light": "green" if has_dmarc else "yellow", + "reason": "dmarc_ok" if has_dmarc else "dmarc_missing", + "down_since": None, + "code": 0, + "elapsed": 0.0, + "error": None, + "url": None, + "endpoints": [], + "endpoints_summary": summarize_endpoints([]), + } + ) + # Per mail host: A/AAAA + PTR + for h in row.get("mail_hosts") or []: + addrs = dns_a_aaaa(h) + a, aaaa = addrs.get("a") or [], addrs.get("aaaa") or [] + if not a and not aaaa: + members.append( + { + "role": "dns", + "host": h, + "ok": False, + "light": "red", + "reason": "dns_missing", + "down_since": None, + "code": 0, + "elapsed": 0.0, + "error": None, + "url": None, + "endpoints": [], + "endpoints_summary": summarize_endpoints([]), + } + ) + continue + light = "green" + reason = "dns_ok" + if a and not aaaa: + light, reason = "yellow", "no_ipv6" + members.append( + { + "role": "dns", + "host": h, + "ok": True, + "light": light, + "reason": reason, + "down_since": None, + "code": 0, + "elapsed": 0.0, + "error": None, + "url": None, + "endpoints": [], + "endpoints_summary": summarize_endpoints([]), + "a": a[:4], + "aaaa": aaaa[:4], + } + ) + # rDNS on first A + if a: + ptrs = dns_ptr(a[0]) + ptr_ok = bool(ptrs) + members.append( + { + "role": "rdns", + "host": a[0], + "ok": ptr_ok, + "light": "green" if ptr_ok else "yellow", + "reason": ("rdns:" + ",".join(ptrs[:2])) if ptr_ok else "rdns_missing", + "down_since": None, + "code": 0, + "elapsed": 0.0, + "error": None, + "url": None, + "endpoints": [], + "endpoints_summary": summarize_endpoints([]), + } + ) + if MAIL_PORTS_ENABLED and h not in port_done: + port_done.add(h) + for p in row.get("smtp_ports") or []: + open_ok = tcp_port_open(h, p) + # 587 often filtered on pixel → yellow warn, not red + if open_ok: + light_p, reason_p = "green", f"smtp_{p}_open" + elif str(p) == "587": + light_p, reason_p = "yellow", f"smtp_{p}_closed" + else: + light_p, reason_p = "orange", f"smtp_{p}_closed" + members.append( + { + "role": "smtp", + "host": f"{h}:{p}", + "ok": open_ok, + "light": light_p, + "reason": reason_p, + "down_since": None, + "code": int(p) if str(p).isdigit() else 0, + "elapsed": 0.0, + "error": None, + "url": None, + "endpoints": [], + "endpoints_summary": summarize_endpoints([]), + } + ) + for p in row.get("imap_ports") or []: + open_ok = tcp_port_open(h, p) + members.append( + { + "role": "imap", + "host": f"{h}:{p}", + "ok": open_ok, + "light": "green" if open_ok else "orange", + "reason": f"imap_{p}_open" if open_ok else f"imap_{p}_closed", + "down_since": None, + "code": int(p) if str(p).isdigit() else 0, + "elapsed": 0.0, + "error": None, + "url": None, + "endpoints": [], + "endpoints_summary": summarize_endpoints([]), + } + ) + elif MAIL_PORTS_ENABLED and h in port_done: + members.append( + { + "role": "smtp", + "host": h, + "ok": True, + "light": "blue", + "reason": "ports_probed_earlier", + "down_since": None, + "code": 0, + "elapsed": 0.0, + "error": None, + "url": None, + "endpoints": [], + "endpoints_summary": summarize_endpoints([]), + } + ) + # Deliverability (regular, cached) — missing/stale → yellow; send fail → orange/red + mx_targets = list(mx) or list(row.get("mail_hosts") or []) + deliver = ensure_deliverability( + domain, mx_hosts=mx_targets, now=now, state=deliver_state + ) + members.append( + { + "role": "deliver", + "host": deliver.get("mx") or (mx_targets[0] if mx_targets else domain), + "ok": bool(deliver.get("ok")), + "light": deliver.get("light") or "yellow", + "reason": deliver.get("reason") or "deliver_unknown", + "down_since": None, + "code": 0, + "elapsed": float(deliver.get("elapsed") or 0.0), + "error": deliver.get("detail") or None, + "url": None, + "endpoints": [], + "endpoints_summary": summarize_endpoints([]), + } + ) + light, intensity = aggregate_stack(members) + software = colorize_software( + [ + item + for h in (row.get("mail_hosts") or [])[:2] + for item in smtp_banner_software(h) + ], + published=None, + ) + return { + "id": domain, + "currency": "", + "currency_info": None, + "kind": "mail", + "service_label": row.get("label") or domain, + "commercial_prod": False, + "bank_backend": "", + "wire_methods": [], + "local": False, + "light": light, + "intensity": intensity, + "members": members, + "stats": None, + "software": software, + "deliverability": deliver, + "network": {"zone": "mail", "label": "mail plane", "ssh": {"expected": False}}, + "vulns": {"enabled": False, "worst": "", "light": None, "hits": [], "servers": []}, + "breakdown": aggregate_breakdown(members), + "tls": {}, + "orphan": False, + } + + +def _run_git_ls_remote(url: str, *, timeout_s: float) -> dict[str, Any]: + """Soft git probe via `git ls-remote` (no checkout; no leftover worktree).""" + t0 = time.monotonic() + env = os.environ.copy() + env["GIT_TERMINAL_PROMPT"] = "0" + env["GIT_SSH_COMMAND"] = ( + "ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new " + "-o ConnectTimeout=5 -o IdentitiesOnly=yes" + ) + try: + r = subprocess.run( + ["git", "ls-remote", url], + capture_output=True, + timeout=max(2.0, float(timeout_s)), + check=False, + env=env, + ) + elapsed = round(time.monotonic() - t0, 3) + out = (r.stdout or b"").decode("utf-8", "replace").strip() + err = (r.stderr or b"").decode("utf-8", "replace").strip()[:200] + cmd = f"git ls-remote {url}" + if r.returncode == 0: + return { + "ok": True, + "light": "green", + "reason": "ls-remote_ok", + "elapsed": elapsed, + "detail": (out.splitlines()[0][:120] if out else "empty_list"), + "command": cmd, + } + # Auth / no key / unreachable → soft (yellow); keep orange for hard refuse + soft = ( + r.returncode in (2, 128) + or "Permission denied" in err + or "Host key" in err + or "Could not resolve" in err + or "unable to access" in err.lower() + ) + return { + "ok": False, + "light": "yellow" if soft else "orange", + "reason": err or f"ls-remote_rc_{r.returncode}", + "elapsed": elapsed, + "detail": "", + "command": cmd, + } + except FileNotFoundError: + return { + "ok": False, + "light": "blue", + "reason": "git_not_installed", + "elapsed": round(time.monotonic() - t0, 3), + "detail": "", + "command": f"git ls-remote {url}", + } + except subprocess.TimeoutExpired: + return { + "ok": False, + "light": "yellow", + "reason": "ls-remote_timeout", + "elapsed": round(time.monotonic() - t0, 3), + "detail": "", + "command": f"git ls-remote {url}", + } + + +def load_git_probe_state() -> dict[str, Any]: + path = git_probe_state_path() + if not path.is_file(): + return {"schema": 1, "hosts": {}} + try: + doc = json.loads(path.read_text(encoding="utf-8")) + if isinstance(doc, dict): + doc.setdefault("hosts", {}) + return doc + except (OSError, json.JSONDecodeError): + pass + return {"schema": 1, "hosts": {}} + + +def save_git_probe_state(doc: dict[str, Any]) -> None: + path = git_probe_state_path() + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(f".{path.name}.tmp.{os.getpid()}") + try: + tmp.write_text(json.dumps(doc, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + os.replace(tmp, path) + finally: + if tmp.exists(): + try: + tmp.unlink() + except OSError: + pass + + +def collect_git_probe( + host: str, + *, + now: float | None = None, + force: bool = False, +) -> dict[str, Any]: + """Infrequent HTTPS+SSH git ls-remote; cached under monitord state.""" + now = time.time() if now is None else float(now) + host = _host_only(host) + empty = { + "enabled": False, + "ok": None, + "light": "blue", + "reason": "git_probes_disabled", + "https": None, + "ssh": None, + "age_s": None, + "stale": False, + } + if not GIT_PROBE_ENABLED or not host: + return empty + state = load_git_probe_state() + prev = (state.get("hosts") or {}).get(host) or {} + prev_ts = prev.get("ts") + try: + age = None if prev_ts is None else now - float(prev_ts) + except (TypeError, ValueError): + age = None + due = force or prev_ts is None or age is None or age >= GIT_PROBE_INTERVAL_S + if not due and prev: + stale = age is None or age > GIT_PROBE_STALE_S + light = str(prev.get("light") or "green") + if stale and light == "green": + light = "yellow" + return { + "enabled": True, + "ok": bool(prev.get("ok")), + "light": light, + "reason": ("git_stale:" if stale else "") + str(prev.get("reason") or ""), + "https": prev.get("https"), + "ssh": prev.get("ssh"), + "age_s": age, + "stale": stale, + "ts": prev_ts, + } + https = _run_git_ls_remote(f"https://{host}/", timeout_s=GIT_PROBE_TIMEOUT_S) + ssh = _run_git_ls_remote(f"git@{host}:", timeout_s=GIT_PROBE_TIMEOUT_S) + # Aggregate: worst soft colour; both fail → orange max for soft probe + lights = [str(https.get("light") or "yellow"), str(ssh.get("light") or "yellow")] + worst = lights[0] + for Lgt in lights[1:]: + if SEVERITY.get(Lgt, 0) > SEVERITY.get(worst, 0): + worst = Lgt + if worst in ("red", "black"): + worst = "orange" + ok = bool(https.get("ok") or ssh.get("ok")) + if ok and worst in ("orange", "red"): + worst = "yellow" # one plane OK → soft at worst + reason = "git_ok" if ok else "git_unreachable" + if https.get("ok") and not ssh.get("ok"): + reason = "git_https_ok_ssh_soft" + elif ssh.get("ok") and not https.get("ok"): + reason = "git_ssh_ok_https_soft" + entry = { + "ts": now, + "ok": ok, + "light": worst if not ok else "green", + "reason": reason, + "https": https, + "ssh": ssh, + } + state.setdefault("hosts", {})[host] = entry + save_git_probe_state(state) + return { + "enabled": True, + "ok": ok, + "light": entry["light"], + "reason": reason, + "https": https, + "ssh": ssh, + "age_s": 0.0, + "stale": False, + "ts": now, + } + + +def payability_state_path() -> Path: + return monitord_state_dir() / "payability.json" + + +def load_payability_state() -> dict[str, Any]: + path = payability_state_path() + if not path.is_file(): + return {"schema": 1, "domains": {}} + try: + doc = json.loads(path.read_text(encoding="utf-8")) + if isinstance(doc, dict): + doc.setdefault("domains", {}) + return doc + except (OSError, json.JSONDecodeError): + pass + return {"schema": 1, "domains": {}} + + +def save_payability_state(doc: dict[str, Any]) -> None: + path = payability_state_path() + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(f".{path.name}.tmp.{os.getpid()}") + try: + tmp.write_text(json.dumps(doc, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + os.replace(tmp, path) + finally: + if tmp.exists(): + try: + tmp.unlink() + except OSError: + pass + + +def _suite_root() -> Path: + raw = (os.environ.get("SUITE_DIR") or "").strip() + if raw: + return Path(raw) + return Path(__file__).resolve().parents[1] + + +def _run_zero_pay(domain: str, *, timeout_s: float) -> dict[str, Any]: + """Run suite zero-pay COUNT=1 for domain (needs WALLET_DB / funded wallet).""" + root = _suite_root() + script = root / "taler-monitoring.sh" + cmd = [ + "bash", + str(script), + "-d", + domain, + "zero-pay", + ] + env = os.environ.copy() + env["COUNT"] = "1" + env["STRICT_EXIT"] = env.get("STRICT_EXIT", "1") + t0 = time.monotonic() + try: + r = subprocess.run( + cmd, + capture_output=True, + timeout=max(30.0, float(timeout_s)), + check=False, + env=env, + cwd=str(root), + ) + elapsed = round(time.monotonic() - t0, 3) + err = (r.stderr or b"").decode("utf-8", "replace").strip()[-300:] + out = (r.stdout or b"").decode("utf-8", "replace").strip()[-300:] + ok = r.returncode == 0 + return { + "ok": ok, + "light": "green" if ok else "orange", + "reason": "zero_pay_ok" if ok else (err or out or f"zero_pay_rc_{r.returncode}"), + "elapsed": elapsed, + "command": f"COUNT=1 ./taler-monitoring.sh -d {domain} zero-pay", + "rc": r.returncode, + } + except FileNotFoundError: + return { + "ok": False, + "light": "blue", + "reason": "suite_missing", + "elapsed": 0.0, + "command": f"COUNT=1 ./taler-monitoring.sh -d {domain} zero-pay", + "rc": 127, + } + except subprocess.TimeoutExpired: + return { + "ok": False, + "light": "orange", + "reason": "zero_pay_timeout", + "elapsed": round(time.monotonic() - t0, 3), + "command": f"COUNT=1 ./taler-monitoring.sh -d {domain} zero-pay", + "rc": 124, + } + + +def collect_payability( + domain: str, + *, + now: float | None = None, + force: bool = False, +) -> dict[str, Any]: + """Cached payability smoke for a stack domain (few×/day when PAY_RUN=1).""" + now = time.time() if now is None else float(now) + domain = str(domain or "").strip() + empty = { + "enabled": False, + "ok": None, + "light": "blue", + "reason": "pay_disabled", + "age_s": None, + "stale": False, + "command": "", + } + if not PAY_ENABLED or not domain: + return empty + allowed = set(PAY_DOMAINS) + if domain not in allowed and not any( + domain == d or domain.endswith("." + d) or d.endswith("." + domain) + for d in allowed + ): + return {**empty, "reason": "pay_domain_skip"} + state = load_payability_state() + prev = (state.get("domains") or {}).get(domain) or {} + prev_ts = prev.get("ts") + try: + age = None if prev_ts is None else now - float(prev_ts) + except (TypeError, ValueError): + age = None + due = force or prev_ts is None or age is None or age >= PAY_INTERVAL_S + if not due and prev: + stale = age is None or age > PAY_STALE_S + light = str(prev.get("light") or "green") + if stale and light == "green": + light = "yellow" + return { + "enabled": True, + "ok": bool(prev.get("ok")), + "light": light, + "reason": ("pay_stale:" if stale else "") + str(prev.get("reason") or ""), + "age_s": age, + "stale": stale, + "command": prev.get("command") or "", + "ts": prev_ts, + } + if not PAY_RUN: + # enabled but operator did not allow live wallet run this tick + if prev: + stale = age is None or age > PAY_STALE_S + light = str(prev.get("light") or "yellow") + if stale: + light = "yellow" + return { + "enabled": True, + "ok": bool(prev.get("ok")), + "light": light, + "reason": ("pay_stale:" if stale else "") + str(prev.get("reason") or "pay_cached"), + "age_s": age, + "stale": bool(stale), + "command": prev.get("command") or "", + "ts": prev_ts, + } + return { + "enabled": True, + "ok": False, + "light": "yellow", + "reason": "pay_never_run", + "age_s": None, + "stale": True, + "command": f"COUNT=1 ./taler-monitoring.sh -d {domain} zero-pay", + "ts": None, + } + result = _run_zero_pay(domain, timeout_s=PAY_TIMEOUT_S) + entry = { + "ts": now, + "ok": bool(result.get("ok")), + "light": result.get("light") or "orange", + "reason": result.get("reason") or "", + "command": result.get("command") or "", + "elapsed": result.get("elapsed"), + "rc": result.get("rc"), + } + state.setdefault("domains", {})[domain] = entry + save_payability_state(state) + return { + "enabled": True, + "ok": entry["ok"], + "light": entry["light"], + "reason": entry["reason"], + "age_s": 0.0, + "stale": False, + "command": entry["command"], + "ts": now, + } + + +def build_service_status( + row: dict[str, Any], + *, + prev_hosts: dict[str, dict[str, Any]], + now: float, + acute_s: float, + probe_fn: Callable[[str], dict[str, Any]], + osv_cache: dict[str, list], + published: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Probe a non-Taler company/service host (apex HTTPS).""" + host = row["host"] + url = f"https://{host}/" + raw = probe_fn(url) + classified = classify_probe(raw, prev=prev_hosts.get(host), now=now, acute_s=acute_s) + member = { + "role": "site", + "host": host, + **classified, + "endpoints": [], + "endpoints_summary": summarize_endpoints([]), + } + members = [member] + tls: dict[str, Any] = {} + if CERT_PROBES_ENABLED: + tls = probe_tls_cert(host) + if tls.get("light") and SEVERITY.get(str(tls["light"]), 0) > SEVERITY.get( + member.get("light") or "green", 0 + ): + members.append( + { + "role": "tls", + "host": host, + "ok": bool(tls.get("ok")), + "light": tls.get("light") or "yellow", + "reason": tls.get("reason") or "cert", + "down_since": None, + "code": 0, + "elapsed": 0.0, + "error": tls.get("error"), + "url": None, + "endpoints": [], + "endpoints_summary": summarize_endpoints([]), + } + ) + label = str(row.get("service_label") or "") + git_probe: dict[str, Any] | None = None + if label.lower() == "git" or host.startswith("git."): + git_probe = collect_git_probe(host, now=now) + if git_probe.get("enabled"): + members.append( + { + "role": "git", + "host": host, + "ok": bool(git_probe.get("ok")), + "light": git_probe.get("light") or "yellow", + "reason": git_probe.get("reason") or "git", + "down_since": None, + "code": 0, + "elapsed": 0.0, + "error": None, + "url": None, + "endpoints": [], + "endpoints_summary": summarize_endpoints([]), + "command": ((git_probe.get("https") or {}).get("command") or ""), + } + ) + light, intensity = aggregate_stack(members) + probe_cache = {url: raw} + vulns = assess_external_vulns( + probe_cache=probe_cache, members=members, osv_cache=osv_cache + ) + light, intensity, members = apply_vuln_to_stack(light, intensity, members, vulns) + software = collect_software_from_probes( + probe_cache=probe_cache, + members=members, + vulns=vulns, + smtp=False, + published=published, + ) + return { + "id": row.get("id") or host, + "currency": "", + "currency_info": None, + "kind": "service", + "service_label": row.get("service_label") or "service", + "commercial_prod": False, + "bank_backend": "", + "wire_methods": [], + "local": False, + "light": light, + "intensity": intensity, + "members": members, + "stats": None, + "software": software, + "git": git_probe or {}, + "network": { + "zone": row.get("zone") or "public", + "label": row.get("zone") or "public", + "ssh": {"expected": False}, + }, + "vulns": { + "enabled": vulns.get("enabled"), + "worst": vulns.get("worst") or "", + "light": vulns.get("light"), + "hits": [ + { + "id": h.get("id"), + "severity": h.get("severity"), + "package": h.get("package"), + "version": h.get("version"), + "host": h.get("host"), + "summary": h.get("summary") or "", + } + for h in (vulns.get("hits") or [])[:8] + ], + "servers": vulns.get("servers") or [], + }, + "breakdown": aggregate_breakdown(members), + "tls": { + "days_left": tls.get("days_left"), + "not_after": tls.get("not_after"), + "sans": tls.get("sans") or [], + "reason": tls.get("reason"), + } + if tls + else {}, + "orphan": False, + } + + +def _host_covered_by_known(host: str, known_hosts: set[str]) -> bool: + """True if host is known or a subdomain/parent of a monitored host (cert sibling).""" + h = _host_only(host).lower() + if not h or h in known_hosts: + return True + for k in known_hosts: + k = (k or "").lower() + if not k or k.upper() == "IBAN": + continue + if h.endswith("." + k) or k.endswith("." + h): + return True + return False + + +def collect_orphan_hosts( + *, + known_hosts: set[str], + tls_sans: list[str], + probe_fn: Callable[[str], dict[str, Any]], +) -> list[dict[str, Any]]: + """Flag SANs that look abandoned: DNS missing (default). + + Skips wildcard expansions already covered as siblings of monitored hosts + (e.g. donations.demo.taler.net when demo.taler.net is known). HTTP probe + optional via TALER_MONITORD_ORPHAN_HTTP=1 (off by default — CDN noise). + """ + if not ORPHAN_PROBES_ENABLED: + return [] + known = {_host_only(x).lower() for x in known_hosts if x} + candidates: list[str] = [] + for raw_san in tls_sans: + san = str(raw_san or "").strip().lower() + if not san or san.startswith("*."): + # Wildcard SAN ≠ concrete orphan candidate + continue + h = _host_only(san) + if not h or _host_covered_by_known(h, known): + continue + if h not in candidates: + candidates.append(h) + orphans: list[dict[str, Any]] = [] + for h in candidates[:25]: + addrs = dns_a_aaaa(h) + if not (addrs.get("a") or addrs.get("aaaa")): + orphans.append({"host": h, "reason": "dns_missing", "light": "orange"}) + continue + if not ORPHAN_HTTP_ENABLED: + continue + raw = probe_fn(f"https://{h}/") + if not raw.get("ok") and int(raw.get("code") or 0) == 0: + orphans.append( + { + "host": h, + "reason": f"http_unreachable:{raw.get('error')}", + "light": "yellow", + } + ) + return orphans + + +def scan_ports_list() -> list[int]: + raw = (os.environ.get("TALER_MONITORD_PORTSCAN_PORTS") or "").strip() + if raw: + out: list[int] = [] + for part in re.split(r"[\s,]+", raw): + if not part: + continue + try: + out.append(int(part)) + except ValueError: + continue + return sorted(set(out)) + return list(_DEFAULT_SCAN_PORTS) + + +def parse_ports_whitelist(path: Path | None = None) -> list[dict[str, Any]]: + """Whitelist lines: host_or_pattern port[,port…] [# note] + + Patterns: exact host, ``*.example.com``, or ``*`` (any host). + """ + path = path or default_ports_whitelist_conf() + rows: list[dict[str, Any]] = [] + if not path.is_file(): + return rows + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.split("#", 1)[0].strip() + if not line: + continue + parts = line.split() + if len(parts) < 2: + continue + pattern = parts[0].lower().rstrip(".") + ports: list[int] = [] + for tok in parts[1].split(","): + tok = tok.strip() + if not tok: + continue + try: + ports.append(int(tok)) + except ValueError: + continue + if ports: + rows.append({"pattern": pattern, "ports": ports}) + return rows + + +def whitelist_ports_for_host(host: str, rules: list[dict[str, Any]]) -> set[int]: + h = _host_only(host).lower() + allowed: set[int] = set() + for rule in rules or []: + pat = str(rule.get("pattern") or "").lower() + ports = set(int(p) for p in (rule.get("ports") or [])) + if not pat or not ports: + continue + if pat == "*" or pat == h: + allowed |= ports + elif pat.startswith("*.") and (h.endswith(pat[1:]) or h == pat[2:]): + allowed |= ports + return allowed + + +def baseline_ports_for_host(host: str, *, mail_hosts: set[str] | None = None) -> set[int]: + """Default expected open ports before whitelist.""" + h = _host_only(host).lower() + allowed = set(_BASELINE_WEB_PORTS) + if mail_hosts and h in mail_hosts: + allowed |= set(_BASELINE_MAIL_PORTS) + return allowed + + +def tcp_scan_host( + host: str, + ports: list[int], + *, + timeout_s: float = PORT_SCAN_TIMEOUT_S, + workers: int = PORT_SCAN_WORKERS, +) -> list[int]: + """Return sorted list of open TCP ports (banner-less connect).""" + h = _host_only(host) + if not h or h.upper() == "IBAN": + return [] + open_ports: list[int] = [] + + def _one(p: int) -> int | None: + return p if tcp_port_open(h, p, timeout_s=timeout_s) else None + + with ThreadPoolExecutor(max_workers=max(4, min(int(workers), len(ports) or 4))) as pool: + futs = [pool.submit(_one, p) for p in ports] + for fut in as_completed(futs): + try: + got = fut.result() + except Exception: + got = None + if got is not None: + open_ports.append(int(got)) + return sorted(set(open_ports)) + + +def assess_host_ports( + host: str, + *, + ports: list[int] | None = None, + whitelist_rules: list[dict[str, Any]] | None = None, + mail_hosts: set[str] | None = None, + open_ports: list[int] | None = None, +) -> dict[str, Any]: + """Scan (or reuse) open ports and classify expected vs suspicious.""" + h = _host_only(host) + ports = ports if ports is not None else scan_ports_list() + rules = whitelist_rules if whitelist_rules is not None else parse_ports_whitelist() + opened = list(open_ports) if open_ports is not None else tcp_scan_host(h, ports) + allowed = baseline_ports_for_host(h, mail_hosts=mail_hosts) | whitelist_ports_for_host(h, rules) + suspicious = [p for p in opened if p not in allowed] + expected_hit = [p for p in opened if p in allowed] + # hard = raise card colour; soft = list only (SSH/mail/http-alt often shared). + high = {21, 23, 135, 139, 445, 1433, 1521, 2049, 3306, 3389, 5432, 5900, 6379, 9200, 11211, 27017} + soft_set = {22, 25, 110, 143, 465, 587, 993, 995, 8080, 8443} + items: list[dict[str, Any]] = [] + for p in suspicious: + if p in high: + sev = "hard" + pl = "red" if p in (23, 445, 3389, 27017) else "orange" + else: + sev, pl = "soft", "yellow" + if p not in soft_set: + sev, pl = "soft", "yellow" + items.append({"port": p, "label": _PORT_LABELS.get(p, str(p)), "severity": sev, "light": pl}) + light = "green" + for it in items: + if SEVERITY.get(str(it["light"]), 0) > SEVERITY.get(light, 0): + light = str(it["light"]) + return { + "host": h, + "scanned": list(ports), + "open": opened, + "allowed": sorted(allowed), + "expected_open": expected_hit, + "suspicious": items, + "light": light, + "ok": not any(it.get("severity") == "hard" for it in items), + } + + +def collect_port_scan( + hosts: list[str], + *, + mail_hosts: set[str] | None = None, + whitelist_rules: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Scan unique hosts; return page-level findings.""" + if not PORT_SCAN_ENABLED: + return {"enabled": False, "hosts": [], "suspicious": [], "light": None} + ports = scan_ports_list() + rules = whitelist_rules if whitelist_rules is not None else parse_ports_whitelist() + mail_hosts = { _host_only(x).lower() for x in (mail_hosts or set()) } + uniq: list[str] = [] + seen: set[str] = set() + for h in hosts: + hh = _host_only(h).lower() + if not hh or hh.upper() == "IBAN" or hh in seen: + continue + # Skip pure IPs from rdns rows unless explicitly wanted + if re.match(r"^\d+\.\d+\.\d+\.\d+$", hh): + continue + seen.add(hh) + uniq.append(hh) + host_results: list[dict[str, Any]] = [] + # Scan hosts concurrently (each host scans ports internally) + def _scan(h: str) -> dict[str, Any]: + return assess_host_ports(h, ports=ports, whitelist_rules=rules, mail_hosts=mail_hosts) + + with ThreadPoolExecutor(max_workers=max(4, min(12, len(uniq) or 4))) as pool: + futs = {pool.submit(_scan, h): h for h in uniq} + for fut in as_completed(futs): + try: + host_results.append(fut.result()) + except Exception as e: + host_results.append( + { + "host": futs[fut], + "scanned": ports, + "open": [], + "allowed": [], + "expected_open": [], + "suspicious": [], + "light": "yellow", + "ok": True, + "error": type(e).__name__, + } + ) + host_results.sort(key=lambda r: r.get("host") or "") + suspicious_flat: list[dict[str, Any]] = [] + worst = "green" + for r in host_results: + for s in r.get("suspicious") or []: + suspicious_flat.append( + { + "host": r.get("host"), + "port": s.get("port"), + "label": s.get("label"), + "severity": s.get("severity") or "soft", + "light": s.get("light") or "yellow", + } + ) + Lgt = r.get("light") or "green" + if SEVERITY.get(str(Lgt), 0) > SEVERITY.get(worst, 0): + worst = str(Lgt) + hard = [s for s in suspicious_flat if s.get("severity") == "hard"] + if not suspicious_flat: + worst = "green" + elif not hard: + worst = "yellow" + return { + "enabled": True, + "ports": ports, + "hosts": host_results, + "suspicious": suspicious_flat, + "n_hosts": len(host_results), + "n_suspicious": len(suspicious_flat), + "n_hard": len(hard), + "light": worst if suspicious_flat else "green", + } + + +def build_stack_status( + stacks: list[dict[str, Any]], + *, + prev_doc: dict[str, Any] | None = None, + now: float | None = None, + acute_s: float = ACUTE_DOWN_S, + probe_fn: Callable[[str], dict[str, Any]] | None = None, + public_only: bool = True, +) -> dict[str, Any]: + """Probe all stacks and build status document.""" + now = time.time() if now is None else now + prev_doc = prev_doc or {} + prev_hosts: dict[str, dict[str, Any]] = {} + for s in prev_doc.get("stacks") or []: + for m in s.get("members") or []: + h = m.get("host") + if h: + prev_hosts[h] = m + + probe = probe_fn or (lambda url: probe_http(url)) + out_stacks: list[dict[str, Any]] = [] + ssh_cache: dict[str, dict[str, Any]] = {} + osv_cache: dict[str, list] = {} + published_doc = fetch_published_taler_packages() + # Previous endpoint rows keyed by host+path for down_since continuity + prev_eps: dict[str, dict[str, Any]] = {} + for s in prev_doc.get("stacks") or []: + for m in s.get("members") or []: + h = m.get("host") or "" + for ep in m.get("endpoints") or []: + p = ep.get("path") + if h and p: + prev_eps[f"{h}{p}"] = ep + + for st in stacks: + if public_only and st.get("local"): + continue + members_out: list[dict[str, Any]] = [] + kind = st.get("kind") or ("taler" if st.get("currency") else "static_pending") + has_intro = stack_has_custom_intro(st) + legal_sev = legal_doc_severity(st) + bank_configured = st.get("bank_configured") + if bank_configured is None: + bank_configured = bank_host_configured(st.get("bank") or "") + bank_backend = "unknown" + wire_methods: set[str] = set() + currency_info: dict[str, Any] | None = None + keys_body_for_denoms: bytes | str | None = None + probe_cache: dict[str, dict[str, Any]] = {} + + def cached_probe(url: str) -> dict[str, Any]: + if url not in probe_cache: + probe_cache[url] = probe(url) + return probe_cache[url] + + if kind == "static_pending": + # Apex static site (prod Taler not up yet) + site_host = _host_only(st["id"]) + site_url = f"https://{site_host}/" + site_raw = cached_probe(site_url) + site_c = classify_probe(site_raw, prev=prev_hosts.get(site_host), now=now, acute_s=acute_s) + if site_c.get("ok"): + site_c["light"] = "green" + site_c["reason"] = "static_ok" + site_member = {"role": "site", "host": site_host, **site_c} + site_prev = {k[len(site_host) :]: v for k, v in prev_eps.items() if k.startswith(site_host)} + site_eps = probe_role_endpoints( + "site", + site_host, + has_intro=False, + legal_severity=legal_sev, + probe_fn=cached_probe, + cache=probe_cache, + prev_by_path=site_prev, + now=now, + acute_s=acute_s, + ) + members_out.append(merge_member_endpoint_light(site_member, site_eps)) + + # Probe exchange before bank so /keys payto://… can decide EBICS vs x-taler-bank. + role_hosts: list[tuple[str, str]] = [ + ("exchange", st["exchange"]), + ("merchant", st["merchant"]), + ] + if bank_configured: + role_hosts.insert(0, ("bank", st.get("bank") or "")) + + classified_by_role: dict[str, dict[str, Any]] = {} + # Always resolve exchange (+ merchant) first for wire detection, then bank. + probe_order = ["exchange", "merchant", "bank"] if bank_configured else ["exchange", "merchant"] + host_by_role = {r: h for r, h in role_hosts} + + for role in probe_order: + if role not in host_by_role: + continue + host = host_by_role[role] + if role == "bank": + # After exchange: skip public bank.* when wire is IBAN/EBICS-only. + bank_backend = classify_bank_backend( + wire_methods, bank_configured=bool(bank_configured) + ) + if bank_backend == "ebics": + classified_by_role[role] = { + "ok": True, + "light": "blue", + "reason": "ebics_iban", + "down_since": None, + "code": 0, + "elapsed": 0.0, + "error": None, + "url": None, + "ts": now, + # No public bank.* — show wire kind in the host column + "host": "IBAN", + "role": role, + } + continue + if role == "exchange": + # Prefer /keys (accounts/payto); fall back to /config for liveness. + keys_url = f"https://{_host_only(host)}/keys" + raw = cached_probe(keys_url) + cfg_data: dict[str, Any] | None = None + if not raw.get("ok"): + alt = member_probe_url(role, host) + raw2 = cached_probe(alt) + if raw2.get("ok") or int(raw2.get("code") or 0) > int(raw.get("code") or 0): + raw = raw2 + cfg_data = _json_body(raw) + else: + keys_body_for_denoms = raw.get("body") + wire_methods = payto_wire_methods(keys_body_for_denoms) + bank_backend = classify_bank_backend( + wire_methods, bank_configured=bool(bank_configured) + ) + # Separate /config for currency_specification (small JSON). + cfg_raw = cached_probe(member_probe_url(role, host)) + if cfg_raw.get("ok"): + cfg_data = _json_body(cfg_raw) + if cfg_data: + cs = cfg_data.get("currency_specification") + if not isinstance(cs, dict): + cs = {} + currency_info = summarize_currency_spec( + cs, + keys_body=keys_body_for_denoms, + fallback_currency=str( + cfg_data.get("currency") or st.get("currency") or "" + ), + ) + elif role == "bank": + url = member_probe_url(role, host) + raw = cached_probe(url) + if not raw.get("ok") and int(raw.get("code") or 0) in (0, 404): + alt = f"https://{_host_only(host)}/config" + raw2 = cached_probe(alt) + if raw2.get("ok") or int(raw2.get("code") or 0) > int(raw.get("code") or 0): + raw = raw2 + else: + raw = cached_probe(member_probe_url(role, host)) + + classified = classify_probe(raw, prev=prev_hosts.get(host), now=now, acute_s=acute_s) + if kind == "static_pending" and not classified.get("ok"): + # Expected: no bank/exchange/merchant prod yet — not an outage. + classified = { + **classified, + "ok": True, + "light": "blue", + "reason": "no_prod_yet", + "down_since": None, + } + classified_by_role[role] = {"role": role, "host": host, **classified} + + if not bank_configured: + bank_backend = "ebics" + classified_by_role["bank"] = { + "role": "bank", + "host": "IBAN", + "ok": True, + "light": "blue", + "reason": "ebics_iban", + "down_since": None, + "code": 0, + "elapsed": 0.0, + "error": None, + "url": None, + "ts": now, + } + + # Stable member order: site (optional) already appended; then bank → exchange → merchant + for role in ("bank", "exchange", "merchant"): + if role not in classified_by_role: + continue + member = classified_by_role[role] + host = member.get("host") or "" + # EBICS placeholder — no public endpoint fan-out + if role == "bank" and ( + bank_backend == "ebics" or (member.get("reason") or "") in _AGG_SKIP_REASONS + ): + members_out.append( + { + **member, + "endpoints": [], + "endpoints_summary": summarize_endpoints([]), + } + ) + continue + if kind == "static_pending" and (member.get("reason") or "") == "no_prod_yet": + members_out.append( + { + **member, + "endpoints": [], + "endpoints_summary": summarize_endpoints([]), + } + ) + continue + host_prev = {k[len(host) :]: v for k, v in prev_eps.items() if host and k.startswith(host)} + eps = probe_role_endpoints( + role, + host, + has_intro=has_intro, + legal_severity=legal_sev, + probe_fn=cached_probe, + cache=probe_cache, + prev_by_path=host_prev, + now=now, + acute_s=acute_s, + ) + members_out.append(merge_member_endpoint_light(member, eps)) + + light, intensity = aggregate_stack(members_out) + stats = fetch_stack_stats( + st, bank_backend=bank_backend, probe_fn=cached_probe, now=now, stale_s=STATS_STALE_S + ) + vulns = assess_external_vulns( + probe_cache=probe_cache, members=members_out, osv_cache=osv_cache + ) + light, intensity, members_out = apply_vuln_to_stack(light, intensity, members_out, vulns) + + tls: dict[str, Any] = {} + primary_host = "" + for role in ("merchant", "exchange", "bank"): + m = classified_by_role.get(role) or {} + h = str(m.get("host") or "") + if h and h.upper() != "IBAN": + primary_host = h + break + if not primary_host: + primary_host = _host_only(str(st.get("id") or "")) + if CERT_PROBES_ENABLED and primary_host: + tls = probe_tls_cert(primary_host) + if tls.get("light") and SEVERITY.get(str(tls["light"]), 0) > SEVERITY.get(light, 0): + members_out.append( + { + "role": "tls", + "host": primary_host, + "ok": bool(tls.get("ok")), + "light": tls.get("light") or "yellow", + "reason": tls.get("reason") or "cert", + "down_since": None, + "code": 0, + "elapsed": 0.0, + "error": tls.get("error"), + "url": None, + "endpoints": [], + "endpoints_summary": summarize_endpoints([]), + } + ) + light, intensity = aggregate_stack(members_out) + + # Prefer live exchange currency; keep domains.conf as fallback label. + detected_currency = "" + if currency_info and currency_info.get("currency"): + detected_currency = str(currency_info["currency"]) + elif st.get("currency"): + detected_currency = str(st.get("currency")) + software = collect_software_from_probes( + probe_cache=probe_cache, + members=members_out, + vulns=vulns, + smtp=False, + published=published_doc, + ) + pay: dict[str, Any] = {} + zone = classify_network_zone(st)["zone"] + pay_id = str(st.get("id") or "") + if PAY_ENABLED and zone in ("tops-stage", "tops-prod", "goa-local", "fp-stage"): + pay = collect_payability(pay_id, now=now) + if pay.get("enabled") and pay.get("light") not in (None, "blue"): + members_out.append( + { + "role": "payability", + "host": pay_id, + "ok": bool(pay.get("ok")), + "light": pay.get("light") or "yellow", + "reason": pay.get("reason") or "pay", + "down_since": None, + "code": 0, + "elapsed": 0.0, + "error": None, + "url": None, + "endpoints": [], + "endpoints_summary": summarize_endpoints([]), + "command": pay.get("command") or "", + } + ) + light, intensity = aggregate_stack(members_out) + out_stacks.append( + { + "id": st["id"], + "currency": detected_currency or (st.get("currency") or ""), + "currency_info": currency_info, + "kind": kind, + "commercial_prod": stack_is_commercial_prod(st), + "bank_backend": bank_backend, + "wire_methods": sorted(wire_methods), + "local": bool(st.get("local")), + "light": light, + "intensity": intensity, + "members": members_out, + "stats": stats, + "software": software, + "payability": pay, + "network": build_network_status( + st, members=members_out, ssh_cache=ssh_cache + ), + "vulns": { + "enabled": vulns.get("enabled"), + "worst": vulns.get("worst") or "", + "light": vulns.get("light"), + "hits": [ + { + "id": h.get("id"), + "severity": h.get("severity"), + "package": h.get("package"), + "version": h.get("version"), + "host": h.get("host"), + "summary": h.get("summary") or "", + } + for h in (vulns.get("hits") or [])[:8] + ], + "servers": vulns.get("servers") or [], + }, + "breakdown": aggregate_breakdown(members_out), + "tls": { + "days_left": tls.get("days_left"), + "not_after": tls.get("not_after"), + "sans": tls.get("sans") or [], + "reason": tls.get("reason"), + } + if tls + else {}, + "orphan": False, + } + ) + + out_services: list[dict[str, Any]] = [] + if SERVICES_ENABLED: + for row in parse_services_conf(): + out_services.append( + build_service_status( + row, + prev_hosts=prev_hosts, + now=now, + acute_s=acute_s, + probe_fn=probe, + osv_cache=osv_cache, + published=published_doc, + ) + ) + + out_mail: list[dict[str, Any]] = [] + if MAIL_ENABLED: + mail_port_done: set[str] = set() + deliver_state = load_deliverability_state() if DELIVER_ENABLED else {"domains": {}} + for row in parse_mail_catalog(): + out_mail.append( + assess_mail_domain( + row, + port_done=mail_port_done, + deliver_state=deliver_state, + now=now, + ) + ) + + known_hosts: set[str] = set() + tls_sans: list[str] = [] + scan_hosts: list[str] = [] + for group in (out_stacks, out_services): + for st_out in group: + for m in st_out.get("members") or []: + h = _host_only(str(m.get("host") or "")) + if h and h.upper() != "IBAN": + known_hosts.add(h) + scan_hosts.append(h) + for san in (st_out.get("tls") or {}).get("sans") or []: + tls_sans.append(str(san)) + known_hosts.add(_host_only(str(st_out.get("id") or ""))) + scan_hosts.append(_host_only(str(st_out.get("id") or ""))) + # Also scan env-mapped SSH targets — findings may be hidden hosts + for st_cfg in stacks: + if public_only and st_cfg.get("local"): + continue + for h in ssh_targets_for_stack(st_cfg).get("hosts") or []: + scan_hosts.append(h) + mail_hosts: set[str] = set() + for row in parse_mail_catalog() if MAIL_ENABLED else []: + for h in row.get("mail_hosts") or []: + mail_hosts.add(_host_only(h).lower()) + scan_hosts.append(h) + orphans = collect_orphan_hosts( + known_hosts=known_hosts, tls_sans=tls_sans, probe_fn=probe + ) + ports_doc = collect_port_scan(scan_hosts, mail_hosts=mail_hosts) + # Attach per-card port slice + optional synthetic member when suspicious + by_host = {str(r.get("host") or ""): r for r in (ports_doc.get("hosts") or [])} + for group in (out_stacks, out_services): + for st_out in group: + ssh_meta = st_out.get("network") or {} + ssh_block = ssh_meta.get("ssh") or {} + # UI clears candidates when hide_host; recover env-mapped hosts for port attach + hidden_ssh: set[str] = set() + if ssh_block.get("hide_host"): + zone = str(ssh_meta.get("zone") or "") + env_map = { + "goa-local": "TALER_MONITORD_SSH_GOA", + "fp-stage": "TALER_MONITORD_SSH_FP", + "tops-stage": "TALER_MONITORD_SSH_TOPS", + "tops-prod": "TALER_MONITORD_SSH_TOPS_PROD", + } + if zone in env_map: + hidden_ssh = { + _host_only(x).lower() + for x in _env_ssh_hosts(env_map[zone]) + if x + } + card_hosts: list[str] = [] + for m in st_out.get("members") or []: + h = _host_only(str(m.get("host") or "")) + if h and h.upper() != "IBAN": + card_hosts.append(h) + sid = _host_only(str(st_out.get("id") or "")) + if sid: + card_hosts.append(sid) + card_hosts.extend(hidden_ssh) + sus: list[dict[str, Any]] = [] + opens: list[dict[str, Any]] = [] + worst = "green" + for h in dict.fromkeys(_host_only(x).lower() for x in card_hosts if x): + r = by_host.get(h) + if not r: + continue + hide = h in hidden_ssh + for s in r.get("suspicious") or []: + sus.append( + { + "host": "" if hide else h, + "port": s.get("port"), + "label": s.get("label"), + "severity": s.get("severity") or "soft", + "light": s.get("light") or "yellow", + } + ) + if r.get("open"): + opens.append( + { + "host": "" if hide else h, + "open": r.get("open"), + "suspicious": [s.get("port") for s in (r.get("suspicious") or [])], + } + ) + for s in r.get("suspicious") or []: + if s.get("severity") != "hard": + continue + Lgt = s.get("light") or "orange" + if SEVERITY.get(str(Lgt), 0) > SEVERITY.get(worst, 0): + worst = str(Lgt) + hard_sus = [s for s in sus if s.get("severity") == "hard"] + st_out["ports"] = { + "suspicious": sus, + "hosts": opens, + "light": worst if hard_sus else ("yellow" if sus else "green"), + "n_suspicious": len(sus), + "n_hard": len(hard_sus), + } + # Only hard findings pull the card colour (soft stays informational). + if hard_sus: + members = list(st_out.get("members") or []) + members.append( + { + "role": "ports", + "host": ",".join( + f"{s.get('label') or s.get('port')}" + + (f"@{s.get('host')}" if s.get("host") else "") + for s in hard_sus[:6] + ), + "ok": False, + "light": worst, + "reason": f"suspicious_ports_hard:{len(hard_sus)}", + "down_since": None, + "code": 0, + "elapsed": 0.0, + "error": None, + "url": None, + "endpoints": [], + "endpoints_summary": summarize_endpoints([]), + } + ) + st_out["members"] = members + light2, intensity2 = aggregate_stack(members) + st_out["light"] = light2 + st_out["intensity"] = intensity2 + st_out["breakdown"] = aggregate_breakdown(members) + return { + "schema": SCHEMA, + "ts": now, + "recorder": os.environ.get("TALER_MONITORD_RECORDER") or socket.gethostname(), + "acute_down_s": acute_s, + "legend": dict(LEGEND), + "stacks": out_stacks, + "services": out_services, + "mail": out_mail, + "orphans": orphans, + "ports": ports_doc, + "published": { + "enabled": published_doc.get("enabled"), + "suite": published_doc.get("suite"), + "error": published_doc.get("error"), + "url": published_doc.get("url"), + "packages": { + k: published_doc.get("packages", {}).get(k) + for k in ( + "taler-exchange", + "taler-merchant", + "libeufin-bank", + "taler-merchant-webui", + "taler-wallet-cli", + ) + if (published_doc.get("packages") or {}).get(k) + }, + "apps": published_doc.get("apps") or [], + }, + "score": compute_total_score(out_stacks, out_services, out_mail), + } + + +def read_status(path: Path | None = None) -> dict[str, Any]: + paths = [path] if path else status_json_paths() + for p in paths: + if p and p.is_file(): + try: + return json.loads(p.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + return {} + + +def atomic_write_json(path: Path, doc: dict[str, Any]) -> None: + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + text = json.dumps(doc, ensure_ascii=False, indent=2) + "\n" + tmp = path.with_name(f".{path.name}.tmp.{os.getpid()}") + try: + tmp.write_text(text, encoding="utf-8") + os.replace(tmp, path) + finally: + if tmp.exists(): + try: + tmp.unlink() + except OSError: + pass + + +def write_stack_status(doc: dict[str, Any], paths: list[Path] | None = None) -> Path: + paths = paths or status_json_paths() + primary = paths[0] + for p in paths: + atomic_write_json(p, doc) + return primary diff --git a/site-gen/taler-monitord b/site-gen/taler-monitord new file mode 100755 index 0000000..8704e62 --- /dev/null +++ b/site-gen/taler-monitord @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +"""taler-monitord — probe domain stacks on an interval; UI reads JSON only.""" +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import stack_status_html as H # noqa: E402 +import stack_status_lib as L # noqa: E402 + +# Match lib.sh: load ~/.config/taler-monitoring/env (or TALER_MONITORING_ENV), then +# refresh module flags that were read at import time. +_ENV_LOADED = L.load_monitoring_env() +if _ENV_LOADED: + L.refresh_runtime_flags_from_env() + + +def _pid_path() -> Path: + return L.monitord_state_dir() / "taler-monitord.pid" + + +def _write_pid() -> None: + _pid_path().write_text(str(os.getpid()) + "\n", encoding="utf-8") + + +def _clear_pid() -> None: + p = _pid_path() + try: + p.unlink() + except OSError: + pass + + +def _publish_one(local: Path, ssh_host: str, remote: str) -> str | None: + remote_dir = str(Path(remote).parent) + mk = subprocess.run( + [ + "ssh", + "-o", + "BatchMode=yes", + "-o", + "ConnectTimeout=8", + ssh_host, + f"mkdir -p -- {remote_dir!s}", + ], + capture_output=True, + timeout=15, + check=False, + ) + if mk.returncode != 0: + err = (mk.stderr or b"").decode("utf-8", "replace").strip()[:200] + print(f"publish mkdir fail ({remote}): {err}", file=sys.stderr, flush=True) + return None + r = subprocess.run( + [ + "scp", + "-o", + "BatchMode=yes", + "-o", + "ConnectTimeout=8", + str(local), + f"{ssh_host}:{remote}", + ], + capture_output=True, + timeout=30, + check=False, + ) + if r.returncode != 0: + err = (r.stderr or b"").decode("utf-8", "replace").strip()[:200] + print(f"publish fail ({remote}): {err}", file=sys.stderr, flush=True) + return None + if "/var/www/monitoring-sites/" in remote: + subprocess.run( + [ + "ssh", + "-o", + "BatchMode=yes", + "-o", + "ConnectTimeout=8", + ssh_host, + ( + f"chmod 755 -- {remote_dir!s} && chmod 644 -- {remote!s} && " + f"setfacl -m u:caddy:rx,m::r-x -- {remote_dir!s} && " + f"setfacl -m u:caddy:r,m::r-- -- {remote!s} || true" + ), + ], + capture_output=True, + timeout=15, + check=False, + ) + else: + subprocess.run( + [ + "ssh", + "-o", + "BatchMode=yes", + "-o", + "ConnectTimeout=8", + ssh_host, + f"chmod 755 -- {remote_dir!s} && chmod 644 -- {remote!s} || true", + ], + capture_output=True, + timeout=15, + check=False, + ) + return f"{ssh_host}:{remote}" + + +def publish_html(doc: dict) -> str | None: + """Render HTML and scp to Cyon (hacktivism) paths. + + Env: + TALER_MONITORD_PUBLISH=0 to disable + TALER_MONITORD_PUBLISH_SSH=hacktivism + TALER_MONITORD_PUBLISH_PATH=… single path (overrides multi default) + TALER_MONITORD_PUBLISH_PATHS=comma-separated paths + Default paths: public_html/stacks + public_html/monitoring.hacktivism.ch + """ + if os.environ.get("TALER_MONITORD_PUBLISH", "1") in ("0", "false", "no", ""): + return None + ssh_host = os.environ.get("TALER_MONITORD_PUBLISH_SSH", "hacktivism").strip() or "hacktivism" + single = (os.environ.get("TALER_MONITORD_PUBLISH_PATH") or "").strip() + multi = (os.environ.get("TALER_MONITORD_PUBLISH_PATHS") or "").strip() + if single: + remotes = [single] + elif multi: + remotes = [p.strip() for p in multi.split(",") if p.strip()] + else: + remotes = [ + "/home/hacktivi/public_html/stacks/index.html", + "/home/hacktivi/public_html/monitoring.hacktivism.ch/index.html", + ] + html_text = H.render_html(doc) + with tempfile.NamedTemporaryFile("w", encoding="utf-8", suffix=".html", delete=False) as f: + f.write(html_text) + local = Path(f.name) + published: list[str] = [] + try: + for remote in remotes: + got = _publish_one(local, ssh_host, remote) + if got: + published.append(got) + return "; ".join(published) if published else None + except (OSError, subprocess.TimeoutExpired) as e: + print(f"publish error: {e}", file=sys.stderr, flush=True) + return None + finally: + try: + local.unlink() + except OSError: + pass + + +def run_once(*, domains: Path, out_paths: list[Path] | None, public_only: bool) -> Path: + stacks = L.parse_domains_conf(domains) + prev = L.read_status(out_paths[0] if out_paths else None) + if not prev and out_paths: + for p in out_paths: + prev = L.read_status(p) + if prev: + break + doc = L.build_stack_status(stacks, prev_doc=prev, public_only=public_only) + paths = out_paths or L.status_json_paths() + path = L.write_stack_status(doc, paths) + # always keep a local HTML copy too + local_html = L.monitord_state_dir() / "index.html" + local_html.write_text(H.render_html(doc), encoding="utf-8") + published = publish_html(doc) + if published: + print(f"published {published}", flush=True) + return path + + +def main() -> int: + ap = argparse.ArgumentParser(description="Collect Taler stack status for the detached UI") + ap.add_argument("--interval", type=float, default=float(os.environ.get("TALER_MONITORD_INTERVAL", "20"))) + ap.add_argument("--once", action="store_true", help="Single probe cycle then exit") + ap.add_argument("--domains-conf", type=Path, default=None) + ap.add_argument("--out", type=Path, action="append", default=None, help="Output JSON path (repeatable)") + # Default on: include GOA / hacktivism.ch (taler.hacktivism.ch merchant). Opt out: --no-include-local + _incl = os.environ.get("TALER_MONITORD_INCLUDE_LOCAL", "1") not in ("0", "false", "no", "") + ap.add_argument( + "--include-local", + action=argparse.BooleanOptionalAction, + default=_incl, + help="Probe local=1 stacks (hacktivism GOA). Default on; --no-include-local to skip", + ) + args = ap.parse_args() + if _ENV_LOADED: + print(f"env {_ENV_LOADED}", flush=True) + domains = args.domains_conf or L.default_domains_conf() + if not domains.is_file(): + print(f"domains.conf not found: {domains}", file=sys.stderr) + return 2 + public_only = not bool(args.include_local) + out_paths = list(args.out) if args.out else None + + if args.once: + path = run_once(domains=domains, out_paths=out_paths, public_only=public_only) + print(path) + return 0 + + _write_pid() + try: + while True: + try: + path = run_once(domains=domains, out_paths=out_paths, public_only=public_only) + print(f"ok {path}", flush=True) + except Exception as e: # keep loop alive + print(f"error {type(e).__name__}: {e}", file=sys.stderr, flush=True) + time.sleep(max(1.0, float(args.interval))) + finally: + _clear_pid() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_stack_status.py b/tests/test_stack_status.py new file mode 100644 index 0000000..560c8c4 --- /dev/null +++ b/tests/test_stack_status.py @@ -0,0 +1,1095 @@ +#!/usr/bin/env python3 +"""Offline unit tests for stack_status_lib (no network).""" +from __future__ import annotations + +import json +import os +import sys +import tempfile +import time +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "site-gen")) + +import stack_status_lib as L # noqa: E402 + + +def _disable_extra_planes() -> None: + """Keep offline unit tests off dig/TLS/catalog network work.""" + L.SERVICES_ENABLED = False + L.MAIL_ENABLED = False + L.CERT_PROBES_ENABLED = False + L.ORPHAN_PROBES_ENABLED = False + L.MAIL_PORTS_ENABLED = False + L.ORPHAN_HTTP_ENABLED = False + L.PORT_SCAN_ENABLED = False + L.DELIVER_ENABLED = False + + +_disable_extra_planes() + + +class AggregateTests(unittest.TestCase): + def test_all_green_ok(self): + members = [{"light": "green"}] * 3 + light, intensity = L.aggregate_stack(members) + self.assertEqual(light, "green") + self.assertEqual(intensity, "ok") + + def test_taler_software_from_config(self): + body = json.dumps( + {"name": "taler-exchange", "build_version": "1.6.15-dev.1", "version": "39:0:5"} + ).encode() + items = L.software_from_taler_config(body, host="exchange.demo.taler.net", role="exchange") + self.assertEqual(len(items), 1) + self.assertEqual(items[0]["package"], "taler-exchange") + self.assertEqual(items[0]["version"], "1.6.15-dev.1") + colored = L.colorize_software( + items + L.software_from_server_header("nginx/1.24.0", host="exchange.demo.taler.net"), + vulns={"hits": [{"package": "nginx", "version": "1.24.0", "severity": "HIGH"}]}, + ) + by = {c["package"]: c for c in colored} + self.assertEqual(by["taler-exchange"]["light"], "green") + self.assertEqual(by["nginx"]["light"], "orange") + + def test_deliverability_stale_and_fail(self): + now = 1_000_000.0 + old_en = L.DELIVER_ENABLED + L.DELIVER_ENABLED = True + try: + state = { + "domains": { + "example.test": { + "ok": True, + "light": "green", + "reason": "rcpt_accepted", + "ts": now - 100, + "mx": "mx.example.test", + "rcpt": "postmaster@example.test", + } + } + } + got = L.ensure_deliverability( + "example.test", mx_hosts=["mx.example.test"], now=now, state=state, force=False + ) + self.assertTrue(got["ok"]) + self.assertEqual(got["light"], "green") + # stale success → yellow (prevent re-probe by raising interval) + state["domains"]["example.test"]["ts"] = now - (L.DELIVER_STALE_S + 10) + old_i = L.DELIVER_INTERVAL_S + L.DELIVER_INTERVAL_S = L.DELIVER_STALE_S + 1000 + try: + got = L.ensure_deliverability( + "example.test", mx_hosts=["mx.example.test"], now=now, state=state, force=False + ) + finally: + L.DELIVER_INTERVAL_S = old_i + self.assertFalse(got["ok"]) + self.assertEqual(got["light"], "yellow") + self.assertTrue(got["stale"]) + # hard fail stored + state["domains"]["example.test"] = { + "ok": False, + "light": "orange", + "reason": "rcpt_reject:550", + "ts": now - 60, + "mx": "mx.example.test", + "rcpt": "postmaster@example.test", + } + old_i = L.DELIVER_INTERVAL_S + L.DELIVER_INTERVAL_S = 10_000 + try: + got = L.ensure_deliverability( + "example.test", mx_hosts=["mx.example.test"], now=now, state=state, force=False + ) + finally: + L.DELIVER_INTERVAL_S = old_i + self.assertEqual(got["light"], "orange") + self.assertIn("rcpt_reject", got["reason"]) + finally: + L.DELIVER_ENABLED = old_en + + def test_published_currency_behind(self): + self.assertEqual(L.normalize_build_version("1.6.15-dev.1"), "1.6.15~dev1") + pub = { + "enabled": True, + "packages": {"taler-merchant": "1.6.22-0+trixie", "taler-exchange": "1.6.15~dev1-0+trixie"}, + } + items = L.apply_published_currency( + [ + {"package": "taler-merchant", "version": "1.6.21", "light": "green"}, + {"package": "taler-exchange", "version": "1.6.15-dev.1", "light": "green"}, + ], + pub, + ) + by = {i["package"]: i for i in items} + self.assertEqual(by["taler-merchant"]["currency_status"], "behind") + self.assertEqual(by["taler-merchant"]["behind"], "1.6.22") + self.assertEqual(by["taler-exchange"]["currency_status"], "current") + self.assertGreater(L.deb_version_cmp("1.6.22-0+trixie", "1.6.21-0+trixie"), 0) + + def test_port_whitelist_and_suspicious(self): + rules = L.parse_ports_whitelist # noqa: ensure exists + self.assertTrue(callable(rules)) + wl = [{"pattern": "*.hacktivism.ch", "ports": [22, 8080, 9200]}, {"pattern": "*", "ports": [8080]}] + self.assertIn(22, L.whitelist_ports_for_host("taler.hacktivism.ch", wl)) + self.assertIn(9200, L.whitelist_ports_for_host("git.hacktivism.ch", wl)) + self.assertIn(8080, L.whitelist_ports_for_host("anything.example", wl)) + # Intentional koopa public ports must not be suspicious / orange + koopa = L.assess_host_ports( + "git.hacktivism.ch", + ports=[22, 80, 443, 8080, 9200, 3306], + whitelist_rules=wl, + open_ports=[22, 80, 443, 8080, 9200], + ) + self.assertEqual(koopa["suspicious"], []) + self.assertTrue(koopa["ok"]) + self.assertEqual(koopa["light"], "green") + got = L.assess_host_ports( + "edge.example", + ports=[22, 80, 443, 3306], + whitelist_rules=[], + open_ports=[80, 443, 3306], + ) + self.assertEqual([s["port"] for s in got["suspicious"]], [3306]) + self.assertEqual(got["suspicious"][0]["severity"], "hard") + self.assertEqual(got["light"], "orange") + got2 = L.assess_host_ports( + "edge.example", + ports=[22, 80, 443], + whitelist_rules=[{"pattern": "edge.example", "ports": [22]}], + open_ports=[22, 443], + ) + self.assertEqual(got2["suspicious"], []) + self.assertTrue(got2["ok"]) + soft = L.assess_host_ports( + "edge.example", + ports=[22, 80, 443], + whitelist_rules=[], + open_ports=[22, 443], + ) + self.assertEqual(soft["suspicious"][0]["severity"], "soft") + self.assertTrue(soft["ok"]) # soft does not fail ok + + def test_orphan_skips_siblings_and_wildcards(self): + known = {"demo.taler.net", "backend.demo.taler.net"} + orphans = L.collect_orphan_hosts( + known_hosts=known, + tls_sans=["*.demo.taler.net", "donations.demo.taler.net", "gone.example.invalid"], + probe_fn=lambda url: {"ok": False, "code": 0, "error": "URLError"}, + ) + hosts = [o["host"] for o in orphans] + self.assertNotIn("demo.taler.net", hosts) + self.assertNotIn("donations.demo.taler.net", hosts) # sibling + # gone.example.invalid may appear only if dig says no DNS — skip assert on network + self.assertTrue(all(not h.startswith("*.") for h in hosts)) + + def test_total_score_thresholds(self): + # 10 green of 10 → green + s = L.compute_total_score([{"light": "green"}] * 10) + self.assertEqual(s["green"], 10) + self.assertEqual(s["total"], 10) + self.assertEqual(s["light"], "green") + # 9/10 → blue (≥0.85) + s = L.compute_total_score([{"light": "green"}] * 9 + [{"light": "orange"}]) + self.assertEqual(s["light"], "blue") + # 6/10 → yellow + s = L.compute_total_score([{"light": "green"}] * 6 + [{"light": "red"}] * 4) + self.assertEqual(s["light"], "yellow") + # 4/10 → orange + s = L.compute_total_score([{"light": "green"}] * 4 + [{"light": "red"}] * 6) + self.assertEqual(s["light"], "orange") + # 1/10 → red + s = L.compute_total_score([{"light": "green"}] + [{"light": "orange"}] * 9) + self.assertEqual(s["light"], "red") + # 0 green mixed → white (gone) + s = L.compute_total_score([{"light": "red"}, {"light": "orange"}]) + self.assertEqual(s["green"], 0) + self.assertEqual(s["light"], "white") + # all black → black + s = L.compute_total_score([{"light": "black"}] * 3) + self.assertEqual(s["light"], "black") + + def test_breakdown_counts(self): + members = [ + {"role": "bank", "host": "a", "light": "green", "reason": "ok"}, + {"role": "exchange", "host": "b", "light": "green", "reason": "ok"}, + {"role": "merchant", "host": "c", "light": "orange", "reason": "http_501"}, + {"role": "bank", "host": "IBAN", "light": "blue", "reason": "ebics_iban"}, + ] + bd = L.aggregate_breakdown(members) + self.assertEqual(bd["by_light"].get("green"), 2) + self.assertEqual(bd["by_light"].get("orange"), 1) + self.assertNotIn("blue", bd["by_light"]) # skipped informational + self.assertEqual(bd["worst"], "orange") + self.assertEqual(bd["counted"], 3) + + def test_blue_present_ok(self): + members = [{"light": "green"}, {"light": "blue"}, {"light": "blue"}] + light, intensity = L.aggregate_stack(members) + self.assertEqual(light, "blue") + self.assertEqual(intensity, "ok") + + def test_one_yellow_muted(self): + members = [ + {"light": "green"}, + {"light": "yellow"}, + {"light": "green"}, + ] + light, intensity = L.aggregate_stack(members) + self.assertEqual(light, "yellow") + self.assertEqual(intensity, "muted") + + def test_one_red_muted(self): + members = [ + {"light": "green"}, + {"light": "red"}, + {"light": "green"}, + ] + light, intensity = L.aggregate_stack(members) + self.assertEqual(light, "red") + self.assertEqual(intensity, "muted") + + def test_all_red_full(self): + members = [{"light": "red"}] * 3 + light, intensity = L.aggregate_stack(members) + self.assertEqual(light, "red") + self.assertEqual(intensity, "full") + + def test_all_black_full(self): + members = [{"light": "black"}] * 3 + light, intensity = L.aggregate_stack(members) + self.assertEqual(light, "black") + self.assertEqual(intensity, "full") + + def test_mixed_orange_green_muted(self): + members = [ + {"light": "orange"}, + {"light": "green"}, + {"light": "green"}, + ] + light, intensity = L.aggregate_stack(members) + self.assertEqual(light, "orange") + self.assertEqual(intensity, "muted") + + def test_all_yellow_still_muted(self): + # Full punch only for red/black total outage + members = [{"light": "yellow"}] * 3 + light, intensity = L.aggregate_stack(members) + self.assertEqual(light, "yellow") + self.assertEqual(intensity, "muted") + + def test_weakest_wins_black_over_red(self): + members = [ + {"light": "red"}, + {"light": "black"}, + {"light": "green"}, + ] + light, intensity = L.aggregate_stack(members) + self.assertEqual(light, "black") + self.assertEqual(intensity, "muted") + + +class ClassifyTests(unittest.TestCase): + def test_ok_green(self): + c = L.classify_probe({"ok": True, "code": 200, "elapsed": 0.2, "error": None}, now=1000.0) + self.assertEqual(c["light"], "green") + self.assertIsNone(c["down_since"]) + + def test_slow_yellow(self): + c = L.classify_probe({"ok": True, "code": 200, "elapsed": 3.0, "error": None}, now=1000.0) + self.assertEqual(c["light"], "yellow") + + def test_acute_red(self): + c = L.classify_probe( + {"ok": False, "code": 0, "elapsed": 1.0, "error": "URLError"}, + prev={"ok": True}, + now=1000.0, + acute_s=90, + ) + self.assertEqual(c["light"], "red") + self.assertEqual(c["down_since"], 1000.0) + + def test_long_black(self): + c = L.classify_probe( + {"ok": False, "code": 0, "elapsed": 1.0, "error": "URLError"}, + prev={"ok": False, "down_since": 1000.0}, + now=1200.0, + acute_s=90, + ) + self.assertEqual(c["light"], "black") + self.assertEqual(c["down_since"], 1000.0) + + def test_timeout_orange_when_fresh(self): + c = L.classify_probe( + {"ok": False, "code": 0, "elapsed": 5.0, "error": "TimeoutError"}, + prev={"ok": True}, + now=1000.0, + acute_s=90, + ) + self.assertEqual(c["light"], "orange") + + +class EndpointTests(unittest.TestCase): + def test_legal_missing_is_medium_orange(self): + for label in ("privacy", "terms"): + c = L.classify_endpoint_probe( + {"ok": False, "code": 404, "elapsed": 0.1, "error": "http_404", "body": b""}, + expect=(200,), + severity="medium", + now=1000.0, + ) + self.assertEqual(c["light"], "orange", label) + self.assertFalse(c["expected"], label) + + def test_summarize_dots(self): + eps = [ + {"light": "green", "skipped": False}, + {"light": "yellow", "skipped": False}, + {"light": "blue", "skipped": True}, + ] + s = L.summarize_endpoints(eps) + self.assertEqual(s["ok"], 1) + self.assertEqual(s["soft"], 1) + self.assertEqual(s["skipped"], 1) + self.assertEqual(s["light"], "yellow") + self.assertEqual(s["dots"], ["green", "yellow"]) # skips omitted from strip + + def test_probe_role_endpoints_intro_skip(self): + calls = [] + + def probe(url: str): + calls.append(url) + return {"ok": True, "code": 200, "elapsed": 0.05, "error": None, "body": b"{}", "url": url} + + eps = L.probe_role_endpoints( + "bank", + "bank.example.test", + has_intro=False, + probe_fn=probe, + now=1000.0, + ) + ids = {e["id"]: e for e in eps} + self.assertTrue(ids["intro"]["skipped"]) + self.assertEqual(ids["intro"]["reason"], "no_intro") + self.assertFalse(ids["config"]["skipped"]) + self.assertTrue(any(u.endswith("/config") for u in calls)) + self.assertFalse(any("/intro/" in u for u in calls)) + + def test_custom_intro_stacks_only(self): + self.assertTrue( + L.stack_has_custom_intro({"currency": "GOA", "id": "hacktivism.ch", "kind": "taler"}) + ) + self.assertTrue( + L.stack_has_custom_intro( + {"currency": "TESTPAYSAN", "id": "stage.lefrancpaysan.ch", "kind": "taler"} + ) + ) + self.assertFalse( + L.stack_has_custom_intro({"currency": "KUDOS", "id": "demo.taler.net", "kind": "taler"}) + ) + self.assertFalse( + L.stack_has_custom_intro({"currency": "CHF", "id": "taler-ops.ch", "kind": "taler"}) + ) + self.assertFalse( + L.stack_has_custom_intro( + {"currency": "", "id": "lefrancpaysan.ch", "kind": "static_pending"} + ) + ) + + def test_server_header_parse(self): + self.assertEqual(L.guess_http_package("nginx/1.22.1"), "nginx") + self.assertEqual(L.extract_http_version("nginx/1.22.1"), "1.22.1") + self.assertEqual(L.guess_http_package("Caddy"), "caddy") + + def test_vuln_raises_box_colour(self): + members = [ + {"role": "exchange", "host": "ex.example", "light": "green", "reason": "ok"}, + {"role": "merchant", "host": "m.example", "light": "green", "reason": "ok"}, + ] + vulns = { + "enabled": True, + "worst": "CRITICAL", + "light": "red", + "hits": [ + { + "id": "CVE-2099-0001", + "severity": "CRITICAL", + "package": "nginx", + "version": "1.0.0", + "host": "ex.example", + } + ], + } + light, intensity, members2 = L.apply_vuln_to_stack("green", "ok", members, vulns) + self.assertEqual(light, "red") + self.assertEqual(intensity, "muted") + self.assertEqual(members2[-1]["role"], "cve") + + def test_network_zones_and_ssh(self): + self.assertEqual( + L.classify_network_zone( + {"id": "hacktivism.ch", "currency": "GOA", "local": True, "kind": "taler"} + )["zone"], + "goa-local", + ) + self.assertEqual( + L.classify_network_zone( + {"id": "demo.taler.net", "currency": "KUDOS", "local": False, "kind": "taler"} + )["zone"], + "public-demo", + ) + targets = L.ssh_targets_for_stack( + { + "id": "hacktivism.ch", + "currency": "GOA", + "local": True, + "kind": "taler", + "merchant": "taler.hacktivism.ch", + } + ) + self.assertTrue(targets["expected"]) + self.assertTrue(any("hacktivism" in h for h in targets["hosts"])) + + def fake_ssh(host: str): + return { + "ok": True, + "light": "green", + "reason": "SSH-2.0-OpenSSH_test", + "host": host, + "elapsed": 0.05, + "banner": "SSH-2.0-OpenSSH_test", + } + + net = L.build_network_status( + { + "id": "hacktivism.ch", + "currency": "GOA", + "local": True, + "kind": "taler", + "merchant": "taler.hacktivism.ch", + }, + members=[{"light": "green", "reason": "ok"}], + ssh_probe_fn=fake_ssh, + ) + self.assertEqual(net["zone"], "goa-local") + self.assertTrue(net["ssh"]["ok"]) + self.assertIn("hacktivism", net["ssh"]["host"]) + + def test_tops_stage_ssh_is_env_mapped_hidden(self): + st = { + "id": "stage.taler-ops.ch", + "currency": "CHF", + "kind": "taler", + "local": False, + "merchant": "stage.my.taler-ops.ch", + "exchange": "exchange.stage.taler-ops.ch", + } + self.assertEqual(L.classify_network_zone(st)["zone"], "tops-stage") + with mock.patch.dict( + os.environ, {"TALER_MONITORD_SSH_TOPS": "betel.taler-systems.com"} + ): + targets = L.ssh_targets_for_stack(st) + self.assertEqual(targets["hosts"], ["betel.taler-systems.com"]) + self.assertTrue(targets.get("hide_host")) + self.assertNotIn("stage.taler-ops.ch", targets["hosts"]) + + def fake_ssh(host: str): + self.assertEqual(host, "betel.taler-systems.com") + return { + "ok": True, + "light": "green", + "reason": "SSH-2.0-OpenSSH_test", + "host": host, + "elapsed": 0.05, + "banner": "SSH-2.0-OpenSSH_test", + } + + net = L.build_network_status( + st, members=[{"light": "green", "reason": "ok"}], ssh_probe_fn=fake_ssh + ) + self.assertTrue(net["ssh"]["ok"]) + # UI/JSON must not advertise env-mapped host + self.assertEqual(net["ssh"]["host"], "") + self.assertEqual(net["ssh"]["candidates"], []) + + def test_tops_stage_ssh_without_env_uses_public_fqdn(self): + st = { + "id": "stage.taler-ops.ch", + "currency": "CHF", + "kind": "taler", + "local": False, + "merchant": "stage.my.taler-ops.ch", + "exchange": "exchange.stage.taler-ops.ch", + } + cleared = { + k: "" + for k in ( + "TALER_MONITORD_SSH_TOPS", + "TALER_MONITORD_SSH_FP", + "TALER_MONITORD_SSH_GOA", + "TALER_MONITORD_SSH_TOPS_PROD", + ) + } + with mock.patch.dict(os.environ, cleared): + targets = L.ssh_targets_for_stack(st) + self.assertEqual(targets["hosts"][0], "stage.taler-ops.ch") + self.assertFalse(targets.get("hide_host")) + + def test_fp_stage_ssh_is_env_mapped_hidden(self): + st = { + "id": "stage.lefrancpaysan.ch", + "currency": "TESTPAYSAN", + "kind": "taler", + "local": False, + "merchant": "stage.monnaie.lefrancpaysan.ch", + "exchange": "stage.exchange.lefrancpaysan.ch", + "bank": "stage.bank.lefrancpaysan.ch", + } + self.assertEqual(L.classify_network_zone(st)["zone"], "fp-stage") + with mock.patch.dict( + os.environ, {"TALER_MONITORD_SSH_FP": "ov-1a5ee8.infomaniak.ch"} + ): + targets = L.ssh_targets_for_stack(st) + self.assertEqual(targets["hosts"], ["ov-1a5ee8.infomaniak.ch"]) + self.assertTrue(targets.get("hide_host")) + self.assertNotIn("lefrancpaysan", " ".join(targets["hosts"])) + + def fake_ssh(host: str): + self.assertEqual(host, "ov-1a5ee8.infomaniak.ch") + return { + "ok": True, + "light": "green", + "reason": "SSH-2.0-OpenSSH_test", + "host": host, + "elapsed": 0.05, + "banner": "SSH-2.0-OpenSSH_test", + } + + net = L.build_network_status( + st, members=[{"light": "green", "reason": "ok"}], ssh_probe_fn=fake_ssh + ) + self.assertTrue(net["ssh"]["ok"]) + self.assertEqual(net["ssh"]["host"], "") + self.assertEqual(net["ssh"]["candidates"], []) + + def test_load_monitoring_env_expands_home(self): + with tempfile.TemporaryDirectory() as td: + p = Path(td) / "env" + p.write_text( + "TALER_MONITORD_SSH_TOPS=betel.example\n" + "SUITE_DIR=$HOME/git/taler-monitoring\n" + "# comment\n" + "export TALER_MONITORD_SSH_FP=fp.example\n", + encoding="utf-8", + ) + with mock.patch.dict(os.environ, {}, clear=False): + for k in ( + "TALER_MONITORD_SSH_TOPS", + "TALER_MONITORD_SSH_FP", + "SUITE_DIR", + ): + os.environ.pop(k, None) + got = L.load_monitoring_env(path=p) + self.assertEqual(got, p) + self.assertEqual(os.environ["TALER_MONITORD_SSH_TOPS"], "betel.example") + self.assertEqual(os.environ["TALER_MONITORD_SSH_FP"], "fp.example") + self.assertTrue(os.environ["SUITE_DIR"].endswith("/git/taler-monitoring")) + + def test_commercial_prod_tops_only(self): + self.assertTrue( + L.stack_is_commercial_prod( + {"id": "taler-ops.ch", "currency": "CHF", "kind": "taler", "local": False} + ) + ) + self.assertEqual( + L.legal_doc_severity( + {"id": "taler-ops.ch", "currency": "CHF", "kind": "taler", "local": False} + ), + "medium", + ) + self.assertFalse( + L.stack_is_commercial_prod( + {"id": "stage.taler-ops.ch", "currency": "CHF", "kind": "taler", "local": False} + ) + ) + self.assertEqual( + L.legal_doc_severity( + {"id": "stage.taler-ops.ch", "currency": "CHF", "kind": "taler", "local": False} + ), + "soft", + ) + self.assertFalse( + L.stack_is_commercial_prod( + {"id": "hacktivism.ch", "currency": "GOA", "kind": "taler", "local": True} + ) + ) + self.assertFalse( + L.stack_is_commercial_prod( + {"id": "demo.taler.net", "currency": "KUDOS", "kind": "taler", "local": False} + ) + ) + + +class CurrencySpecTests(unittest.TestCase): + def test_summarize_chf(self): + cs = { + "name": "Swiss francs", + "currency": "CHF", + "alt_unit_names": {"0": "Fr.", "-2": "Rp."}, + "common_amounts": ["CHF:5", "CHF:10", "CHF:25", "CHF:50"], + } + info = L.summarize_currency_spec(cs) + self.assertEqual(info["currency"], "CHF") + self.assertEqual(info["symbol"], "Fr.") + self.assertEqual(info["units_label"], "Fr. / Rp.") + self.assertEqual(info["denoms"], ["5", "10", "25", "50"]) + + def test_denoms_from_keys_when_common_empty(self): + cs = {"currency": "FLODOS", "alt_unit_names": {"0": "FLODOS"}, "common_amounts": []} + body = b'{"denominations":[{"value":"FLODOS:1"},{"value":"FLODOS:5"},{"value":"FLODOS:0"}]}' + info = L.summarize_currency_spec(cs, keys_body=body) + self.assertEqual(info["denoms"], ["1", "5"]) + + +class WireBackendTests(unittest.TestCase): + def test_payto_methods(self): + blob = b'{"accounts":[{"payto_uri":"payto://iban/CH9300762011623852957"},{"payto_uri":"payto://x-taler-bank/bank.example/ex"}]}' + self.assertEqual(L.payto_wire_methods(blob), {"iban", "x-taler-bank"}) + + def test_classify_ebics_iban_only(self): + self.assertEqual(L.classify_bank_backend({"iban"}), "ebics") + + def test_classify_xtaler_wins_mixed(self): + self.assertEqual(L.classify_bank_backend({"iban", "x-taler-bank"}), "x-taler-bank") + + def test_classify_conf_skip(self): + self.assertEqual(L.classify_bank_backend(set(), bank_configured=False), "ebics") + self.assertFalse(L.bank_host_configured("-")) + self.assertFalse(L.bank_host_configured("ebics")) + self.assertTrue(L.bank_host_configured("bank.example.test")) + + +class ParseAndBuildTests(unittest.TestCase): + def setUp(self): + _disable_extra_planes() + + def test_parse_stage_fp(self): + conf = ROOT / "domains.conf" + stacks = L.parse_domains_conf(conf) + ids = {s["id"] for s in stacks} + self.assertIn("stage.lefrancpaysan.ch", ids) + stage = next(s for s in stacks if s["id"] == "stage.lefrancpaysan.ch") + self.assertEqual(stage["bank"], "stage.bank.lefrancpaysan.ch") + self.assertEqual(stage["merchant"], "stage.monnaie.lefrancpaysan.ch") + self.assertEqual(stage["currency"], "TESTPAYSAN") + self.assertEqual(stage["kind"], "taler") + prod = next(s for s in stacks if s["id"] == "lefrancpaysan.ch") + self.assertEqual(prod["kind"], "static_pending") + self.assertEqual(prod["currency"], "") + + def test_ebics_skips_bank_probe(self): + stacks = [ + { + "id": "tops.example", + "bank": "bank.tops.example", + "exchange": "exchange.tops.example", + "merchant": "my.tops.example", + "currency": "CHF", + "kind": "taler", + "local": False, + } + ] + probed = [] + + def probe(url: str): + probed.append(url) + if "/keys" in url: + body = b'{"accounts":[{"payto_uri":"payto://iban/CH6808573105529100001?receiver-name=X"}]}' + return {"ok": True, "code": 200, "elapsed": 0.1, "error": None, "body": body, "url": url} + if "bank." in url: + return {"ok": False, "code": 0, "elapsed": 0.1, "error": "URLError", "body": b"", "url": url} + return {"ok": True, "code": 200, "elapsed": 0.1, "error": None, "body": b"{}", "url": url} + + doc = L.build_stack_status(stacks, probe_fn=probe, now=5000.0, public_only=False) + st = doc["stacks"][0] + self.assertEqual(st["bank_backend"], "ebics") + self.assertEqual(st["light"], "green") + self.assertEqual(st["intensity"], "ok") + roles = {m["role"]: m for m in st["members"]} + self.assertEqual(roles["bank"]["reason"], "ebics_iban") + self.assertEqual(roles["bank"]["light"], "blue") + self.assertEqual(roles["bank"]["host"], "IBAN") + self.assertTrue(all("bank.tops" not in u for u in probed)) + + def test_xtaler_bank_still_probed(self): + stacks = [ + { + "id": "demo.example", + "bank": "bank.demo.example", + "exchange": "exchange.demo.example", + "merchant": "backend.demo.example", + "currency": "KUDOS", + "kind": "taler", + "local": False, + } + ] + + def probe(url: str): + if "/keys" in url: + body = b'{"accounts":[{"payto_uri":"payto://x-taler-bank/bank.demo.example/exchange"}]}' + return {"ok": True, "code": 200, "elapsed": 0.1, "error": None, "body": body, "url": url} + if "bank." in url: + return {"ok": False, "code": 0, "elapsed": 0.1, "error": "URLError", "body": b"", "url": url} + return {"ok": True, "code": 200, "elapsed": 0.1, "error": None, "body": b"{}", "url": url} + + doc = L.build_stack_status(stacks, probe_fn=probe, now=5000.0, public_only=False) + st = doc["stacks"][0] + self.assertEqual(st["bank_backend"], "x-taler-bank") + self.assertEqual(st["light"], "red") + self.assertEqual(st["intensity"], "muted") + roles = {m["role"]: m for m in st["members"]} + self.assertEqual(roles["bank"]["light"], "red") + + def test_domains_conf_bank_dash(self): + with tempfile.TemporaryDirectory() as td: + conf = Path(td) / "domains.conf" + conf.write_text( + "ebics.example - exchange.ebics.example my.ebics.example CHF 0 0 en de-CH\n", + encoding="utf-8", + ) + stacks = L.parse_domains_conf(conf) + self.assertEqual(len(stacks), 1) + self.assertFalse(stacks[0]["bank_configured"]) + self.assertEqual(stacks[0]["bank"], "") + + def test_landing_stats_summary(self): + stacks = [ + { + "id": "stage.example", + "bank": "bank.stage.example", + "exchange": "exchange.stage.example", + "merchant": "monnaie.stage.example", + "currency": "TESTPAYSAN", + "kind": "taler", + "local": False, + } + ] + stats_body = json.dumps( + { + "ok": True, + "currency": "TESTPAYSAN", + "generated_at_unix": 5000.0 - 120, + "generated_at_human": "just now", + "source": "stage-shared", + "bank_accounts": {"total": 36, "users": 34}, + "withdraws": {"count": 45, "total_amount": "TESTPAYSAN:12"}, + } + ).encode() + + cfg_body = json.dumps( + { + "currency": "TESTPAYSAN", + "currency_specification": { + "name": "TESTPAYSAN stage currency", + "currency": "TESTPAYSAN", + "alt_unit_names": {"0": "TESTPAYSAN", "-2": "ct"}, + "common_amounts": [ + "TESTPAYSAN:5", + "TESTPAYSAN:10", + "TESTPAYSAN:20", + "TESTPAYSAN:50", + "TESTPAYSAN:100", + ], + }, + } + ).encode() + + def probe(url: str): + if url.endswith("/intro/stats.json") and "bank.stage" in url: + return { + "ok": True, + "code": 200, + "elapsed": 0.05, + "error": None, + "body": stats_body, + "url": url, + } + if "/keys" in url: + body = b'{"accounts":[{"payto_uri":"payto://x-taler-bank/bank.stage.example/exchange"}]}' + return {"ok": True, "code": 200, "elapsed": 0.1, "error": None, "body": body, "url": url} + if url.endswith("/config") and "exchange.stage" in url: + return { + "ok": True, + "code": 200, + "elapsed": 0.05, + "error": None, + "body": cfg_body, + "url": url, + } + return {"ok": True, "code": 200, "elapsed": 0.1, "error": None, "body": b"{}", "url": url} + + doc = L.build_stack_status(stacks, probe_fn=probe, now=5000.0, public_only=False) + st = doc["stacks"][0] + self.assertIsNotNone(st.get("stats")) + self.assertTrue(st["stats"]["present"]) + self.assertEqual(st["stats"]["age_s"], 120) + self.assertIn("accounts 36", st["stats"]["lines"]) + self.assertFalse(st["stats"]["stale"]) + self.assertEqual(st["currency"], "TESTPAYSAN") + self.assertEqual(st["currency_info"]["symbol"], "TESTPAYSAN") + self.assertEqual(st["currency_info"]["units_label"], "TESTPAYSAN / ct") + self.assertEqual(st["currency_info"]["denoms"], ["5", "10", "20", "50", "100"]) + + def test_static_pending_softens_api_down(self): + stacks = [ + { + "id": "lefrancpaysan.ch", + "bank": "bank.lefrancpaysan.ch", + "exchange": "exchange.lefrancpaysan.ch", + "merchant": "monnaie.lefrancpaysan.ch", + "currency": "", + "kind": "static_pending", + "local": False, + } + ] + + def probe(url: str): + if url.rstrip("/").endswith("lefrancpaysan.ch"): + return {"ok": True, "code": 200, "elapsed": 0.1, "error": None, "url": url} + return {"ok": False, "code": 502, "elapsed": 0.1, "error": "http_502", "url": url} + + doc = L.build_stack_status(stacks, probe_fn=probe, now=5000.0, public_only=False) + st = doc["stacks"][0] + self.assertEqual(st["kind"], "static_pending") + self.assertEqual(st["light"], "blue") + self.assertEqual(st["intensity"], "ok") + roles = {m["role"]: m for m in st["members"]} + self.assertEqual(roles["site"]["light"], "green") + self.assertEqual(roles["bank"]["reason"], "no_prod_yet") + self.assertEqual(roles["bank"]["light"], "blue") + + def test_build_with_mock_probe(self): + stacks = [ + { + "id": "example.test", + "bank": "bank.example.test", + "exchange": "exchange.example.test", + "merchant": "merch.example.test", + "currency": "X", + "local": False, + } + ] + + def probe(url: str): + if "exchange" in url: + return {"ok": False, "code": 0, "elapsed": 0.1, "error": "URLError", "url": url} + return {"ok": True, "code": 200, "elapsed": 0.1, "error": None, "url": url} + + doc = L.build_stack_status(stacks, probe_fn=probe, now=5000.0, public_only=False) + self.assertEqual(len(doc["stacks"]), 1) + st = doc["stacks"][0] + self.assertEqual(st["light"], "red") + self.assertEqual(st["intensity"], "muted") + self.assertIn("breakdown", st) + self.assertEqual(doc.get("schema"), L.SCHEMA) + + def test_atomic_write_roundtrip(self): + with tempfile.TemporaryDirectory() as td: + path = Path(td) / "stack-status.json" + doc = {"schema": 1, "stacks": []} + L.atomic_write_json(path, doc) + self.assertTrue(path.is_file()) + got = L.read_status(path) + self.assertEqual(got["schema"], 1) + + def test_parse_services_conf(self): + with tempfile.TemporaryDirectory() as td: + conf = Path(td) / "services-catalog.conf" + conf.write_text( + "bugs.gnunet.org bugs.gnunet.org MantisBT firefly\n" + "erp.stage.taler-systems.com erp.stage.taler-systems.com ERPnext pixel\n", + encoding="utf-8", + ) + rows = L.parse_services_conf(conf) + self.assertEqual(len(rows), 2) + self.assertEqual(rows[0]["service_label"], "MantisBT") + self.assertEqual(rows[1]["zone"], "pixel") + + +class HtmlRenderTests(unittest.TestCase): + def test_degraded_why_and_cve_link(self): + import stack_status_html as H + + doc = { + "schema": L.SCHEMA, + "ts": time.time(), + "acute_down_s": 90, + "stacks": [ + { + "id": "my.taler-ops.ch", + "light": "orange", + "intensity": "muted", + "currency": "CHF", + "bank_backend": "ebics", + "kind": "taler", + "breakdown": {"by_light": {"green": 4, "orange": 3}, "worst": "orange", "counted": 7}, + "members": [ + {"role": "bank", "host": "IBAN", "light": "blue", "reason": "ebics_iban"}, + {"role": "exchange", "host": "exchange.taler-ops.ch", "light": "green", "reason": "ok"}, + { + "role": "merchant", + "host": "my.taler-ops.ch", + "light": "orange", + "reason": "http_501", + }, + ], + "vulns": { + "enabled": True, + "worst": "HIGH", + "hits": [ + { + "id": "CVE-2024-1234", + "severity": "HIGH", + "package": "nginx", + "version": "1.24.0", + "summary": "test vuln", + "host": "my.taler-ops.ch", + } + ], + "servers": [ + {"host": "my.taler-ops.ch", "package": "nginx", "version": "1.24.0", "server": "nginx/1.24.0"} + ], + }, + "network": {"zone": "tops-prod", "ssh": {"expected": False}}, + "tls": {"days_left": 60, "sans": ["my.taler-ops.ch"]}, + } + ], + "services": [], + "mail": [], + "orphans": [], + } + html = H.render_html(doc) + self.assertIn("4× OK", html) + self.assertIn("3× Degraded", html) + self.assertIn("osv.dev/vulnerability/CVE-2024-1234", html) + self.assertIn("openStack", html) + self.assertIn("aria-label=\"nginx\"", html) + self.assertIn("0/1 green", html) # orange stack → not green + self.assertIn("Gone", html) # or Degraded label in score — 0 green → white/Gone + + def test_git_ls_remote_soft_probe_mocked(self): + calls: list[list[str]] = [] + + def fake_run(cmd, **kwargs): + calls.append(list(cmd)) + + class R: + returncode = 0 + stdout = b"abc123\trefs/heads/main\n" + stderr = b"" + + return R() + + with tempfile.TemporaryDirectory() as td: + with mock.patch.dict( + os.environ, {"TALER_MONITORD_STATE": td, "TALER_MONITORD_GIT": "1"} + ): + L.refresh_runtime_flags_from_env() + with mock.patch("stack_status_lib.subprocess.run", side_effect=fake_run): + got = L.collect_git_probe("git.example", now=1000.0, force=True) + self.assertTrue(got["ok"]) + self.assertEqual(got["light"], "green") + self.assertGreaterEqual(len(calls), 2) # https + ssh + n = len(calls) + with mock.patch("stack_status_lib.subprocess.run", side_effect=fake_run): + cached = L.collect_git_probe("git.example", now=1100.0, force=False) + self.assertTrue(cached["ok"]) + self.assertEqual(len(calls), n) # interval cache hit + + def test_result_index_optional_by_colour_and_cmd(self): + import stack_status_html as H + + doc = { + "schema": L.SCHEMA, + "ts": time.time(), + "acute_down_s": 90, + "stacks": [ + { + "id": "stage.example", + "light": "yellow", + "intensity": "muted", + "currency": "CHF", + "kind": "taler", + "members": [ + { + "role": "exchange", + "host": "exchange.stage.example", + "light": "green", + "reason": "ok", + "endpoints": [ + { + "id": "config", + "path": "/config", + "url": "https://exchange.stage.example/config", + "light": "green", + "reason": "ok", + "skipped": False, + }, + { + "id": "keys", + "path": "/keys", + "url": "https://exchange.stage.example/keys", + "light": "yellow", + "reason": "slow", + "skipped": False, + }, + ], + }, + { + "role": "bank", + "host": "IBAN", + "light": "blue", + "reason": "ebics_iban", + }, + ], + "network": { + "zone": "tops-stage", + "ssh": { + "expected": True, + "ok": True, + "light": "green", + "host": "", + "candidates": [], + "reason": "SSH-2.0-OpenSSH_test", + }, + }, + } + ], + "services": [], + "mail": [], + } + off = H.render_html(doc) + self.assertNotIn('id="result-index"', off) + + colors = H._render_result_index(doc, mode="colors") + self.assertIn('id="result-index"', colors) + self.assertIn("all probes by colour", colors) + self.assertIn("exchange/config", colors) + self.assertIn("exchange/keys", colors) + self.assertIn("bank", colors) + self.assertIn("ssh", colors) + self.assertNotIn("ri-cmd", colors) + + cmd = H._render_result_index(doc, mode="cmd") + self.assertIn("with command", cmd) + self.assertIn("GET https://exchange.stage.example/config", cmd) + self.assertIn("GET https://exchange.stage.example/keys", cmd) + self.assertIn("TCP", cmd) + + +if __name__ == "__main__": + unittest.main()