#!/usr/bin/env python3 """Stack traffic lights for taler-monitoring (daemon → JSON; UI reads only).""" from __future__ import annotations import json import os import re import socket import ssl import subprocess import time import urllib.error import urllib.request from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from typing import Any, Callable SCHEMA = 8 LIGHTS = ("green", "blue", "yellow", "orange", "red", "black") SEVERITY = {name: i for i, name in enumerate(LIGHTS)} # Soft endpoint fails (intro/terms/…) cap at yellow unless hard=True. ENDPOINT_SOFT_MAX = "yellow" def _env_truthy(name: str, default: str = "1") -> bool: return os.environ.get(name, default) not in ("0", "false", "no", "") def refresh_runtime_flags_from_env() -> None: """Re-read module toggles after load_monitoring_env() (import-time values go stale).""" global PUBLISHED_ENABLED, PUBLISHED_SUITE, PUBLISHED_APT_BASE global ACUTE_DOWN_S, PROBE_TIMEOUT_S, STATS_STALE_S, SSH_PROBE_TIMEOUT_S, TLS_VERIFY global SSH_PROBES_ENABLED, CVE_PROBES_ENABLED, SERVICES_ENABLED, MAIL_ENABLED global CERT_PROBES_ENABLED, ORPHAN_PROBES_ENABLED, ORPHAN_HTTP_ENABLED global MAIL_PORTS_ENABLED, MAIL_PORT_TIMEOUT_S global DELIVER_ENABLED, DELIVER_INTERVAL_S, DELIVER_STALE_S, DELIVER_TIMEOUT_S global PORT_SCAN_ENABLED, PORT_SCAN_TIMEOUT_S, PORT_SCAN_WORKERS global CERT_WARN_DAYS, CERT_CRIT_DAYS global SCORE_RATIO_GREEN, SCORE_RATIO_BLUE, SCORE_RATIO_YELLOW global SCORE_RATIO_ORANGE, SCORE_RATIO_RED global RESULT_INDEX_MODE global GIT_PROBE_ENABLED, GIT_PROBE_INTERVAL_S, GIT_PROBE_STALE_S, GIT_PROBE_TIMEOUT_S global PAY_ENABLED, PAY_RUN, PAY_INTERVAL_S, PAY_STALE_S, PAY_TIMEOUT_S, PAY_DOMAINS PUBLISHED_ENABLED = _env_truthy("TALER_MONITORD_PUBLISHED") PUBLISHED_SUITE = os.environ.get("TALER_MONITORD_APT_SUITE", "trixie") PUBLISHED_APT_BASE = os.environ.get( "TALER_MONITORD_APT_BASE", "https://deb.taler.net/apt/debian" ).rstrip("/") ACUTE_DOWN_S = float(os.environ.get("TALER_MONITORD_ACUTE_DOWN_S", "90")) PROBE_TIMEOUT_S = float(os.environ.get("TALER_MONITORD_PROBE_TIMEOUT_S", "5")) STATS_STALE_S = float(os.environ.get("TALER_MONITORD_STATS_STALE_S", str(6 * 3600))) SSH_PROBE_TIMEOUT_S = float(os.environ.get("TALER_MONITORD_SSH_TIMEOUT_S", "4")) TLS_VERIFY = _env_truthy("TALER_MONITORD_TLS_VERIFY", "0") SSH_PROBES_ENABLED = _env_truthy("TALER_MONITORD_SSH") CVE_PROBES_ENABLED = _env_truthy("TALER_MONITORD_CVE") SERVICES_ENABLED = _env_truthy("TALER_MONITORD_SERVICES") MAIL_ENABLED = _env_truthy("TALER_MONITORD_MAIL") CERT_PROBES_ENABLED = _env_truthy("TALER_MONITORD_CERT") ORPHAN_PROBES_ENABLED = _env_truthy("TALER_MONITORD_ORPHAN") ORPHAN_HTTP_ENABLED = _env_truthy("TALER_MONITORD_ORPHAN_HTTP", "0") MAIL_PORTS_ENABLED = _env_truthy("TALER_MONITORD_MAIL_PORTS") MAIL_PORT_TIMEOUT_S = float(os.environ.get("TALER_MONITORD_MAIL_PORT_TIMEOUT_S", "4")) DELIVER_ENABLED = _env_truthy("TALER_MONITORD_DELIVER") DELIVER_INTERVAL_S = float( os.environ.get("TALER_MONITORD_DELIVER_INTERVAL_S", str(6 * 3600)) ) DELIVER_STALE_S = float( os.environ.get("TALER_MONITORD_DELIVER_STALE_S", str(30 * 3600)) ) DELIVER_TIMEOUT_S = float(os.environ.get("TALER_MONITORD_DELIVER_TIMEOUT_S", "12")) PORT_SCAN_ENABLED = _env_truthy("TALER_MONITORD_PORTSCAN") PORT_SCAN_TIMEOUT_S = float(os.environ.get("TALER_MONITORD_PORTSCAN_TIMEOUT_S", "0.8")) PORT_SCAN_WORKERS = int(os.environ.get("TALER_MONITORD_PORTSCAN_WORKERS", "24")) CERT_WARN_DAYS = int(os.environ.get("TALER_MONITORD_CERT_WARN_DAYS", "30")) CERT_CRIT_DAYS = int(os.environ.get("TALER_MONITORD_CERT_CRIT_DAYS", "14")) SCORE_RATIO_GREEN = float(os.environ.get("TALER_MONITORD_SCORE_GREEN", "1.0")) SCORE_RATIO_BLUE = float(os.environ.get("TALER_MONITORD_SCORE_BLUE", "0.85")) SCORE_RATIO_YELLOW = float(os.environ.get("TALER_MONITORD_SCORE_YELLOW", "0.60")) SCORE_RATIO_ORANGE = float(os.environ.get("TALER_MONITORD_SCORE_ORANGE", "0.35")) SCORE_RATIO_RED = float(os.environ.get("TALER_MONITORD_SCORE_RED", "0.01")) # Optional bottom index: 0/off | 1/colors | cmd/full (+ concrete probe command) raw_idx = (os.environ.get("TALER_MONITORD_RESULT_INDEX") or "0").strip().lower() if raw_idx in ("0", "false", "no", "off", ""): RESULT_INDEX_MODE = "off" elif raw_idx in ("cmd", "full", "commands", "2"): RESULT_INDEX_MODE = "cmd" else: RESULT_INDEX_MODE = "colors" if RESULT_INDEX_MODE == "colors" and _env_truthy( "TALER_MONITORD_RESULT_INDEX_CMD", "0" ): RESULT_INDEX_MODE = "cmd" GIT_PROBE_ENABLED = _env_truthy("TALER_MONITORD_GIT") GIT_PROBE_INTERVAL_S = float( os.environ.get("TALER_MONITORD_GIT_INTERVAL_S", str(6 * 3600)) ) GIT_PROBE_STALE_S = float( os.environ.get("TALER_MONITORD_GIT_STALE_S", str(30 * 3600)) ) GIT_PROBE_TIMEOUT_S = float(os.environ.get("TALER_MONITORD_GIT_TIMEOUT_S", "12")) # Payability (wallet-cli zero-pay) — opt-in; not every monitord tick PAY_ENABLED = _env_truthy("TALER_MONITORD_PAY", "0") PAY_RUN = _env_truthy("TALER_MONITORD_PAY_RUN", "0") PAY_INTERVAL_S = float(os.environ.get("TALER_MONITORD_PAY_INTERVAL_S", str(4 * 3600))) PAY_STALE_S = float(os.environ.get("TALER_MONITORD_PAY_STALE_S", str(30 * 3600))) PAY_TIMEOUT_S = float(os.environ.get("TALER_MONITORD_PAY_TIMEOUT_S", "180")) raw_pay = (os.environ.get("TALER_MONITORD_PAY_DOMAINS") or "").strip() PAY_DOMAINS = [ p.strip() for p in re.split(r"[\s,]+", raw_pay) if p.strip() ] or ["stage.taler-ops.ch"] # Placeholders; filled by refresh_runtime_flags_from_env() below. PUBLISHED_ENABLED = True PUBLISHED_SUITE = "trixie" PUBLISHED_APT_BASE = "https://deb.taler.net/apt/debian" ACUTE_DOWN_S = 90.0 PROBE_TIMEOUT_S = 5.0 STATS_STALE_S = float(6 * 3600) SSH_PROBE_TIMEOUT_S = 4.0 TLS_VERIFY = False SSH_PROBES_ENABLED = True CVE_PROBES_ENABLED = True SERVICES_ENABLED = True MAIL_ENABLED = True CERT_PROBES_ENABLED = True ORPHAN_PROBES_ENABLED = True ORPHAN_HTTP_ENABLED = False MAIL_PORTS_ENABLED = True MAIL_PORT_TIMEOUT_S = 4.0 DELIVER_ENABLED = True DELIVER_INTERVAL_S = float(6 * 3600) DELIVER_STALE_S = float(30 * 3600) DELIVER_TIMEOUT_S = 12.0 # Outside-in notable port scan (public TCP only). Suspicious = open ∩ ¬whitelist ∩ ¬baseline. # Whitelist = operator-known intentional public opens — not an inventory of private topology. PORT_SCAN_ENABLED = True PORT_SCAN_TIMEOUT_S = 0.8 PORT_SCAN_WORKERS = 24 CERT_WARN_DAYS = 30 CERT_CRIT_DAYS = 14 SCORE_RATIO_GREEN = 1.0 SCORE_RATIO_BLUE = 0.85 SCORE_RATIO_YELLOW = 0.60 SCORE_RATIO_ORANGE = 0.35 SCORE_RATIO_RED = 0.01 RESULT_INDEX_MODE = "off" GIT_PROBE_ENABLED = True GIT_PROBE_INTERVAL_S = float(6 * 3600) GIT_PROBE_STALE_S = float(30 * 3600) GIT_PROBE_TIMEOUT_S = 12.0 PAY_ENABLED = False PAY_RUN = False PAY_INTERVAL_S = float(4 * 3600) PAY_STALE_S = float(30 * 3600) PAY_TIMEOUT_S = 180.0 PAY_DOMAINS = ["stage.taler-ops.ch"] refresh_runtime_flags_from_env() # Notable TCP ports (not a full nmap range). Override: TALER_MONITORD_PORTSCAN_PORTS=22,80,443,… _DEFAULT_SCAN_PORTS = ( 21, 22, 23, 25, 80, 110, 135, 139, 143, 443, 445, 465, 587, 993, 995, 1433, 1521, 1789, 1790, 2049, 3306, 3389, 5432, 5900, 6379, 8080, 8443, 9200, 9418, 11211, 23230, 23235, 23236, 23237, 23238, 27017, 52323, ) # Baseline always-ok for public HTTPS endpoints (still overridable by denying via empty whitelist only — # whitelist *adds* allowance; baseline is the default allow for web). _BASELINE_WEB_PORTS = frozenset({80, 443}) _BASELINE_MAIL_PORTS = frozenset({25, 110, 143, 465, 587, 993, 995}) # Port → short label for UI _PORT_LABELS = { 21: "ftp", 22: "ssh", 23: "telnet", 25: "smtp", 80: "http", 110: "pop3", 135: "msrpc", 139: "netbios", 143: "imap", 443: "https", 445: "smb", 465: "smtps", 587: "submission", 993: "imaps", 995: "pop3s", 1433: "mssql", 1521: "oracle", 2049: "nfs", 3306: "mysql", 3389: "rdp", 5432: "postgres", 5900: "vnc", 6379: "redis", 8080: "http-alt", 8443: "https-alt", 1789: "nym-sphinx", 1790: "nym-verloc", 9200: "git-ssh", 9418: "git-daemon", 11211: "memcached", 23230: "ssh-dnat", 23235: "ssh-dnat", 23236: "ssh-dnat", 23237: "ssh-dnat", 23238: "ssh-dnat", 27017: "mongodb", 52323: "tortuna", } # Member reasons that are informational only (do not weaken the stack box). _AGG_SKIP_REASONS = frozenset({"ebics_iban", "ebics_no_bank"}) # Suite-like public endpoints per machine (paths relative to https://host). # need_intro: custom /intro stack only (GOA + FrancPaysan/TESTPAYSAN stage). # severity: soft (≤yellow) | medium (missing → orange) | hard (full classify). ROLE_ENDPOINTS: dict[str, list[dict[str, Any]]] = { "bank": [ {"id": "integration", "path": "/taler-integration/config", "expect": (200,), "severity": "hard"}, {"id": "config", "path": "/config", "expect": (200,), "severity": "hard"}, {"id": "webui", "path": "/webui/", "expect": (200, 301, 302), "severity": "soft"}, {"id": "intro", "path": "/intro/", "expect": (200, 301, 302), "severity": "soft", "need_intro": True}, {"id": "stats", "path": "/intro/stats.json", "expect": (200,), "severity": "soft", "need_intro": True}, # Legal docs: severity overridden at probe time (medium only for commercial prod). {"id": "terms", "path": "/terms", "expect": (200,), "severity": "soft", "need_intro": True, "legal": True}, {"id": "privacy", "path": "/privacy", "expect": (200,), "severity": "soft", "need_intro": True, "legal": True}, ], "exchange": [ {"id": "config", "path": "/config", "expect": (200,), "severity": "hard"}, {"id": "keys", "path": "/keys", "expect": (200,), "severity": "hard"}, {"id": "terms", "path": "/terms", "expect": (200,), "severity": "soft", "legal": True}, {"id": "privacy", "path": "/privacy", "expect": (200,), "severity": "soft", "legal": True}, {"id": "intro", "path": "/intro/", "expect": (200, 301, 302), "severity": "soft", "need_intro": True}, ], "merchant": [ {"id": "config", "path": "/config", "expect": (200,), "severity": "hard"}, {"id": "webui", "path": "/webui/", "expect": (200, 301, 302), "severity": "soft"}, {"id": "intro", "path": "/intro/", "expect": (200, 301, 302), "severity": "soft", "need_intro": True}, {"id": "stats", "path": "/intro/stats.json", "expect": (200,), "severity": "soft", "need_intro": True}, {"id": "terms", "path": "/terms", "expect": (200,), "severity": "soft", "legal": True}, {"id": "privacy", "path": "/privacy", "expect": (200,), "severity": "soft", "legal": True}, ], "site": [ {"id": "apex", "path": "/", "expect": (200, 301, 302), "severity": "hard"}, ], } def stack_has_custom_intro(st: dict[str, Any]) -> bool: """True only for custom /intro landings: GOA, FrancPaysan, TESTPAYSAN stage. Prefer domains.conf landing=1; also recognize by currency / hostname so demo/test/regio/TOPS never get /intro probes by accident. """ cur = str(st.get("currency") or "").upper() if cur in ("GOA", "TESTPAYSAN"): return True blob = " ".join( [ str(st.get("id") or ""), " ".join(str(a) for a in (st.get("aliases") or [])), str(st.get("bank") or ""), str(st.get("exchange") or ""), str(st.get("merchant") or ""), ] ).lower() if "hacktivism" in blob or "lefrancpaysan" in blob or "francpaysan" in blob: # static_pending FP prod has no Taler /intro APIs yet if (st.get("kind") or "") == "static_pending" or not cur: return False return True # Explicit landing=1 only if it is one of the known custom stacks above return False def stack_is_commercial_prod(st: dict[str, Any]) -> bool: """Commercial production (legal docs must be solid) — not stage/demo/test/GOA. Today: TOPS CHF prod (taler-ops.ch / my.taler-ops.ch). Stage and sandboxes stay soft. """ if st.get("local"): return False if (st.get("kind") or "") == "static_pending": return False sid = str(st.get("id") or "").lower() blob = " ".join( [ sid, " ".join(str(a) for a in (st.get("aliases") or [])), str(st.get("merchant") or ""), str(st.get("exchange") or ""), ] ).lower() if "stage." in sid or sid.startswith("stage.") or ".stage." in sid: return False if "stage." in blob and "taler-ops" in blob: return False # TOPS production CHF if "taler-ops.ch" in blob and "stage" not in sid: cur = str(st.get("currency") or "").upper() if cur in ("", "CHF"): return True return False def legal_doc_severity(st: dict[str, Any]) -> str: """terms/privacy: medium (orange) only for commercial production.""" return "medium" if stack_is_commercial_prod(st) else "soft" def _stack_blob(st: dict[str, Any]) -> str: return " ".join( [ str(st.get("id") or ""), " ".join(str(a) for a in (st.get("aliases") or [])), str(st.get("bank") or ""), str(st.get("exchange") or ""), str(st.get("merchant") or ""), str(st.get("currency") or ""), ] ).lower() def classify_network_zone(st: dict[str, Any]) -> dict[str, str]: """Map a stack to a coarse network area (not a full topology).""" blob = _stack_blob(st) cur = str(st.get("currency") or "").upper() if st.get("local") or "hacktivism" in blob or cur == "GOA": return {"zone": "goa-local", "label": "GOA / local (koopa)"} if cur == "TESTPAYSAN" or "stage.lefrancpaysan" in blob: return {"zone": "fp-stage", "label": "FrancPaysan stage"} if (st.get("kind") or "") == "static_pending" or ( "lefrancpaysan" in blob and not cur ): return {"zone": "fp-edge", "label": "FrancPaysan edge (static)"} if stack_is_commercial_prod(st): return {"zone": "tops-prod", "label": "TOPS prod CHF"} if "stage.taler-ops" in blob or "stage.my.taler-ops" in blob: return {"zone": "tops-stage", "label": "TOPS stage"} if cur in ("KUDOS", "TESTKUDOS", "FLODOS") or "demo.taler.net" in blob or "test.taler.net" in blob: return {"zone": "public-demo", "label": "public demo/test"} if "taler-systems" in blob or "anastasis" in blob or "pixel" in blob: return {"zone": "pixel", "label": "pixel / TSA / Anastasis"} if "gnunet.org" in blob or (st.get("kind") or "") == "mail": zone = str(st.get("zone") or "") if zone == "mail" or (st.get("kind") or "") == "mail": return {"zone": "mail", "label": "mail plane"} return {"zone": "firefly", "label": "firefly / gnunet"} return {"zone": "public", "label": "public HTTPS"} def _env_ssh_hosts(*keys: str) -> list[str]: out: list[str] = [] for k in keys: raw = (os.environ.get(k) or "").strip() if not raw: continue for part in re.split(r"[\s,]+", raw): if part and part not in out: out.append(part) return out def monitoring_env_path() -> Path: """Path to taler-monitoring-env install (~/.config/taler-monitoring/env).""" raw = (os.environ.get("TALER_MONITORING_ENV") or "").strip() if raw: return Path(raw) xdg = os.environ.get("XDG_CONFIG_HOME") or str(Path.home() / ".config") return Path(xdg) / "taler-monitoring" / "env" def _expand_env_value(val: str) -> str: """Expand $VAR / ${VAR} using current environ (shell-like, for env profiles).""" def repl(m: re.Match[str]) -> str: name = m.group(1) or m.group(2) return os.environ.get(name, "") return re.sub( r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}|\$([A-Za-z_][A-Za-z0-9_]*)", repl, val, ) def load_monitoring_env(*, path: Path | None = None, override: bool = True) -> Path | None: """Load KEY=value from taler-monitoring-env profile into os.environ (like lib.sh). Skips comments/blanks. Expands $HOME / $VAR. Returns path if loaded, else None. """ p = path or monitoring_env_path() if not p.is_file(): return None try: text = p.read_text(encoding="utf-8") except OSError: return None for line in text.splitlines(): s = line.strip() if not s or s.startswith("#"): continue if s.startswith("export "): s = s[7:].lstrip() if "=" not in s: continue key, _, val = s.partition("=") key = key.strip() if not key or not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", key): continue if not override and key in os.environ: continue val = val.strip() if len(val) >= 2 and val[0] == val[-1] and val[0] in ("'", '"'): val = val[1:-1] os.environ[key] = _expand_env_value(val) return p def ssh_targets_for_stack(st: dict[str, Any]) -> dict[str, Any]: """Hostnames to probe for an SSH daemon (port 22 banner) — not jump-host login. Zone-mapped SSH targets come only from taler-monitoring-env (`TALER_MONITORD_SSH_TOPS` / `_FP` / `_GOA` / `_TOPS_PROD`). No suite hardcodes. When env supplies hosts, hide them in UI/JSON display fields. """ zone = classify_network_zone(st)["zone"] # Prefer apex / merchant hostname that is publicly meaningful candidates: list[str] = [] for key in ("id", "merchant", "exchange", "bank"): h = _host_only(str(st.get(key) or "")) if h and h.upper() != "IBAN" and h not in candidates: candidates.append(h) env_map = { "goa-local": "TALER_MONITORD_SSH_GOA", "fp-stage": "TALER_MONITORD_SSH_FP", "tops-stage": "TALER_MONITORD_SSH_TOPS", "tops-prod": "TALER_MONITORD_SSH_TOPS_PROD", } env_hosts = _env_ssh_hosts(env_map[zone]) if zone in env_map else [] if env_hosts: hosts = env_hosts hide_host = True else: hosts = candidates[:2] hide_host = False # Public demos: still check if SSH is exposed (security signal) expected = True if zone in ("fp-edge",): expected = bool(hosts) return { "expected": expected and bool(hosts), "hosts": hosts, "via": "tcp22-banner", "hide_host": hide_host, } def probe_ssh_host(host: str, *, timeout_s: float = SSH_PROBE_TIMEOUT_S) -> dict[str, Any]: """Check whether an SSH service answers (banner), not whether our keys work. SSH-2.0-… banner ⇒ present (ok/green). Connection refused / timeout ⇒ absent. """ host = _host_only(host) if not host: return {"ok": False, "light": "blue", "reason": "no_host", "host": "", "elapsed": 0.0} t0 = time.monotonic() try: with socket.create_connection((host, 22), timeout=max(1.0, float(timeout_s))) as sock: sock.settimeout(max(1.0, float(timeout_s))) banner = sock.recv(256).decode("utf-8", "replace").strip().splitlines() line = banner[0] if banner else "" elapsed = time.monotonic() - t0 if line.upper().startswith("SSH-"): return { "ok": True, "light": "green", "reason": line[:80], "host": host, "elapsed": round(elapsed, 3), "banner": line[:120], } return { "ok": False, "light": "orange", "reason": f"no_ssh_banner:{line[:60]}", "host": host, "elapsed": round(elapsed, 3), } except (socket.timeout, TimeoutError): return { "ok": False, "light": "orange", "reason": "timeout", "host": host, "elapsed": round(time.monotonic() - t0, 3), } except OSError as e: # refused / unreachable = no public SSH (often OK for hardened edge) return { "ok": False, "light": "blue", "reason": type(e).__name__, "host": host, "elapsed": round(time.monotonic() - t0, 3), } def build_network_status( st: dict[str, Any], *, members: list[dict[str, Any]] | None = None, ssh_cache: dict[str, dict[str, Any]] | None = None, ssh_probe_fn: Callable[[str], dict[str, Any]] | None = None, ) -> dict[str, Any]: """Zone + public HTTPS summary + optional SSH reachability for the bundle.""" zone = classify_network_zone(st) members = members or [] # Public path: worst non-informational member light pub_lights = [ m.get("light") or "black" for m in members if (m.get("reason") or "") not in _AGG_SKIP_REASONS ] if not pub_lights: pub_lights = [m.get("light") or "black" for m in members] or ["blue"] pub_worst = LIGHTS[max(SEVERITY.get(L, 5) for L in pub_lights)] targets = ssh_targets_for_stack(st) hide_host = bool(targets.get("hide_host")) ssh_block: dict[str, Any] = { "expected": bool(targets.get("expected")), "via": targets.get("via") or "", "candidates": [] if hide_host else list(targets.get("hosts") or []), "ok": None, "host": "", "light": "blue", "reason": "not_expected", "elapsed": 0.0, } if not SSH_PROBES_ENABLED: ssh_block["reason"] = "ssh_probes_disabled" ssh_block["light"] = "blue" elif targets.get("expected") and targets.get("hosts"): cache = ssh_cache if ssh_cache is not None else {} probe = ssh_probe_fn or probe_ssh_host best: dict[str, Any] | None = None for h in targets["hosts"]: if h not in cache: cache[h] = probe(h) got = cache[h] if got.get("ok"): best = got break if best is None: best = got if best: # Keep real probe host out of UI for mapped zones (e.g. betel ≠ stage FQDN). shown_host = "" if targets.get("hide_host") else (best.get("host") or "") reason = best.get("reason") or "" if targets.get("hide_host") and reason and not str(reason).upper().startswith("SSH-"): # Drop accidental hostname leakage in soft errors; keep SSH banner text. if shown_host and shown_host in str(reason): reason = "ssh_probe" ssh_block.update( { "ok": bool(best.get("ok")), "host": shown_host, "light": best.get("light") or ("green" if best.get("ok") else "orange"), "reason": reason, "elapsed": best.get("elapsed") or 0.0, } ) return { "zone": zone["zone"], "label": zone["label"], "public_light": pub_worst, "ssh": ssh_block, } LEGEND = { "green": "ok", "blue": "present", "yellow": "degraded", "orange": "recent_or_hard_degraded", "red": "down_acute", "black": "down_long", } def monitord_state_dir() -> Path: base = os.environ.get("TALER_MONITORD_STATE") or os.environ.get("XDG_RUNTIME_DIR") if base: d = Path(base) / "taler-monitord" else: d = Path.home() / ".local" / "state" / "taler-monitord" d.mkdir(parents=True, exist_ok=True) return d def monitord_data_dir() -> Path: raw = os.environ.get("TALER_MONITORD_DATA") if raw: d = Path(raw) else: d = Path.home() / ".local" / "share" / "taler-monitord" d.mkdir(parents=True, exist_ok=True) return d def status_json_paths() -> list[Path]: return [monitord_state_dir() / "stack-status.json", monitord_data_dir() / "stack-status.json"] def default_domains_conf() -> Path: env = os.environ.get("TALER_DOMAINS_CONF") or "" if env and Path(env).is_file(): return Path(env) here = Path(__file__).resolve().parent for p in (here.parent / "domains.conf", Path("domains.conf")): if p.is_file(): return p return here.parent / "domains.conf" def _repo_conf(name: str, env_key: str) -> Path: env = os.environ.get(env_key) or "" if env and Path(env).is_file(): return Path(env) here = Path(__file__).resolve().parent for p in (here.parent / name, Path(name)): if p.is_file(): return p return here.parent / name def default_services_conf() -> Path: return _repo_conf("services-catalog.conf", "TALER_SERVICES_CONF") def default_mail_catalog() -> Path: return _repo_conf("mail-catalog.conf", "MAIL_CATALOG") def default_ports_whitelist_conf() -> Path: return _repo_conf("ports-whitelist.conf", "TALER_PORTS_WHITELIST") def _host_only(s: str) -> str: s = (s or "").strip() s = s.removeprefix("https://").removeprefix("http://") return s.split("/")[0].split(":")[0] # domains.conf bank column: no public libeufin / x-taler-bank host expected _BANK_SKIP_TOKENS = frozenset({"-", ".", "none", "n/a", "na", "ebics", "iban", "skip", "off"}) def bank_host_configured(bank: str) -> bool: """False when domains.conf marks bank as absent (EBICS / IBAN wire).""" b = (bank or "").strip().lower() return bool(b) and b not in _BANK_SKIP_TOKENS def payto_wire_methods(blob: bytes | str | None) -> set[str]: """Extract payto://METHOD/… wire methods from exchange /keys (or similar) body.""" if blob is None: return set() if isinstance(blob, bytes): text = blob.decode("utf-8", "replace") else: text = str(blob) return {m.lower() for m in re.findall(r"payto://([a-z0-9-]+)/", text, flags=re.I)} def classify_bank_backend(methods: set[str], *, bank_configured: bool = True) -> str: """Return x-taler-bank | ebics | unknown. EBICS/IBAN stacks (e.g. TOPS CHF) have no public bank.* — only payto://iban. Demo/stage libeufin stacks advertise payto://x-taler-bank/…. Mixed (regio: both) still expects bank.* because x-taler-bank is present. """ if not bank_configured: return "ebics" m = {x.lower() for x in (methods or set())} if "x-taler-bank" in m: return "x-taler-bank" if "iban" in m: return "ebics" return "unknown" def _looks_locale(tok: str) -> bool: t = tok.lower().replace("_", "-") return bool(t in ("fr-ch", "de-ch", "en-ch", "ch-fr", "ch-de", "ch-en") or ( len(t) == 5 and t[2] == "-" and t[:2].isalpha() and t[3:].isalpha() )) def _looks_lang(tok: str) -> bool: return tok.lower() in ("en", "fr", "de", "eng", "fra", "deu", "ger", "german", "deutsch", "french", "english") def parse_domains_conf(path: Path | None = None) -> list[dict[str, Any]]: """Return unique stacks keyed by (bank, exchange, merchant).""" path = path or default_domains_conf() by_endpoints: dict[tuple[str, str, str], dict[str, Any]] = {} if not path.is_file(): return [] for raw in path.read_text(encoding="utf-8").splitlines(): line = raw.split("#", 1)[0].strip() if not line: continue parts = line.split() if len(parts) < 5: continue name, bank, exchange, merchant, currency = parts[0], parts[1], parts[2], parts[3], parts[4] local_stack = parts[5] if len(parts) > 5 else "0" landing = parts[6] if len(parts) > 6 else "" rest = parts[7:] # Legacy: 7th field was canonical (contains a dot, not 0/1) if landing and landing not in ("0", "1") and "." in landing: rest = [landing] + rest landing = "" canon = name if rest: if _looks_lang(rest[0]): if len(rest) >= 2 and _looks_locale(rest[1]): canon = rest[2] if len(rest) >= 3 else name elif len(rest) >= 2 and "." in rest[1]: canon = rest[1] elif "." in rest[0]: canon = rest[0] bank_configured = bank_host_configured(bank) bank_h = _host_only(bank) if bank_configured else "" ex_h, mer_h = _host_only(exchange), _host_only(merchant) # Keep skip-token in key so EBICS rows do not collide with empty bank hosts. key = (bank_h or (bank.strip().lower() if bank.strip() else "-"), ex_h, mer_h) cur = currency if currency not in ("-", "") else "" # No currency ⇒ static site / Taler prod not provisioned yet (e.g. lefrancpaysan.ch). kind = "taler" if cur else "static_pending" row = { "id": canon, "bank": bank_h, "bank_configured": bank_configured, "exchange": ex_h, "merchant": mer_h, "currency": cur, "kind": kind, "local": local_stack in ("1", "true", "yes"), "landing": landing in ("1", "true", "yes"), "aliases": [name], } if key not in by_endpoints: by_endpoints[key] = row else: prev = by_endpoints[key] prev["aliases"].append(name) # Prefer non-service hostname as id when alias list grows if prev["id"] in (bank_h, ex_h, mer_h) and canon not in (bank_h, ex_h, mer_h): prev["id"] = canon elif canon == name and name not in (bank_h, ex_h, mer_h) and len(name) <= len(prev["id"]): # keep shorter apex-like id when equal quality if prev["id"] in prev["aliases"] and name.count(".") <= prev["id"].count("."): prev["id"] = canon if cur and not prev["currency"]: prev["currency"] = cur prev["kind"] = "taler" return sorted(by_endpoints.values(), key=lambda s: s["id"]) def member_probe_url(role: str, host: str) -> str: host = _host_only(host) if role == "bank": # Prefer integration config (wallet-facing); fallback handled in probe return f"https://{host}/taler-integration/config" return f"https://{host}/config" def stats_probe_url(host: str) -> str: return f"https://{_host_only(host)}/intro/stats.json" def _amount_value(amount: str) -> str: """Strip CUR: prefix from Taler amount string.""" s = (amount or "").strip() if ":" in s: return s.split(":", 1)[1] return s def _amount_sort_key(val: str) -> tuple: try: return (0, float(val)) except (TypeError, ValueError): return (1, str(val)) def extract_denom_values(blob: bytes | str | None, currency: str = "") -> list[str]: """Unique denomination values from exchange /keys body (best-effort).""" if blob is None: return [] text = blob.decode("utf-8", "replace") if isinstance(blob, bytes) else str(blob) vals: set[str] = set() # "value": "CUR:N" or amount fields for m in re.finditer(r'"value"\s*:\s*"([^"]+)"', text): raw = m.group(1) if currency and raw.startswith(currency + ":"): vals.add(_amount_value(raw)) elif ":" in raw and re.match(r"^[A-Z][A-Z0-9_]+:", raw): vals.add(_amount_value(raw)) # Prefer positive coin values; drop zero-only noise cleaned = [v for v in vals if v not in ("", "0", "0.0", "0.00")] return sorted(cleaned, key=_amount_sort_key)[:32] def summarize_currency_spec( cs: dict[str, Any] | None, *, keys_body: bytes | str | None = None, fallback_currency: str = "", ) -> dict[str, Any] | None: """Currency code, coin symbols (alt_unit_names), denomination list for UI.""" cs = cs if isinstance(cs, dict) else {} currency = str(cs.get("currency") or fallback_currency or "").strip() alt_raw = cs.get("alt_unit_names") if isinstance(cs.get("alt_unit_names"), dict) else {} alt = {str(k): str(v) for k, v in alt_raw.items() if str(v).strip()} if not currency and not alt: return None symbol = alt.get("0") or currency # Subunits: "-2" → Rp./ct — sorted by exponent def _exp_key(k: str) -> int: try: return int(k) except ValueError: return 0 subunits: list[dict[str, str]] = [] for k in sorted((x for x in alt if x != "0"), key=_exp_key, reverse=True): subunits.append({"exp": k, "symbol": alt[k]}) denoms: list[str] = [] for a in cs.get("common_amounts") or []: v = _amount_value(str(a)) if v and v not in denoms: denoms.append(v) if not denoms and keys_body is not None: denoms = extract_denom_values(keys_body, currency) unit_bits = [symbol] + [s["symbol"] for s in subunits] # unique preserve order seen: set[str] = set() units: list[str] = [] for u in unit_bits: if u and u not in seen: seen.add(u) units.append(u) return { "currency": currency, "name": str(cs.get("name") or ""), "symbol": symbol, "alt_unit_names": alt, "subunits": subunits, "units_label": " / ".join(units) if units else currency, "denoms": denoms, "denoms_label": " · ".join(denoms[:16]), } def _json_body(raw: dict[str, Any] | None) -> dict[str, Any] | None: if not raw: return None body = raw.get("body") if body is None: return None if isinstance(body, bytes): text = body.decode("utf-8", "replace") else: text = str(body) text = text.strip() if not text: return None try: data = json.loads(text) except json.JSONDecodeError: return None return data if isinstance(data, dict) else None def summarize_landing_stats( data: dict[str, Any], *, now: float | None = None, stale_s: float = STATS_STALE_S, role: str = "", host: str = "", url: str = "", ) -> dict[str, Any]: """Compact, UI-friendly summary of public …/intro/stats.json (no full dump).""" now = time.time() if now is None else now gu = data.get("generated_at_unix") age: int | None = None if gu is not None and str(gu).strip() != "": try: src_u = int(float(gu)) if src_u > 10_000_000_000: src_u //= 1000 age = int(now - src_u) except (TypeError, ValueError): age = None lines: list[str] = [] ba = data.get("bank_accounts") if isinstance(data.get("bank_accounts"), dict) else {} wd = data.get("withdraws") if isinstance(data.get("withdraws"), dict) else {} if ba or wd: n_acc = ba.get("total") if ba.get("total") is not None else ba.get("users") if n_acc is not None: lines.append(f"accounts {n_acc}") if wd.get("count") is not None: lines.append(f"withdraws {wd.get('count')}") amt = wd.get("total_amount_alt") or wd.get("total_amount") if amt: lines.append(str(amt)) if "reserves" in data: lines.append(f"reserves {data.get('reserves')}") if data.get("wire_in_count") is not None: lines.append(f"wire_in {data.get('wire_in_count')}") coins = data.get("known_coins") if coins is None: coins = data.get("coins_live") if coins is not None: lines.append(f"coins {coins}") if data.get("instances") is not None: lines.append(f"instances {data.get('instances')}") if data.get("orders") is not None: lines.append(f"orders {data.get('orders')}") bal = data.get("balance_explorer_alt") or data.get("balance_explorer") if bal and not any(str(bal) in x for x in lines): lines.append(f"explorer {bal}") return { "present": True, "ok": data.get("ok") is not False, "age_s": age, "stale": bool(age is not None and age > stale_s), "generated_at_human": data.get("generated_at_human") or data.get("generated_at") or "", "source": str(data.get("source") or "")[:72], "currency": data.get("currency") or "", "lines": lines[:6], "role": role, "host": host, "url": url, } def fetch_stack_stats( st: dict[str, Any], *, bank_backend: str, probe_fn: Callable[[str], dict[str, Any]], now: float | None = None, stale_s: float = STATS_STALE_S, ) -> dict[str, Any] | None: """Try public /intro/stats.json (custom intro stacks only). Soft-miss → None.""" now = time.time() if now is None else now kind = st.get("kind") or "taler" if kind == "static_pending": return None if not stack_has_custom_intro(st): return None candidates: list[tuple[str, str]] = [] if bank_backend == "x-taler-bank" and bank_host_configured(st.get("bank") or ""): candidates.append(("bank", _host_only(st["bank"]))) if st.get("merchant"): candidates.append(("merchant", _host_only(st["merchant"]))) if st.get("exchange"): candidates.append(("exchange", _host_only(st["exchange"]))) # de-dupe hosts while keeping order seen: set[str] = set() ordered: list[tuple[str, str]] = [] for role, host in candidates: if not host or host in seen: continue seen.add(host) ordered.append((role, host)) for role, host in ordered: url = stats_probe_url(host) raw = probe_fn(url) if not raw.get("ok"): continue data = _json_body(raw) if not data: continue return summarize_landing_stats( data, now=now, stale_s=stale_s, role=role, host=host, url=url ) return None def _ssl_context() -> ssl.SSLContext | None: if TLS_VERIFY: return None ctx = ssl.create_default_context() ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE return ctx def _header_server(headers: Any) -> str: if headers is None: return "" try: return str(headers.get("Server") or headers.get("server") or "").strip() except Exception: return "" def probe_http(url: str, *, timeout: float = PROBE_TIMEOUT_S) -> dict[str, Any]: """Probe one URL. Returns ok/code/elapsed/error/server (no light yet).""" t0 = time.monotonic() req = urllib.request.Request(url, method="GET", headers={"User-Agent": "taler-monitord/1"}) try: with urllib.request.urlopen(req, timeout=timeout, context=_ssl_context()) as resp: body = resp.read(65536) code = getattr(resp, "status", None) or resp.getcode() elapsed = time.monotonic() - t0 return { "ok": 200 <= int(code) < 400, "code": int(code), "elapsed": elapsed, "error": None, "bytes": len(body), "body": body, "server": _header_server(getattr(resp, "headers", None)), "url": url, } except urllib.error.HTTPError as e: elapsed = time.monotonic() - t0 try: err_body = e.read(65536) or b"" except Exception: err_body = b"" return { "ok": False, "code": int(e.code), "elapsed": elapsed, "error": f"http_{e.code}", "bytes": len(err_body), "body": err_body, "server": _header_server(getattr(e, "headers", None)), "url": url, } except (urllib.error.URLError, TimeoutError, socket.timeout, OSError) as e: elapsed = time.monotonic() - t0 err = type(e).__name__ reason = getattr(e, "reason", None) if reason is not None: err = f"{err}:{reason}" return { "ok": False, "code": 0, "elapsed": elapsed, "error": err, "bytes": 0, "body": b"", "server": "", "url": url, } def guess_http_package(server: str) -> str: s = (server or "").lower() if "nginx" in s: return "nginx" if "apache" in s or "httpd" in s: return "apache" if "caddy" in s: return "caddy" return "" def extract_http_version(server: str) -> str: m = re.search(r"/([0-9]+(?:\.[0-9]+)*)", server or "") return m.group(1) if m else "" def _taler_pkg_from_name(name: str) -> str: n = (name or "").lower() if "exchange" in n: return "taler-exchange" if "merchant" in n: return "taler-merchant" if "corebank" in n or n.endswith("bank") or "libeufin" in n: return "taler-bank" if "bank-integration" in n or "integration" in n: return "taler-bank" if n.startswith("taler"): return "taler" return "" def software_from_taler_config(body: bytes | str | dict | None, *, host: str = "", role: str = "") -> list[dict[str, Any]]: """Extract Taler component name + build_version from /config JSON.""" if isinstance(body, dict): data = body elif body is None: data = None else: data = _json_body({"body": body}) if not isinstance(data, dict): return [] name = str(data.get("name") or "") pkg = _taler_pkg_from_name(name) if not pkg and role in ("exchange", "merchant", "bank"): pkg = {"exchange": "taler-exchange", "merchant": "taler-merchant", "bank": "taler-bank"}[role] if not pkg: return [] build = str(data.get("build_version") or "").strip() api = str(data.get("version") or "").strip() ver = build or api return [ { "package": pkg, "label": name or pkg, "version": ver, "api_version": api, "build_version": build, "host": _host_only(host), "source": "taler-config", "role": role, } ] def software_from_server_header(server: str, *, host: str = "") -> list[dict[str, Any]]: pkg = guess_http_package(server) if not pkg: return [] ver = extract_http_version(server) return [ { "package": pkg, "label": pkg, "version": ver, "host": _host_only(host), "source": "server-header", "server": (server or "")[:80], } ] def smtp_banner_software(host: str, *, timeout_s: float = 4.0) -> list[dict[str, Any]]: """Read SMTP 220 banner; detect Postfix (+ version if present).""" h = _host_only(host) if not h: return [] try: with socket.create_connection((h, 25), timeout=max(1.0, float(timeout_s))) as sock: sock.settimeout(max(1.0, float(timeout_s))) banner = sock.recv(512).decode("utf-8", "replace").strip().splitlines() line = banner[0] if banner else "" try: sock.sendall(b"QUIT\r\n") except OSError: pass except OSError: return [] low = line.lower() if "postfix" not in low: if "esmtp" in low or line.startswith("220"): return [ { "package": "mta", "label": "SMTP", "version": "", "host": h, "source": "smtp-banner", "banner": line[:100], } ] return [] ver = "" m = re.search(r"postfix[^\d]*([0-9]+(?:\.[0-9]+)*)", line, flags=re.I) if m: ver = m.group(1) return [ { "package": "postfix", "label": "Postfix", "version": ver, "host": h, "source": "smtp-banner", "banner": line[:100], } ] def _sev_to_light(sev: str) -> str: s = (sev or "").upper() if s == "CRITICAL": return "red" if s == "HIGH": return "orange" if s == "MEDIUM": return "yellow" if s == "LOW": return "blue" return "green" _PUBLISHED_CACHE: dict[str, Any] = {"ts": 0.0, "pkgs": {}, "apps": [], "suite": "", "error": None} # inventory package → deb.taler.net package name(s) _SOFT_TO_DEB = { "taler-exchange": ("taler-exchange",), "taler-merchant": ("taler-merchant",), "taler-bank": ("libeufin-bank",), "taler": ("taler-exchange",), } # Published apps / SPAs we surface even without a live probe _APP_CATALOG = ( {"package": "taler-merchant-webui", "label": "Merchant WebUI", "kind": "webui", "logo": "webui"}, {"package": "libeufin-bank-webui", "label": "Bank WebUI", "kind": "webui", "logo": "webui"}, {"package": "taler-exchange-kyc-webui", "label": "KYC WebUI", "kind": "webui", "logo": "webui"}, {"package": "taler-exchange-aml-webui", "label": "AML WebUI", "kind": "webui", "logo": "webui"}, {"package": "taler-wallet-cli", "label": "Wallet CLI", "kind": "wallet", "logo": "wallet"}, ) def _deb_ord_ch(c: str) -> int: if c == "~": return -1 if c.isdigit(): return 0 if c.isalpha(): return ord(c) return ord(c) + 256 def _deb_split(v: str) -> tuple[int, str, str]: epoch = 0 if ":" in v: e, v = v.split(":", 1) try: epoch = int(e) except ValueError: epoch = 0 if "-" in v: upstream, deb = v.rsplit("-", 1) else: upstream, deb = v, "" return epoch, upstream, deb def _deb_cmp_part(a: str, b: str) -> int: ia = ib = 0 while ia < len(a) or ib < len(b): while True: ca = a[ia] if ia < len(a) and not a[ia].isdigit() else "" cb = b[ib] if ib < len(b) and not b[ib].isdigit() else "" if ca == "" and cb == "": break if ca == "" and cb: return -1 if _deb_ord_ch(cb) > 0 else (1 if cb == "~" else -1) if cb == "" and ca: return 1 if _deb_ord_ch(ca) > 0 else (-1 if ca == "~" else 1) oa, ob = _deb_ord_ch(ca), _deb_ord_ch(cb) ia += 1 ib += 1 if oa != ob: return (oa > ob) - (oa < ob) sa = sb = "" while ia < len(a) and a[ia].isdigit(): sa += a[ia] ia += 1 while ib < len(b) and b[ib].isdigit(): sb += b[ib] ib += 1 na = int(sa) if sa else 0 nb = int(sb) if sb else 0 if na != nb: return (na > nb) - (na < nb) return 0 def deb_version_cmp(a: str, b: str) -> int: """Debian-policy-ish version compare (subset used by check_versions.sh).""" if a == b: return 0 ea, ua, da = _deb_split(a or "") eb, ub, db = _deb_split(b or "") if ea != eb: return (ea > eb) - (ea < eb) c = _deb_cmp_part(ua, ub) if c: return c return _deb_cmp_part(da, db) def normalize_build_version(v: str) -> str: """Map Taler build_version toward deb upstream form for comparison. Examples: ``X.Y.Z-dev.1`` → ``X.Y.Z~dev1``; plain ``X.Y.Z`` stays. """ s = (v or "").strip() if not s: return "" # API versions like 39:0:5 are not package versions if re.fullmatch(r"\d+:\d+:\d+", s): return "" s = re.sub(r"-dev\.(\d+)", r"~dev\1", s, flags=re.I) s = re.sub(r"-dev(\d+)", r"~dev\1", s, flags=re.I) s = s.replace("-dev", "~dev") return s def parse_apt_packages_index(text: str) -> dict[str, str]: """Return package → highest Version from a Packages file body.""" pkgs: dict[str, str] = {} cur = ver = None for line in (text or "").splitlines(): if line.startswith("Package: "): if cur and ver and (cur not in pkgs or deb_version_cmp(ver, pkgs[cur]) > 0): pkgs[cur] = ver cur = line[9:].strip() ver = None elif line.startswith("Version: ") and cur: ver = line[9:].strip() elif line == "" and cur: if ver and (cur not in pkgs or deb_version_cmp(ver, pkgs[cur]) > 0): pkgs[cur] = ver cur = ver = None if cur and ver and (cur not in pkgs or deb_version_cmp(ver, pkgs[cur]) > 0): pkgs[cur] = ver return pkgs def fetch_published_taler_packages( *, suite: str | None = None, apt_base: str | None = None, cache_s: float = 3600.0, ) -> dict[str, Any]: """Fetch/cached deb.taler.net Packages for published Taler versions.""" if not PUBLISHED_ENABLED: return {"enabled": False, "suite": suite or PUBLISHED_SUITE, "packages": {}, "apps": []} suite = suite or PUBLISHED_SUITE apt_base = (apt_base or PUBLISHED_APT_BASE).rstrip("/") now = time.time() if ( _PUBLISHED_CACHE.get("pkgs") and _PUBLISHED_CACHE.get("suite") == suite and now - float(_PUBLISHED_CACHE.get("ts") or 0) < cache_s ): return { "enabled": True, "suite": suite, "packages": dict(_PUBLISHED_CACHE["pkgs"]), "apps": list(_PUBLISHED_CACHE.get("apps") or []), "error": _PUBLISHED_CACHE.get("error"), "cached": True, } url = f"{apt_base}/dists/{suite}/main/binary-amd64/Packages" # Full Packages index is >>64KiB (probe_http truncates); fetch with a larger cap. pkgs: dict[str, str] = {} err = None try: req = urllib.request.Request(url, method="GET", headers={"User-Agent": "taler-monitord/1"}) with urllib.request.urlopen(req, timeout=20, context=_ssl_context()) as resp: body = resp.read(8_000_000) code = getattr(resp, "status", None) or resp.getcode() if 200 <= int(code) < 400 and body: text = body.decode("utf-8", "replace") pkgs = parse_apt_packages_index(text) else: err = f"http_{code}" except Exception as e: err = f"{type(e).__name__}:{e}" # fallback tiny probe (may be incomplete) raw = probe_http(url, timeout=max(8.0, PROBE_TIMEOUT_S)) if raw.get("ok") and raw.get("body"): body = raw["body"] text = body.decode("utf-8", "replace") if isinstance(body, bytes) else str(body) pkgs = parse_apt_packages_index(text) apps = [] for app in _APP_CATALOG: pub = pkgs.get(app["package"]) or "" apps.append( { **app, "published": pub, "published_upstream": _deb_split(pub)[1] if pub else "", "light": "green" if pub else "yellow", "status": "published" if pub else "missing", } ) # Wallet tarball from GNU FTP (best-effort latest numbered) try: ftp = probe_http("https://ftp.gnu.org/gnu/taler/", timeout=8.0) if ftp.get("ok") and ftp.get("body"): html = ftp["body"].decode("utf-8", "replace") if isinstance(ftp["body"], bytes) else str(ftp["body"]) vers = re.findall(r"taler-wallet-([0-9]+\.[0-9]+(?:\.[0-9]+)?)\.tar\.gz", html) if vers: latest = sorted(vers, key=lambda v: _ver_tuple(v))[-1] apps.append( { "package": "taler-wallet", "label": "Wallet (GNU FTP)", "kind": "wallet", "logo": "wallet", "published": latest, "published_upstream": latest, "light": "green", "status": "published", "source": "ftp.gnu.org", } ) except Exception: pass _PUBLISHED_CACHE.update({"ts": now, "pkgs": pkgs, "apps": apps, "suite": suite, "error": err}) return { "enabled": True, "suite": suite, "packages": pkgs, "apps": apps, "error": err, "cached": False, "url": url, } def soft_to_deb_names(item: dict[str, Any]) -> list[str]: pkg = str(item.get("package") or "") label = str(item.get("label") or "").lower() if pkg == "taler-bank" and "integration" in label: return [] # API version only, not a deb package build return list(_SOFT_TO_DEB.get(pkg) or ()) def version_currency( installed: str, published_deb: str, ) -> dict[str, Any]: """Compare installed build_version to published deb version; estimate lag.""" inst_n = normalize_build_version(installed) pub_up = _deb_split(published_deb)[1] if published_deb else "" if not inst_n or not pub_up: return { "status": "unknown", "light": "blue", "published": published_deb, "published_upstream": pub_up, "behind": None, "delta": None, } # Compare upstream-like strings via deb rules cmp = deb_version_cmp(inst_n, pub_up) if cmp == 0: return { "status": "current", "light": "green", "published": published_deb, "published_upstream": pub_up, "behind": None, "delta": 0, } if cmp > 0: return { "status": "ahead", "light": "blue", "published": published_deb, "published_upstream": pub_up, "behind": None, "delta": None, } # behind — try numeric distance on dotted prefix ti, tp = _ver_tuple(inst_n.replace("~", ".")), _ver_tuple(pub_up.replace("~", ".")) delta = None if ti and tp and len(ti) >= 2 and len(tp) >= 2: # rough component-wise lag on last differing field n = max(len(ti), len(tp)) ti = ti + (0,) * (n - len(ti)) tp = tp + (0,) * (n - len(tp)) for i in range(n): if tp[i] > ti[i]: delta = int(tp[i] - ti[i]) break return { "status": "behind", "light": "orange" if (delta is None or delta > 0) else "yellow", "published": published_deb, "published_upstream": pub_up, "behind": pub_up, "delta": delta, } def apply_published_currency( items: list[dict[str, Any]], published: dict[str, Any] | None, ) -> list[dict[str, Any]]: """Attach published/current/behind fields; raise light when behind.""" pkgs = (published or {}).get("packages") or {} out = [] for it in items: row = dict(it) deb_names = soft_to_deb_names(row) best = None for dn in deb_names: pub = pkgs.get(dn) or "" if not pub: continue cur = version_currency(str(row.get("version") or ""), pub) cur["deb_package"] = dn if best is None or ( cur.get("status") == "behind" and best.get("status") != "behind" ): best = cur elif cur.get("status") == "behind" and best.get("status") == "behind": # keep higher published upstream if deb_version_cmp(str(cur.get("published") or ""), str(best.get("published") or "")) > 0: best = cur if best: row.update( { "published": best.get("published"), "published_upstream": best.get("published_upstream"), "currency_status": best.get("status"), "behind": best.get("behind"), "delta": best.get("delta"), "deb_package": best.get("deb_package"), } ) # CVE colour wins if worse; else use currency light if best.get("status") == "behind": if SEVERITY.get(str(best.get("light")), 0) > SEVERITY.get(str(row.get("light") or "green"), 0): row["light"] = best.get("light") elif best.get("status") == "current" and row.get("light") in ("green", "blue", None): row["light"] = "green" else: row.setdefault("currency_status", "n/a") out.append(row) return out def colorize_software( items: list[dict[str, Any]], vulns: dict[str, Any] | None = None, published: dict[str, Any] | None = None, ) -> list[dict[str, Any]]: """Dedupe + CVE colour + published currency (current/behind/ahead).""" hits = (vulns or {}).get("hits") or [] by_pkg_ver: dict[str, str] = {} for h in hits: pkg = str(h.get("package") or "") ver = str(h.get("version") or "") sev = str(h.get("severity") or "") key = f"{pkg}|{ver}" light = _sev_to_light(sev) if key not in by_pkg_ver or SEVERITY.get(light, 0) > SEVERITY.get(by_pkg_ver[key], 0): by_pkg_ver[key] = light out: list[dict[str, Any]] = [] seen: set[str] = set() for it in items: pkg = str(it.get("package") or "") ver = str(it.get("version") or "") host = str(it.get("host") or "") key = f"{pkg}|{ver}|{host}|{it.get('source')}" if not pkg or key in seen: continue seen.add(key) light = by_pkg_ver.get(f"{pkg}|{ver}") or ("green" if ver else "blue") out.append({**it, "light": light}) out = apply_published_currency(out, published) order = { "taler-exchange": 0, "taler-merchant": 1, "taler-bank": 2, "taler": 3, "nginx": 10, "apache": 11, "caddy": 12, "postfix": 20, "mta": 21, } out.sort(key=lambda x: (order.get(str(x.get("package")), 50), str(x.get("package")), str(x.get("host")))) return out def collect_software_from_probes( *, probe_cache: dict[str, dict[str, Any]], members: list[dict[str, Any]], vulns: dict[str, Any] | None = None, mail_hosts: list[str] | None = None, smtp: bool = False, published: dict[str, Any] | None = None, ) -> list[dict[str, Any]]: """Assemble software inventory for one card from HTTP/Taler/SMTP signals.""" items: list[dict[str, Any]] = [] for url, raw in (probe_cache or {}).items(): host = _host_only(url) srv = str((raw or {}).get("server") or "") items.extend(software_from_server_header(srv, host=host)) # Taler /config or /keys-adjacent config bodies path = url.split(host, 1)[-1] if host and host in url else "" if path.endswith("/config") or "/taler-integration/config" in path: role = "" for m in members or []: if _host_only(str(m.get("host") or "")) == host: role = str(m.get("role") or "") break items.extend(software_from_taler_config((raw or {}).get("body"), host=host, role=role)) if smtp: for h in mail_hosts or []: items.extend(smtp_banner_software(h)) return colorize_software(items, vulns, published=published) def _ver_tuple(s: str) -> tuple[int, ...]: nums = [int(x) for x in re.findall(r"\d+", s or "")[:5]] return tuple(nums) if nums else () def _vt_cmp(a: tuple[int, ...], b: tuple[int, ...]) -> int: n = max(len(a), len(b)) a = a + (0,) * (n - len(a)) b = b + (0,) * (n - len(b)) return (a > b) - (a < b) def _in_semver_events(ver: str, events: list) -> bool: vt = _ver_tuple(ver) if not vt: return False intro = None for e in events or []: if "introduced" in e: intro = e.get("introduced") elif "fixed" in e or "last_affected" in e: fixed = e.get("fixed") last = e.get("last_affected") lo = _ver_tuple("0" if intro in (None, "0") else str(intro)) if _vt_cmp(vt, lo) < 0: intro = None continue if fixed is not None: if _vt_cmp(vt, _ver_tuple(str(fixed))) < 0: return True elif last is not None: if _vt_cmp(vt, _ver_tuple(str(last))) <= 0: return True intro = None return False _OSV_SEED = { "nginx": [ "CVE-2025-23419", "CVE-2025-53859", "CVE-2024-7347", "CVE-2024-34161", "CVE-2024-32760", "CVE-2024-31079", "CVE-2024-24989", "CVE-2024-24990", ], "apache": [ "CVE-2024-38474", "CVE-2024-38476", "CVE-2024-38477", "CVE-2023-31122", "CVE-2023-43622", ], "caddy": ["CVE-2022-29718", "CVE-2023-50463"], } def _osv_get(vid: str) -> dict[str, Any]: try: with urllib.request.urlopen(f"https://api.osv.dev/v1/vulns/{vid}", timeout=12) as r: return json.loads(r.read().decode("utf-8", "replace")) except Exception: return {} def _severity_rank(sev: str) -> int: s = (sev or "").upper() if s == "CRITICAL": return 3 if s == "HIGH": return 2 if s == "MEDIUM": return 1 return 0 def osv_hits_for_server(pkg: str, ver: str, *, cache: dict[str, list] | None = None) -> list[dict[str, Any]]: """Actionable OSV hits for a bare upstream Server header version (no Debian rev). Same policy as check_surface.sh: use SEMVER extracted_events on known CVE seeds — avoids Debian ecosystem false positives on bare versions. """ if not pkg or not ver: return [] cache = cache if cache is not None else {} key = f"{pkg}|{ver}" if key in cache: return list(cache[key]) hits: list[dict[str, Any]] = [] for vid in _OSV_SEED.get(pkg, []): doc = _osv_get(vid) if not doc: continue ok_hit = False for a in doc.get("affected") or []: for rg in a.get("ranges") or []: db = rg.get("database_specific") or {} extracted = db.get("extracted_events") or [] if extracted and _in_semver_events(ver, extracted): blob = json.dumps(a).lower() if pkg == "nginx" and "nginx" not in blob and "f5" not in blob: continue ok_hit = True break if rg.get("type") == "SEMVER" and _in_semver_events(ver, rg.get("events") or []): ok_hit = True break if ok_hit: break if not ok_hit: continue sev = "" for s in doc.get("severity") or []: if isinstance(s, dict) and s.get("type") in ("CVSS_V3", "CVSS_V4", "CVSS_V2"): # score-based score = s.get("score") try: sc = float(str(score).split("/")[0]) if score else 0.0 except ValueError: sc = 0.0 if sc >= 9.0: sev = "CRITICAL" elif sc >= 7.0: sev = "HIGH" elif sc >= 4.0: sev = "MEDIUM" break dbsev = ((doc.get("database_specific") or {}).get("severity") or "").upper() if dbsev in ("CRITICAL", "HIGH", "MEDIUM", "LOW") and _severity_rank(dbsev) > _severity_rank(sev): sev = dbsev if not sev: sev = "HIGH" # known actionable seed hit without score → treat serious hits.append( { "id": vid, "severity": sev, "package": pkg, "version": ver, "summary": str(doc.get("summary") or "")[:160], } ) # keep critical/high only for box colour; still return medium for display cache[key] = hits return list(hits) def assess_external_vulns( *, probe_cache: dict[str, dict[str, Any]], members: list[dict[str, Any]], osv_cache: dict[str, list] | None = None, ) -> dict[str, Any]: """Scan outside-visible Server headers from probe cache; critical/high → box colour.""" if not CVE_PROBES_ENABLED: return {"enabled": False, "hits": [], "worst": "", "light": None, "servers": []} seen_srv: dict[str, str] = {} # Prefer apex/config responses already in cache for url, raw in (probe_cache or {}).items(): srv = str((raw or {}).get("server") or "").strip() if not srv: continue host = _host_only(url) if host and host not in seen_srv: seen_srv[host] = srv # Also ensure we looked at each member host apex once for m in members or []: host = _host_only(str(m.get("host") or "")) if not host or host.upper() == "IBAN" or host in seen_srv: continue url = f"https://{host}/" if url not in probe_cache: probe_cache[url] = probe_http(url) srv = str((probe_cache[url] or {}).get("server") or "").strip() if srv: seen_srv[host] = srv servers_out = [] all_hits: list[dict[str, Any]] = [] osv_cache = osv_cache if osv_cache is not None else {} for host, srv in sorted(seen_srv.items()): pkg = guess_http_package(srv) ver = extract_http_version(srv) entry = {"host": host, "server": srv, "package": pkg, "version": ver, "hits": []} if pkg and ver: hits = osv_hits_for_server(pkg, ver, cache=osv_cache) for h in hits: h2 = {**h, "host": host, "server": srv} entry["hits"].append(h2) all_hits.append(h2) servers_out.append(entry) worst = "" light = None for h in all_hits: sev = h.get("severity") or "" if _severity_rank(sev) > _severity_rank(worst): worst = sev if worst == "CRITICAL": light = "red" elif worst == "HIGH": light = "orange" elif worst == "MEDIUM": light = "yellow" return { "enabled": True, "hits": all_hits[:20], "worst": worst, "light": light, "servers": servers_out, } def apply_vuln_to_stack( light: str, intensity: str, members: list[dict[str, Any]], vulns: dict[str, Any], ) -> tuple[str, str, list[dict[str, Any]]]: """Raise stack colour for critical/high outside-visible CVEs; add quiet member row.""" vlight = vulns.get("light") hits = vulns.get("hits") or [] if not vlight or not hits: return light, intensity, members # synthetic member so UI/aggregation stay consistent top = hits[0] reason = f"{top.get('id')} {top.get('severity')} ({top.get('package')}/{top.get('version')})" members = list(members) + [ { "role": "cve", "host": top.get("host") or "edge", "ok": False, "light": vlight, "reason": reason, "down_since": None, "code": 0, "elapsed": 0.0, "error": None, "url": None, "endpoints": [], "endpoints_summary": {"n": 0, "ok": 0, "soft": 0, "bad": 0, "skipped": 0, "light": "green", "dots": []}, } ] # re-aggregate including CVE member light2, intensity2 = aggregate_stack(members) return light2, intensity2, members def classify_probe( result: dict[str, Any], *, prev: dict[str, Any] | None = None, now: float | None = None, acute_s: float = ACUTE_DOWN_S, soft_s: float = 2.5, ) -> dict[str, Any]: """Map probe result + previous down_since → light/reason.""" now = time.time() if now is None else now prev = prev or {} ok = bool(result.get("ok")) code = int(result.get("code") or 0) elapsed = float(result.get("elapsed") or 0) error = result.get("error") down_since = prev.get("down_since") if ok: down_since = None if elapsed >= soft_s: light, reason = "yellow", "slow" else: light, reason = "green", "ok" else: # Reachable but bad status → degraded rather than offline if code >= 400: down_since = None if code in (401, 403): light, reason = "blue", "auth_present" elif code >= 500: light, reason = "orange", f"http_{code}" else: light, reason = "yellow", f"http_{code}" else: prev_ok = prev.get("ok") try: inherited = float(prev["down_since"]) if prev.get("down_since") is not None else None except (TypeError, ValueError): inherited = None if prev_ok or inherited is None: down_since = now else: down_since = inherited age = now - float(down_since) err_l = str(error or "").lower() # Timeouts / reset often mean DoS or broken path — orange if still "fresh" if age < acute_s and ("timeout" in err_l or "timedout" in err_l or "reset" in err_l): light, reason = "orange", "hard_degraded" elif age < acute_s: light, reason = "red", "down_acute" else: light, reason = "black", "down_long" return { "ok": ok, "light": light, "reason": reason, "down_since": down_since, "code": code, "elapsed": round(elapsed, 3), "error": error, "url": result.get("url"), "ts": now, } def _cap_light(light: str, max_light: str) -> str: if SEVERITY.get(light, 5) > SEVERITY.get(max_light, 5): return max_light return light def summarize_endpoints(endpoints: list[dict[str, Any]]) -> dict[str, Any]: """Compact rollup for UI: counts + worst light (dots = probed only, not skips).""" if not endpoints: return {"n": 0, "ok": 0, "soft": 0, "bad": 0, "skipped": 0, "light": "green", "dots": []} dots: list[str] = [] ok = soft = bad = skipped = 0 worst = "green" for ep in endpoints: if ep.get("skipped"): skipped += 1 continue L = ep.get("light") or "black" dots.append(L) if SEVERITY.get(L, 5) > SEVERITY.get(worst, 0): worst = L if SEVERITY.get(L, 5) <= SEVERITY["blue"]: ok += 1 elif SEVERITY.get(L, 5) <= SEVERITY["yellow"]: soft += 1 else: bad += 1 return { "n": len(endpoints), "ok": ok, "soft": soft, "bad": bad, "skipped": skipped, "light": worst if dots else "green", "dots": dots, } def classify_endpoint_probe( raw: dict[str, Any], *, expect: tuple[int, ...], severity: str = "soft", hard: bool | None = None, prev: dict[str, Any] | None = None, now: float | None = None, acute_s: float = ACUTE_DOWN_S, ) -> dict[str, Any]: """Map HTTP probe to light. soft → fail capped at yellow medium → missing/wrong is at least orange (mittelschwer), not red for 4xx hard → full classify_probe severity """ if hard is not None and not severity: severity = "hard" if hard else "soft" sev = (severity or "soft").lower() code = int(raw.get("code") or 0) if code in expect: c = classify_probe(raw, prev=prev, now=now, acute_s=acute_s) return {**c, "expected": True, "severity": sev} fake = {**raw, "ok": False} if code >= 400: fake["error"] = f"http_{code}" elif not fake.get("error"): fake["error"] = "unreachable" c = classify_probe(fake, prev=prev, now=now, acute_s=acute_s) reason = c.get("reason") or (f"http_{code}" if code else "unreachable") if sev == "soft": c = { **c, "light": _cap_light(c.get("light") or "yellow", ENDPOINT_SOFT_MAX), "reason": reason, } elif sev == "medium": # At least orange; keep 5xx/unreachable from going black via acute age — still medium. light = c.get("light") or "orange" if SEVERITY.get(light, 0) < SEVERITY["orange"]: light = "orange" elif SEVERITY.get(light, 0) > SEVERITY["orange"]: light = "orange" c = {**c, "light": light, "reason": reason, "down_since": None} # hard: keep classify_probe result return {**c, "expected": False, "severity": sev} def probe_role_endpoints( role: str, host: str, *, has_intro: bool = False, landing: bool | None = None, legal_severity: str = "soft", probe_fn: Callable[[str], dict[str, Any]], cache: dict[str, dict[str, Any]] | None = None, prev_by_path: dict[str, dict[str, Any]] | None = None, now: float | None = None, acute_s: float = ACUTE_DOWN_S, ) -> list[dict[str, Any]]: """Probe suite-like public endpoints for one machine; return compact rows.""" host = _host_only(host) if not host or host.upper() == "IBAN": return [] # Compat: older callers passed landing= if landing is not None: has_intro = bool(landing) catalog = ROLE_ENDPOINTS.get(role) or [] cache = cache if cache is not None else {} prev_by_path = prev_by_path or {} now = time.time() if now is None else now out: list[dict[str, Any]] = [] for spec in catalog: if spec.get("need_intro") and not has_intro: out.append( { "id": spec["id"], "path": spec["path"], "skipped": True, "light": "blue", "reason": "no_intro", "code": 0, "elapsed": 0.0, } ) continue url = f"https://{host}{spec['path']}" if url in cache: raw = cache[url] else: raw = probe_fn(url) cache[url] = raw sev = str(spec.get("severity") or ("hard" if spec.get("hard") else "soft")) if spec.get("legal"): sev = legal_severity or sev classified = classify_endpoint_probe( raw, expect=tuple(spec.get("expect") or (200,)), severity=sev, prev=prev_by_path.get(spec["path"]), now=now, acute_s=acute_s, ) out.append( { "id": spec["id"], "path": spec["path"], "url": url, "skipped": False, "light": classified.get("light") or "black", "reason": classified.get("reason") or "", "code": classified.get("code") or 0, "elapsed": classified.get("elapsed") or 0.0, "ok": bool(classified.get("ok")) and bool(classified.get("expected")), "severity": sev, # medium/hard pull machine colour past soft yellow "hard": sev in ("hard", "medium"), } ) return out def merge_member_endpoint_light(member: dict[str, Any], endpoints: list[dict[str, Any]]) -> dict[str, Any]: """Attach endpoints + summary; raise member light to worst *hard* endpoint issue.""" summary = summarize_endpoints(endpoints) member = {**member, "endpoints": endpoints, "endpoints_summary": summary} # Informational members stay as-is if (member.get("reason") or "") in _AGG_SKIP_REASONS: return member base = member.get("light") or "black" # Only hard endpoint failures pull the machine light past yellow via soft cap already applied worst = base for ep in endpoints: if ep.get("skipped"): continue L = ep.get("light") or "black" if ep.get("hard") is False and SEVERITY.get(L, 0) <= SEVERITY["yellow"]: # soft noise shouldn't turn machine red if primary is green — still show yellow if soft warn if SEVERITY.get(L, 0) > SEVERITY.get(worst, 0) and SEVERITY.get(L, 0) <= SEVERITY["yellow"]: worst = L continue if SEVERITY.get(L, 0) > SEVERITY.get(worst, 0): worst = L member["light"] = worst return member def score_light_from_ratio(ratio: float, *, all_black: bool = False) -> str: """Map green/total ratio to overall traffic light (incl. white = gone).""" if ratio <= 0: return "black" if all_black else "white" if ratio >= SCORE_RATIO_GREEN: return "green" if ratio >= SCORE_RATIO_BLUE: return "blue" if ratio >= SCORE_RATIO_YELLOW: return "yellow" if ratio >= SCORE_RATIO_ORANGE: return "orange" if ratio >= SCORE_RATIO_RED: return "red" return "white" def compute_total_score( stacks: list[dict[str, Any]], services: list[dict[str, Any]] | None = None, mail: list[dict[str, Any]] | None = None, ) -> dict[str, Any]: """Page-level score: how many boxes are green out of all scored boxes. One point = one stack / service / mail card. Strict green only (blue ≠ green). """ boxes: list[dict[str, Any]] = [] for group in (stacks or [], services or [], mail or []): boxes.extend(group) total = len(boxes) by_light: dict[str, int] = {} for b in boxes: Lgt = str(b.get("light") or "black") by_light[Lgt] = by_light.get(Lgt, 0) + 1 green = int(by_light.get("green") or 0) ratio = (green / total) if total else 0.0 all_black = total > 0 and int(by_light.get("black") or 0) == total light = "white" if total == 0 else score_light_from_ratio(ratio, all_black=all_black) return { "green": green, "total": total, "ratio": round(ratio, 4), "pct": int(round(ratio * 100)) if total else 0, "light": light, "by_light": by_light, "thresholds": { "green": SCORE_RATIO_GREEN, "blue": SCORE_RATIO_BLUE, "yellow": SCORE_RATIO_YELLOW, "orange": SCORE_RATIO_ORANGE, "red": SCORE_RATIO_RED, }, } def aggregate_breakdown(members: list[dict[str, Any]]) -> dict[str, Any]: """Counts + parts so UI can explain e.g. 4× OK, 3× Soft → Degraded.""" counted = [m for m in members if (m.get("reason") or "") not in _AGG_SKIP_REASONS] if not counted: counted = list(members) by_light: dict[str, int] = {} parts: list[dict[str, Any]] = [] for m in counted: Lgt = str(m.get("light") or "black") by_light[Lgt] = by_light.get(Lgt, 0) + 1 parts.append( { "role": m.get("role"), "host": m.get("host"), "light": Lgt, "reason": m.get("reason"), } ) worst = "black" if by_light: worst = LIGHTS[max(SEVERITY.get(k, 5) for k in by_light)] return { "by_light": by_light, "worst": worst, "counted": len(counted), "parts": parts, } def aggregate_stack(members: list[dict[str, Any]]) -> tuple[str, str]: """Return (light, intensity) for a stack box. Logical color = weakest *expected* member. Intensity muted unless every counted member is at worst severity and that severity is red/black (full). Informational rows (e.g. ebics_no_bank) are shown but ignored for the box. """ counted = [m for m in members if (m.get("reason") or "") not in _AGG_SKIP_REASONS] if not counted: counted = list(members) if not counted: return "black", "full" lights = [m.get("light") or "black" for m in counted] worst_rank = max(SEVERITY.get(L, 5) for L in lights) worst = LIGHTS[worst_rank] n = len(counted) k_worst = sum(1 for L in lights if SEVERITY.get(L, 5) == worst_rank) # green/blue = healthy or expected-present (e.g. static_pending / no_prod_yet) if worst_rank <= SEVERITY["blue"]: return worst, "ok" if worst_rank >= SEVERITY["red"] and k_worst == n: return worst, "full" return worst, "muted" def _dig_lines(rrtype: str, name: str) -> list[str]: """Best-effort dig +short; empty if dig missing/fails.""" try: r = subprocess.run( ["dig", "+short", "+time=2", "+tries=1", rrtype, name], capture_output=True, timeout=6, check=False, ) except (OSError, subprocess.TimeoutExpired): return [] out = (r.stdout or b"").decode("utf-8", "replace") return [ln.strip().rstrip(".") for ln in out.splitlines() if ln.strip()] def dns_a_aaaa(host: str) -> dict[str, list[str]]: h = _host_only(host) a = [x for x in _dig_lines("A", h) if re.match(r"^\d+\.\d+\.\d+\.\d+$", x)] aaaa = [x for x in _dig_lines("AAAA", h) if ":" in x] return {"a": a, "aaaa": aaaa} def dns_mx(domain: str) -> list[str]: lines = _dig_lines("MX", domain) hosts: list[str] = [] for ln in lines: parts = ln.split() if len(parts) >= 2: hosts.append(parts[-1].lower().rstrip(".")) elif parts: hosts.append(parts[0].lower().rstrip(".")) return hosts def dns_txt(name: str) -> str: return " ".join(_dig_lines("TXT", name)).replace('"', "") def dns_ptr(ip: str) -> list[str]: """Reverse DNS via dig -x.""" try: r = subprocess.run( ["dig", "+short", "+time=2", "+tries=1", "-x", ip], capture_output=True, timeout=6, check=False, ) except (OSError, subprocess.TimeoutExpired): return [] out = (r.stdout or b"").decode("utf-8", "replace") return [ln.strip().rstrip(".") for ln in out.splitlines() if ln.strip()] def probe_tls_cert(host: str, *, timeout: float = PROBE_TIMEOUT_S) -> dict[str, Any]: """Fetch peer cert notAfter + SANs (outside-in).""" h = _host_only(host) if not h or h.upper() == "IBAN": return {"ok": False, "host": h, "error": "no_host"} ctx = ssl.create_default_context() if not TLS_VERIFY: ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE try: with socket.create_connection((h, 443), timeout=timeout) as sock: with ctx.wrap_socket(sock, server_hostname=h) as ssock: cert = ssock.getpeercert() except OSError as e: return {"ok": False, "host": h, "error": type(e).__name__} if not cert: # CERT_NONE often yields empty dict — retry binary DER via getpeercert(True) try: with socket.create_connection((h, 443), timeout=timeout) as sock: with ctx.wrap_socket(sock, server_hostname=h) as ssock: der = ssock.getpeercert(binary_form=True) if not der: return {"ok": False, "host": h, "error": "no_cert"} # Parse notAfter via openssl if available return _parse_der_cert(h, der) except OSError as e: return {"ok": False, "host": h, "error": type(e).__name__} return _cert_from_dict(h, cert) def _cert_from_dict(host: str, cert: dict[str, Any]) -> dict[str, Any]: not_after = cert.get("notAfter") or "" days_left = None try: # OpenSSL ASN.1: 'Jun 15 12:00:00 2027 GMT' exp = time.strptime(str(not_after), "%b %d %H:%M:%S %Y %Z") days_left = int((time.mktime(exp) - time.time()) / 86400) except (TypeError, ValueError, OverflowError, OSError): days_left = None sans: list[str] = [] for typ, val in cert.get("subjectAltName") or []: if typ.lower() == "dns" and val and val not in sans: sans.append(str(val).lower()) light = "green" reason = "cert_ok" if days_left is None: light, reason = "yellow", "cert_parse" elif days_left < 0: light, reason = "red", "cert_expired" elif days_left < CERT_CRIT_DAYS: light, reason = "orange", "cert_expiring" elif days_left < CERT_WARN_DAYS: light, reason = "yellow", "cert_warn" return { "ok": days_left is not None and days_left >= 0, "host": host, "not_after": not_after, "days_left": days_left, "sans": sans[:32], "light": light, "reason": reason, "error": None, } def _parse_der_cert(host: str, der: bytes) -> dict[str, Any]: """Fallback: openssl x509 -noout -dates -ext subjectAltName.""" try: r = subprocess.run( ["openssl", "x509", "-inform", "DER", "-noout", "-enddate", "-ext", "subjectAltName"], input=der, capture_output=True, timeout=6, check=False, ) except (OSError, subprocess.TimeoutExpired): return {"ok": False, "host": host, "error": "openssl_fail"} text = (r.stdout or b"").decode("utf-8", "replace") not_after = "" for ln in text.splitlines(): if ln.startswith("notAfter="): not_after = ln.split("=", 1)[1].strip() sans: list[str] = [] for m in re.finditer(r"DNS:([A-Za-z0-9_.*-]+)", text): v = m.group(1).lower() if v not in sans: sans.append(v) fake = {"notAfter": not_after, "subjectAltName": [("DNS", s) for s in sans]} out = _cert_from_dict(host, fake) if out.get("days_left") is None and not_after: out["error"] = "cert_parse" return out def parse_services_conf(path: Path | None = None) -> list[dict[str, Any]]: """Company/service hosts (non-Taler): id host label zone.""" path = path or default_services_conf() rows: list[dict[str, Any]] = [] if not path.is_file(): return rows seen: set[str] = set() for raw in path.read_text(encoding="utf-8").splitlines(): line = raw.split("#", 1)[0].strip() if not line: continue parts = line.split() if len(parts) < 2: continue sid, host = parts[0], _host_only(parts[1]) label = parts[2] if len(parts) > 2 else "service" zone = parts[3] if len(parts) > 3 else "public" if not host or host in seen: continue seen.add(host) rows.append( { "id": sid, "host": host, "service_label": label, "zone": zone, "kind": "service", } ) return rows def parse_mail_catalog(path: Path | None = None) -> list[dict[str, Any]]: path = path or default_mail_catalog() rows: list[dict[str, Any]] = [] if not path.is_file(): return rows for raw in path.read_text(encoding="utf-8").splitlines(): line = raw.split("#", 1)[0].strip() if not line: continue parts = line.split() if len(parts) < 2: continue domain = parts[0] expect_mx = [x.strip().lower() for x in parts[1].split(",") if x.strip()] mail_hosts = ( [x.strip().lower() for x in parts[2].split(",") if x.strip()] if len(parts) > 2 else list(expect_mx) ) smtp_ports = [ p.strip() for p in (parts[3] if len(parts) > 3 else "25,465,587").split(",") if p.strip() ] imap_ports = [ p.strip() for p in (parts[4] if len(parts) > 4 else "143,993").split(",") if p.strip() ] label = parts[5] if len(parts) > 5 else domain rows.append( { "domain": domain, "expect_mx": expect_mx, "mail_hosts": mail_hosts, "smtp_ports": smtp_ports, "imap_ports": imap_ports, "label": label, } ) return rows def tcp_port_open(host: str, port: int | str, *, timeout_s: float = MAIL_PORT_TIMEOUT_S) -> bool: try: with socket.create_connection((_host_only(host), int(port)), timeout=max(1.0, float(timeout_s))): return True except (OSError, ValueError, TypeError): return False def git_probe_state_path() -> Path: return monitord_state_dir() / "git-probes.json" def deliverability_state_path() -> Path: return monitord_state_dir() / "mail-deliverability.json" def load_deliverability_state() -> dict[str, Any]: path = deliverability_state_path() if not path.is_file(): return {"schema": 1, "domains": {}} try: doc = json.loads(path.read_text(encoding="utf-8")) if isinstance(doc, dict): doc.setdefault("domains", {}) return doc except (OSError, json.JSONDecodeError): pass return {"schema": 1, "domains": {}} def save_deliverability_state(doc: dict[str, Any]) -> None: path = deliverability_state_path() atomic_write_json = globals().get("atomic_write_json") # atomic_write_json is defined later; fall back to simple write if importing early try: from typing import cast _ = cast(Any, None) except Exception: pass path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_name(f".{path.name}.tmp.{os.getpid()}") text = json.dumps(doc, ensure_ascii=False, indent=2) + "\n" try: tmp.write_text(text, encoding="utf-8") os.replace(tmp, path) finally: if tmp.exists(): try: tmp.unlink() except OSError: pass def _smtp_read_reply(sock: socket.socket, timeout_s: float) -> tuple[int, str]: sock.settimeout(timeout_s) data = b"" while True: chunk = sock.recv(4096) if not chunk: break data += chunk text = data.decode("utf-8", "replace") lines = text.splitlines() if not lines: continue # multi-line: last line is "NNN " (space) not "NNN-" last = lines[-1] if len(last) >= 4 and last[0:3].isdigit() and last[3] == " ": return int(last[0:3]), text.strip() if len(data) > 65536: break text = data.decode("utf-8", "replace").strip() code = int(text[:3]) if len(text) >= 3 and text[:3].isdigit() else 0 return code, text def smtp_deliverability_probe( mx_host: str, *, domain: str, mail_from: str | None = None, rcpt_to: str | None = None, timeout_s: float = DELIVER_TIMEOUT_S, ) -> dict[str, Any]: """Outside-in SMTP deliverability smoke: banner → EHLO → MAIL FROM → RCPT TO → RSET/QUIT. Does not send DATA/body. Acceptance of RCPT for postmaster@domain (or configured recipient) counts as deliverable. Soft-fail/greylist may still return 4xx. """ h = _host_only(mx_host) mail_from = mail_from or os.environ.get("TALER_MONITORD_DELIVER_FROM", "monitor@taler-monitoring.invalid") rcpt_to = rcpt_to or os.environ.get("TALER_MONITORD_DELIVER_RCPT") or f"postmaster@{domain}" t0 = time.monotonic() steps: list[str] = [] try: with socket.create_connection((h, 25), timeout=max(2.0, float(timeout_s))) as sock: code, text = _smtp_read_reply(sock, timeout_s) steps.append(f"banner:{code}") if code != 220: return { "ok": False, "light": "orange", "reason": f"bad_banner:{code}", "detail": text[:160], "mx": h, "rcpt": rcpt_to, "elapsed": round(time.monotonic() - t0, 3), "steps": steps, } sock.sendall(b"EHLO taler-monitoring.invalid\r\n") code, text = _smtp_read_reply(sock, timeout_s) steps.append(f"ehlo:{code}") if code != 250: return { "ok": False, "light": "orange", "reason": f"ehlo_fail:{code}", "detail": text[:160], "mx": h, "rcpt": rcpt_to, "elapsed": round(time.monotonic() - t0, 3), "steps": steps, } sock.sendall(f"MAIL FROM:<{mail_from}>\r\n".encode()) code, text = _smtp_read_reply(sock, timeout_s) steps.append(f"mail:{code}") if code not in (250, 251): return { "ok": False, "light": "orange", "reason": f"mail_from_reject:{code}", "detail": text[:160], "mx": h, "rcpt": rcpt_to, "elapsed": round(time.monotonic() - t0, 3), "steps": steps, } sock.sendall(f"RCPT TO:<{rcpt_to}>\r\n".encode()) code, text = _smtp_read_reply(sock, timeout_s) steps.append(f"rcpt:{code}") try: sock.sendall(b"RSET\r\n") _smtp_read_reply(sock, min(3.0, timeout_s)) sock.sendall(b"QUIT\r\n") except OSError: pass elapsed = round(time.monotonic() - t0, 3) if code in (250, 251): return { "ok": True, "light": "green", "reason": "rcpt_accepted", "detail": text[:160], "mx": h, "rcpt": rcpt_to, "elapsed": elapsed, "steps": steps, } if 400 <= code < 500: # greylist / temp fail — soft degrade return { "ok": False, "light": "yellow", "reason": f"rcpt_tempfail:{code}", "detail": text[:160], "mx": h, "rcpt": rcpt_to, "elapsed": elapsed, "steps": steps, } return { "ok": False, "light": "orange", "reason": f"rcpt_reject:{code}", "detail": text[:160], "mx": h, "rcpt": rcpt_to, "elapsed": elapsed, "steps": steps, } except (socket.timeout, TimeoutError): return { "ok": False, "light": "orange", "reason": "smtp_timeout", "detail": "", "mx": h, "rcpt": rcpt_to, "elapsed": round(time.monotonic() - t0, 3), "steps": steps, } except OSError as e: return { "ok": False, "light": "red" if "refused" in str(e).lower() else "orange", "reason": f"smtp_connect:{type(e).__name__}", "detail": str(e)[:120], "mx": h, "rcpt": rcpt_to, "elapsed": round(time.monotonic() - t0, 3), "steps": steps, } def ensure_deliverability( domain: str, *, mx_hosts: list[str], now: float | None = None, force: bool = False, state: dict[str, Any] | None = None, ) -> dict[str, Any]: """Return deliverability status for domain; refresh on interval if due. Missing/stale results → yellow (no recent test). Send/server failures → orange/red. """ now = time.time() if now is None else now if not DELIVER_ENABLED: return { "ok": True, "light": "blue", "reason": "deliver_disabled", "ts": None, "age_s": None, "stale": False, } st = state if state is not None else load_deliverability_state() domains = st.setdefault("domains", {}) prev = domains.get(domain) if isinstance(domains.get(domain), dict) else {} prev_ts = prev.get("ts") try: age = None if prev_ts is None else now - float(prev_ts) except (TypeError, ValueError): age = None due = force or prev_ts is None or age is None or age >= DELIVER_INTERVAL_S if due and mx_hosts: mx = _host_only(mx_hosts[0]) result = smtp_deliverability_probe(mx, domain=domain) result["ts"] = now result["domain"] = domain domains[domain] = result st["ts"] = now try: save_deliverability_state(st) except OSError: pass prev = result age = 0.0 elif not prev: # never ran and no MX to probe return { "ok": False, "light": "yellow", "reason": "deliver_never_run" if mx_hosts else "deliver_no_mx", "ts": None, "age_s": None, "stale": True, "mx": "", "rcpt": f"postmaster@{domain}", } # Classify freshness try: age_s = None if prev.get("ts") is None else now - float(prev["ts"]) except (TypeError, ValueError): age_s = None stale = age_s is None or age_s > DELIVER_STALE_S light = str(prev.get("light") or "yellow") reason = str(prev.get("reason") or "deliver_unknown") ok = bool(prev.get("ok")) if stale and ok: light, reason, ok = "yellow", "deliver_stale", False elif stale and not ok: # keep failure colour but mark stale if light in ("green", "blue"): light = "yellow" reason = f"deliver_stale:{reason}" return { "ok": ok and not stale, "light": light, "reason": reason, "ts": prev.get("ts"), "age_s": None if age_s is None else round(age_s, 1), "stale": stale, "mx": prev.get("mx") or "", "rcpt": prev.get("rcpt") or "", "detail": prev.get("detail") or "", "elapsed": prev.get("elapsed"), "steps": prev.get("steps") or [], } def assess_mail_domain( row: dict[str, Any], *, port_done: set[str] | None = None, deliver_state: dict[str, Any] | None = None, now: float | None = None, ) -> dict[str, Any]: """Outside-in MX/SPF/DMARC/A/AAAA/rDNS (+ ports + deliverability) for one catalog domain.""" domain = row["domain"] members: list[dict[str, Any]] = [] port_done = port_done if port_done is not None else set() now = time.time() if now is None else now mx = dns_mx(domain) expect = row.get("expect_mx") or [] mx_ok = bool(mx) mx_match = True if expect and mx: mx_match = any( any(m == w or m.endswith("." + w) or w in m for m in mx) for w in expect ) elif expect and not mx: mx_match = False members.append( { "role": "mx", "host": ",".join(mx[:3]) or domain, "ok": mx_ok and mx_match, "light": "green" if (mx_ok and mx_match) else ("orange" if mx_ok else "red"), "reason": "mx_ok" if (mx_ok and mx_match) else ("mx_mismatch" if mx_ok else "mx_missing"), "down_since": None, "code": 0, "elapsed": 0.0, "error": None, "url": None, "endpoints": [], "endpoints_summary": summarize_endpoints([]), } ) spf = dns_txt(domain) has_spf = "v=spf1" in spf.lower() members.append( { "role": "spf", "host": domain, "ok": has_spf, "light": "green" if has_spf else "yellow", "reason": "spf_ok" if has_spf else "spf_missing", "down_since": None, "code": 0, "elapsed": 0.0, "error": None, "url": None, "endpoints": [], "endpoints_summary": summarize_endpoints([]), } ) dmarc = dns_txt(f"_dmarc.{domain}") has_dmarc = "v=dmarc1" in dmarc.lower() members.append( { "role": "dmarc", "host": f"_dmarc.{domain}", "ok": has_dmarc, "light": "green" if has_dmarc else "yellow", "reason": "dmarc_ok" if has_dmarc else "dmarc_missing", "down_since": None, "code": 0, "elapsed": 0.0, "error": None, "url": None, "endpoints": [], "endpoints_summary": summarize_endpoints([]), } ) # Per mail host: A/AAAA + PTR for h in row.get("mail_hosts") or []: addrs = dns_a_aaaa(h) a, aaaa = addrs.get("a") or [], addrs.get("aaaa") or [] if not a and not aaaa: members.append( { "role": "dns", "host": h, "ok": False, "light": "red", "reason": "dns_missing", "down_since": None, "code": 0, "elapsed": 0.0, "error": None, "url": None, "endpoints": [], "endpoints_summary": summarize_endpoints([]), } ) continue light = "green" reason = "dns_ok" if a and not aaaa: light, reason = "yellow", "no_ipv6" members.append( { "role": "dns", "host": h, "ok": True, "light": light, "reason": reason, "down_since": None, "code": 0, "elapsed": 0.0, "error": None, "url": None, "endpoints": [], "endpoints_summary": summarize_endpoints([]), "a": a[:4], "aaaa": aaaa[:4], } ) # rDNS on first A if a: ptrs = dns_ptr(a[0]) ptr_ok = bool(ptrs) members.append( { "role": "rdns", "host": a[0], "ok": ptr_ok, "light": "green" if ptr_ok else "yellow", "reason": ("rdns:" + ",".join(ptrs[:2])) if ptr_ok else "rdns_missing", "down_since": None, "code": 0, "elapsed": 0.0, "error": None, "url": None, "endpoints": [], "endpoints_summary": summarize_endpoints([]), } ) if MAIL_PORTS_ENABLED and h not in port_done: port_done.add(h) for p in row.get("smtp_ports") or []: open_ok = tcp_port_open(h, p) # 587 often filtered on pixel → yellow warn, not red if open_ok: light_p, reason_p = "green", f"smtp_{p}_open" elif str(p) == "587": light_p, reason_p = "yellow", f"smtp_{p}_closed" else: light_p, reason_p = "orange", f"smtp_{p}_closed" members.append( { "role": "smtp", "host": f"{h}:{p}", "ok": open_ok, "light": light_p, "reason": reason_p, "down_since": None, "code": int(p) if str(p).isdigit() else 0, "elapsed": 0.0, "error": None, "url": None, "endpoints": [], "endpoints_summary": summarize_endpoints([]), } ) for p in row.get("imap_ports") or []: open_ok = tcp_port_open(h, p) members.append( { "role": "imap", "host": f"{h}:{p}", "ok": open_ok, "light": "green" if open_ok else "orange", "reason": f"imap_{p}_open" if open_ok else f"imap_{p}_closed", "down_since": None, "code": int(p) if str(p).isdigit() else 0, "elapsed": 0.0, "error": None, "url": None, "endpoints": [], "endpoints_summary": summarize_endpoints([]), } ) elif MAIL_PORTS_ENABLED and h in port_done: members.append( { "role": "smtp", "host": h, "ok": True, "light": "blue", "reason": "ports_probed_earlier", "down_since": None, "code": 0, "elapsed": 0.0, "error": None, "url": None, "endpoints": [], "endpoints_summary": summarize_endpoints([]), } ) # Deliverability (regular, cached) — missing/stale → yellow; send fail → orange/red mx_targets = list(mx) or list(row.get("mail_hosts") or []) deliver = ensure_deliverability( domain, mx_hosts=mx_targets, now=now, state=deliver_state ) members.append( { "role": "deliver", "host": deliver.get("mx") or (mx_targets[0] if mx_targets else domain), "ok": bool(deliver.get("ok")), "light": deliver.get("light") or "yellow", "reason": deliver.get("reason") or "deliver_unknown", "down_since": None, "code": 0, "elapsed": float(deliver.get("elapsed") or 0.0), "error": deliver.get("detail") or None, "url": None, "endpoints": [], "endpoints_summary": summarize_endpoints([]), } ) light, intensity = aggregate_stack(members) software = colorize_software( [ item for h in (row.get("mail_hosts") or [])[:2] for item in smtp_banner_software(h) ], published=None, ) return { "id": domain, "currency": "", "currency_info": None, "kind": "mail", "service_label": row.get("label") or domain, "commercial_prod": False, "bank_backend": "", "wire_methods": [], "local": False, "light": light, "intensity": intensity, "members": members, "stats": None, "software": software, "deliverability": deliver, "network": {"zone": "mail", "label": "mail plane", "ssh": {"expected": False}}, "vulns": {"enabled": False, "worst": "", "light": None, "hits": [], "servers": []}, "breakdown": aggregate_breakdown(members), "tls": {}, "orphan": False, } def _run_git_ls_remote(url: str, *, timeout_s: float) -> dict[str, Any]: """Soft git probe via `git ls-remote` (no checkout; no leftover worktree).""" t0 = time.monotonic() env = os.environ.copy() env["GIT_TERMINAL_PROMPT"] = "0" env["GIT_SSH_COMMAND"] = ( "ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new " "-o ConnectTimeout=5 -o IdentitiesOnly=yes" ) try: r = subprocess.run( ["git", "ls-remote", url], capture_output=True, timeout=max(2.0, float(timeout_s)), check=False, env=env, ) elapsed = round(time.monotonic() - t0, 3) out = (r.stdout or b"").decode("utf-8", "replace").strip() err = (r.stderr or b"").decode("utf-8", "replace").strip()[:200] cmd = f"git ls-remote {url}" if r.returncode == 0: return { "ok": True, "light": "green", "reason": "ls-remote_ok", "elapsed": elapsed, "detail": (out.splitlines()[0][:120] if out else "empty_list"), "command": cmd, } # Auth / no key / unreachable → soft (yellow); keep orange for hard refuse soft = ( r.returncode in (2, 128) or "Permission denied" in err or "Host key" in err or "Could not resolve" in err or "unable to access" in err.lower() ) return { "ok": False, "light": "yellow" if soft else "orange", "reason": err or f"ls-remote_rc_{r.returncode}", "elapsed": elapsed, "detail": "", "command": cmd, } except FileNotFoundError: return { "ok": False, "light": "blue", "reason": "git_not_installed", "elapsed": round(time.monotonic() - t0, 3), "detail": "", "command": f"git ls-remote {url}", } except subprocess.TimeoutExpired: return { "ok": False, "light": "yellow", "reason": "ls-remote_timeout", "elapsed": round(time.monotonic() - t0, 3), "detail": "", "command": f"git ls-remote {url}", } def load_git_probe_state() -> dict[str, Any]: path = git_probe_state_path() if not path.is_file(): return {"schema": 1, "hosts": {}} try: doc = json.loads(path.read_text(encoding="utf-8")) if isinstance(doc, dict): doc.setdefault("hosts", {}) return doc except (OSError, json.JSONDecodeError): pass return {"schema": 1, "hosts": {}} def save_git_probe_state(doc: dict[str, Any]) -> None: path = git_probe_state_path() path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_name(f".{path.name}.tmp.{os.getpid()}") try: tmp.write_text(json.dumps(doc, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") os.replace(tmp, path) finally: if tmp.exists(): try: tmp.unlink() except OSError: pass def collect_git_probe( host: str, *, now: float | None = None, force: bool = False, ) -> dict[str, Any]: """Infrequent HTTPS+SSH git ls-remote; cached under monitord state.""" now = time.time() if now is None else float(now) host = _host_only(host) empty = { "enabled": False, "ok": None, "light": "blue", "reason": "git_probes_disabled", "https": None, "ssh": None, "age_s": None, "stale": False, } if not GIT_PROBE_ENABLED or not host: return empty state = load_git_probe_state() prev = (state.get("hosts") or {}).get(host) or {} prev_ts = prev.get("ts") try: age = None if prev_ts is None else now - float(prev_ts) except (TypeError, ValueError): age = None due = force or prev_ts is None or age is None or age >= GIT_PROBE_INTERVAL_S if not due and prev: stale = age is None or age > GIT_PROBE_STALE_S light = str(prev.get("light") or "green") if stale and light == "green": light = "yellow" return { "enabled": True, "ok": bool(prev.get("ok")), "light": light, "reason": ("git_stale:" if stale else "") + str(prev.get("reason") or ""), "https": prev.get("https"), "ssh": prev.get("ssh"), "age_s": age, "stale": stale, "ts": prev_ts, } https = _run_git_ls_remote(f"https://{host}/", timeout_s=GIT_PROBE_TIMEOUT_S) ssh = _run_git_ls_remote(f"git@{host}:", timeout_s=GIT_PROBE_TIMEOUT_S) # Aggregate: worst soft colour; both fail → orange max for soft probe lights = [str(https.get("light") or "yellow"), str(ssh.get("light") or "yellow")] worst = lights[0] for Lgt in lights[1:]: if SEVERITY.get(Lgt, 0) > SEVERITY.get(worst, 0): worst = Lgt if worst in ("red", "black"): worst = "orange" ok = bool(https.get("ok") or ssh.get("ok")) if ok and worst in ("orange", "red"): worst = "yellow" # one plane OK → soft at worst reason = "git_ok" if ok else "git_unreachable" if https.get("ok") and not ssh.get("ok"): reason = "git_https_ok_ssh_soft" elif ssh.get("ok") and not https.get("ok"): reason = "git_ssh_ok_https_soft" entry = { "ts": now, "ok": ok, "light": worst if not ok else "green", "reason": reason, "https": https, "ssh": ssh, } state.setdefault("hosts", {})[host] = entry save_git_probe_state(state) return { "enabled": True, "ok": ok, "light": entry["light"], "reason": reason, "https": https, "ssh": ssh, "age_s": 0.0, "stale": False, "ts": now, } def payability_state_path() -> Path: return monitord_state_dir() / "payability.json" def load_payability_state() -> dict[str, Any]: path = payability_state_path() if not path.is_file(): return {"schema": 1, "domains": {}} try: doc = json.loads(path.read_text(encoding="utf-8")) if isinstance(doc, dict): doc.setdefault("domains", {}) return doc except (OSError, json.JSONDecodeError): pass return {"schema": 1, "domains": {}} def save_payability_state(doc: dict[str, Any]) -> None: path = payability_state_path() path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_name(f".{path.name}.tmp.{os.getpid()}") try: tmp.write_text(json.dumps(doc, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") os.replace(tmp, path) finally: if tmp.exists(): try: tmp.unlink() except OSError: pass def _suite_root() -> Path: raw = (os.environ.get("SUITE_DIR") or "").strip() if raw: return Path(raw) return Path(__file__).resolve().parents[1] def _run_zero_pay(domain: str, *, timeout_s: float) -> dict[str, Any]: """Run suite zero-pay COUNT=1 for domain (needs WALLET_DB / funded wallet).""" root = _suite_root() script = root / "taler-monitoring.sh" cmd = [ "bash", str(script), "-d", domain, "zero-pay", ] env = os.environ.copy() env["COUNT"] = "1" env["STRICT_EXIT"] = env.get("STRICT_EXIT", "1") t0 = time.monotonic() try: r = subprocess.run( cmd, capture_output=True, timeout=max(30.0, float(timeout_s)), check=False, env=env, cwd=str(root), ) elapsed = round(time.monotonic() - t0, 3) err = (r.stderr or b"").decode("utf-8", "replace").strip()[-300:] out = (r.stdout or b"").decode("utf-8", "replace").strip()[-300:] ok = r.returncode == 0 return { "ok": ok, "light": "green" if ok else "orange", "reason": "zero_pay_ok" if ok else (err or out or f"zero_pay_rc_{r.returncode}"), "elapsed": elapsed, "command": f"COUNT=1 ./taler-monitoring.sh -d {domain} zero-pay", "rc": r.returncode, } except FileNotFoundError: return { "ok": False, "light": "blue", "reason": "suite_missing", "elapsed": 0.0, "command": f"COUNT=1 ./taler-monitoring.sh -d {domain} zero-pay", "rc": 127, } except subprocess.TimeoutExpired: return { "ok": False, "light": "orange", "reason": "zero_pay_timeout", "elapsed": round(time.monotonic() - t0, 3), "command": f"COUNT=1 ./taler-monitoring.sh -d {domain} zero-pay", "rc": 124, } def collect_payability( domain: str, *, now: float | None = None, force: bool = False, ) -> dict[str, Any]: """Cached payability smoke for a stack domain (few×/day when PAY_RUN=1).""" now = time.time() if now is None else float(now) domain = str(domain or "").strip() empty = { "enabled": False, "ok": None, "light": "blue", "reason": "pay_disabled", "age_s": None, "stale": False, "command": "", } if not PAY_ENABLED or not domain: return empty allowed = set(PAY_DOMAINS) if domain not in allowed and not any( domain == d or domain.endswith("." + d) or d.endswith("." + domain) for d in allowed ): return {**empty, "reason": "pay_domain_skip"} state = load_payability_state() prev = (state.get("domains") or {}).get(domain) or {} prev_ts = prev.get("ts") try: age = None if prev_ts is None else now - float(prev_ts) except (TypeError, ValueError): age = None due = force or prev_ts is None or age is None or age >= PAY_INTERVAL_S if not due and prev: stale = age is None or age > PAY_STALE_S light = str(prev.get("light") or "green") if stale and light == "green": light = "yellow" return { "enabled": True, "ok": bool(prev.get("ok")), "light": light, "reason": ("pay_stale:" if stale else "") + str(prev.get("reason") or ""), "age_s": age, "stale": stale, "command": prev.get("command") or "", "ts": prev_ts, } if not PAY_RUN: # enabled but operator did not allow live wallet run this tick if prev: stale = age is None or age > PAY_STALE_S light = str(prev.get("light") or "yellow") if stale: light = "yellow" return { "enabled": True, "ok": bool(prev.get("ok")), "light": light, "reason": ("pay_stale:" if stale else "") + str(prev.get("reason") or "pay_cached"), "age_s": age, "stale": bool(stale), "command": prev.get("command") or "", "ts": prev_ts, } return { "enabled": True, "ok": False, "light": "yellow", "reason": "pay_never_run", "age_s": None, "stale": True, "command": f"COUNT=1 ./taler-monitoring.sh -d {domain} zero-pay", "ts": None, } result = _run_zero_pay(domain, timeout_s=PAY_TIMEOUT_S) entry = { "ts": now, "ok": bool(result.get("ok")), "light": result.get("light") or "orange", "reason": result.get("reason") or "", "command": result.get("command") or "", "elapsed": result.get("elapsed"), "rc": result.get("rc"), } state.setdefault("domains", {})[domain] = entry save_payability_state(state) return { "enabled": True, "ok": entry["ok"], "light": entry["light"], "reason": entry["reason"], "age_s": 0.0, "stale": False, "command": entry["command"], "ts": now, } def build_service_status( row: dict[str, Any], *, prev_hosts: dict[str, dict[str, Any]], now: float, acute_s: float, probe_fn: Callable[[str], dict[str, Any]], osv_cache: dict[str, list], published: dict[str, Any] | None = None, ) -> dict[str, Any]: """Probe a non-Taler company/service host (apex HTTPS).""" host = row["host"] url = f"https://{host}/" raw = probe_fn(url) classified = classify_probe(raw, prev=prev_hosts.get(host), now=now, acute_s=acute_s) member = { "role": "site", "host": host, **classified, "endpoints": [], "endpoints_summary": summarize_endpoints([]), } members = [member] tls: dict[str, Any] = {} if CERT_PROBES_ENABLED: tls = probe_tls_cert(host) if tls.get("light") and SEVERITY.get(str(tls["light"]), 0) > SEVERITY.get( member.get("light") or "green", 0 ): members.append( { "role": "tls", "host": host, "ok": bool(tls.get("ok")), "light": tls.get("light") or "yellow", "reason": tls.get("reason") or "cert", "down_since": None, "code": 0, "elapsed": 0.0, "error": tls.get("error"), "url": None, "endpoints": [], "endpoints_summary": summarize_endpoints([]), } ) label = str(row.get("service_label") or "") git_probe: dict[str, Any] | None = None if label.lower() == "git" or host.startswith("git."): git_probe = collect_git_probe(host, now=now) if git_probe.get("enabled"): members.append( { "role": "git", "host": host, "ok": bool(git_probe.get("ok")), "light": git_probe.get("light") or "yellow", "reason": git_probe.get("reason") or "git", "down_since": None, "code": 0, "elapsed": 0.0, "error": None, "url": None, "endpoints": [], "endpoints_summary": summarize_endpoints([]), "command": ((git_probe.get("https") or {}).get("command") or ""), } ) light, intensity = aggregate_stack(members) probe_cache = {url: raw} vulns = assess_external_vulns( probe_cache=probe_cache, members=members, osv_cache=osv_cache ) light, intensity, members = apply_vuln_to_stack(light, intensity, members, vulns) software = collect_software_from_probes( probe_cache=probe_cache, members=members, vulns=vulns, smtp=False, published=published, ) return { "id": row.get("id") or host, "currency": "", "currency_info": None, "kind": "service", "service_label": row.get("service_label") or "service", "commercial_prod": False, "bank_backend": "", "wire_methods": [], "local": False, "light": light, "intensity": intensity, "members": members, "stats": None, "software": software, "git": git_probe or {}, "network": { "zone": row.get("zone") or "public", "label": row.get("zone") or "public", "ssh": {"expected": False}, }, "vulns": { "enabled": vulns.get("enabled"), "worst": vulns.get("worst") or "", "light": vulns.get("light"), "hits": [ { "id": h.get("id"), "severity": h.get("severity"), "package": h.get("package"), "version": h.get("version"), "host": h.get("host"), "summary": h.get("summary") or "", } for h in (vulns.get("hits") or [])[:8] ], "servers": vulns.get("servers") or [], }, "breakdown": aggregate_breakdown(members), "tls": { "days_left": tls.get("days_left"), "not_after": tls.get("not_after"), "sans": tls.get("sans") or [], "reason": tls.get("reason"), } if tls else {}, "orphan": False, } def _host_covered_by_known(host: str, known_hosts: set[str]) -> bool: """True if host is known or a subdomain/parent of a monitored host (cert sibling).""" h = _host_only(host).lower() if not h or h in known_hosts: return True for k in known_hosts: k = (k or "").lower() if not k or k.upper() == "IBAN": continue if h.endswith("." + k) or k.endswith("." + h): return True return False def collect_orphan_hosts( *, known_hosts: set[str], tls_sans: list[str], probe_fn: Callable[[str], dict[str, Any]], ) -> list[dict[str, Any]]: """Flag SANs that look abandoned: DNS missing (default). Skips wildcard expansions already covered as siblings of monitored hosts (e.g. donations.demo.taler.net when demo.taler.net is known). HTTP probe optional via TALER_MONITORD_ORPHAN_HTTP=1 (off by default — CDN noise). """ if not ORPHAN_PROBES_ENABLED: return [] known = {_host_only(x).lower() for x in known_hosts if x} candidates: list[str] = [] for raw_san in tls_sans: san = str(raw_san or "").strip().lower() if not san or san.startswith("*."): # Wildcard SAN ≠ concrete orphan candidate continue h = _host_only(san) if not h or _host_covered_by_known(h, known): continue if h not in candidates: candidates.append(h) orphans: list[dict[str, Any]] = [] for h in candidates[:25]: addrs = dns_a_aaaa(h) if not (addrs.get("a") or addrs.get("aaaa")): orphans.append({"host": h, "reason": "dns_missing", "light": "orange"}) continue if not ORPHAN_HTTP_ENABLED: continue raw = probe_fn(f"https://{h}/") if not raw.get("ok") and int(raw.get("code") or 0) == 0: orphans.append( { "host": h, "reason": f"http_unreachable:{raw.get('error')}", "light": "yellow", } ) return orphans def scan_ports_list() -> list[int]: raw = (os.environ.get("TALER_MONITORD_PORTSCAN_PORTS") or "").strip() if raw: out: list[int] = [] for part in re.split(r"[\s,]+", raw): if not part: continue try: out.append(int(part)) except ValueError: continue return sorted(set(out)) return list(_DEFAULT_SCAN_PORTS) def parse_ports_whitelist(path: Path | None = None) -> list[dict[str, Any]]: """Whitelist lines: host_or_pattern port[,port…] [# note] Patterns: exact host, ``*.example.com``, or ``*`` (any host). """ path = path or default_ports_whitelist_conf() rows: list[dict[str, Any]] = [] if not path.is_file(): return rows for raw in path.read_text(encoding="utf-8").splitlines(): line = raw.split("#", 1)[0].strip() if not line: continue parts = line.split() if len(parts) < 2: continue pattern = parts[0].lower().rstrip(".") ports: list[int] = [] for tok in parts[1].split(","): tok = tok.strip() if not tok: continue try: ports.append(int(tok)) except ValueError: continue if ports: rows.append({"pattern": pattern, "ports": ports}) return rows def whitelist_ports_for_host(host: str, rules: list[dict[str, Any]]) -> set[int]: h = _host_only(host).lower() allowed: set[int] = set() for rule in rules or []: pat = str(rule.get("pattern") or "").lower() ports = set(int(p) for p in (rule.get("ports") or [])) if not pat or not ports: continue if pat == "*" or pat == h: allowed |= ports elif pat.startswith("*.") and (h.endswith(pat[1:]) or h == pat[2:]): allowed |= ports return allowed def baseline_ports_for_host(host: str, *, mail_hosts: set[str] | None = None) -> set[int]: """Default expected open ports before whitelist.""" h = _host_only(host).lower() allowed = set(_BASELINE_WEB_PORTS) if mail_hosts and h in mail_hosts: allowed |= set(_BASELINE_MAIL_PORTS) return allowed def tcp_scan_host( host: str, ports: list[int], *, timeout_s: float = PORT_SCAN_TIMEOUT_S, workers: int = PORT_SCAN_WORKERS, ) -> list[int]: """Return sorted list of open TCP ports (banner-less connect).""" h = _host_only(host) if not h or h.upper() == "IBAN": return [] open_ports: list[int] = [] def _one(p: int) -> int | None: return p if tcp_port_open(h, p, timeout_s=timeout_s) else None with ThreadPoolExecutor(max_workers=max(4, min(int(workers), len(ports) or 4))) as pool: futs = [pool.submit(_one, p) for p in ports] for fut in as_completed(futs): try: got = fut.result() except Exception: got = None if got is not None: open_ports.append(int(got)) return sorted(set(open_ports)) def assess_host_ports( host: str, *, ports: list[int] | None = None, whitelist_rules: list[dict[str, Any]] | None = None, mail_hosts: set[str] | None = None, open_ports: list[int] | None = None, ) -> dict[str, Any]: """Scan (or reuse) open ports and classify expected vs suspicious.""" h = _host_only(host) ports = ports if ports is not None else scan_ports_list() rules = whitelist_rules if whitelist_rules is not None else parse_ports_whitelist() opened = list(open_ports) if open_ports is not None else tcp_scan_host(h, ports) allowed = baseline_ports_for_host(h, mail_hosts=mail_hosts) | whitelist_ports_for_host(h, rules) suspicious = [p for p in opened if p not in allowed] expected_hit = [p for p in opened if p in allowed] # hard = raise card colour; soft = list only (SSH/mail/http-alt often shared). high = {21, 23, 135, 139, 445, 1433, 1521, 2049, 3306, 3389, 5432, 5900, 6379, 9200, 11211, 27017} soft_set = {22, 25, 110, 143, 465, 587, 993, 995, 8080, 8443} items: list[dict[str, Any]] = [] for p in suspicious: if p in high: sev = "hard" pl = "red" if p in (23, 445, 3389, 27017) else "orange" else: sev, pl = "soft", "yellow" if p not in soft_set: sev, pl = "soft", "yellow" items.append({"port": p, "label": _PORT_LABELS.get(p, str(p)), "severity": sev, "light": pl}) light = "green" for it in items: if SEVERITY.get(str(it["light"]), 0) > SEVERITY.get(light, 0): light = str(it["light"]) return { "host": h, "scanned": list(ports), "open": opened, "allowed": sorted(allowed), "expected_open": expected_hit, "suspicious": items, "light": light, "ok": not any(it.get("severity") == "hard" for it in items), } def collect_port_scan( hosts: list[str], *, mail_hosts: set[str] | None = None, whitelist_rules: list[dict[str, Any]] | None = None, ) -> dict[str, Any]: """Scan unique hosts; return page-level findings.""" if not PORT_SCAN_ENABLED: return {"enabled": False, "hosts": [], "suspicious": [], "light": None} ports = scan_ports_list() rules = whitelist_rules if whitelist_rules is not None else parse_ports_whitelist() mail_hosts = { _host_only(x).lower() for x in (mail_hosts or set()) } uniq: list[str] = [] seen: set[str] = set() for h in hosts: hh = _host_only(h).lower() if not hh or hh.upper() == "IBAN" or hh in seen: continue # Skip pure IPs from rdns rows unless explicitly wanted if re.match(r"^\d+\.\d+\.\d+\.\d+$", hh): continue seen.add(hh) uniq.append(hh) host_results: list[dict[str, Any]] = [] # Scan hosts concurrently (each host scans ports internally) def _scan(h: str) -> dict[str, Any]: return assess_host_ports(h, ports=ports, whitelist_rules=rules, mail_hosts=mail_hosts) with ThreadPoolExecutor(max_workers=max(4, min(12, len(uniq) or 4))) as pool: futs = {pool.submit(_scan, h): h for h in uniq} for fut in as_completed(futs): try: host_results.append(fut.result()) except Exception as e: host_results.append( { "host": futs[fut], "scanned": ports, "open": [], "allowed": [], "expected_open": [], "suspicious": [], "light": "yellow", "ok": True, "error": type(e).__name__, } ) host_results.sort(key=lambda r: r.get("host") or "") suspicious_flat: list[dict[str, Any]] = [] worst = "green" for r in host_results: for s in r.get("suspicious") or []: suspicious_flat.append( { "host": r.get("host"), "port": s.get("port"), "label": s.get("label"), "severity": s.get("severity") or "soft", "light": s.get("light") or "yellow", } ) Lgt = r.get("light") or "green" if SEVERITY.get(str(Lgt), 0) > SEVERITY.get(worst, 0): worst = str(Lgt) hard = [s for s in suspicious_flat if s.get("severity") == "hard"] if not suspicious_flat: worst = "green" elif not hard: worst = "yellow" return { "enabled": True, "ports": ports, "hosts": host_results, "suspicious": suspicious_flat, "n_hosts": len(host_results), "n_suspicious": len(suspicious_flat), "n_hard": len(hard), "light": worst if suspicious_flat else "green", } def build_stack_status( stacks: list[dict[str, Any]], *, prev_doc: dict[str, Any] | None = None, now: float | None = None, acute_s: float = ACUTE_DOWN_S, probe_fn: Callable[[str], dict[str, Any]] | None = None, public_only: bool = True, ) -> dict[str, Any]: """Probe all stacks and build status document.""" now = time.time() if now is None else now prev_doc = prev_doc or {} prev_hosts: dict[str, dict[str, Any]] = {} for s in prev_doc.get("stacks") or []: for m in s.get("members") or []: h = m.get("host") if h: prev_hosts[h] = m probe = probe_fn or (lambda url: probe_http(url)) out_stacks: list[dict[str, Any]] = [] ssh_cache: dict[str, dict[str, Any]] = {} osv_cache: dict[str, list] = {} published_doc = fetch_published_taler_packages() # Previous endpoint rows keyed by host+path for down_since continuity prev_eps: dict[str, dict[str, Any]] = {} for s in prev_doc.get("stacks") or []: for m in s.get("members") or []: h = m.get("host") or "" for ep in m.get("endpoints") or []: p = ep.get("path") if h and p: prev_eps[f"{h}{p}"] = ep for st in stacks: if public_only and st.get("local"): continue members_out: list[dict[str, Any]] = [] kind = st.get("kind") or ("taler" if st.get("currency") else "static_pending") has_intro = stack_has_custom_intro(st) legal_sev = legal_doc_severity(st) bank_configured = st.get("bank_configured") if bank_configured is None: bank_configured = bank_host_configured(st.get("bank") or "") bank_backend = "unknown" wire_methods: set[str] = set() currency_info: dict[str, Any] | None = None keys_body_for_denoms: bytes | str | None = None probe_cache: dict[str, dict[str, Any]] = {} def cached_probe(url: str) -> dict[str, Any]: if url not in probe_cache: probe_cache[url] = probe(url) return probe_cache[url] if kind == "static_pending": # Apex static site (prod Taler not up yet) site_host = _host_only(st["id"]) site_url = f"https://{site_host}/" site_raw = cached_probe(site_url) site_c = classify_probe(site_raw, prev=prev_hosts.get(site_host), now=now, acute_s=acute_s) if site_c.get("ok"): site_c["light"] = "green" site_c["reason"] = "static_ok" site_member = {"role": "site", "host": site_host, **site_c} site_prev = {k[len(site_host) :]: v for k, v in prev_eps.items() if k.startswith(site_host)} site_eps = probe_role_endpoints( "site", site_host, has_intro=False, legal_severity=legal_sev, probe_fn=cached_probe, cache=probe_cache, prev_by_path=site_prev, now=now, acute_s=acute_s, ) members_out.append(merge_member_endpoint_light(site_member, site_eps)) # Probe exchange before bank so /keys payto://… can decide EBICS vs x-taler-bank. role_hosts: list[tuple[str, str]] = [ ("exchange", st["exchange"]), ("merchant", st["merchant"]), ] if bank_configured: role_hosts.insert(0, ("bank", st.get("bank") or "")) classified_by_role: dict[str, dict[str, Any]] = {} # Always resolve exchange (+ merchant) first for wire detection, then bank. probe_order = ["exchange", "merchant", "bank"] if bank_configured else ["exchange", "merchant"] host_by_role = {r: h for r, h in role_hosts} for role in probe_order: if role not in host_by_role: continue host = host_by_role[role] if role == "bank": # After exchange: skip public bank.* when wire is IBAN/EBICS-only. bank_backend = classify_bank_backend( wire_methods, bank_configured=bool(bank_configured) ) if bank_backend == "ebics": classified_by_role[role] = { "ok": True, "light": "blue", "reason": "ebics_iban", "down_since": None, "code": 0, "elapsed": 0.0, "error": None, "url": None, "ts": now, # No public bank.* — show wire kind in the host column "host": "IBAN", "role": role, } continue if role == "exchange": # Prefer /keys (accounts/payto); fall back to /config for liveness. keys_url = f"https://{_host_only(host)}/keys" raw = cached_probe(keys_url) cfg_data: dict[str, Any] | None = None if not raw.get("ok"): alt = member_probe_url(role, host) raw2 = cached_probe(alt) if raw2.get("ok") or int(raw2.get("code") or 0) > int(raw.get("code") or 0): raw = raw2 cfg_data = _json_body(raw) else: keys_body_for_denoms = raw.get("body") wire_methods = payto_wire_methods(keys_body_for_denoms) bank_backend = classify_bank_backend( wire_methods, bank_configured=bool(bank_configured) ) # Separate /config for currency_specification (small JSON). cfg_raw = cached_probe(member_probe_url(role, host)) if cfg_raw.get("ok"): cfg_data = _json_body(cfg_raw) if cfg_data: cs = cfg_data.get("currency_specification") if not isinstance(cs, dict): cs = {} currency_info = summarize_currency_spec( cs, keys_body=keys_body_for_denoms, fallback_currency=str( cfg_data.get("currency") or st.get("currency") or "" ), ) elif role == "bank": url = member_probe_url(role, host) raw = cached_probe(url) if not raw.get("ok") and int(raw.get("code") or 0) in (0, 404): alt = f"https://{_host_only(host)}/config" raw2 = cached_probe(alt) if raw2.get("ok") or int(raw2.get("code") or 0) > int(raw.get("code") or 0): raw = raw2 else: raw = cached_probe(member_probe_url(role, host)) classified = classify_probe(raw, prev=prev_hosts.get(host), now=now, acute_s=acute_s) if kind == "static_pending" and not classified.get("ok"): # Expected: no bank/exchange/merchant prod yet — not an outage. classified = { **classified, "ok": True, "light": "blue", "reason": "no_prod_yet", "down_since": None, } classified_by_role[role] = {"role": role, "host": host, **classified} if not bank_configured: bank_backend = "ebics" classified_by_role["bank"] = { "role": "bank", "host": "IBAN", "ok": True, "light": "blue", "reason": "ebics_iban", "down_since": None, "code": 0, "elapsed": 0.0, "error": None, "url": None, "ts": now, } # Stable member order: site (optional) already appended; then bank → exchange → merchant for role in ("bank", "exchange", "merchant"): if role not in classified_by_role: continue member = classified_by_role[role] host = member.get("host") or "" # EBICS placeholder — no public endpoint fan-out if role == "bank" and ( bank_backend == "ebics" or (member.get("reason") or "") in _AGG_SKIP_REASONS ): members_out.append( { **member, "endpoints": [], "endpoints_summary": summarize_endpoints([]), } ) continue if kind == "static_pending" and (member.get("reason") or "") == "no_prod_yet": members_out.append( { **member, "endpoints": [], "endpoints_summary": summarize_endpoints([]), } ) continue host_prev = {k[len(host) :]: v for k, v in prev_eps.items() if host and k.startswith(host)} eps = probe_role_endpoints( role, host, has_intro=has_intro, legal_severity=legal_sev, probe_fn=cached_probe, cache=probe_cache, prev_by_path=host_prev, now=now, acute_s=acute_s, ) members_out.append(merge_member_endpoint_light(member, eps)) light, intensity = aggregate_stack(members_out) stats = fetch_stack_stats( st, bank_backend=bank_backend, probe_fn=cached_probe, now=now, stale_s=STATS_STALE_S ) vulns = assess_external_vulns( probe_cache=probe_cache, members=members_out, osv_cache=osv_cache ) light, intensity, members_out = apply_vuln_to_stack(light, intensity, members_out, vulns) tls: dict[str, Any] = {} primary_host = "" for role in ("merchant", "exchange", "bank"): m = classified_by_role.get(role) or {} h = str(m.get("host") or "") if h and h.upper() != "IBAN": primary_host = h break if not primary_host: primary_host = _host_only(str(st.get("id") or "")) if CERT_PROBES_ENABLED and primary_host: tls = probe_tls_cert(primary_host) if tls.get("light") and SEVERITY.get(str(tls["light"]), 0) > SEVERITY.get(light, 0): members_out.append( { "role": "tls", "host": primary_host, "ok": bool(tls.get("ok")), "light": tls.get("light") or "yellow", "reason": tls.get("reason") or "cert", "down_since": None, "code": 0, "elapsed": 0.0, "error": tls.get("error"), "url": None, "endpoints": [], "endpoints_summary": summarize_endpoints([]), } ) light, intensity = aggregate_stack(members_out) # Prefer live exchange currency; keep domains.conf as fallback label. detected_currency = "" if currency_info and currency_info.get("currency"): detected_currency = str(currency_info["currency"]) elif st.get("currency"): detected_currency = str(st.get("currency")) software = collect_software_from_probes( probe_cache=probe_cache, members=members_out, vulns=vulns, smtp=False, published=published_doc, ) pay: dict[str, Any] = {} zone = classify_network_zone(st)["zone"] pay_id = str(st.get("id") or "") if PAY_ENABLED and zone in ("tops-stage", "tops-prod", "goa-local", "fp-stage"): pay = collect_payability(pay_id, now=now) if pay.get("enabled") and pay.get("light") not in (None, "blue"): members_out.append( { "role": "payability", "host": pay_id, "ok": bool(pay.get("ok")), "light": pay.get("light") or "yellow", "reason": pay.get("reason") or "pay", "down_since": None, "code": 0, "elapsed": 0.0, "error": None, "url": None, "endpoints": [], "endpoints_summary": summarize_endpoints([]), "command": pay.get("command") or "", } ) light, intensity = aggregate_stack(members_out) out_stacks.append( { "id": st["id"], "currency": detected_currency or (st.get("currency") or ""), "currency_info": currency_info, "kind": kind, "commercial_prod": stack_is_commercial_prod(st), "bank_backend": bank_backend, "wire_methods": sorted(wire_methods), "local": bool(st.get("local")), "light": light, "intensity": intensity, "members": members_out, "stats": stats, "software": software, "payability": pay, "network": build_network_status( st, members=members_out, ssh_cache=ssh_cache ), "vulns": { "enabled": vulns.get("enabled"), "worst": vulns.get("worst") or "", "light": vulns.get("light"), "hits": [ { "id": h.get("id"), "severity": h.get("severity"), "package": h.get("package"), "version": h.get("version"), "host": h.get("host"), "summary": h.get("summary") or "", } for h in (vulns.get("hits") or [])[:8] ], "servers": vulns.get("servers") or [], }, "breakdown": aggregate_breakdown(members_out), "tls": { "days_left": tls.get("days_left"), "not_after": tls.get("not_after"), "sans": tls.get("sans") or [], "reason": tls.get("reason"), } if tls else {}, "orphan": False, } ) out_services: list[dict[str, Any]] = [] if SERVICES_ENABLED: for row in parse_services_conf(): out_services.append( build_service_status( row, prev_hosts=prev_hosts, now=now, acute_s=acute_s, probe_fn=probe, osv_cache=osv_cache, published=published_doc, ) ) out_mail: list[dict[str, Any]] = [] if MAIL_ENABLED: mail_port_done: set[str] = set() deliver_state = load_deliverability_state() if DELIVER_ENABLED else {"domains": {}} for row in parse_mail_catalog(): out_mail.append( assess_mail_domain( row, port_done=mail_port_done, deliver_state=deliver_state, now=now, ) ) known_hosts: set[str] = set() tls_sans: list[str] = [] scan_hosts: list[str] = [] for group in (out_stacks, out_services): for st_out in group: for m in st_out.get("members") or []: h = _host_only(str(m.get("host") or "")) if h and h.upper() != "IBAN": known_hosts.add(h) scan_hosts.append(h) for san in (st_out.get("tls") or {}).get("sans") or []: tls_sans.append(str(san)) known_hosts.add(_host_only(str(st_out.get("id") or ""))) scan_hosts.append(_host_only(str(st_out.get("id") or ""))) # Also scan env-mapped SSH targets — findings may be hidden hosts for st_cfg in stacks: if public_only and st_cfg.get("local"): continue for h in ssh_targets_for_stack(st_cfg).get("hosts") or []: scan_hosts.append(h) mail_hosts: set[str] = set() for row in parse_mail_catalog() if MAIL_ENABLED else []: for h in row.get("mail_hosts") or []: mail_hosts.add(_host_only(h).lower()) scan_hosts.append(h) orphans = collect_orphan_hosts( known_hosts=known_hosts, tls_sans=tls_sans, probe_fn=probe ) ports_doc = collect_port_scan(scan_hosts, mail_hosts=mail_hosts) # Attach per-card port slice + optional synthetic member when suspicious by_host = {str(r.get("host") or ""): r for r in (ports_doc.get("hosts") or [])} for group in (out_stacks, out_services): for st_out in group: ssh_meta = st_out.get("network") or {} ssh_block = ssh_meta.get("ssh") or {} # UI clears candidates when hide_host; recover env-mapped hosts for port attach hidden_ssh: set[str] = set() if ssh_block.get("hide_host"): zone = str(ssh_meta.get("zone") or "") env_map = { "goa-local": "TALER_MONITORD_SSH_GOA", "fp-stage": "TALER_MONITORD_SSH_FP", "tops-stage": "TALER_MONITORD_SSH_TOPS", "tops-prod": "TALER_MONITORD_SSH_TOPS_PROD", } if zone in env_map: hidden_ssh = { _host_only(x).lower() for x in _env_ssh_hosts(env_map[zone]) if x } card_hosts: list[str] = [] for m in st_out.get("members") or []: h = _host_only(str(m.get("host") or "")) if h and h.upper() != "IBAN": card_hosts.append(h) sid = _host_only(str(st_out.get("id") or "")) if sid: card_hosts.append(sid) card_hosts.extend(hidden_ssh) sus: list[dict[str, Any]] = [] opens: list[dict[str, Any]] = [] worst = "green" for h in dict.fromkeys(_host_only(x).lower() for x in card_hosts if x): r = by_host.get(h) if not r: continue hide = h in hidden_ssh for s in r.get("suspicious") or []: sus.append( { "host": "" if hide else h, "port": s.get("port"), "label": s.get("label"), "severity": s.get("severity") or "soft", "light": s.get("light") or "yellow", } ) if r.get("open"): opens.append( { "host": "" if hide else h, "open": r.get("open"), "suspicious": [s.get("port") for s in (r.get("suspicious") or [])], } ) for s in r.get("suspicious") or []: if s.get("severity") != "hard": continue Lgt = s.get("light") or "orange" if SEVERITY.get(str(Lgt), 0) > SEVERITY.get(worst, 0): worst = str(Lgt) hard_sus = [s for s in sus if s.get("severity") == "hard"] st_out["ports"] = { "suspicious": sus, "hosts": opens, "light": worst if hard_sus else ("yellow" if sus else "green"), "n_suspicious": len(sus), "n_hard": len(hard_sus), } # Only hard findings pull the card colour (soft stays informational). if hard_sus: members = list(st_out.get("members") or []) members.append( { "role": "ports", "host": ",".join( f"{s.get('label') or s.get('port')}" + (f"@{s.get('host')}" if s.get("host") else "") for s in hard_sus[:6] ), "ok": False, "light": worst, "reason": f"suspicious_ports_hard:{len(hard_sus)}", "down_since": None, "code": 0, "elapsed": 0.0, "error": None, "url": None, "endpoints": [], "endpoints_summary": summarize_endpoints([]), } ) st_out["members"] = members light2, intensity2 = aggregate_stack(members) st_out["light"] = light2 st_out["intensity"] = intensity2 st_out["breakdown"] = aggregate_breakdown(members) return { "schema": SCHEMA, "ts": now, "recorder": os.environ.get("TALER_MONITORD_RECORDER") or socket.gethostname(), "acute_down_s": acute_s, "legend": dict(LEGEND), "stacks": out_stacks, "services": out_services, "mail": out_mail, "orphans": orphans, "ports": ports_doc, "published": { "enabled": published_doc.get("enabled"), "suite": published_doc.get("suite"), "error": published_doc.get("error"), "url": published_doc.get("url"), "packages": { k: published_doc.get("packages", {}).get(k) for k in ( "taler-exchange", "taler-merchant", "libeufin-bank", "taler-merchant-webui", "taler-wallet-cli", ) if (published_doc.get("packages") or {}).get(k) }, "apps": published_doc.get("apps") or [], }, "score": compute_total_score(out_stacks, out_services, out_mail), } def read_status(path: Path | None = None) -> dict[str, Any]: paths = [path] if path else status_json_paths() for p in paths: if p and p.is_file(): try: return json.loads(p.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue return {} def atomic_write_json(path: Path, doc: dict[str, Any]) -> None: path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) text = json.dumps(doc, ensure_ascii=False, indent=2) + "\n" tmp = path.with_name(f".{path.name}.tmp.{os.getpid()}") try: tmp.write_text(text, encoding="utf-8") os.replace(tmp, path) finally: if tmp.exists(): try: tmp.unlink() except OSError: pass def write_stack_status(doc: dict[str, Any], paths: list[Path] | None = None) -> Path: paths = paths or status_json_paths() primary = paths[0] for p in paths: atomic_write_json(p, doc) return primary