4410 lines
155 KiB
Python
4410 lines
155 KiB
Python
#!/usr/bin/env python3
|
||
"""Stack traffic lights for taler-monitoring (daemon → JSON; UI reads only)."""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
import socket
|
||
import ssl
|
||
import subprocess
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
from pathlib import Path
|
||
from typing import Any, Callable
|
||
|
||
SCHEMA = 8
|
||
LIGHTS = ("green", "blue", "yellow", "orange", "red", "black")
|
||
SEVERITY = {name: i for i, name in enumerate(LIGHTS)}
|
||
# Soft endpoint fails (intro/terms/…) cap at yellow unless hard=True.
|
||
ENDPOINT_SOFT_MAX = "yellow"
|
||
|
||
|
||
def _env_truthy(name: str, default: str = "1") -> bool:
|
||
return os.environ.get(name, default) not in ("0", "false", "no", "")
|
||
|
||
|
||
def refresh_runtime_flags_from_env() -> None:
|
||
"""Re-read module toggles after load_monitoring_env() (import-time values go stale)."""
|
||
global PUBLISHED_ENABLED, PUBLISHED_SUITE, PUBLISHED_APT_BASE
|
||
global ACUTE_DOWN_S, PROBE_TIMEOUT_S, STATS_STALE_S, SSH_PROBE_TIMEOUT_S, TLS_VERIFY
|
||
global SSH_PROBES_ENABLED, CVE_PROBES_ENABLED, SERVICES_ENABLED, MAIL_ENABLED
|
||
global CERT_PROBES_ENABLED, ORPHAN_PROBES_ENABLED, ORPHAN_HTTP_ENABLED
|
||
global MAIL_PORTS_ENABLED, MAIL_PORT_TIMEOUT_S
|
||
global DELIVER_ENABLED, DELIVER_INTERVAL_S, DELIVER_STALE_S, DELIVER_TIMEOUT_S
|
||
global PORT_SCAN_ENABLED, PORT_SCAN_TIMEOUT_S, PORT_SCAN_WORKERS
|
||
global CERT_WARN_DAYS, CERT_CRIT_DAYS
|
||
global SCORE_RATIO_GREEN, SCORE_RATIO_BLUE, SCORE_RATIO_YELLOW
|
||
global SCORE_RATIO_ORANGE, SCORE_RATIO_RED
|
||
global RESULT_INDEX_MODE
|
||
global GIT_PROBE_ENABLED, GIT_PROBE_INTERVAL_S, GIT_PROBE_STALE_S, GIT_PROBE_TIMEOUT_S
|
||
global PAY_ENABLED, PAY_RUN, PAY_INTERVAL_S, PAY_STALE_S, PAY_TIMEOUT_S, PAY_DOMAINS
|
||
PUBLISHED_ENABLED = _env_truthy("TALER_MONITORD_PUBLISHED")
|
||
PUBLISHED_SUITE = os.environ.get("TALER_MONITORD_APT_SUITE", "trixie")
|
||
PUBLISHED_APT_BASE = os.environ.get(
|
||
"TALER_MONITORD_APT_BASE", "https://deb.taler.net/apt/debian"
|
||
).rstrip("/")
|
||
ACUTE_DOWN_S = float(os.environ.get("TALER_MONITORD_ACUTE_DOWN_S", "90"))
|
||
PROBE_TIMEOUT_S = float(os.environ.get("TALER_MONITORD_PROBE_TIMEOUT_S", "5"))
|
||
STATS_STALE_S = float(os.environ.get("TALER_MONITORD_STATS_STALE_S", str(6 * 3600)))
|
||
SSH_PROBE_TIMEOUT_S = float(os.environ.get("TALER_MONITORD_SSH_TIMEOUT_S", "4"))
|
||
TLS_VERIFY = _env_truthy("TALER_MONITORD_TLS_VERIFY", "0")
|
||
SSH_PROBES_ENABLED = _env_truthy("TALER_MONITORD_SSH")
|
||
CVE_PROBES_ENABLED = _env_truthy("TALER_MONITORD_CVE")
|
||
SERVICES_ENABLED = _env_truthy("TALER_MONITORD_SERVICES")
|
||
MAIL_ENABLED = _env_truthy("TALER_MONITORD_MAIL")
|
||
CERT_PROBES_ENABLED = _env_truthy("TALER_MONITORD_CERT")
|
||
ORPHAN_PROBES_ENABLED = _env_truthy("TALER_MONITORD_ORPHAN")
|
||
ORPHAN_HTTP_ENABLED = _env_truthy("TALER_MONITORD_ORPHAN_HTTP", "0")
|
||
MAIL_PORTS_ENABLED = _env_truthy("TALER_MONITORD_MAIL_PORTS")
|
||
MAIL_PORT_TIMEOUT_S = float(os.environ.get("TALER_MONITORD_MAIL_PORT_TIMEOUT_S", "4"))
|
||
DELIVER_ENABLED = _env_truthy("TALER_MONITORD_DELIVER")
|
||
DELIVER_INTERVAL_S = float(
|
||
os.environ.get("TALER_MONITORD_DELIVER_INTERVAL_S", str(6 * 3600))
|
||
)
|
||
DELIVER_STALE_S = float(
|
||
os.environ.get("TALER_MONITORD_DELIVER_STALE_S", str(30 * 3600))
|
||
)
|
||
DELIVER_TIMEOUT_S = float(os.environ.get("TALER_MONITORD_DELIVER_TIMEOUT_S", "12"))
|
||
PORT_SCAN_ENABLED = _env_truthy("TALER_MONITORD_PORTSCAN")
|
||
PORT_SCAN_TIMEOUT_S = float(os.environ.get("TALER_MONITORD_PORTSCAN_TIMEOUT_S", "0.8"))
|
||
PORT_SCAN_WORKERS = int(os.environ.get("TALER_MONITORD_PORTSCAN_WORKERS", "24"))
|
||
CERT_WARN_DAYS = int(os.environ.get("TALER_MONITORD_CERT_WARN_DAYS", "30"))
|
||
CERT_CRIT_DAYS = int(os.environ.get("TALER_MONITORD_CERT_CRIT_DAYS", "14"))
|
||
SCORE_RATIO_GREEN = float(os.environ.get("TALER_MONITORD_SCORE_GREEN", "1.0"))
|
||
SCORE_RATIO_BLUE = float(os.environ.get("TALER_MONITORD_SCORE_BLUE", "0.85"))
|
||
SCORE_RATIO_YELLOW = float(os.environ.get("TALER_MONITORD_SCORE_YELLOW", "0.60"))
|
||
SCORE_RATIO_ORANGE = float(os.environ.get("TALER_MONITORD_SCORE_ORANGE", "0.35"))
|
||
SCORE_RATIO_RED = float(os.environ.get("TALER_MONITORD_SCORE_RED", "0.01"))
|
||
# Optional bottom index: 0/off | 1/colors | cmd/full (+ concrete probe command)
|
||
raw_idx = (os.environ.get("TALER_MONITORD_RESULT_INDEX") or "0").strip().lower()
|
||
if raw_idx in ("0", "false", "no", "off", ""):
|
||
RESULT_INDEX_MODE = "off"
|
||
elif raw_idx in ("cmd", "full", "commands", "2"):
|
||
RESULT_INDEX_MODE = "cmd"
|
||
else:
|
||
RESULT_INDEX_MODE = "colors"
|
||
if RESULT_INDEX_MODE == "colors" and _env_truthy(
|
||
"TALER_MONITORD_RESULT_INDEX_CMD", "0"
|
||
):
|
||
RESULT_INDEX_MODE = "cmd"
|
||
GIT_PROBE_ENABLED = _env_truthy("TALER_MONITORD_GIT")
|
||
GIT_PROBE_INTERVAL_S = float(
|
||
os.environ.get("TALER_MONITORD_GIT_INTERVAL_S", str(6 * 3600))
|
||
)
|
||
GIT_PROBE_STALE_S = float(
|
||
os.environ.get("TALER_MONITORD_GIT_STALE_S", str(30 * 3600))
|
||
)
|
||
GIT_PROBE_TIMEOUT_S = float(os.environ.get("TALER_MONITORD_GIT_TIMEOUT_S", "12"))
|
||
# Payability (wallet-cli zero-pay) — opt-in; not every monitord tick
|
||
PAY_ENABLED = _env_truthy("TALER_MONITORD_PAY", "0")
|
||
PAY_RUN = _env_truthy("TALER_MONITORD_PAY_RUN", "0")
|
||
PAY_INTERVAL_S = float(os.environ.get("TALER_MONITORD_PAY_INTERVAL_S", str(4 * 3600)))
|
||
PAY_STALE_S = float(os.environ.get("TALER_MONITORD_PAY_STALE_S", str(30 * 3600)))
|
||
PAY_TIMEOUT_S = float(os.environ.get("TALER_MONITORD_PAY_TIMEOUT_S", "180"))
|
||
raw_pay = (os.environ.get("TALER_MONITORD_PAY_DOMAINS") or "").strip()
|
||
PAY_DOMAINS = [
|
||
p.strip()
|
||
for p in re.split(r"[\s,]+", raw_pay)
|
||
if p.strip()
|
||
] or ["stage.taler-ops.ch"]
|
||
|
||
|
||
# Placeholders; filled by refresh_runtime_flags_from_env() below.
|
||
PUBLISHED_ENABLED = True
|
||
PUBLISHED_SUITE = "trixie"
|
||
PUBLISHED_APT_BASE = "https://deb.taler.net/apt/debian"
|
||
ACUTE_DOWN_S = 90.0
|
||
PROBE_TIMEOUT_S = 5.0
|
||
STATS_STALE_S = float(6 * 3600)
|
||
SSH_PROBE_TIMEOUT_S = 4.0
|
||
TLS_VERIFY = False
|
||
SSH_PROBES_ENABLED = True
|
||
CVE_PROBES_ENABLED = True
|
||
SERVICES_ENABLED = True
|
||
MAIL_ENABLED = True
|
||
CERT_PROBES_ENABLED = True
|
||
ORPHAN_PROBES_ENABLED = True
|
||
ORPHAN_HTTP_ENABLED = False
|
||
MAIL_PORTS_ENABLED = True
|
||
MAIL_PORT_TIMEOUT_S = 4.0
|
||
DELIVER_ENABLED = True
|
||
DELIVER_INTERVAL_S = float(6 * 3600)
|
||
DELIVER_STALE_S = float(30 * 3600)
|
||
DELIVER_TIMEOUT_S = 12.0
|
||
# Outside-in notable port scan (public TCP only). Suspicious = open ∩ ¬whitelist ∩ ¬baseline.
|
||
# Whitelist = operator-known intentional public opens — not an inventory of private topology.
|
||
PORT_SCAN_ENABLED = True
|
||
PORT_SCAN_TIMEOUT_S = 0.8
|
||
PORT_SCAN_WORKERS = 24
|
||
CERT_WARN_DAYS = 30
|
||
CERT_CRIT_DAYS = 14
|
||
SCORE_RATIO_GREEN = 1.0
|
||
SCORE_RATIO_BLUE = 0.85
|
||
SCORE_RATIO_YELLOW = 0.60
|
||
SCORE_RATIO_ORANGE = 0.35
|
||
SCORE_RATIO_RED = 0.01
|
||
RESULT_INDEX_MODE = "off"
|
||
GIT_PROBE_ENABLED = True
|
||
GIT_PROBE_INTERVAL_S = float(6 * 3600)
|
||
GIT_PROBE_STALE_S = float(30 * 3600)
|
||
GIT_PROBE_TIMEOUT_S = 12.0
|
||
PAY_ENABLED = False
|
||
PAY_RUN = False
|
||
PAY_INTERVAL_S = float(4 * 3600)
|
||
PAY_STALE_S = float(30 * 3600)
|
||
PAY_TIMEOUT_S = 180.0
|
||
PAY_DOMAINS = ["stage.taler-ops.ch"]
|
||
refresh_runtime_flags_from_env()
|
||
# Notable TCP ports (not a full nmap range). Override: TALER_MONITORD_PORTSCAN_PORTS=22,80,443,…
|
||
_DEFAULT_SCAN_PORTS = (
|
||
21,
|
||
22,
|
||
23,
|
||
25,
|
||
80,
|
||
110,
|
||
135,
|
||
139,
|
||
143,
|
||
443,
|
||
445,
|
||
465,
|
||
587,
|
||
993,
|
||
995,
|
||
1433,
|
||
1521,
|
||
1789,
|
||
1790,
|
||
2049,
|
||
3306,
|
||
3389,
|
||
5432,
|
||
5900,
|
||
6379,
|
||
8080,
|
||
8443,
|
||
9200,
|
||
9418,
|
||
11211,
|
||
23230,
|
||
23235,
|
||
23236,
|
||
23237,
|
||
23238,
|
||
27017,
|
||
52323,
|
||
)
|
||
# Baseline always-ok for public HTTPS endpoints (still overridable by denying via empty whitelist only —
|
||
# whitelist *adds* allowance; baseline is the default allow for web).
|
||
_BASELINE_WEB_PORTS = frozenset({80, 443})
|
||
_BASELINE_MAIL_PORTS = frozenset({25, 110, 143, 465, 587, 993, 995})
|
||
# Port → short label for UI
|
||
_PORT_LABELS = {
|
||
21: "ftp",
|
||
22: "ssh",
|
||
23: "telnet",
|
||
25: "smtp",
|
||
80: "http",
|
||
110: "pop3",
|
||
135: "msrpc",
|
||
139: "netbios",
|
||
143: "imap",
|
||
443: "https",
|
||
445: "smb",
|
||
465: "smtps",
|
||
587: "submission",
|
||
993: "imaps",
|
||
995: "pop3s",
|
||
1433: "mssql",
|
||
1521: "oracle",
|
||
2049: "nfs",
|
||
3306: "mysql",
|
||
3389: "rdp",
|
||
5432: "postgres",
|
||
5900: "vnc",
|
||
6379: "redis",
|
||
8080: "http-alt",
|
||
8443: "https-alt",
|
||
1789: "nym-sphinx",
|
||
1790: "nym-verloc",
|
||
9200: "git-ssh",
|
||
9418: "git-daemon",
|
||
11211: "memcached",
|
||
23230: "ssh-dnat",
|
||
23235: "ssh-dnat",
|
||
23236: "ssh-dnat",
|
||
23237: "ssh-dnat",
|
||
23238: "ssh-dnat",
|
||
27017: "mongodb",
|
||
52323: "tortuna",
|
||
}
|
||
# Member reasons that are informational only (do not weaken the stack box).
|
||
_AGG_SKIP_REASONS = frozenset({"ebics_iban", "ebics_no_bank"})
|
||
|
||
# Suite-like public endpoints per machine (paths relative to https://host).
|
||
# need_intro: custom /intro stack only (GOA + FrancPaysan/TESTPAYSAN stage).
|
||
# severity: soft (≤yellow) | medium (missing → orange) | hard (full classify).
|
||
ROLE_ENDPOINTS: dict[str, list[dict[str, Any]]] = {
|
||
"bank": [
|
||
{"id": "integration", "path": "/taler-integration/config", "expect": (200,), "severity": "hard"},
|
||
{"id": "config", "path": "/config", "expect": (200,), "severity": "hard"},
|
||
{"id": "webui", "path": "/webui/", "expect": (200, 301, 302), "severity": "soft"},
|
||
{"id": "intro", "path": "/intro/", "expect": (200, 301, 302), "severity": "soft", "need_intro": True},
|
||
{"id": "stats", "path": "/intro/stats.json", "expect": (200,), "severity": "soft", "need_intro": True},
|
||
# Legal docs: severity overridden at probe time (medium only for commercial prod).
|
||
{"id": "terms", "path": "/terms", "expect": (200,), "severity": "soft", "need_intro": True, "legal": True},
|
||
{"id": "privacy", "path": "/privacy", "expect": (200,), "severity": "soft", "need_intro": True, "legal": True},
|
||
],
|
||
"exchange": [
|
||
{"id": "config", "path": "/config", "expect": (200,), "severity": "hard"},
|
||
{"id": "keys", "path": "/keys", "expect": (200,), "severity": "hard"},
|
||
{"id": "terms", "path": "/terms", "expect": (200,), "severity": "soft", "legal": True},
|
||
{"id": "privacy", "path": "/privacy", "expect": (200,), "severity": "soft", "legal": True},
|
||
{"id": "intro", "path": "/intro/", "expect": (200, 301, 302), "severity": "soft", "need_intro": True},
|
||
],
|
||
"merchant": [
|
||
{"id": "config", "path": "/config", "expect": (200,), "severity": "hard"},
|
||
{"id": "webui", "path": "/webui/", "expect": (200, 301, 302), "severity": "soft"},
|
||
{"id": "intro", "path": "/intro/", "expect": (200, 301, 302), "severity": "soft", "need_intro": True},
|
||
{"id": "stats", "path": "/intro/stats.json", "expect": (200,), "severity": "soft", "need_intro": True},
|
||
{"id": "terms", "path": "/terms", "expect": (200,), "severity": "soft", "legal": True},
|
||
{"id": "privacy", "path": "/privacy", "expect": (200,), "severity": "soft", "legal": True},
|
||
],
|
||
"site": [
|
||
{"id": "apex", "path": "/", "expect": (200, 301, 302), "severity": "hard"},
|
||
],
|
||
}
|
||
|
||
|
||
def stack_has_custom_intro(st: dict[str, Any]) -> bool:
|
||
"""True only for custom /intro landings: GOA, FrancPaysan, TESTPAYSAN stage.
|
||
|
||
Prefer domains.conf landing=1; also recognize by currency / hostname so
|
||
demo/test/regio/TOPS never get /intro probes by accident.
|
||
"""
|
||
cur = str(st.get("currency") or "").upper()
|
||
if cur in ("GOA", "TESTPAYSAN"):
|
||
return True
|
||
blob = " ".join(
|
||
[
|
||
str(st.get("id") or ""),
|
||
" ".join(str(a) for a in (st.get("aliases") or [])),
|
||
str(st.get("bank") or ""),
|
||
str(st.get("exchange") or ""),
|
||
str(st.get("merchant") or ""),
|
||
]
|
||
).lower()
|
||
if "hacktivism" in blob or "lefrancpaysan" in blob or "francpaysan" in blob:
|
||
# static_pending FP prod has no Taler /intro APIs yet
|
||
if (st.get("kind") or "") == "static_pending" or not cur:
|
||
return False
|
||
return True
|
||
# Explicit landing=1 only if it is one of the known custom stacks above
|
||
return False
|
||
|
||
|
||
def stack_is_commercial_prod(st: dict[str, Any]) -> bool:
|
||
"""Commercial production (legal docs must be solid) — not stage/demo/test/GOA.
|
||
|
||
Today: TOPS CHF prod (taler-ops.ch / my.taler-ops.ch). Stage and sandboxes stay soft.
|
||
"""
|
||
if st.get("local"):
|
||
return False
|
||
if (st.get("kind") or "") == "static_pending":
|
||
return False
|
||
sid = str(st.get("id") or "").lower()
|
||
blob = " ".join(
|
||
[
|
||
sid,
|
||
" ".join(str(a) for a in (st.get("aliases") or [])),
|
||
str(st.get("merchant") or ""),
|
||
str(st.get("exchange") or ""),
|
||
]
|
||
).lower()
|
||
if "stage." in sid or sid.startswith("stage.") or ".stage." in sid:
|
||
return False
|
||
if "stage." in blob and "taler-ops" in blob:
|
||
return False
|
||
# TOPS production CHF
|
||
if "taler-ops.ch" in blob and "stage" not in sid:
|
||
cur = str(st.get("currency") or "").upper()
|
||
if cur in ("", "CHF"):
|
||
return True
|
||
return False
|
||
|
||
|
||
def legal_doc_severity(st: dict[str, Any]) -> str:
|
||
"""terms/privacy: medium (orange) only for commercial production."""
|
||
return "medium" if stack_is_commercial_prod(st) else "soft"
|
||
|
||
|
||
def _stack_blob(st: dict[str, Any]) -> str:
|
||
return " ".join(
|
||
[
|
||
str(st.get("id") or ""),
|
||
" ".join(str(a) for a in (st.get("aliases") or [])),
|
||
str(st.get("bank") or ""),
|
||
str(st.get("exchange") or ""),
|
||
str(st.get("merchant") or ""),
|
||
str(st.get("currency") or ""),
|
||
]
|
||
).lower()
|
||
|
||
|
||
def classify_network_zone(st: dict[str, Any]) -> dict[str, str]:
|
||
"""Map a stack to a coarse network area (not a full topology)."""
|
||
blob = _stack_blob(st)
|
||
cur = str(st.get("currency") or "").upper()
|
||
if st.get("local") or "hacktivism" in blob or cur == "GOA":
|
||
return {"zone": "goa-local", "label": "GOA / local (koopa)"}
|
||
if cur == "TESTPAYSAN" or "stage.lefrancpaysan" in blob:
|
||
return {"zone": "fp-stage", "label": "FrancPaysan stage"}
|
||
if (st.get("kind") or "") == "static_pending" or (
|
||
"lefrancpaysan" in blob and not cur
|
||
):
|
||
return {"zone": "fp-edge", "label": "FrancPaysan edge (static)"}
|
||
if stack_is_commercial_prod(st):
|
||
return {"zone": "tops-prod", "label": "TOPS prod CHF"}
|
||
if "stage.taler-ops" in blob or "stage.my.taler-ops" in blob:
|
||
return {"zone": "tops-stage", "label": "TOPS stage"}
|
||
if cur in ("KUDOS", "TESTKUDOS", "FLODOS") or "demo.taler.net" in blob or "test.taler.net" in blob:
|
||
return {"zone": "public-demo", "label": "public demo/test"}
|
||
if "taler-systems" in blob or "anastasis" in blob or "pixel" in blob:
|
||
return {"zone": "pixel", "label": "pixel / TSA / Anastasis"}
|
||
if "gnunet.org" in blob or (st.get("kind") or "") == "mail":
|
||
zone = str(st.get("zone") or "")
|
||
if zone == "mail" or (st.get("kind") or "") == "mail":
|
||
return {"zone": "mail", "label": "mail plane"}
|
||
return {"zone": "firefly", "label": "firefly / gnunet"}
|
||
return {"zone": "public", "label": "public HTTPS"}
|
||
|
||
|
||
def _env_ssh_hosts(*keys: str) -> list[str]:
|
||
out: list[str] = []
|
||
for k in keys:
|
||
raw = (os.environ.get(k) or "").strip()
|
||
if not raw:
|
||
continue
|
||
for part in re.split(r"[\s,]+", raw):
|
||
if part and part not in out:
|
||
out.append(part)
|
||
return out
|
||
|
||
|
||
def monitoring_env_path() -> Path:
|
||
"""Path to taler-monitoring-env install (~/.config/taler-monitoring/env)."""
|
||
raw = (os.environ.get("TALER_MONITORING_ENV") or "").strip()
|
||
if raw:
|
||
return Path(raw)
|
||
xdg = os.environ.get("XDG_CONFIG_HOME") or str(Path.home() / ".config")
|
||
return Path(xdg) / "taler-monitoring" / "env"
|
||
|
||
|
||
def _expand_env_value(val: str) -> str:
|
||
"""Expand $VAR / ${VAR} using current environ (shell-like, for env profiles)."""
|
||
|
||
def repl(m: re.Match[str]) -> str:
|
||
name = m.group(1) or m.group(2)
|
||
return os.environ.get(name, "")
|
||
|
||
return re.sub(
|
||
r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}|\$([A-Za-z_][A-Za-z0-9_]*)",
|
||
repl,
|
||
val,
|
||
)
|
||
|
||
|
||
def load_monitoring_env(*, path: Path | None = None, override: bool = True) -> Path | None:
|
||
"""Load KEY=value from taler-monitoring-env profile into os.environ (like lib.sh).
|
||
|
||
Skips comments/blanks. Expands $HOME / $VAR. Returns path if loaded, else None.
|
||
"""
|
||
p = path or monitoring_env_path()
|
||
if not p.is_file():
|
||
return None
|
||
try:
|
||
text = p.read_text(encoding="utf-8")
|
||
except OSError:
|
||
return None
|
||
for line in text.splitlines():
|
||
s = line.strip()
|
||
if not s or s.startswith("#"):
|
||
continue
|
||
if s.startswith("export "):
|
||
s = s[7:].lstrip()
|
||
if "=" not in s:
|
||
continue
|
||
key, _, val = s.partition("=")
|
||
key = key.strip()
|
||
if not key or not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", key):
|
||
continue
|
||
if not override and key in os.environ:
|
||
continue
|
||
val = val.strip()
|
||
if len(val) >= 2 and val[0] == val[-1] and val[0] in ("'", '"'):
|
||
val = val[1:-1]
|
||
os.environ[key] = _expand_env_value(val)
|
||
return p
|
||
|
||
|
||
def ssh_targets_for_stack(st: dict[str, Any]) -> dict[str, Any]:
|
||
"""Hostnames to probe for an SSH daemon (port 22 banner) — not jump-host login.
|
||
|
||
Zone-mapped SSH targets come only from taler-monitoring-env
|
||
(`TALER_MONITORD_SSH_TOPS` / `_FP` / `_GOA` / `_TOPS_PROD`). No suite hardcodes.
|
||
When env supplies hosts, hide them in UI/JSON display fields.
|
||
"""
|
||
zone = classify_network_zone(st)["zone"]
|
||
# Prefer apex / merchant hostname that is publicly meaningful
|
||
candidates: list[str] = []
|
||
for key in ("id", "merchant", "exchange", "bank"):
|
||
h = _host_only(str(st.get(key) or ""))
|
||
if h and h.upper() != "IBAN" and h not in candidates:
|
||
candidates.append(h)
|
||
env_map = {
|
||
"goa-local": "TALER_MONITORD_SSH_GOA",
|
||
"fp-stage": "TALER_MONITORD_SSH_FP",
|
||
"tops-stage": "TALER_MONITORD_SSH_TOPS",
|
||
"tops-prod": "TALER_MONITORD_SSH_TOPS_PROD",
|
||
}
|
||
env_hosts = _env_ssh_hosts(env_map[zone]) if zone in env_map else []
|
||
if env_hosts:
|
||
hosts = env_hosts
|
||
hide_host = True
|
||
else:
|
||
hosts = candidates[:2]
|
||
hide_host = False
|
||
# Public demos: still check if SSH is exposed (security signal)
|
||
expected = True
|
||
if zone in ("fp-edge",):
|
||
expected = bool(hosts)
|
||
return {
|
||
"expected": expected and bool(hosts),
|
||
"hosts": hosts,
|
||
"via": "tcp22-banner",
|
||
"hide_host": hide_host,
|
||
}
|
||
|
||
|
||
def probe_ssh_host(host: str, *, timeout_s: float = SSH_PROBE_TIMEOUT_S) -> dict[str, Any]:
|
||
"""Check whether an SSH service answers (banner), not whether our keys work.
|
||
|
||
SSH-2.0-… banner ⇒ present (ok/green). Connection refused / timeout ⇒ absent.
|
||
"""
|
||
host = _host_only(host)
|
||
if not host:
|
||
return {"ok": False, "light": "blue", "reason": "no_host", "host": "", "elapsed": 0.0}
|
||
t0 = time.monotonic()
|
||
try:
|
||
with socket.create_connection((host, 22), timeout=max(1.0, float(timeout_s))) as sock:
|
||
sock.settimeout(max(1.0, float(timeout_s)))
|
||
banner = sock.recv(256).decode("utf-8", "replace").strip().splitlines()
|
||
line = banner[0] if banner else ""
|
||
elapsed = time.monotonic() - t0
|
||
if line.upper().startswith("SSH-"):
|
||
return {
|
||
"ok": True,
|
||
"light": "green",
|
||
"reason": line[:80],
|
||
"host": host,
|
||
"elapsed": round(elapsed, 3),
|
||
"banner": line[:120],
|
||
}
|
||
return {
|
||
"ok": False,
|
||
"light": "orange",
|
||
"reason": f"no_ssh_banner:{line[:60]}",
|
||
"host": host,
|
||
"elapsed": round(elapsed, 3),
|
||
}
|
||
except (socket.timeout, TimeoutError):
|
||
return {
|
||
"ok": False,
|
||
"light": "orange",
|
||
"reason": "timeout",
|
||
"host": host,
|
||
"elapsed": round(time.monotonic() - t0, 3),
|
||
}
|
||
except OSError as e:
|
||
# refused / unreachable = no public SSH (often OK for hardened edge)
|
||
return {
|
||
"ok": False,
|
||
"light": "blue",
|
||
"reason": type(e).__name__,
|
||
"host": host,
|
||
"elapsed": round(time.monotonic() - t0, 3),
|
||
}
|
||
|
||
|
||
def build_network_status(
|
||
st: dict[str, Any],
|
||
*,
|
||
members: list[dict[str, Any]] | None = None,
|
||
ssh_cache: dict[str, dict[str, Any]] | None = None,
|
||
ssh_probe_fn: Callable[[str], dict[str, Any]] | None = None,
|
||
) -> dict[str, Any]:
|
||
"""Zone + public HTTPS summary + optional SSH reachability for the bundle."""
|
||
zone = classify_network_zone(st)
|
||
members = members or []
|
||
# Public path: worst non-informational member light
|
||
pub_lights = [
|
||
m.get("light") or "black"
|
||
for m in members
|
||
if (m.get("reason") or "") not in _AGG_SKIP_REASONS
|
||
]
|
||
if not pub_lights:
|
||
pub_lights = [m.get("light") or "black" for m in members] or ["blue"]
|
||
pub_worst = LIGHTS[max(SEVERITY.get(L, 5) for L in pub_lights)]
|
||
targets = ssh_targets_for_stack(st)
|
||
hide_host = bool(targets.get("hide_host"))
|
||
ssh_block: dict[str, Any] = {
|
||
"expected": bool(targets.get("expected")),
|
||
"via": targets.get("via") or "",
|
||
"candidates": [] if hide_host else list(targets.get("hosts") or []),
|
||
"ok": None,
|
||
"host": "",
|
||
"light": "blue",
|
||
"reason": "not_expected",
|
||
"elapsed": 0.0,
|
||
}
|
||
if not SSH_PROBES_ENABLED:
|
||
ssh_block["reason"] = "ssh_probes_disabled"
|
||
ssh_block["light"] = "blue"
|
||
elif targets.get("expected") and targets.get("hosts"):
|
||
cache = ssh_cache if ssh_cache is not None else {}
|
||
probe = ssh_probe_fn or probe_ssh_host
|
||
best: dict[str, Any] | None = None
|
||
for h in targets["hosts"]:
|
||
if h not in cache:
|
||
cache[h] = probe(h)
|
||
got = cache[h]
|
||
if got.get("ok"):
|
||
best = got
|
||
break
|
||
if best is None:
|
||
best = got
|
||
if best:
|
||
# Keep real probe host out of UI for mapped zones (e.g. betel ≠ stage FQDN).
|
||
shown_host = "" if targets.get("hide_host") else (best.get("host") or "")
|
||
reason = best.get("reason") or ""
|
||
if targets.get("hide_host") and reason and not str(reason).upper().startswith("SSH-"):
|
||
# Drop accidental hostname leakage in soft errors; keep SSH banner text.
|
||
if shown_host and shown_host in str(reason):
|
||
reason = "ssh_probe"
|
||
ssh_block.update(
|
||
{
|
||
"ok": bool(best.get("ok")),
|
||
"host": shown_host,
|
||
"light": best.get("light") or ("green" if best.get("ok") else "orange"),
|
||
"reason": reason,
|
||
"elapsed": best.get("elapsed") or 0.0,
|
||
}
|
||
)
|
||
return {
|
||
"zone": zone["zone"],
|
||
"label": zone["label"],
|
||
"public_light": pub_worst,
|
||
"ssh": ssh_block,
|
||
}
|
||
|
||
LEGEND = {
|
||
"green": "ok",
|
||
"blue": "present",
|
||
"yellow": "degraded",
|
||
"orange": "recent_or_hard_degraded",
|
||
"red": "down_acute",
|
||
"black": "down_long",
|
||
}
|
||
|
||
|
||
def monitord_state_dir() -> Path:
|
||
base = os.environ.get("TALER_MONITORD_STATE") or os.environ.get("XDG_RUNTIME_DIR")
|
||
if base:
|
||
d = Path(base) / "taler-monitord"
|
||
else:
|
||
d = Path.home() / ".local" / "state" / "taler-monitord"
|
||
d.mkdir(parents=True, exist_ok=True)
|
||
return d
|
||
|
||
|
||
def monitord_data_dir() -> Path:
|
||
raw = os.environ.get("TALER_MONITORD_DATA")
|
||
if raw:
|
||
d = Path(raw)
|
||
else:
|
||
d = Path.home() / ".local" / "share" / "taler-monitord"
|
||
d.mkdir(parents=True, exist_ok=True)
|
||
return d
|
||
|
||
|
||
def status_json_paths() -> list[Path]:
|
||
return [monitord_state_dir() / "stack-status.json", monitord_data_dir() / "stack-status.json"]
|
||
|
||
|
||
def default_domains_conf() -> Path:
|
||
env = os.environ.get("TALER_DOMAINS_CONF") or ""
|
||
if env and Path(env).is_file():
|
||
return Path(env)
|
||
here = Path(__file__).resolve().parent
|
||
for p in (here.parent / "domains.conf", Path("domains.conf")):
|
||
if p.is_file():
|
||
return p
|
||
return here.parent / "domains.conf"
|
||
|
||
|
||
def _repo_conf(name: str, env_key: str) -> Path:
|
||
env = os.environ.get(env_key) or ""
|
||
if env and Path(env).is_file():
|
||
return Path(env)
|
||
here = Path(__file__).resolve().parent
|
||
for p in (here.parent / name, Path(name)):
|
||
if p.is_file():
|
||
return p
|
||
return here.parent / name
|
||
|
||
|
||
def default_services_conf() -> Path:
|
||
return _repo_conf("services-catalog.conf", "TALER_SERVICES_CONF")
|
||
|
||
|
||
def default_mail_catalog() -> Path:
|
||
return _repo_conf("mail-catalog.conf", "MAIL_CATALOG")
|
||
|
||
|
||
def default_ports_whitelist_conf() -> Path:
|
||
return _repo_conf("ports-whitelist.conf", "TALER_PORTS_WHITELIST")
|
||
|
||
|
||
def _host_only(s: str) -> str:
|
||
s = (s or "").strip()
|
||
s = s.removeprefix("https://").removeprefix("http://")
|
||
return s.split("/")[0].split(":")[0]
|
||
|
||
|
||
# domains.conf bank column: no public libeufin / x-taler-bank host expected
|
||
_BANK_SKIP_TOKENS = frozenset({"-", ".", "none", "n/a", "na", "ebics", "iban", "skip", "off"})
|
||
|
||
|
||
def bank_host_configured(bank: str) -> bool:
|
||
"""False when domains.conf marks bank as absent (EBICS / IBAN wire)."""
|
||
b = (bank or "").strip().lower()
|
||
return bool(b) and b not in _BANK_SKIP_TOKENS
|
||
|
||
|
||
def payto_wire_methods(blob: bytes | str | None) -> set[str]:
|
||
"""Extract payto://METHOD/… wire methods from exchange /keys (or similar) body."""
|
||
if blob is None:
|
||
return set()
|
||
if isinstance(blob, bytes):
|
||
text = blob.decode("utf-8", "replace")
|
||
else:
|
||
text = str(blob)
|
||
return {m.lower() for m in re.findall(r"payto://([a-z0-9-]+)/", text, flags=re.I)}
|
||
|
||
|
||
def classify_bank_backend(methods: set[str], *, bank_configured: bool = True) -> str:
|
||
"""Return x-taler-bank | ebics | unknown.
|
||
|
||
EBICS/IBAN stacks (e.g. TOPS CHF) have no public bank.* — only payto://iban.
|
||
Demo/stage libeufin stacks advertise payto://x-taler-bank/….
|
||
Mixed (regio: both) still expects bank.* because x-taler-bank is present.
|
||
"""
|
||
if not bank_configured:
|
||
return "ebics"
|
||
m = {x.lower() for x in (methods or set())}
|
||
if "x-taler-bank" in m:
|
||
return "x-taler-bank"
|
||
if "iban" in m:
|
||
return "ebics"
|
||
return "unknown"
|
||
|
||
|
||
def _looks_locale(tok: str) -> bool:
|
||
t = tok.lower().replace("_", "-")
|
||
return bool(t in ("fr-ch", "de-ch", "en-ch", "ch-fr", "ch-de", "ch-en") or (
|
||
len(t) == 5 and t[2] == "-" and t[:2].isalpha() and t[3:].isalpha()
|
||
))
|
||
|
||
|
||
def _looks_lang(tok: str) -> bool:
|
||
return tok.lower() in ("en", "fr", "de", "eng", "fra", "deu", "ger", "german", "deutsch", "french", "english")
|
||
|
||
|
||
def parse_domains_conf(path: Path | None = None) -> list[dict[str, Any]]:
|
||
"""Return unique stacks keyed by (bank, exchange, merchant)."""
|
||
path = path or default_domains_conf()
|
||
by_endpoints: dict[tuple[str, str, str], dict[str, Any]] = {}
|
||
if not path.is_file():
|
||
return []
|
||
|
||
for raw in path.read_text(encoding="utf-8").splitlines():
|
||
line = raw.split("#", 1)[0].strip()
|
||
if not line:
|
||
continue
|
||
parts = line.split()
|
||
if len(parts) < 5:
|
||
continue
|
||
name, bank, exchange, merchant, currency = parts[0], parts[1], parts[2], parts[3], parts[4]
|
||
local_stack = parts[5] if len(parts) > 5 else "0"
|
||
landing = parts[6] if len(parts) > 6 else ""
|
||
rest = parts[7:]
|
||
|
||
# Legacy: 7th field was canonical (contains a dot, not 0/1)
|
||
if landing and landing not in ("0", "1") and "." in landing:
|
||
rest = [landing] + rest
|
||
landing = ""
|
||
|
||
canon = name
|
||
if rest:
|
||
if _looks_lang(rest[0]):
|
||
if len(rest) >= 2 and _looks_locale(rest[1]):
|
||
canon = rest[2] if len(rest) >= 3 else name
|
||
elif len(rest) >= 2 and "." in rest[1]:
|
||
canon = rest[1]
|
||
elif "." in rest[0]:
|
||
canon = rest[0]
|
||
|
||
bank_configured = bank_host_configured(bank)
|
||
bank_h = _host_only(bank) if bank_configured else ""
|
||
ex_h, mer_h = _host_only(exchange), _host_only(merchant)
|
||
# Keep skip-token in key so EBICS rows do not collide with empty bank hosts.
|
||
key = (bank_h or (bank.strip().lower() if bank.strip() else "-"), ex_h, mer_h)
|
||
cur = currency if currency not in ("-", "") else ""
|
||
# No currency ⇒ static site / Taler prod not provisioned yet (e.g. lefrancpaysan.ch).
|
||
kind = "taler" if cur else "static_pending"
|
||
row = {
|
||
"id": canon,
|
||
"bank": bank_h,
|
||
"bank_configured": bank_configured,
|
||
"exchange": ex_h,
|
||
"merchant": mer_h,
|
||
"currency": cur,
|
||
"kind": kind,
|
||
"local": local_stack in ("1", "true", "yes"),
|
||
"landing": landing in ("1", "true", "yes"),
|
||
"aliases": [name],
|
||
}
|
||
if key not in by_endpoints:
|
||
by_endpoints[key] = row
|
||
else:
|
||
prev = by_endpoints[key]
|
||
prev["aliases"].append(name)
|
||
# Prefer non-service hostname as id when alias list grows
|
||
if prev["id"] in (bank_h, ex_h, mer_h) and canon not in (bank_h, ex_h, mer_h):
|
||
prev["id"] = canon
|
||
elif canon == name and name not in (bank_h, ex_h, mer_h) and len(name) <= len(prev["id"]):
|
||
# keep shorter apex-like id when equal quality
|
||
if prev["id"] in prev["aliases"] and name.count(".") <= prev["id"].count("."):
|
||
prev["id"] = canon
|
||
if cur and not prev["currency"]:
|
||
prev["currency"] = cur
|
||
prev["kind"] = "taler"
|
||
|
||
return sorted(by_endpoints.values(), key=lambda s: s["id"])
|
||
|
||
|
||
def member_probe_url(role: str, host: str) -> str:
|
||
host = _host_only(host)
|
||
if role == "bank":
|
||
# Prefer integration config (wallet-facing); fallback handled in probe
|
||
return f"https://{host}/taler-integration/config"
|
||
return f"https://{host}/config"
|
||
|
||
|
||
def stats_probe_url(host: str) -> str:
|
||
return f"https://{_host_only(host)}/intro/stats.json"
|
||
|
||
|
||
def _amount_value(amount: str) -> str:
|
||
"""Strip CUR: prefix from Taler amount string."""
|
||
s = (amount or "").strip()
|
||
if ":" in s:
|
||
return s.split(":", 1)[1]
|
||
return s
|
||
|
||
|
||
def _amount_sort_key(val: str) -> tuple:
|
||
try:
|
||
return (0, float(val))
|
||
except (TypeError, ValueError):
|
||
return (1, str(val))
|
||
|
||
|
||
def extract_denom_values(blob: bytes | str | None, currency: str = "") -> list[str]:
|
||
"""Unique denomination values from exchange /keys body (best-effort)."""
|
||
if blob is None:
|
||
return []
|
||
text = blob.decode("utf-8", "replace") if isinstance(blob, bytes) else str(blob)
|
||
vals: set[str] = set()
|
||
# "value": "CUR:N" or amount fields
|
||
for m in re.finditer(r'"value"\s*:\s*"([^"]+)"', text):
|
||
raw = m.group(1)
|
||
if currency and raw.startswith(currency + ":"):
|
||
vals.add(_amount_value(raw))
|
||
elif ":" in raw and re.match(r"^[A-Z][A-Z0-9_]+:", raw):
|
||
vals.add(_amount_value(raw))
|
||
# Prefer positive coin values; drop zero-only noise
|
||
cleaned = [v for v in vals if v not in ("", "0", "0.0", "0.00")]
|
||
return sorted(cleaned, key=_amount_sort_key)[:32]
|
||
|
||
|
||
def summarize_currency_spec(
|
||
cs: dict[str, Any] | None,
|
||
*,
|
||
keys_body: bytes | str | None = None,
|
||
fallback_currency: str = "",
|
||
) -> dict[str, Any] | None:
|
||
"""Currency code, coin symbols (alt_unit_names), denomination list for UI."""
|
||
cs = cs if isinstance(cs, dict) else {}
|
||
currency = str(cs.get("currency") or fallback_currency or "").strip()
|
||
alt_raw = cs.get("alt_unit_names") if isinstance(cs.get("alt_unit_names"), dict) else {}
|
||
alt = {str(k): str(v) for k, v in alt_raw.items() if str(v).strip()}
|
||
if not currency and not alt:
|
||
return None
|
||
symbol = alt.get("0") or currency
|
||
# Subunits: "-2" → Rp./ct — sorted by exponent
|
||
def _exp_key(k: str) -> int:
|
||
try:
|
||
return int(k)
|
||
except ValueError:
|
||
return 0
|
||
|
||
subunits: list[dict[str, str]] = []
|
||
for k in sorted((x for x in alt if x != "0"), key=_exp_key, reverse=True):
|
||
subunits.append({"exp": k, "symbol": alt[k]})
|
||
|
||
denoms: list[str] = []
|
||
for a in cs.get("common_amounts") or []:
|
||
v = _amount_value(str(a))
|
||
if v and v not in denoms:
|
||
denoms.append(v)
|
||
if not denoms and keys_body is not None:
|
||
denoms = extract_denom_values(keys_body, currency)
|
||
|
||
unit_bits = [symbol] + [s["symbol"] for s in subunits]
|
||
# unique preserve order
|
||
seen: set[str] = set()
|
||
units: list[str] = []
|
||
for u in unit_bits:
|
||
if u and u not in seen:
|
||
seen.add(u)
|
||
units.append(u)
|
||
|
||
return {
|
||
"currency": currency,
|
||
"name": str(cs.get("name") or ""),
|
||
"symbol": symbol,
|
||
"alt_unit_names": alt,
|
||
"subunits": subunits,
|
||
"units_label": " / ".join(units) if units else currency,
|
||
"denoms": denoms,
|
||
"denoms_label": " · ".join(denoms[:16]),
|
||
}
|
||
|
||
|
||
def _json_body(raw: dict[str, Any] | None) -> dict[str, Any] | None:
|
||
if not raw:
|
||
return None
|
||
body = raw.get("body")
|
||
if body is None:
|
||
return None
|
||
if isinstance(body, bytes):
|
||
text = body.decode("utf-8", "replace")
|
||
else:
|
||
text = str(body)
|
||
text = text.strip()
|
||
if not text:
|
||
return None
|
||
try:
|
||
data = json.loads(text)
|
||
except json.JSONDecodeError:
|
||
return None
|
||
return data if isinstance(data, dict) else None
|
||
|
||
|
||
def summarize_landing_stats(
|
||
data: dict[str, Any],
|
||
*,
|
||
now: float | None = None,
|
||
stale_s: float = STATS_STALE_S,
|
||
role: str = "",
|
||
host: str = "",
|
||
url: str = "",
|
||
) -> dict[str, Any]:
|
||
"""Compact, UI-friendly summary of public …/intro/stats.json (no full dump)."""
|
||
now = time.time() if now is None else now
|
||
gu = data.get("generated_at_unix")
|
||
age: int | None = None
|
||
if gu is not None and str(gu).strip() != "":
|
||
try:
|
||
src_u = int(float(gu))
|
||
if src_u > 10_000_000_000:
|
||
src_u //= 1000
|
||
age = int(now - src_u)
|
||
except (TypeError, ValueError):
|
||
age = None
|
||
|
||
lines: list[str] = []
|
||
ba = data.get("bank_accounts") if isinstance(data.get("bank_accounts"), dict) else {}
|
||
wd = data.get("withdraws") if isinstance(data.get("withdraws"), dict) else {}
|
||
if ba or wd:
|
||
n_acc = ba.get("total") if ba.get("total") is not None else ba.get("users")
|
||
if n_acc is not None:
|
||
lines.append(f"accounts {n_acc}")
|
||
if wd.get("count") is not None:
|
||
lines.append(f"withdraws {wd.get('count')}")
|
||
amt = wd.get("total_amount_alt") or wd.get("total_amount")
|
||
if amt:
|
||
lines.append(str(amt))
|
||
if "reserves" in data:
|
||
lines.append(f"reserves {data.get('reserves')}")
|
||
if data.get("wire_in_count") is not None:
|
||
lines.append(f"wire_in {data.get('wire_in_count')}")
|
||
coins = data.get("known_coins")
|
||
if coins is None:
|
||
coins = data.get("coins_live")
|
||
if coins is not None:
|
||
lines.append(f"coins {coins}")
|
||
if data.get("instances") is not None:
|
||
lines.append(f"instances {data.get('instances')}")
|
||
if data.get("orders") is not None:
|
||
lines.append(f"orders {data.get('orders')}")
|
||
bal = data.get("balance_explorer_alt") or data.get("balance_explorer")
|
||
if bal and not any(str(bal) in x for x in lines):
|
||
lines.append(f"explorer {bal}")
|
||
|
||
return {
|
||
"present": True,
|
||
"ok": data.get("ok") is not False,
|
||
"age_s": age,
|
||
"stale": bool(age is not None and age > stale_s),
|
||
"generated_at_human": data.get("generated_at_human") or data.get("generated_at") or "",
|
||
"source": str(data.get("source") or "")[:72],
|
||
"currency": data.get("currency") or "",
|
||
"lines": lines[:6],
|
||
"role": role,
|
||
"host": host,
|
||
"url": url,
|
||
}
|
||
|
||
|
||
def fetch_stack_stats(
|
||
st: dict[str, Any],
|
||
*,
|
||
bank_backend: str,
|
||
probe_fn: Callable[[str], dict[str, Any]],
|
||
now: float | None = None,
|
||
stale_s: float = STATS_STALE_S,
|
||
) -> dict[str, Any] | None:
|
||
"""Try public /intro/stats.json (custom intro stacks only). Soft-miss → None."""
|
||
now = time.time() if now is None else now
|
||
kind = st.get("kind") or "taler"
|
||
if kind == "static_pending":
|
||
return None
|
||
if not stack_has_custom_intro(st):
|
||
return None
|
||
|
||
candidates: list[tuple[str, str]] = []
|
||
if bank_backend == "x-taler-bank" and bank_host_configured(st.get("bank") or ""):
|
||
candidates.append(("bank", _host_only(st["bank"])))
|
||
if st.get("merchant"):
|
||
candidates.append(("merchant", _host_only(st["merchant"])))
|
||
if st.get("exchange"):
|
||
candidates.append(("exchange", _host_only(st["exchange"])))
|
||
# de-dupe hosts while keeping order
|
||
seen: set[str] = set()
|
||
ordered: list[tuple[str, str]] = []
|
||
for role, host in candidates:
|
||
if not host or host in seen:
|
||
continue
|
||
seen.add(host)
|
||
ordered.append((role, host))
|
||
|
||
for role, host in ordered:
|
||
url = stats_probe_url(host)
|
||
raw = probe_fn(url)
|
||
if not raw.get("ok"):
|
||
continue
|
||
data = _json_body(raw)
|
||
if not data:
|
||
continue
|
||
return summarize_landing_stats(
|
||
data, now=now, stale_s=stale_s, role=role, host=host, url=url
|
||
)
|
||
return None
|
||
|
||
|
||
def _ssl_context() -> ssl.SSLContext | None:
|
||
if TLS_VERIFY:
|
||
return None
|
||
ctx = ssl.create_default_context()
|
||
ctx.check_hostname = False
|
||
ctx.verify_mode = ssl.CERT_NONE
|
||
return ctx
|
||
|
||
|
||
def _header_server(headers: Any) -> str:
|
||
if headers is None:
|
||
return ""
|
||
try:
|
||
return str(headers.get("Server") or headers.get("server") or "").strip()
|
||
except Exception:
|
||
return ""
|
||
|
||
|
||
def probe_http(url: str, *, timeout: float = PROBE_TIMEOUT_S) -> dict[str, Any]:
|
||
"""Probe one URL. Returns ok/code/elapsed/error/server (no light yet)."""
|
||
t0 = time.monotonic()
|
||
req = urllib.request.Request(url, method="GET", headers={"User-Agent": "taler-monitord/1"})
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=timeout, context=_ssl_context()) as resp:
|
||
body = resp.read(65536)
|
||
code = getattr(resp, "status", None) or resp.getcode()
|
||
elapsed = time.monotonic() - t0
|
||
return {
|
||
"ok": 200 <= int(code) < 400,
|
||
"code": int(code),
|
||
"elapsed": elapsed,
|
||
"error": None,
|
||
"bytes": len(body),
|
||
"body": body,
|
||
"server": _header_server(getattr(resp, "headers", None)),
|
||
"url": url,
|
||
}
|
||
except urllib.error.HTTPError as e:
|
||
elapsed = time.monotonic() - t0
|
||
try:
|
||
err_body = e.read(65536) or b""
|
||
except Exception:
|
||
err_body = b""
|
||
return {
|
||
"ok": False,
|
||
"code": int(e.code),
|
||
"elapsed": elapsed,
|
||
"error": f"http_{e.code}",
|
||
"bytes": len(err_body),
|
||
"body": err_body,
|
||
"server": _header_server(getattr(e, "headers", None)),
|
||
"url": url,
|
||
}
|
||
except (urllib.error.URLError, TimeoutError, socket.timeout, OSError) as e:
|
||
elapsed = time.monotonic() - t0
|
||
err = type(e).__name__
|
||
reason = getattr(e, "reason", None)
|
||
if reason is not None:
|
||
err = f"{err}:{reason}"
|
||
return {
|
||
"ok": False,
|
||
"code": 0,
|
||
"elapsed": elapsed,
|
||
"error": err,
|
||
"bytes": 0,
|
||
"body": b"",
|
||
"server": "",
|
||
"url": url,
|
||
}
|
||
|
||
|
||
def guess_http_package(server: str) -> str:
|
||
s = (server or "").lower()
|
||
if "nginx" in s:
|
||
return "nginx"
|
||
if "apache" in s or "httpd" in s:
|
||
return "apache"
|
||
if "caddy" in s:
|
||
return "caddy"
|
||
return ""
|
||
|
||
|
||
def extract_http_version(server: str) -> str:
|
||
m = re.search(r"/([0-9]+(?:\.[0-9]+)*)", server or "")
|
||
return m.group(1) if m else ""
|
||
|
||
|
||
def _taler_pkg_from_name(name: str) -> str:
|
||
n = (name or "").lower()
|
||
if "exchange" in n:
|
||
return "taler-exchange"
|
||
if "merchant" in n:
|
||
return "taler-merchant"
|
||
if "corebank" in n or n.endswith("bank") or "libeufin" in n:
|
||
return "taler-bank"
|
||
if "bank-integration" in n or "integration" in n:
|
||
return "taler-bank"
|
||
if n.startswith("taler"):
|
||
return "taler"
|
||
return ""
|
||
|
||
|
||
def software_from_taler_config(body: bytes | str | dict | None, *, host: str = "", role: str = "") -> list[dict[str, Any]]:
|
||
"""Extract Taler component name + build_version from /config JSON."""
|
||
if isinstance(body, dict):
|
||
data = body
|
||
elif body is None:
|
||
data = None
|
||
else:
|
||
data = _json_body({"body": body})
|
||
if not isinstance(data, dict):
|
||
return []
|
||
name = str(data.get("name") or "")
|
||
pkg = _taler_pkg_from_name(name)
|
||
if not pkg and role in ("exchange", "merchant", "bank"):
|
||
pkg = {"exchange": "taler-exchange", "merchant": "taler-merchant", "bank": "taler-bank"}[role]
|
||
if not pkg:
|
||
return []
|
||
build = str(data.get("build_version") or "").strip()
|
||
api = str(data.get("version") or "").strip()
|
||
ver = build or api
|
||
return [
|
||
{
|
||
"package": pkg,
|
||
"label": name or pkg,
|
||
"version": ver,
|
||
"api_version": api,
|
||
"build_version": build,
|
||
"host": _host_only(host),
|
||
"source": "taler-config",
|
||
"role": role,
|
||
}
|
||
]
|
||
|
||
|
||
def software_from_server_header(server: str, *, host: str = "") -> list[dict[str, Any]]:
|
||
pkg = guess_http_package(server)
|
||
if not pkg:
|
||
return []
|
||
ver = extract_http_version(server)
|
||
return [
|
||
{
|
||
"package": pkg,
|
||
"label": pkg,
|
||
"version": ver,
|
||
"host": _host_only(host),
|
||
"source": "server-header",
|
||
"server": (server or "")[:80],
|
||
}
|
||
]
|
||
|
||
|
||
def smtp_banner_software(host: str, *, timeout_s: float = 4.0) -> list[dict[str, Any]]:
|
||
"""Read SMTP 220 banner; detect Postfix (+ version if present)."""
|
||
h = _host_only(host)
|
||
if not h:
|
||
return []
|
||
try:
|
||
with socket.create_connection((h, 25), timeout=max(1.0, float(timeout_s))) as sock:
|
||
sock.settimeout(max(1.0, float(timeout_s)))
|
||
banner = sock.recv(512).decode("utf-8", "replace").strip().splitlines()
|
||
line = banner[0] if banner else ""
|
||
try:
|
||
sock.sendall(b"QUIT\r\n")
|
||
except OSError:
|
||
pass
|
||
except OSError:
|
||
return []
|
||
low = line.lower()
|
||
if "postfix" not in low:
|
||
if "esmtp" in low or line.startswith("220"):
|
||
return [
|
||
{
|
||
"package": "mta",
|
||
"label": "SMTP",
|
||
"version": "",
|
||
"host": h,
|
||
"source": "smtp-banner",
|
||
"banner": line[:100],
|
||
}
|
||
]
|
||
return []
|
||
ver = ""
|
||
m = re.search(r"postfix[^\d]*([0-9]+(?:\.[0-9]+)*)", line, flags=re.I)
|
||
if m:
|
||
ver = m.group(1)
|
||
return [
|
||
{
|
||
"package": "postfix",
|
||
"label": "Postfix",
|
||
"version": ver,
|
||
"host": h,
|
||
"source": "smtp-banner",
|
||
"banner": line[:100],
|
||
}
|
||
]
|
||
|
||
|
||
def _sev_to_light(sev: str) -> str:
|
||
s = (sev or "").upper()
|
||
if s == "CRITICAL":
|
||
return "red"
|
||
if s == "HIGH":
|
||
return "orange"
|
||
if s == "MEDIUM":
|
||
return "yellow"
|
||
if s == "LOW":
|
||
return "blue"
|
||
return "green"
|
||
|
||
|
||
_PUBLISHED_CACHE: dict[str, Any] = {"ts": 0.0, "pkgs": {}, "apps": [], "suite": "", "error": None}
|
||
|
||
# inventory package → deb.taler.net package name(s)
|
||
_SOFT_TO_DEB = {
|
||
"taler-exchange": ("taler-exchange",),
|
||
"taler-merchant": ("taler-merchant",),
|
||
"taler-bank": ("libeufin-bank",),
|
||
"taler": ("taler-exchange",),
|
||
}
|
||
|
||
# Published apps / SPAs we surface even without a live probe
|
||
_APP_CATALOG = (
|
||
{"package": "taler-merchant-webui", "label": "Merchant WebUI", "kind": "webui", "logo": "webui"},
|
||
{"package": "libeufin-bank-webui", "label": "Bank WebUI", "kind": "webui", "logo": "webui"},
|
||
{"package": "taler-exchange-kyc-webui", "label": "KYC WebUI", "kind": "webui", "logo": "webui"},
|
||
{"package": "taler-exchange-aml-webui", "label": "AML WebUI", "kind": "webui", "logo": "webui"},
|
||
{"package": "taler-wallet-cli", "label": "Wallet CLI", "kind": "wallet", "logo": "wallet"},
|
||
)
|
||
|
||
|
||
def _deb_ord_ch(c: str) -> int:
|
||
if c == "~":
|
||
return -1
|
||
if c.isdigit():
|
||
return 0
|
||
if c.isalpha():
|
||
return ord(c)
|
||
return ord(c) + 256
|
||
|
||
|
||
def _deb_split(v: str) -> tuple[int, str, str]:
|
||
epoch = 0
|
||
if ":" in v:
|
||
e, v = v.split(":", 1)
|
||
try:
|
||
epoch = int(e)
|
||
except ValueError:
|
||
epoch = 0
|
||
if "-" in v:
|
||
upstream, deb = v.rsplit("-", 1)
|
||
else:
|
||
upstream, deb = v, ""
|
||
return epoch, upstream, deb
|
||
|
||
|
||
def _deb_cmp_part(a: str, b: str) -> int:
|
||
ia = ib = 0
|
||
while ia < len(a) or ib < len(b):
|
||
while True:
|
||
ca = a[ia] if ia < len(a) and not a[ia].isdigit() else ""
|
||
cb = b[ib] if ib < len(b) and not b[ib].isdigit() else ""
|
||
if ca == "" and cb == "":
|
||
break
|
||
if ca == "" and cb:
|
||
return -1 if _deb_ord_ch(cb) > 0 else (1 if cb == "~" else -1)
|
||
if cb == "" and ca:
|
||
return 1 if _deb_ord_ch(ca) > 0 else (-1 if ca == "~" else 1)
|
||
oa, ob = _deb_ord_ch(ca), _deb_ord_ch(cb)
|
||
ia += 1
|
||
ib += 1
|
||
if oa != ob:
|
||
return (oa > ob) - (oa < ob)
|
||
sa = sb = ""
|
||
while ia < len(a) and a[ia].isdigit():
|
||
sa += a[ia]
|
||
ia += 1
|
||
while ib < len(b) and b[ib].isdigit():
|
||
sb += b[ib]
|
||
ib += 1
|
||
na = int(sa) if sa else 0
|
||
nb = int(sb) if sb else 0
|
||
if na != nb:
|
||
return (na > nb) - (na < nb)
|
||
return 0
|
||
|
||
|
||
def deb_version_cmp(a: str, b: str) -> int:
|
||
"""Debian-policy-ish version compare (subset used by check_versions.sh)."""
|
||
if a == b:
|
||
return 0
|
||
ea, ua, da = _deb_split(a or "")
|
||
eb, ub, db = _deb_split(b or "")
|
||
if ea != eb:
|
||
return (ea > eb) - (ea < eb)
|
||
c = _deb_cmp_part(ua, ub)
|
||
if c:
|
||
return c
|
||
return _deb_cmp_part(da, db)
|
||
|
||
|
||
def normalize_build_version(v: str) -> str:
|
||
"""Map Taler build_version toward deb upstream form for comparison.
|
||
|
||
Examples: ``X.Y.Z-dev.1`` → ``X.Y.Z~dev1``; plain ``X.Y.Z`` stays.
|
||
"""
|
||
s = (v or "").strip()
|
||
if not s:
|
||
return ""
|
||
# API versions like 39:0:5 are not package versions
|
||
if re.fullmatch(r"\d+:\d+:\d+", s):
|
||
return ""
|
||
s = re.sub(r"-dev\.(\d+)", r"~dev\1", s, flags=re.I)
|
||
s = re.sub(r"-dev(\d+)", r"~dev\1", s, flags=re.I)
|
||
s = s.replace("-dev", "~dev")
|
||
return s
|
||
|
||
|
||
def parse_apt_packages_index(text: str) -> dict[str, str]:
|
||
"""Return package → highest Version from a Packages file body."""
|
||
pkgs: dict[str, str] = {}
|
||
cur = ver = None
|
||
for line in (text or "").splitlines():
|
||
if line.startswith("Package: "):
|
||
if cur and ver and (cur not in pkgs or deb_version_cmp(ver, pkgs[cur]) > 0):
|
||
pkgs[cur] = ver
|
||
cur = line[9:].strip()
|
||
ver = None
|
||
elif line.startswith("Version: ") and cur:
|
||
ver = line[9:].strip()
|
||
elif line == "" and cur:
|
||
if ver and (cur not in pkgs or deb_version_cmp(ver, pkgs[cur]) > 0):
|
||
pkgs[cur] = ver
|
||
cur = ver = None
|
||
if cur and ver and (cur not in pkgs or deb_version_cmp(ver, pkgs[cur]) > 0):
|
||
pkgs[cur] = ver
|
||
return pkgs
|
||
|
||
|
||
def fetch_published_taler_packages(
|
||
*,
|
||
suite: str | None = None,
|
||
apt_base: str | None = None,
|
||
cache_s: float = 3600.0,
|
||
) -> dict[str, Any]:
|
||
"""Fetch/cached deb.taler.net Packages for published Taler versions."""
|
||
if not PUBLISHED_ENABLED:
|
||
return {"enabled": False, "suite": suite or PUBLISHED_SUITE, "packages": {}, "apps": []}
|
||
suite = suite or PUBLISHED_SUITE
|
||
apt_base = (apt_base or PUBLISHED_APT_BASE).rstrip("/")
|
||
now = time.time()
|
||
if (
|
||
_PUBLISHED_CACHE.get("pkgs")
|
||
and _PUBLISHED_CACHE.get("suite") == suite
|
||
and now - float(_PUBLISHED_CACHE.get("ts") or 0) < cache_s
|
||
):
|
||
return {
|
||
"enabled": True,
|
||
"suite": suite,
|
||
"packages": dict(_PUBLISHED_CACHE["pkgs"]),
|
||
"apps": list(_PUBLISHED_CACHE.get("apps") or []),
|
||
"error": _PUBLISHED_CACHE.get("error"),
|
||
"cached": True,
|
||
}
|
||
url = f"{apt_base}/dists/{suite}/main/binary-amd64/Packages"
|
||
# Full Packages index is >>64KiB (probe_http truncates); fetch with a larger cap.
|
||
pkgs: dict[str, str] = {}
|
||
err = None
|
||
try:
|
||
req = urllib.request.Request(url, method="GET", headers={"User-Agent": "taler-monitord/1"})
|
||
with urllib.request.urlopen(req, timeout=20, context=_ssl_context()) as resp:
|
||
body = resp.read(8_000_000)
|
||
code = getattr(resp, "status", None) or resp.getcode()
|
||
if 200 <= int(code) < 400 and body:
|
||
text = body.decode("utf-8", "replace")
|
||
pkgs = parse_apt_packages_index(text)
|
||
else:
|
||
err = f"http_{code}"
|
||
except Exception as e:
|
||
err = f"{type(e).__name__}:{e}"
|
||
# fallback tiny probe (may be incomplete)
|
||
raw = probe_http(url, timeout=max(8.0, PROBE_TIMEOUT_S))
|
||
if raw.get("ok") and raw.get("body"):
|
||
body = raw["body"]
|
||
text = body.decode("utf-8", "replace") if isinstance(body, bytes) else str(body)
|
||
pkgs = parse_apt_packages_index(text)
|
||
apps = []
|
||
for app in _APP_CATALOG:
|
||
pub = pkgs.get(app["package"]) or ""
|
||
apps.append(
|
||
{
|
||
**app,
|
||
"published": pub,
|
||
"published_upstream": _deb_split(pub)[1] if pub else "",
|
||
"light": "green" if pub else "yellow",
|
||
"status": "published" if pub else "missing",
|
||
}
|
||
)
|
||
# Wallet tarball from GNU FTP (best-effort latest numbered)
|
||
try:
|
||
ftp = probe_http("https://ftp.gnu.org/gnu/taler/", timeout=8.0)
|
||
if ftp.get("ok") and ftp.get("body"):
|
||
html = ftp["body"].decode("utf-8", "replace") if isinstance(ftp["body"], bytes) else str(ftp["body"])
|
||
vers = re.findall(r"taler-wallet-([0-9]+\.[0-9]+(?:\.[0-9]+)?)\.tar\.gz", html)
|
||
if vers:
|
||
latest = sorted(vers, key=lambda v: _ver_tuple(v))[-1]
|
||
apps.append(
|
||
{
|
||
"package": "taler-wallet",
|
||
"label": "Wallet (GNU FTP)",
|
||
"kind": "wallet",
|
||
"logo": "wallet",
|
||
"published": latest,
|
||
"published_upstream": latest,
|
||
"light": "green",
|
||
"status": "published",
|
||
"source": "ftp.gnu.org",
|
||
}
|
||
)
|
||
except Exception:
|
||
pass
|
||
_PUBLISHED_CACHE.update({"ts": now, "pkgs": pkgs, "apps": apps, "suite": suite, "error": err})
|
||
return {
|
||
"enabled": True,
|
||
"suite": suite,
|
||
"packages": pkgs,
|
||
"apps": apps,
|
||
"error": err,
|
||
"cached": False,
|
||
"url": url,
|
||
}
|
||
|
||
|
||
def soft_to_deb_names(item: dict[str, Any]) -> list[str]:
|
||
pkg = str(item.get("package") or "")
|
||
label = str(item.get("label") or "").lower()
|
||
if pkg == "taler-bank" and "integration" in label:
|
||
return [] # API version only, not a deb package build
|
||
return list(_SOFT_TO_DEB.get(pkg) or ())
|
||
|
||
|
||
def version_currency(
|
||
installed: str,
|
||
published_deb: str,
|
||
) -> dict[str, Any]:
|
||
"""Compare installed build_version to published deb version; estimate lag."""
|
||
inst_n = normalize_build_version(installed)
|
||
pub_up = _deb_split(published_deb)[1] if published_deb else ""
|
||
if not inst_n or not pub_up:
|
||
return {
|
||
"status": "unknown",
|
||
"light": "blue",
|
||
"published": published_deb,
|
||
"published_upstream": pub_up,
|
||
"behind": None,
|
||
"delta": None,
|
||
}
|
||
# Compare upstream-like strings via deb rules
|
||
cmp = deb_version_cmp(inst_n, pub_up)
|
||
if cmp == 0:
|
||
return {
|
||
"status": "current",
|
||
"light": "green",
|
||
"published": published_deb,
|
||
"published_upstream": pub_up,
|
||
"behind": None,
|
||
"delta": 0,
|
||
}
|
||
if cmp > 0:
|
||
return {
|
||
"status": "ahead",
|
||
"light": "blue",
|
||
"published": published_deb,
|
||
"published_upstream": pub_up,
|
||
"behind": None,
|
||
"delta": None,
|
||
}
|
||
# behind — try numeric distance on dotted prefix
|
||
ti, tp = _ver_tuple(inst_n.replace("~", ".")), _ver_tuple(pub_up.replace("~", "."))
|
||
delta = None
|
||
if ti and tp and len(ti) >= 2 and len(tp) >= 2:
|
||
# rough component-wise lag on last differing field
|
||
n = max(len(ti), len(tp))
|
||
ti = ti + (0,) * (n - len(ti))
|
||
tp = tp + (0,) * (n - len(tp))
|
||
for i in range(n):
|
||
if tp[i] > ti[i]:
|
||
delta = int(tp[i] - ti[i])
|
||
break
|
||
return {
|
||
"status": "behind",
|
||
"light": "orange" if (delta is None or delta > 0) else "yellow",
|
||
"published": published_deb,
|
||
"published_upstream": pub_up,
|
||
"behind": pub_up,
|
||
"delta": delta,
|
||
}
|
||
|
||
|
||
def apply_published_currency(
|
||
items: list[dict[str, Any]],
|
||
published: dict[str, Any] | None,
|
||
) -> list[dict[str, Any]]:
|
||
"""Attach published/current/behind fields; raise light when behind."""
|
||
pkgs = (published or {}).get("packages") or {}
|
||
out = []
|
||
for it in items:
|
||
row = dict(it)
|
||
deb_names = soft_to_deb_names(row)
|
||
best = None
|
||
for dn in deb_names:
|
||
pub = pkgs.get(dn) or ""
|
||
if not pub:
|
||
continue
|
||
cur = version_currency(str(row.get("version") or ""), pub)
|
||
cur["deb_package"] = dn
|
||
if best is None or (
|
||
cur.get("status") == "behind"
|
||
and best.get("status") != "behind"
|
||
):
|
||
best = cur
|
||
elif cur.get("status") == "behind" and best.get("status") == "behind":
|
||
# keep higher published upstream
|
||
if deb_version_cmp(str(cur.get("published") or ""), str(best.get("published") or "")) > 0:
|
||
best = cur
|
||
if best:
|
||
row.update(
|
||
{
|
||
"published": best.get("published"),
|
||
"published_upstream": best.get("published_upstream"),
|
||
"currency_status": best.get("status"),
|
||
"behind": best.get("behind"),
|
||
"delta": best.get("delta"),
|
||
"deb_package": best.get("deb_package"),
|
||
}
|
||
)
|
||
# CVE colour wins if worse; else use currency light
|
||
if best.get("status") == "behind":
|
||
if SEVERITY.get(str(best.get("light")), 0) > SEVERITY.get(str(row.get("light") or "green"), 0):
|
||
row["light"] = best.get("light")
|
||
elif best.get("status") == "current" and row.get("light") in ("green", "blue", None):
|
||
row["light"] = "green"
|
||
else:
|
||
row.setdefault("currency_status", "n/a")
|
||
out.append(row)
|
||
return out
|
||
|
||
|
||
def colorize_software(
|
||
items: list[dict[str, Any]],
|
||
vulns: dict[str, Any] | None = None,
|
||
published: dict[str, Any] | None = None,
|
||
) -> list[dict[str, Any]]:
|
||
"""Dedupe + CVE colour + published currency (current/behind/ahead)."""
|
||
hits = (vulns or {}).get("hits") or []
|
||
by_pkg_ver: dict[str, str] = {}
|
||
for h in hits:
|
||
pkg = str(h.get("package") or "")
|
||
ver = str(h.get("version") or "")
|
||
sev = str(h.get("severity") or "")
|
||
key = f"{pkg}|{ver}"
|
||
light = _sev_to_light(sev)
|
||
if key not in by_pkg_ver or SEVERITY.get(light, 0) > SEVERITY.get(by_pkg_ver[key], 0):
|
||
by_pkg_ver[key] = light
|
||
out: list[dict[str, Any]] = []
|
||
seen: set[str] = set()
|
||
for it in items:
|
||
pkg = str(it.get("package") or "")
|
||
ver = str(it.get("version") or "")
|
||
host = str(it.get("host") or "")
|
||
key = f"{pkg}|{ver}|{host}|{it.get('source')}"
|
||
if not pkg or key in seen:
|
||
continue
|
||
seen.add(key)
|
||
light = by_pkg_ver.get(f"{pkg}|{ver}") or ("green" if ver else "blue")
|
||
out.append({**it, "light": light})
|
||
out = apply_published_currency(out, published)
|
||
order = {
|
||
"taler-exchange": 0,
|
||
"taler-merchant": 1,
|
||
"taler-bank": 2,
|
||
"taler": 3,
|
||
"nginx": 10,
|
||
"apache": 11,
|
||
"caddy": 12,
|
||
"postfix": 20,
|
||
"mta": 21,
|
||
}
|
||
out.sort(key=lambda x: (order.get(str(x.get("package")), 50), str(x.get("package")), str(x.get("host"))))
|
||
return out
|
||
|
||
|
||
def collect_software_from_probes(
|
||
*,
|
||
probe_cache: dict[str, dict[str, Any]],
|
||
members: list[dict[str, Any]],
|
||
vulns: dict[str, Any] | None = None,
|
||
mail_hosts: list[str] | None = None,
|
||
smtp: bool = False,
|
||
published: dict[str, Any] | None = None,
|
||
) -> list[dict[str, Any]]:
|
||
"""Assemble software inventory for one card from HTTP/Taler/SMTP signals."""
|
||
items: list[dict[str, Any]] = []
|
||
for url, raw in (probe_cache or {}).items():
|
||
host = _host_only(url)
|
||
srv = str((raw or {}).get("server") or "")
|
||
items.extend(software_from_server_header(srv, host=host))
|
||
# Taler /config or /keys-adjacent config bodies
|
||
path = url.split(host, 1)[-1] if host and host in url else ""
|
||
if path.endswith("/config") or "/taler-integration/config" in path:
|
||
role = ""
|
||
for m in members or []:
|
||
if _host_only(str(m.get("host") or "")) == host:
|
||
role = str(m.get("role") or "")
|
||
break
|
||
items.extend(software_from_taler_config((raw or {}).get("body"), host=host, role=role))
|
||
if smtp:
|
||
for h in mail_hosts or []:
|
||
items.extend(smtp_banner_software(h))
|
||
return colorize_software(items, vulns, published=published)
|
||
|
||
|
||
def _ver_tuple(s: str) -> tuple[int, ...]:
|
||
nums = [int(x) for x in re.findall(r"\d+", s or "")[:5]]
|
||
return tuple(nums) if nums else ()
|
||
|
||
|
||
def _vt_cmp(a: tuple[int, ...], b: tuple[int, ...]) -> int:
|
||
n = max(len(a), len(b))
|
||
a = a + (0,) * (n - len(a))
|
||
b = b + (0,) * (n - len(b))
|
||
return (a > b) - (a < b)
|
||
|
||
|
||
def _in_semver_events(ver: str, events: list) -> bool:
|
||
vt = _ver_tuple(ver)
|
||
if not vt:
|
||
return False
|
||
intro = None
|
||
for e in events or []:
|
||
if "introduced" in e:
|
||
intro = e.get("introduced")
|
||
elif "fixed" in e or "last_affected" in e:
|
||
fixed = e.get("fixed")
|
||
last = e.get("last_affected")
|
||
lo = _ver_tuple("0" if intro in (None, "0") else str(intro))
|
||
if _vt_cmp(vt, lo) < 0:
|
||
intro = None
|
||
continue
|
||
if fixed is not None:
|
||
if _vt_cmp(vt, _ver_tuple(str(fixed))) < 0:
|
||
return True
|
||
elif last is not None:
|
||
if _vt_cmp(vt, _ver_tuple(str(last))) <= 0:
|
||
return True
|
||
intro = None
|
||
return False
|
||
|
||
|
||
_OSV_SEED = {
|
||
"nginx": [
|
||
"CVE-2025-23419",
|
||
"CVE-2025-53859",
|
||
"CVE-2024-7347",
|
||
"CVE-2024-34161",
|
||
"CVE-2024-32760",
|
||
"CVE-2024-31079",
|
||
"CVE-2024-24989",
|
||
"CVE-2024-24990",
|
||
],
|
||
"apache": [
|
||
"CVE-2024-38474",
|
||
"CVE-2024-38476",
|
||
"CVE-2024-38477",
|
||
"CVE-2023-31122",
|
||
"CVE-2023-43622",
|
||
],
|
||
"caddy": ["CVE-2022-29718", "CVE-2023-50463"],
|
||
}
|
||
|
||
|
||
def _osv_get(vid: str) -> dict[str, Any]:
|
||
try:
|
||
with urllib.request.urlopen(f"https://api.osv.dev/v1/vulns/{vid}", timeout=12) as r:
|
||
return json.loads(r.read().decode("utf-8", "replace"))
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
def _severity_rank(sev: str) -> int:
|
||
s = (sev or "").upper()
|
||
if s == "CRITICAL":
|
||
return 3
|
||
if s == "HIGH":
|
||
return 2
|
||
if s == "MEDIUM":
|
||
return 1
|
||
return 0
|
||
|
||
|
||
def osv_hits_for_server(pkg: str, ver: str, *, cache: dict[str, list] | None = None) -> list[dict[str, Any]]:
|
||
"""Actionable OSV hits for a bare upstream Server header version (no Debian rev).
|
||
|
||
Same policy as check_surface.sh: use SEMVER extracted_events on known CVE seeds
|
||
— avoids Debian ecosystem false positives on bare versions.
|
||
"""
|
||
if not pkg or not ver:
|
||
return []
|
||
cache = cache if cache is not None else {}
|
||
key = f"{pkg}|{ver}"
|
||
if key in cache:
|
||
return list(cache[key])
|
||
hits: list[dict[str, Any]] = []
|
||
for vid in _OSV_SEED.get(pkg, []):
|
||
doc = _osv_get(vid)
|
||
if not doc:
|
||
continue
|
||
ok_hit = False
|
||
for a in doc.get("affected") or []:
|
||
for rg in a.get("ranges") or []:
|
||
db = rg.get("database_specific") or {}
|
||
extracted = db.get("extracted_events") or []
|
||
if extracted and _in_semver_events(ver, extracted):
|
||
blob = json.dumps(a).lower()
|
||
if pkg == "nginx" and "nginx" not in blob and "f5" not in blob:
|
||
continue
|
||
ok_hit = True
|
||
break
|
||
if rg.get("type") == "SEMVER" and _in_semver_events(ver, rg.get("events") or []):
|
||
ok_hit = True
|
||
break
|
||
if ok_hit:
|
||
break
|
||
if not ok_hit:
|
||
continue
|
||
sev = ""
|
||
for s in doc.get("severity") or []:
|
||
if isinstance(s, dict) and s.get("type") in ("CVSS_V3", "CVSS_V4", "CVSS_V2"):
|
||
# score-based
|
||
score = s.get("score")
|
||
try:
|
||
sc = float(str(score).split("/")[0]) if score else 0.0
|
||
except ValueError:
|
||
sc = 0.0
|
||
if sc >= 9.0:
|
||
sev = "CRITICAL"
|
||
elif sc >= 7.0:
|
||
sev = "HIGH"
|
||
elif sc >= 4.0:
|
||
sev = "MEDIUM"
|
||
break
|
||
dbsev = ((doc.get("database_specific") or {}).get("severity") or "").upper()
|
||
if dbsev in ("CRITICAL", "HIGH", "MEDIUM", "LOW") and _severity_rank(dbsev) > _severity_rank(sev):
|
||
sev = dbsev
|
||
if not sev:
|
||
sev = "HIGH" # known actionable seed hit without score → treat serious
|
||
hits.append(
|
||
{
|
||
"id": vid,
|
||
"severity": sev,
|
||
"package": pkg,
|
||
"version": ver,
|
||
"summary": str(doc.get("summary") or "")[:160],
|
||
}
|
||
)
|
||
# keep critical/high only for box colour; still return medium for display
|
||
cache[key] = hits
|
||
return list(hits)
|
||
|
||
|
||
def assess_external_vulns(
|
||
*,
|
||
probe_cache: dict[str, dict[str, Any]],
|
||
members: list[dict[str, Any]],
|
||
osv_cache: dict[str, list] | None = None,
|
||
) -> dict[str, Any]:
|
||
"""Scan outside-visible Server headers from probe cache; critical/high → box colour."""
|
||
if not CVE_PROBES_ENABLED:
|
||
return {"enabled": False, "hits": [], "worst": "", "light": None, "servers": []}
|
||
seen_srv: dict[str, str] = {}
|
||
# Prefer apex/config responses already in cache
|
||
for url, raw in (probe_cache or {}).items():
|
||
srv = str((raw or {}).get("server") or "").strip()
|
||
if not srv:
|
||
continue
|
||
host = _host_only(url)
|
||
if host and host not in seen_srv:
|
||
seen_srv[host] = srv
|
||
# Also ensure we looked at each member host apex once
|
||
for m in members or []:
|
||
host = _host_only(str(m.get("host") or ""))
|
||
if not host or host.upper() == "IBAN" or host in seen_srv:
|
||
continue
|
||
url = f"https://{host}/"
|
||
if url not in probe_cache:
|
||
probe_cache[url] = probe_http(url)
|
||
srv = str((probe_cache[url] or {}).get("server") or "").strip()
|
||
if srv:
|
||
seen_srv[host] = srv
|
||
|
||
servers_out = []
|
||
all_hits: list[dict[str, Any]] = []
|
||
osv_cache = osv_cache if osv_cache is not None else {}
|
||
for host, srv in sorted(seen_srv.items()):
|
||
pkg = guess_http_package(srv)
|
||
ver = extract_http_version(srv)
|
||
entry = {"host": host, "server": srv, "package": pkg, "version": ver, "hits": []}
|
||
if pkg and ver:
|
||
hits = osv_hits_for_server(pkg, ver, cache=osv_cache)
|
||
for h in hits:
|
||
h2 = {**h, "host": host, "server": srv}
|
||
entry["hits"].append(h2)
|
||
all_hits.append(h2)
|
||
servers_out.append(entry)
|
||
|
||
worst = ""
|
||
light = None
|
||
for h in all_hits:
|
||
sev = h.get("severity") or ""
|
||
if _severity_rank(sev) > _severity_rank(worst):
|
||
worst = sev
|
||
if worst == "CRITICAL":
|
||
light = "red"
|
||
elif worst == "HIGH":
|
||
light = "orange"
|
||
elif worst == "MEDIUM":
|
||
light = "yellow"
|
||
return {
|
||
"enabled": True,
|
||
"hits": all_hits[:20],
|
||
"worst": worst,
|
||
"light": light,
|
||
"servers": servers_out,
|
||
}
|
||
|
||
|
||
def apply_vuln_to_stack(
|
||
light: str,
|
||
intensity: str,
|
||
members: list[dict[str, Any]],
|
||
vulns: dict[str, Any],
|
||
) -> tuple[str, str, list[dict[str, Any]]]:
|
||
"""Raise stack colour for critical/high outside-visible CVEs; add quiet member row."""
|
||
vlight = vulns.get("light")
|
||
hits = vulns.get("hits") or []
|
||
if not vlight or not hits:
|
||
return light, intensity, members
|
||
# synthetic member so UI/aggregation stay consistent
|
||
top = hits[0]
|
||
reason = f"{top.get('id')} {top.get('severity')} ({top.get('package')}/{top.get('version')})"
|
||
members = list(members) + [
|
||
{
|
||
"role": "cve",
|
||
"host": top.get("host") or "edge",
|
||
"ok": False,
|
||
"light": vlight,
|
||
"reason": reason,
|
||
"down_since": None,
|
||
"code": 0,
|
||
"elapsed": 0.0,
|
||
"error": None,
|
||
"url": None,
|
||
"endpoints": [],
|
||
"endpoints_summary": {"n": 0, "ok": 0, "soft": 0, "bad": 0, "skipped": 0, "light": "green", "dots": []},
|
||
}
|
||
]
|
||
# re-aggregate including CVE member
|
||
light2, intensity2 = aggregate_stack(members)
|
||
return light2, intensity2, members
|
||
|
||
|
||
def classify_probe(
|
||
result: dict[str, Any],
|
||
*,
|
||
prev: dict[str, Any] | None = None,
|
||
now: float | None = None,
|
||
acute_s: float = ACUTE_DOWN_S,
|
||
soft_s: float = 2.5,
|
||
) -> dict[str, Any]:
|
||
"""Map probe result + previous down_since → light/reason."""
|
||
now = time.time() if now is None else now
|
||
prev = prev or {}
|
||
ok = bool(result.get("ok"))
|
||
code = int(result.get("code") or 0)
|
||
elapsed = float(result.get("elapsed") or 0)
|
||
error = result.get("error")
|
||
down_since = prev.get("down_since")
|
||
|
||
if ok:
|
||
down_since = None
|
||
if elapsed >= soft_s:
|
||
light, reason = "yellow", "slow"
|
||
else:
|
||
light, reason = "green", "ok"
|
||
else:
|
||
# Reachable but bad status → degraded rather than offline
|
||
if code >= 400:
|
||
down_since = None
|
||
if code in (401, 403):
|
||
light, reason = "blue", "auth_present"
|
||
elif code >= 500:
|
||
light, reason = "orange", f"http_{code}"
|
||
else:
|
||
light, reason = "yellow", f"http_{code}"
|
||
else:
|
||
prev_ok = prev.get("ok")
|
||
try:
|
||
inherited = float(prev["down_since"]) if prev.get("down_since") is not None else None
|
||
except (TypeError, ValueError):
|
||
inherited = None
|
||
if prev_ok or inherited is None:
|
||
down_since = now
|
||
else:
|
||
down_since = inherited
|
||
age = now - float(down_since)
|
||
err_l = str(error or "").lower()
|
||
# Timeouts / reset often mean DoS or broken path — orange if still "fresh"
|
||
if age < acute_s and ("timeout" in err_l or "timedout" in err_l or "reset" in err_l):
|
||
light, reason = "orange", "hard_degraded"
|
||
elif age < acute_s:
|
||
light, reason = "red", "down_acute"
|
||
else:
|
||
light, reason = "black", "down_long"
|
||
|
||
return {
|
||
"ok": ok,
|
||
"light": light,
|
||
"reason": reason,
|
||
"down_since": down_since,
|
||
"code": code,
|
||
"elapsed": round(elapsed, 3),
|
||
"error": error,
|
||
"url": result.get("url"),
|
||
"ts": now,
|
||
}
|
||
|
||
|
||
def _cap_light(light: str, max_light: str) -> str:
|
||
if SEVERITY.get(light, 5) > SEVERITY.get(max_light, 5):
|
||
return max_light
|
||
return light
|
||
|
||
|
||
def summarize_endpoints(endpoints: list[dict[str, Any]]) -> dict[str, Any]:
|
||
"""Compact rollup for UI: counts + worst light (dots = probed only, not skips)."""
|
||
if not endpoints:
|
||
return {"n": 0, "ok": 0, "soft": 0, "bad": 0, "skipped": 0, "light": "green", "dots": []}
|
||
dots: list[str] = []
|
||
ok = soft = bad = skipped = 0
|
||
worst = "green"
|
||
for ep in endpoints:
|
||
if ep.get("skipped"):
|
||
skipped += 1
|
||
continue
|
||
L = ep.get("light") or "black"
|
||
dots.append(L)
|
||
if SEVERITY.get(L, 5) > SEVERITY.get(worst, 0):
|
||
worst = L
|
||
if SEVERITY.get(L, 5) <= SEVERITY["blue"]:
|
||
ok += 1
|
||
elif SEVERITY.get(L, 5) <= SEVERITY["yellow"]:
|
||
soft += 1
|
||
else:
|
||
bad += 1
|
||
return {
|
||
"n": len(endpoints),
|
||
"ok": ok,
|
||
"soft": soft,
|
||
"bad": bad,
|
||
"skipped": skipped,
|
||
"light": worst if dots else "green",
|
||
"dots": dots,
|
||
}
|
||
|
||
|
||
def classify_endpoint_probe(
|
||
raw: dict[str, Any],
|
||
*,
|
||
expect: tuple[int, ...],
|
||
severity: str = "soft",
|
||
hard: bool | None = None,
|
||
prev: dict[str, Any] | None = None,
|
||
now: float | None = None,
|
||
acute_s: float = ACUTE_DOWN_S,
|
||
) -> dict[str, Any]:
|
||
"""Map HTTP probe to light.
|
||
|
||
soft → fail capped at yellow
|
||
medium → missing/wrong is at least orange (mittelschwer), not red for 4xx
|
||
hard → full classify_probe severity
|
||
"""
|
||
if hard is not None and not severity:
|
||
severity = "hard" if hard else "soft"
|
||
sev = (severity or "soft").lower()
|
||
code = int(raw.get("code") or 0)
|
||
if code in expect:
|
||
c = classify_probe(raw, prev=prev, now=now, acute_s=acute_s)
|
||
return {**c, "expected": True, "severity": sev}
|
||
fake = {**raw, "ok": False}
|
||
if code >= 400:
|
||
fake["error"] = f"http_{code}"
|
||
elif not fake.get("error"):
|
||
fake["error"] = "unreachable"
|
||
c = classify_probe(fake, prev=prev, now=now, acute_s=acute_s)
|
||
reason = c.get("reason") or (f"http_{code}" if code else "unreachable")
|
||
if sev == "soft":
|
||
c = {
|
||
**c,
|
||
"light": _cap_light(c.get("light") or "yellow", ENDPOINT_SOFT_MAX),
|
||
"reason": reason,
|
||
}
|
||
elif sev == "medium":
|
||
# At least orange; keep 5xx/unreachable from going black via acute age — still medium.
|
||
light = c.get("light") or "orange"
|
||
if SEVERITY.get(light, 0) < SEVERITY["orange"]:
|
||
light = "orange"
|
||
elif SEVERITY.get(light, 0) > SEVERITY["orange"]:
|
||
light = "orange"
|
||
c = {**c, "light": light, "reason": reason, "down_since": None}
|
||
# hard: keep classify_probe result
|
||
return {**c, "expected": False, "severity": sev}
|
||
|
||
|
||
def probe_role_endpoints(
|
||
role: str,
|
||
host: str,
|
||
*,
|
||
has_intro: bool = False,
|
||
landing: bool | None = None,
|
||
legal_severity: str = "soft",
|
||
probe_fn: Callable[[str], dict[str, Any]],
|
||
cache: dict[str, dict[str, Any]] | None = None,
|
||
prev_by_path: dict[str, dict[str, Any]] | None = None,
|
||
now: float | None = None,
|
||
acute_s: float = ACUTE_DOWN_S,
|
||
) -> list[dict[str, Any]]:
|
||
"""Probe suite-like public endpoints for one machine; return compact rows."""
|
||
host = _host_only(host)
|
||
if not host or host.upper() == "IBAN":
|
||
return []
|
||
# Compat: older callers passed landing=
|
||
if landing is not None:
|
||
has_intro = bool(landing)
|
||
catalog = ROLE_ENDPOINTS.get(role) or []
|
||
cache = cache if cache is not None else {}
|
||
prev_by_path = prev_by_path or {}
|
||
now = time.time() if now is None else now
|
||
out: list[dict[str, Any]] = []
|
||
for spec in catalog:
|
||
if spec.get("need_intro") and not has_intro:
|
||
out.append(
|
||
{
|
||
"id": spec["id"],
|
||
"path": spec["path"],
|
||
"skipped": True,
|
||
"light": "blue",
|
||
"reason": "no_intro",
|
||
"code": 0,
|
||
"elapsed": 0.0,
|
||
}
|
||
)
|
||
continue
|
||
url = f"https://{host}{spec['path']}"
|
||
if url in cache:
|
||
raw = cache[url]
|
||
else:
|
||
raw = probe_fn(url)
|
||
cache[url] = raw
|
||
sev = str(spec.get("severity") or ("hard" if spec.get("hard") else "soft"))
|
||
if spec.get("legal"):
|
||
sev = legal_severity or sev
|
||
classified = classify_endpoint_probe(
|
||
raw,
|
||
expect=tuple(spec.get("expect") or (200,)),
|
||
severity=sev,
|
||
prev=prev_by_path.get(spec["path"]),
|
||
now=now,
|
||
acute_s=acute_s,
|
||
)
|
||
out.append(
|
||
{
|
||
"id": spec["id"],
|
||
"path": spec["path"],
|
||
"url": url,
|
||
"skipped": False,
|
||
"light": classified.get("light") or "black",
|
||
"reason": classified.get("reason") or "",
|
||
"code": classified.get("code") or 0,
|
||
"elapsed": classified.get("elapsed") or 0.0,
|
||
"ok": bool(classified.get("ok")) and bool(classified.get("expected")),
|
||
"severity": sev,
|
||
# medium/hard pull machine colour past soft yellow
|
||
"hard": sev in ("hard", "medium"),
|
||
}
|
||
)
|
||
return out
|
||
|
||
|
||
def merge_member_endpoint_light(member: dict[str, Any], endpoints: list[dict[str, Any]]) -> dict[str, Any]:
|
||
"""Attach endpoints + summary; raise member light to worst *hard* endpoint issue."""
|
||
summary = summarize_endpoints(endpoints)
|
||
member = {**member, "endpoints": endpoints, "endpoints_summary": summary}
|
||
# Informational members stay as-is
|
||
if (member.get("reason") or "") in _AGG_SKIP_REASONS:
|
||
return member
|
||
base = member.get("light") or "black"
|
||
# Only hard endpoint failures pull the machine light past yellow via soft cap already applied
|
||
worst = base
|
||
for ep in endpoints:
|
||
if ep.get("skipped"):
|
||
continue
|
||
L = ep.get("light") or "black"
|
||
if ep.get("hard") is False and SEVERITY.get(L, 0) <= SEVERITY["yellow"]:
|
||
# soft noise shouldn't turn machine red if primary is green — still show yellow if soft warn
|
||
if SEVERITY.get(L, 0) > SEVERITY.get(worst, 0) and SEVERITY.get(L, 0) <= SEVERITY["yellow"]:
|
||
worst = L
|
||
continue
|
||
if SEVERITY.get(L, 0) > SEVERITY.get(worst, 0):
|
||
worst = L
|
||
member["light"] = worst
|
||
return member
|
||
|
||
|
||
def score_light_from_ratio(ratio: float, *, all_black: bool = False) -> str:
|
||
"""Map green/total ratio to overall traffic light (incl. white = gone)."""
|
||
if ratio <= 0:
|
||
return "black" if all_black else "white"
|
||
if ratio >= SCORE_RATIO_GREEN:
|
||
return "green"
|
||
if ratio >= SCORE_RATIO_BLUE:
|
||
return "blue"
|
||
if ratio >= SCORE_RATIO_YELLOW:
|
||
return "yellow"
|
||
if ratio >= SCORE_RATIO_ORANGE:
|
||
return "orange"
|
||
if ratio >= SCORE_RATIO_RED:
|
||
return "red"
|
||
return "white"
|
||
|
||
|
||
def compute_total_score(
|
||
stacks: list[dict[str, Any]],
|
||
services: list[dict[str, Any]] | None = None,
|
||
mail: list[dict[str, Any]] | None = None,
|
||
) -> dict[str, Any]:
|
||
"""Page-level score: how many boxes are green out of all scored boxes.
|
||
|
||
One point = one stack / service / mail card. Strict green only (blue ≠ green).
|
||
"""
|
||
boxes: list[dict[str, Any]] = []
|
||
for group in (stacks or [], services or [], mail or []):
|
||
boxes.extend(group)
|
||
total = len(boxes)
|
||
by_light: dict[str, int] = {}
|
||
for b in boxes:
|
||
Lgt = str(b.get("light") or "black")
|
||
by_light[Lgt] = by_light.get(Lgt, 0) + 1
|
||
green = int(by_light.get("green") or 0)
|
||
ratio = (green / total) if total else 0.0
|
||
all_black = total > 0 and int(by_light.get("black") or 0) == total
|
||
light = "white" if total == 0 else score_light_from_ratio(ratio, all_black=all_black)
|
||
return {
|
||
"green": green,
|
||
"total": total,
|
||
"ratio": round(ratio, 4),
|
||
"pct": int(round(ratio * 100)) if total else 0,
|
||
"light": light,
|
||
"by_light": by_light,
|
||
"thresholds": {
|
||
"green": SCORE_RATIO_GREEN,
|
||
"blue": SCORE_RATIO_BLUE,
|
||
"yellow": SCORE_RATIO_YELLOW,
|
||
"orange": SCORE_RATIO_ORANGE,
|
||
"red": SCORE_RATIO_RED,
|
||
},
|
||
}
|
||
|
||
|
||
def aggregate_breakdown(members: list[dict[str, Any]]) -> dict[str, Any]:
|
||
"""Counts + parts so UI can explain e.g. 4× OK, 3× Soft → Degraded."""
|
||
counted = [m for m in members if (m.get("reason") or "") not in _AGG_SKIP_REASONS]
|
||
if not counted:
|
||
counted = list(members)
|
||
by_light: dict[str, int] = {}
|
||
parts: list[dict[str, Any]] = []
|
||
for m in counted:
|
||
Lgt = str(m.get("light") or "black")
|
||
by_light[Lgt] = by_light.get(Lgt, 0) + 1
|
||
parts.append(
|
||
{
|
||
"role": m.get("role"),
|
||
"host": m.get("host"),
|
||
"light": Lgt,
|
||
"reason": m.get("reason"),
|
||
}
|
||
)
|
||
worst = "black"
|
||
if by_light:
|
||
worst = LIGHTS[max(SEVERITY.get(k, 5) for k in by_light)]
|
||
return {
|
||
"by_light": by_light,
|
||
"worst": worst,
|
||
"counted": len(counted),
|
||
"parts": parts,
|
||
}
|
||
|
||
|
||
def aggregate_stack(members: list[dict[str, Any]]) -> tuple[str, str]:
|
||
"""Return (light, intensity) for a stack box.
|
||
|
||
Logical color = weakest *expected* member. Intensity muted unless every
|
||
counted member is at worst severity and that severity is red/black (full).
|
||
Informational rows (e.g. ebics_no_bank) are shown but ignored for the box.
|
||
"""
|
||
counted = [m for m in members if (m.get("reason") or "") not in _AGG_SKIP_REASONS]
|
||
if not counted:
|
||
counted = list(members)
|
||
if not counted:
|
||
return "black", "full"
|
||
lights = [m.get("light") or "black" for m in counted]
|
||
worst_rank = max(SEVERITY.get(L, 5) for L in lights)
|
||
worst = LIGHTS[worst_rank]
|
||
n = len(counted)
|
||
k_worst = sum(1 for L in lights if SEVERITY.get(L, 5) == worst_rank)
|
||
# green/blue = healthy or expected-present (e.g. static_pending / no_prod_yet)
|
||
if worst_rank <= SEVERITY["blue"]:
|
||
return worst, "ok"
|
||
if worst_rank >= SEVERITY["red"] and k_worst == n:
|
||
return worst, "full"
|
||
return worst, "muted"
|
||
|
||
|
||
def _dig_lines(rrtype: str, name: str) -> list[str]:
|
||
"""Best-effort dig +short; empty if dig missing/fails."""
|
||
try:
|
||
r = subprocess.run(
|
||
["dig", "+short", "+time=2", "+tries=1", rrtype, name],
|
||
capture_output=True,
|
||
timeout=6,
|
||
check=False,
|
||
)
|
||
except (OSError, subprocess.TimeoutExpired):
|
||
return []
|
||
out = (r.stdout or b"").decode("utf-8", "replace")
|
||
return [ln.strip().rstrip(".") for ln in out.splitlines() if ln.strip()]
|
||
|
||
|
||
def dns_a_aaaa(host: str) -> dict[str, list[str]]:
|
||
h = _host_only(host)
|
||
a = [x for x in _dig_lines("A", h) if re.match(r"^\d+\.\d+\.\d+\.\d+$", x)]
|
||
aaaa = [x for x in _dig_lines("AAAA", h) if ":" in x]
|
||
return {"a": a, "aaaa": aaaa}
|
||
|
||
|
||
def dns_mx(domain: str) -> list[str]:
|
||
lines = _dig_lines("MX", domain)
|
||
hosts: list[str] = []
|
||
for ln in lines:
|
||
parts = ln.split()
|
||
if len(parts) >= 2:
|
||
hosts.append(parts[-1].lower().rstrip("."))
|
||
elif parts:
|
||
hosts.append(parts[0].lower().rstrip("."))
|
||
return hosts
|
||
|
||
|
||
def dns_txt(name: str) -> str:
|
||
return " ".join(_dig_lines("TXT", name)).replace('"', "")
|
||
|
||
|
||
def dns_ptr(ip: str) -> list[str]:
|
||
"""Reverse DNS via dig -x."""
|
||
try:
|
||
r = subprocess.run(
|
||
["dig", "+short", "+time=2", "+tries=1", "-x", ip],
|
||
capture_output=True,
|
||
timeout=6,
|
||
check=False,
|
||
)
|
||
except (OSError, subprocess.TimeoutExpired):
|
||
return []
|
||
out = (r.stdout or b"").decode("utf-8", "replace")
|
||
return [ln.strip().rstrip(".") for ln in out.splitlines() if ln.strip()]
|
||
|
||
|
||
def probe_tls_cert(host: str, *, timeout: float = PROBE_TIMEOUT_S) -> dict[str, Any]:
|
||
"""Fetch peer cert notAfter + SANs (outside-in)."""
|
||
h = _host_only(host)
|
||
if not h or h.upper() == "IBAN":
|
||
return {"ok": False, "host": h, "error": "no_host"}
|
||
ctx = ssl.create_default_context()
|
||
if not TLS_VERIFY:
|
||
ctx.check_hostname = False
|
||
ctx.verify_mode = ssl.CERT_NONE
|
||
try:
|
||
with socket.create_connection((h, 443), timeout=timeout) as sock:
|
||
with ctx.wrap_socket(sock, server_hostname=h) as ssock:
|
||
cert = ssock.getpeercert()
|
||
except OSError as e:
|
||
return {"ok": False, "host": h, "error": type(e).__name__}
|
||
if not cert:
|
||
# CERT_NONE often yields empty dict — retry binary DER via getpeercert(True)
|
||
try:
|
||
with socket.create_connection((h, 443), timeout=timeout) as sock:
|
||
with ctx.wrap_socket(sock, server_hostname=h) as ssock:
|
||
der = ssock.getpeercert(binary_form=True)
|
||
if not der:
|
||
return {"ok": False, "host": h, "error": "no_cert"}
|
||
# Parse notAfter via openssl if available
|
||
return _parse_der_cert(h, der)
|
||
except OSError as e:
|
||
return {"ok": False, "host": h, "error": type(e).__name__}
|
||
return _cert_from_dict(h, cert)
|
||
|
||
|
||
def _cert_from_dict(host: str, cert: dict[str, Any]) -> dict[str, Any]:
|
||
not_after = cert.get("notAfter") or ""
|
||
days_left = None
|
||
try:
|
||
# OpenSSL ASN.1: 'Jun 15 12:00:00 2027 GMT'
|
||
exp = time.strptime(str(not_after), "%b %d %H:%M:%S %Y %Z")
|
||
days_left = int((time.mktime(exp) - time.time()) / 86400)
|
||
except (TypeError, ValueError, OverflowError, OSError):
|
||
days_left = None
|
||
sans: list[str] = []
|
||
for typ, val in cert.get("subjectAltName") or []:
|
||
if typ.lower() == "dns" and val and val not in sans:
|
||
sans.append(str(val).lower())
|
||
light = "green"
|
||
reason = "cert_ok"
|
||
if days_left is None:
|
||
light, reason = "yellow", "cert_parse"
|
||
elif days_left < 0:
|
||
light, reason = "red", "cert_expired"
|
||
elif days_left < CERT_CRIT_DAYS:
|
||
light, reason = "orange", "cert_expiring"
|
||
elif days_left < CERT_WARN_DAYS:
|
||
light, reason = "yellow", "cert_warn"
|
||
return {
|
||
"ok": days_left is not None and days_left >= 0,
|
||
"host": host,
|
||
"not_after": not_after,
|
||
"days_left": days_left,
|
||
"sans": sans[:32],
|
||
"light": light,
|
||
"reason": reason,
|
||
"error": None,
|
||
}
|
||
|
||
|
||
def _parse_der_cert(host: str, der: bytes) -> dict[str, Any]:
|
||
"""Fallback: openssl x509 -noout -dates -ext subjectAltName."""
|
||
try:
|
||
r = subprocess.run(
|
||
["openssl", "x509", "-inform", "DER", "-noout", "-enddate", "-ext", "subjectAltName"],
|
||
input=der,
|
||
capture_output=True,
|
||
timeout=6,
|
||
check=False,
|
||
)
|
||
except (OSError, subprocess.TimeoutExpired):
|
||
return {"ok": False, "host": host, "error": "openssl_fail"}
|
||
text = (r.stdout or b"").decode("utf-8", "replace")
|
||
not_after = ""
|
||
for ln in text.splitlines():
|
||
if ln.startswith("notAfter="):
|
||
not_after = ln.split("=", 1)[1].strip()
|
||
sans: list[str] = []
|
||
for m in re.finditer(r"DNS:([A-Za-z0-9_.*-]+)", text):
|
||
v = m.group(1).lower()
|
||
if v not in sans:
|
||
sans.append(v)
|
||
fake = {"notAfter": not_after, "subjectAltName": [("DNS", s) for s in sans]}
|
||
out = _cert_from_dict(host, fake)
|
||
if out.get("days_left") is None and not_after:
|
||
out["error"] = "cert_parse"
|
||
return out
|
||
|
||
|
||
def parse_services_conf(path: Path | None = None) -> list[dict[str, Any]]:
|
||
"""Company/service hosts (non-Taler): id host label zone."""
|
||
path = path or default_services_conf()
|
||
rows: list[dict[str, Any]] = []
|
||
if not path.is_file():
|
||
return rows
|
||
seen: set[str] = set()
|
||
for raw in path.read_text(encoding="utf-8").splitlines():
|
||
line = raw.split("#", 1)[0].strip()
|
||
if not line:
|
||
continue
|
||
parts = line.split()
|
||
if len(parts) < 2:
|
||
continue
|
||
sid, host = parts[0], _host_only(parts[1])
|
||
label = parts[2] if len(parts) > 2 else "service"
|
||
zone = parts[3] if len(parts) > 3 else "public"
|
||
if not host or host in seen:
|
||
continue
|
||
seen.add(host)
|
||
rows.append(
|
||
{
|
||
"id": sid,
|
||
"host": host,
|
||
"service_label": label,
|
||
"zone": zone,
|
||
"kind": "service",
|
||
}
|
||
)
|
||
return rows
|
||
|
||
|
||
def parse_mail_catalog(path: Path | None = None) -> list[dict[str, Any]]:
|
||
path = path or default_mail_catalog()
|
||
rows: list[dict[str, Any]] = []
|
||
if not path.is_file():
|
||
return rows
|
||
for raw in path.read_text(encoding="utf-8").splitlines():
|
||
line = raw.split("#", 1)[0].strip()
|
||
if not line:
|
||
continue
|
||
parts = line.split()
|
||
if len(parts) < 2:
|
||
continue
|
||
domain = parts[0]
|
||
expect_mx = [x.strip().lower() for x in parts[1].split(",") if x.strip()]
|
||
mail_hosts = (
|
||
[x.strip().lower() for x in parts[2].split(",") if x.strip()]
|
||
if len(parts) > 2
|
||
else list(expect_mx)
|
||
)
|
||
smtp_ports = [
|
||
p.strip()
|
||
for p in (parts[3] if len(parts) > 3 else "25,465,587").split(",")
|
||
if p.strip()
|
||
]
|
||
imap_ports = [
|
||
p.strip()
|
||
for p in (parts[4] if len(parts) > 4 else "143,993").split(",")
|
||
if p.strip()
|
||
]
|
||
label = parts[5] if len(parts) > 5 else domain
|
||
rows.append(
|
||
{
|
||
"domain": domain,
|
||
"expect_mx": expect_mx,
|
||
"mail_hosts": mail_hosts,
|
||
"smtp_ports": smtp_ports,
|
||
"imap_ports": imap_ports,
|
||
"label": label,
|
||
}
|
||
)
|
||
return rows
|
||
|
||
|
||
def tcp_port_open(host: str, port: int | str, *, timeout_s: float = MAIL_PORT_TIMEOUT_S) -> bool:
|
||
try:
|
||
with socket.create_connection((_host_only(host), int(port)), timeout=max(1.0, float(timeout_s))):
|
||
return True
|
||
except (OSError, ValueError, TypeError):
|
||
return False
|
||
|
||
|
||
def git_probe_state_path() -> Path:
|
||
return monitord_state_dir() / "git-probes.json"
|
||
|
||
|
||
def deliverability_state_path() -> Path:
|
||
return monitord_state_dir() / "mail-deliverability.json"
|
||
|
||
|
||
def load_deliverability_state() -> dict[str, Any]:
|
||
path = deliverability_state_path()
|
||
if not path.is_file():
|
||
return {"schema": 1, "domains": {}}
|
||
try:
|
||
doc = json.loads(path.read_text(encoding="utf-8"))
|
||
if isinstance(doc, dict):
|
||
doc.setdefault("domains", {})
|
||
return doc
|
||
except (OSError, json.JSONDecodeError):
|
||
pass
|
||
return {"schema": 1, "domains": {}}
|
||
|
||
|
||
def save_deliverability_state(doc: dict[str, Any]) -> None:
|
||
path = deliverability_state_path()
|
||
atomic_write_json = globals().get("atomic_write_json")
|
||
# atomic_write_json is defined later; fall back to simple write if importing early
|
||
try:
|
||
from typing import cast
|
||
|
||
_ = cast(Any, None)
|
||
except Exception:
|
||
pass
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
tmp = path.with_name(f".{path.name}.tmp.{os.getpid()}")
|
||
text = json.dumps(doc, ensure_ascii=False, indent=2) + "\n"
|
||
try:
|
||
tmp.write_text(text, encoding="utf-8")
|
||
os.replace(tmp, path)
|
||
finally:
|
||
if tmp.exists():
|
||
try:
|
||
tmp.unlink()
|
||
except OSError:
|
||
pass
|
||
|
||
|
||
def _smtp_read_reply(sock: socket.socket, timeout_s: float) -> tuple[int, str]:
|
||
sock.settimeout(timeout_s)
|
||
data = b""
|
||
while True:
|
||
chunk = sock.recv(4096)
|
||
if not chunk:
|
||
break
|
||
data += chunk
|
||
text = data.decode("utf-8", "replace")
|
||
lines = text.splitlines()
|
||
if not lines:
|
||
continue
|
||
# multi-line: last line is "NNN " (space) not "NNN-"
|
||
last = lines[-1]
|
||
if len(last) >= 4 and last[0:3].isdigit() and last[3] == " ":
|
||
return int(last[0:3]), text.strip()
|
||
if len(data) > 65536:
|
||
break
|
||
text = data.decode("utf-8", "replace").strip()
|
||
code = int(text[:3]) if len(text) >= 3 and text[:3].isdigit() else 0
|
||
return code, text
|
||
|
||
|
||
def smtp_deliverability_probe(
|
||
mx_host: str,
|
||
*,
|
||
domain: str,
|
||
mail_from: str | None = None,
|
||
rcpt_to: str | None = None,
|
||
timeout_s: float = DELIVER_TIMEOUT_S,
|
||
) -> dict[str, Any]:
|
||
"""Outside-in SMTP deliverability smoke: banner → EHLO → MAIL FROM → RCPT TO → RSET/QUIT.
|
||
|
||
Does not send DATA/body. Acceptance of RCPT for postmaster@domain (or configured
|
||
recipient) counts as deliverable. Soft-fail/greylist may still return 4xx.
|
||
"""
|
||
h = _host_only(mx_host)
|
||
mail_from = mail_from or os.environ.get("TALER_MONITORD_DELIVER_FROM", "monitor@taler-monitoring.invalid")
|
||
rcpt_to = rcpt_to or os.environ.get("TALER_MONITORD_DELIVER_RCPT") or f"postmaster@{domain}"
|
||
t0 = time.monotonic()
|
||
steps: list[str] = []
|
||
try:
|
||
with socket.create_connection((h, 25), timeout=max(2.0, float(timeout_s))) as sock:
|
||
code, text = _smtp_read_reply(sock, timeout_s)
|
||
steps.append(f"banner:{code}")
|
||
if code != 220:
|
||
return {
|
||
"ok": False,
|
||
"light": "orange",
|
||
"reason": f"bad_banner:{code}",
|
||
"detail": text[:160],
|
||
"mx": h,
|
||
"rcpt": rcpt_to,
|
||
"elapsed": round(time.monotonic() - t0, 3),
|
||
"steps": steps,
|
||
}
|
||
sock.sendall(b"EHLO taler-monitoring.invalid\r\n")
|
||
code, text = _smtp_read_reply(sock, timeout_s)
|
||
steps.append(f"ehlo:{code}")
|
||
if code != 250:
|
||
return {
|
||
"ok": False,
|
||
"light": "orange",
|
||
"reason": f"ehlo_fail:{code}",
|
||
"detail": text[:160],
|
||
"mx": h,
|
||
"rcpt": rcpt_to,
|
||
"elapsed": round(time.monotonic() - t0, 3),
|
||
"steps": steps,
|
||
}
|
||
sock.sendall(f"MAIL FROM:<{mail_from}>\r\n".encode())
|
||
code, text = _smtp_read_reply(sock, timeout_s)
|
||
steps.append(f"mail:{code}")
|
||
if code not in (250, 251):
|
||
return {
|
||
"ok": False,
|
||
"light": "orange",
|
||
"reason": f"mail_from_reject:{code}",
|
||
"detail": text[:160],
|
||
"mx": h,
|
||
"rcpt": rcpt_to,
|
||
"elapsed": round(time.monotonic() - t0, 3),
|
||
"steps": steps,
|
||
}
|
||
sock.sendall(f"RCPT TO:<{rcpt_to}>\r\n".encode())
|
||
code, text = _smtp_read_reply(sock, timeout_s)
|
||
steps.append(f"rcpt:{code}")
|
||
try:
|
||
sock.sendall(b"RSET\r\n")
|
||
_smtp_read_reply(sock, min(3.0, timeout_s))
|
||
sock.sendall(b"QUIT\r\n")
|
||
except OSError:
|
||
pass
|
||
elapsed = round(time.monotonic() - t0, 3)
|
||
if code in (250, 251):
|
||
return {
|
||
"ok": True,
|
||
"light": "green",
|
||
"reason": "rcpt_accepted",
|
||
"detail": text[:160],
|
||
"mx": h,
|
||
"rcpt": rcpt_to,
|
||
"elapsed": elapsed,
|
||
"steps": steps,
|
||
}
|
||
if 400 <= code < 500:
|
||
# greylist / temp fail — soft degrade
|
||
return {
|
||
"ok": False,
|
||
"light": "yellow",
|
||
"reason": f"rcpt_tempfail:{code}",
|
||
"detail": text[:160],
|
||
"mx": h,
|
||
"rcpt": rcpt_to,
|
||
"elapsed": elapsed,
|
||
"steps": steps,
|
||
}
|
||
return {
|
||
"ok": False,
|
||
"light": "orange",
|
||
"reason": f"rcpt_reject:{code}",
|
||
"detail": text[:160],
|
||
"mx": h,
|
||
"rcpt": rcpt_to,
|
||
"elapsed": elapsed,
|
||
"steps": steps,
|
||
}
|
||
except (socket.timeout, TimeoutError):
|
||
return {
|
||
"ok": False,
|
||
"light": "orange",
|
||
"reason": "smtp_timeout",
|
||
"detail": "",
|
||
"mx": h,
|
||
"rcpt": rcpt_to,
|
||
"elapsed": round(time.monotonic() - t0, 3),
|
||
"steps": steps,
|
||
}
|
||
except OSError as e:
|
||
return {
|
||
"ok": False,
|
||
"light": "red" if "refused" in str(e).lower() else "orange",
|
||
"reason": f"smtp_connect:{type(e).__name__}",
|
||
"detail": str(e)[:120],
|
||
"mx": h,
|
||
"rcpt": rcpt_to,
|
||
"elapsed": round(time.monotonic() - t0, 3),
|
||
"steps": steps,
|
||
}
|
||
|
||
|
||
def ensure_deliverability(
|
||
domain: str,
|
||
*,
|
||
mx_hosts: list[str],
|
||
now: float | None = None,
|
||
force: bool = False,
|
||
state: dict[str, Any] | None = None,
|
||
) -> dict[str, Any]:
|
||
"""Return deliverability status for domain; refresh on interval if due.
|
||
|
||
Missing/stale results → yellow (no recent test). Send/server failures → orange/red.
|
||
"""
|
||
now = time.time() if now is None else now
|
||
if not DELIVER_ENABLED:
|
||
return {
|
||
"ok": True,
|
||
"light": "blue",
|
||
"reason": "deliver_disabled",
|
||
"ts": None,
|
||
"age_s": None,
|
||
"stale": False,
|
||
}
|
||
st = state if state is not None else load_deliverability_state()
|
||
domains = st.setdefault("domains", {})
|
||
prev = domains.get(domain) if isinstance(domains.get(domain), dict) else {}
|
||
prev_ts = prev.get("ts")
|
||
try:
|
||
age = None if prev_ts is None else now - float(prev_ts)
|
||
except (TypeError, ValueError):
|
||
age = None
|
||
due = force or prev_ts is None or age is None or age >= DELIVER_INTERVAL_S
|
||
if due and mx_hosts:
|
||
mx = _host_only(mx_hosts[0])
|
||
result = smtp_deliverability_probe(mx, domain=domain)
|
||
result["ts"] = now
|
||
result["domain"] = domain
|
||
domains[domain] = result
|
||
st["ts"] = now
|
||
try:
|
||
save_deliverability_state(st)
|
||
except OSError:
|
||
pass
|
||
prev = result
|
||
age = 0.0
|
||
elif not prev:
|
||
# never ran and no MX to probe
|
||
return {
|
||
"ok": False,
|
||
"light": "yellow",
|
||
"reason": "deliver_never_run" if mx_hosts else "deliver_no_mx",
|
||
"ts": None,
|
||
"age_s": None,
|
||
"stale": True,
|
||
"mx": "",
|
||
"rcpt": f"postmaster@{domain}",
|
||
}
|
||
# Classify freshness
|
||
try:
|
||
age_s = None if prev.get("ts") is None else now - float(prev["ts"])
|
||
except (TypeError, ValueError):
|
||
age_s = None
|
||
stale = age_s is None or age_s > DELIVER_STALE_S
|
||
light = str(prev.get("light") or "yellow")
|
||
reason = str(prev.get("reason") or "deliver_unknown")
|
||
ok = bool(prev.get("ok"))
|
||
if stale and ok:
|
||
light, reason, ok = "yellow", "deliver_stale", False
|
||
elif stale and not ok:
|
||
# keep failure colour but mark stale
|
||
if light in ("green", "blue"):
|
||
light = "yellow"
|
||
reason = f"deliver_stale:{reason}"
|
||
return {
|
||
"ok": ok and not stale,
|
||
"light": light,
|
||
"reason": reason,
|
||
"ts": prev.get("ts"),
|
||
"age_s": None if age_s is None else round(age_s, 1),
|
||
"stale": stale,
|
||
"mx": prev.get("mx") or "",
|
||
"rcpt": prev.get("rcpt") or "",
|
||
"detail": prev.get("detail") or "",
|
||
"elapsed": prev.get("elapsed"),
|
||
"steps": prev.get("steps") or [],
|
||
}
|
||
|
||
|
||
def assess_mail_domain(
|
||
row: dict[str, Any],
|
||
*,
|
||
port_done: set[str] | None = None,
|
||
deliver_state: dict[str, Any] | None = None,
|
||
now: float | None = None,
|
||
) -> dict[str, Any]:
|
||
"""Outside-in MX/SPF/DMARC/A/AAAA/rDNS (+ ports + deliverability) for one catalog domain."""
|
||
domain = row["domain"]
|
||
members: list[dict[str, Any]] = []
|
||
port_done = port_done if port_done is not None else set()
|
||
now = time.time() if now is None else now
|
||
mx = dns_mx(domain)
|
||
expect = row.get("expect_mx") or []
|
||
mx_ok = bool(mx)
|
||
mx_match = True
|
||
if expect and mx:
|
||
mx_match = any(
|
||
any(m == w or m.endswith("." + w) or w in m for m in mx) for w in expect
|
||
)
|
||
elif expect and not mx:
|
||
mx_match = False
|
||
members.append(
|
||
{
|
||
"role": "mx",
|
||
"host": ",".join(mx[:3]) or domain,
|
||
"ok": mx_ok and mx_match,
|
||
"light": "green" if (mx_ok and mx_match) else ("orange" if mx_ok else "red"),
|
||
"reason": "mx_ok" if (mx_ok and mx_match) else ("mx_mismatch" if mx_ok else "mx_missing"),
|
||
"down_since": None,
|
||
"code": 0,
|
||
"elapsed": 0.0,
|
||
"error": None,
|
||
"url": None,
|
||
"endpoints": [],
|
||
"endpoints_summary": summarize_endpoints([]),
|
||
}
|
||
)
|
||
spf = dns_txt(domain)
|
||
has_spf = "v=spf1" in spf.lower()
|
||
members.append(
|
||
{
|
||
"role": "spf",
|
||
"host": domain,
|
||
"ok": has_spf,
|
||
"light": "green" if has_spf else "yellow",
|
||
"reason": "spf_ok" if has_spf else "spf_missing",
|
||
"down_since": None,
|
||
"code": 0,
|
||
"elapsed": 0.0,
|
||
"error": None,
|
||
"url": None,
|
||
"endpoints": [],
|
||
"endpoints_summary": summarize_endpoints([]),
|
||
}
|
||
)
|
||
dmarc = dns_txt(f"_dmarc.{domain}")
|
||
has_dmarc = "v=dmarc1" in dmarc.lower()
|
||
members.append(
|
||
{
|
||
"role": "dmarc",
|
||
"host": f"_dmarc.{domain}",
|
||
"ok": has_dmarc,
|
||
"light": "green" if has_dmarc else "yellow",
|
||
"reason": "dmarc_ok" if has_dmarc else "dmarc_missing",
|
||
"down_since": None,
|
||
"code": 0,
|
||
"elapsed": 0.0,
|
||
"error": None,
|
||
"url": None,
|
||
"endpoints": [],
|
||
"endpoints_summary": summarize_endpoints([]),
|
||
}
|
||
)
|
||
# Per mail host: A/AAAA + PTR
|
||
for h in row.get("mail_hosts") or []:
|
||
addrs = dns_a_aaaa(h)
|
||
a, aaaa = addrs.get("a") or [], addrs.get("aaaa") or []
|
||
if not a and not aaaa:
|
||
members.append(
|
||
{
|
||
"role": "dns",
|
||
"host": h,
|
||
"ok": False,
|
||
"light": "red",
|
||
"reason": "dns_missing",
|
||
"down_since": None,
|
||
"code": 0,
|
||
"elapsed": 0.0,
|
||
"error": None,
|
||
"url": None,
|
||
"endpoints": [],
|
||
"endpoints_summary": summarize_endpoints([]),
|
||
}
|
||
)
|
||
continue
|
||
light = "green"
|
||
reason = "dns_ok"
|
||
if a and not aaaa:
|
||
light, reason = "yellow", "no_ipv6"
|
||
members.append(
|
||
{
|
||
"role": "dns",
|
||
"host": h,
|
||
"ok": True,
|
||
"light": light,
|
||
"reason": reason,
|
||
"down_since": None,
|
||
"code": 0,
|
||
"elapsed": 0.0,
|
||
"error": None,
|
||
"url": None,
|
||
"endpoints": [],
|
||
"endpoints_summary": summarize_endpoints([]),
|
||
"a": a[:4],
|
||
"aaaa": aaaa[:4],
|
||
}
|
||
)
|
||
# rDNS on first A
|
||
if a:
|
||
ptrs = dns_ptr(a[0])
|
||
ptr_ok = bool(ptrs)
|
||
members.append(
|
||
{
|
||
"role": "rdns",
|
||
"host": a[0],
|
||
"ok": ptr_ok,
|
||
"light": "green" if ptr_ok else "yellow",
|
||
"reason": ("rdns:" + ",".join(ptrs[:2])) if ptr_ok else "rdns_missing",
|
||
"down_since": None,
|
||
"code": 0,
|
||
"elapsed": 0.0,
|
||
"error": None,
|
||
"url": None,
|
||
"endpoints": [],
|
||
"endpoints_summary": summarize_endpoints([]),
|
||
}
|
||
)
|
||
if MAIL_PORTS_ENABLED and h not in port_done:
|
||
port_done.add(h)
|
||
for p in row.get("smtp_ports") or []:
|
||
open_ok = tcp_port_open(h, p)
|
||
# 587 often filtered on pixel → yellow warn, not red
|
||
if open_ok:
|
||
light_p, reason_p = "green", f"smtp_{p}_open"
|
||
elif str(p) == "587":
|
||
light_p, reason_p = "yellow", f"smtp_{p}_closed"
|
||
else:
|
||
light_p, reason_p = "orange", f"smtp_{p}_closed"
|
||
members.append(
|
||
{
|
||
"role": "smtp",
|
||
"host": f"{h}:{p}",
|
||
"ok": open_ok,
|
||
"light": light_p,
|
||
"reason": reason_p,
|
||
"down_since": None,
|
||
"code": int(p) if str(p).isdigit() else 0,
|
||
"elapsed": 0.0,
|
||
"error": None,
|
||
"url": None,
|
||
"endpoints": [],
|
||
"endpoints_summary": summarize_endpoints([]),
|
||
}
|
||
)
|
||
for p in row.get("imap_ports") or []:
|
||
open_ok = tcp_port_open(h, p)
|
||
members.append(
|
||
{
|
||
"role": "imap",
|
||
"host": f"{h}:{p}",
|
||
"ok": open_ok,
|
||
"light": "green" if open_ok else "orange",
|
||
"reason": f"imap_{p}_open" if open_ok else f"imap_{p}_closed",
|
||
"down_since": None,
|
||
"code": int(p) if str(p).isdigit() else 0,
|
||
"elapsed": 0.0,
|
||
"error": None,
|
||
"url": None,
|
||
"endpoints": [],
|
||
"endpoints_summary": summarize_endpoints([]),
|
||
}
|
||
)
|
||
elif MAIL_PORTS_ENABLED and h in port_done:
|
||
members.append(
|
||
{
|
||
"role": "smtp",
|
||
"host": h,
|
||
"ok": True,
|
||
"light": "blue",
|
||
"reason": "ports_probed_earlier",
|
||
"down_since": None,
|
||
"code": 0,
|
||
"elapsed": 0.0,
|
||
"error": None,
|
||
"url": None,
|
||
"endpoints": [],
|
||
"endpoints_summary": summarize_endpoints([]),
|
||
}
|
||
)
|
||
# Deliverability (regular, cached) — missing/stale → yellow; send fail → orange/red
|
||
mx_targets = list(mx) or list(row.get("mail_hosts") or [])
|
||
deliver = ensure_deliverability(
|
||
domain, mx_hosts=mx_targets, now=now, state=deliver_state
|
||
)
|
||
members.append(
|
||
{
|
||
"role": "deliver",
|
||
"host": deliver.get("mx") or (mx_targets[0] if mx_targets else domain),
|
||
"ok": bool(deliver.get("ok")),
|
||
"light": deliver.get("light") or "yellow",
|
||
"reason": deliver.get("reason") or "deliver_unknown",
|
||
"down_since": None,
|
||
"code": 0,
|
||
"elapsed": float(deliver.get("elapsed") or 0.0),
|
||
"error": deliver.get("detail") or None,
|
||
"url": None,
|
||
"endpoints": [],
|
||
"endpoints_summary": summarize_endpoints([]),
|
||
}
|
||
)
|
||
light, intensity = aggregate_stack(members)
|
||
software = colorize_software(
|
||
[
|
||
item
|
||
for h in (row.get("mail_hosts") or [])[:2]
|
||
for item in smtp_banner_software(h)
|
||
],
|
||
published=None,
|
||
)
|
||
return {
|
||
"id": domain,
|
||
"currency": "",
|
||
"currency_info": None,
|
||
"kind": "mail",
|
||
"service_label": row.get("label") or domain,
|
||
"commercial_prod": False,
|
||
"bank_backend": "",
|
||
"wire_methods": [],
|
||
"local": False,
|
||
"light": light,
|
||
"intensity": intensity,
|
||
"members": members,
|
||
"stats": None,
|
||
"software": software,
|
||
"deliverability": deliver,
|
||
"network": {"zone": "mail", "label": "mail plane", "ssh": {"expected": False}},
|
||
"vulns": {"enabled": False, "worst": "", "light": None, "hits": [], "servers": []},
|
||
"breakdown": aggregate_breakdown(members),
|
||
"tls": {},
|
||
"orphan": False,
|
||
}
|
||
|
||
|
||
def _run_git_ls_remote(url: str, *, timeout_s: float) -> dict[str, Any]:
|
||
"""Soft git probe via `git ls-remote` (no checkout; no leftover worktree)."""
|
||
t0 = time.monotonic()
|
||
env = os.environ.copy()
|
||
env["GIT_TERMINAL_PROMPT"] = "0"
|
||
env["GIT_SSH_COMMAND"] = (
|
||
"ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new "
|
||
"-o ConnectTimeout=5 -o IdentitiesOnly=yes"
|
||
)
|
||
try:
|
||
r = subprocess.run(
|
||
["git", "ls-remote", url],
|
||
capture_output=True,
|
||
timeout=max(2.0, float(timeout_s)),
|
||
check=False,
|
||
env=env,
|
||
)
|
||
elapsed = round(time.monotonic() - t0, 3)
|
||
out = (r.stdout or b"").decode("utf-8", "replace").strip()
|
||
err = (r.stderr or b"").decode("utf-8", "replace").strip()[:200]
|
||
cmd = f"git ls-remote {url}"
|
||
if r.returncode == 0:
|
||
return {
|
||
"ok": True,
|
||
"light": "green",
|
||
"reason": "ls-remote_ok",
|
||
"elapsed": elapsed,
|
||
"detail": (out.splitlines()[0][:120] if out else "empty_list"),
|
||
"command": cmd,
|
||
}
|
||
# Auth / no key / unreachable → soft (yellow); keep orange for hard refuse
|
||
soft = (
|
||
r.returncode in (2, 128)
|
||
or "Permission denied" in err
|
||
or "Host key" in err
|
||
or "Could not resolve" in err
|
||
or "unable to access" in err.lower()
|
||
)
|
||
return {
|
||
"ok": False,
|
||
"light": "yellow" if soft else "orange",
|
||
"reason": err or f"ls-remote_rc_{r.returncode}",
|
||
"elapsed": elapsed,
|
||
"detail": "",
|
||
"command": cmd,
|
||
}
|
||
except FileNotFoundError:
|
||
return {
|
||
"ok": False,
|
||
"light": "blue",
|
||
"reason": "git_not_installed",
|
||
"elapsed": round(time.monotonic() - t0, 3),
|
||
"detail": "",
|
||
"command": f"git ls-remote {url}",
|
||
}
|
||
except subprocess.TimeoutExpired:
|
||
return {
|
||
"ok": False,
|
||
"light": "yellow",
|
||
"reason": "ls-remote_timeout",
|
||
"elapsed": round(time.monotonic() - t0, 3),
|
||
"detail": "",
|
||
"command": f"git ls-remote {url}",
|
||
}
|
||
|
||
|
||
def load_git_probe_state() -> dict[str, Any]:
|
||
path = git_probe_state_path()
|
||
if not path.is_file():
|
||
return {"schema": 1, "hosts": {}}
|
||
try:
|
||
doc = json.loads(path.read_text(encoding="utf-8"))
|
||
if isinstance(doc, dict):
|
||
doc.setdefault("hosts", {})
|
||
return doc
|
||
except (OSError, json.JSONDecodeError):
|
||
pass
|
||
return {"schema": 1, "hosts": {}}
|
||
|
||
|
||
def save_git_probe_state(doc: dict[str, Any]) -> None:
|
||
path = git_probe_state_path()
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
tmp = path.with_name(f".{path.name}.tmp.{os.getpid()}")
|
||
try:
|
||
tmp.write_text(json.dumps(doc, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
os.replace(tmp, path)
|
||
finally:
|
||
if tmp.exists():
|
||
try:
|
||
tmp.unlink()
|
||
except OSError:
|
||
pass
|
||
|
||
|
||
def collect_git_probe(
|
||
host: str,
|
||
*,
|
||
now: float | None = None,
|
||
force: bool = False,
|
||
) -> dict[str, Any]:
|
||
"""Infrequent HTTPS+SSH git ls-remote; cached under monitord state."""
|
||
now = time.time() if now is None else float(now)
|
||
host = _host_only(host)
|
||
empty = {
|
||
"enabled": False,
|
||
"ok": None,
|
||
"light": "blue",
|
||
"reason": "git_probes_disabled",
|
||
"https": None,
|
||
"ssh": None,
|
||
"age_s": None,
|
||
"stale": False,
|
||
}
|
||
if not GIT_PROBE_ENABLED or not host:
|
||
return empty
|
||
state = load_git_probe_state()
|
||
prev = (state.get("hosts") or {}).get(host) or {}
|
||
prev_ts = prev.get("ts")
|
||
try:
|
||
age = None if prev_ts is None else now - float(prev_ts)
|
||
except (TypeError, ValueError):
|
||
age = None
|
||
due = force or prev_ts is None or age is None or age >= GIT_PROBE_INTERVAL_S
|
||
if not due and prev:
|
||
stale = age is None or age > GIT_PROBE_STALE_S
|
||
light = str(prev.get("light") or "green")
|
||
if stale and light == "green":
|
||
light = "yellow"
|
||
return {
|
||
"enabled": True,
|
||
"ok": bool(prev.get("ok")),
|
||
"light": light,
|
||
"reason": ("git_stale:" if stale else "") + str(prev.get("reason") or ""),
|
||
"https": prev.get("https"),
|
||
"ssh": prev.get("ssh"),
|
||
"age_s": age,
|
||
"stale": stale,
|
||
"ts": prev_ts,
|
||
}
|
||
https = _run_git_ls_remote(f"https://{host}/", timeout_s=GIT_PROBE_TIMEOUT_S)
|
||
ssh = _run_git_ls_remote(f"git@{host}:", timeout_s=GIT_PROBE_TIMEOUT_S)
|
||
# Aggregate: worst soft colour; both fail → orange max for soft probe
|
||
lights = [str(https.get("light") or "yellow"), str(ssh.get("light") or "yellow")]
|
||
worst = lights[0]
|
||
for Lgt in lights[1:]:
|
||
if SEVERITY.get(Lgt, 0) > SEVERITY.get(worst, 0):
|
||
worst = Lgt
|
||
if worst in ("red", "black"):
|
||
worst = "orange"
|
||
ok = bool(https.get("ok") or ssh.get("ok"))
|
||
if ok and worst in ("orange", "red"):
|
||
worst = "yellow" # one plane OK → soft at worst
|
||
reason = "git_ok" if ok else "git_unreachable"
|
||
if https.get("ok") and not ssh.get("ok"):
|
||
reason = "git_https_ok_ssh_soft"
|
||
elif ssh.get("ok") and not https.get("ok"):
|
||
reason = "git_ssh_ok_https_soft"
|
||
entry = {
|
||
"ts": now,
|
||
"ok": ok,
|
||
"light": worst if not ok else "green",
|
||
"reason": reason,
|
||
"https": https,
|
||
"ssh": ssh,
|
||
}
|
||
state.setdefault("hosts", {})[host] = entry
|
||
save_git_probe_state(state)
|
||
return {
|
||
"enabled": True,
|
||
"ok": ok,
|
||
"light": entry["light"],
|
||
"reason": reason,
|
||
"https": https,
|
||
"ssh": ssh,
|
||
"age_s": 0.0,
|
||
"stale": False,
|
||
"ts": now,
|
||
}
|
||
|
||
|
||
def payability_state_path() -> Path:
|
||
return monitord_state_dir() / "payability.json"
|
||
|
||
|
||
def load_payability_state() -> dict[str, Any]:
|
||
path = payability_state_path()
|
||
if not path.is_file():
|
||
return {"schema": 1, "domains": {}}
|
||
try:
|
||
doc = json.loads(path.read_text(encoding="utf-8"))
|
||
if isinstance(doc, dict):
|
||
doc.setdefault("domains", {})
|
||
return doc
|
||
except (OSError, json.JSONDecodeError):
|
||
pass
|
||
return {"schema": 1, "domains": {}}
|
||
|
||
|
||
def save_payability_state(doc: dict[str, Any]) -> None:
|
||
path = payability_state_path()
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
tmp = path.with_name(f".{path.name}.tmp.{os.getpid()}")
|
||
try:
|
||
tmp.write_text(json.dumps(doc, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
os.replace(tmp, path)
|
||
finally:
|
||
if tmp.exists():
|
||
try:
|
||
tmp.unlink()
|
||
except OSError:
|
||
pass
|
||
|
||
|
||
def _suite_root() -> Path:
|
||
raw = (os.environ.get("SUITE_DIR") or "").strip()
|
||
if raw:
|
||
return Path(raw)
|
||
return Path(__file__).resolve().parents[1]
|
||
|
||
|
||
def _run_zero_pay(domain: str, *, timeout_s: float) -> dict[str, Any]:
|
||
"""Run suite zero-pay COUNT=1 for domain (needs WALLET_DB / funded wallet)."""
|
||
root = _suite_root()
|
||
script = root / "taler-monitoring.sh"
|
||
cmd = [
|
||
"bash",
|
||
str(script),
|
||
"-d",
|
||
domain,
|
||
"zero-pay",
|
||
]
|
||
env = os.environ.copy()
|
||
env["COUNT"] = "1"
|
||
env["STRICT_EXIT"] = env.get("STRICT_EXIT", "1")
|
||
t0 = time.monotonic()
|
||
try:
|
||
r = subprocess.run(
|
||
cmd,
|
||
capture_output=True,
|
||
timeout=max(30.0, float(timeout_s)),
|
||
check=False,
|
||
env=env,
|
||
cwd=str(root),
|
||
)
|
||
elapsed = round(time.monotonic() - t0, 3)
|
||
err = (r.stderr or b"").decode("utf-8", "replace").strip()[-300:]
|
||
out = (r.stdout or b"").decode("utf-8", "replace").strip()[-300:]
|
||
ok = r.returncode == 0
|
||
return {
|
||
"ok": ok,
|
||
"light": "green" if ok else "orange",
|
||
"reason": "zero_pay_ok" if ok else (err or out or f"zero_pay_rc_{r.returncode}"),
|
||
"elapsed": elapsed,
|
||
"command": f"COUNT=1 ./taler-monitoring.sh -d {domain} zero-pay",
|
||
"rc": r.returncode,
|
||
}
|
||
except FileNotFoundError:
|
||
return {
|
||
"ok": False,
|
||
"light": "blue",
|
||
"reason": "suite_missing",
|
||
"elapsed": 0.0,
|
||
"command": f"COUNT=1 ./taler-monitoring.sh -d {domain} zero-pay",
|
||
"rc": 127,
|
||
}
|
||
except subprocess.TimeoutExpired:
|
||
return {
|
||
"ok": False,
|
||
"light": "orange",
|
||
"reason": "zero_pay_timeout",
|
||
"elapsed": round(time.monotonic() - t0, 3),
|
||
"command": f"COUNT=1 ./taler-monitoring.sh -d {domain} zero-pay",
|
||
"rc": 124,
|
||
}
|
||
|
||
|
||
def collect_payability(
|
||
domain: str,
|
||
*,
|
||
now: float | None = None,
|
||
force: bool = False,
|
||
) -> dict[str, Any]:
|
||
"""Cached payability smoke for a stack domain (few×/day when PAY_RUN=1)."""
|
||
now = time.time() if now is None else float(now)
|
||
domain = str(domain or "").strip()
|
||
empty = {
|
||
"enabled": False,
|
||
"ok": None,
|
||
"light": "blue",
|
||
"reason": "pay_disabled",
|
||
"age_s": None,
|
||
"stale": False,
|
||
"command": "",
|
||
}
|
||
if not PAY_ENABLED or not domain:
|
||
return empty
|
||
allowed = set(PAY_DOMAINS)
|
||
if domain not in allowed and not any(
|
||
domain == d or domain.endswith("." + d) or d.endswith("." + domain)
|
||
for d in allowed
|
||
):
|
||
return {**empty, "reason": "pay_domain_skip"}
|
||
state = load_payability_state()
|
||
prev = (state.get("domains") or {}).get(domain) or {}
|
||
prev_ts = prev.get("ts")
|
||
try:
|
||
age = None if prev_ts is None else now - float(prev_ts)
|
||
except (TypeError, ValueError):
|
||
age = None
|
||
due = force or prev_ts is None or age is None or age >= PAY_INTERVAL_S
|
||
if not due and prev:
|
||
stale = age is None or age > PAY_STALE_S
|
||
light = str(prev.get("light") or "green")
|
||
if stale and light == "green":
|
||
light = "yellow"
|
||
return {
|
||
"enabled": True,
|
||
"ok": bool(prev.get("ok")),
|
||
"light": light,
|
||
"reason": ("pay_stale:" if stale else "") + str(prev.get("reason") or ""),
|
||
"age_s": age,
|
||
"stale": stale,
|
||
"command": prev.get("command") or "",
|
||
"ts": prev_ts,
|
||
}
|
||
if not PAY_RUN:
|
||
# enabled but operator did not allow live wallet run this tick
|
||
if prev:
|
||
stale = age is None or age > PAY_STALE_S
|
||
light = str(prev.get("light") or "yellow")
|
||
if stale:
|
||
light = "yellow"
|
||
return {
|
||
"enabled": True,
|
||
"ok": bool(prev.get("ok")),
|
||
"light": light,
|
||
"reason": ("pay_stale:" if stale else "") + str(prev.get("reason") or "pay_cached"),
|
||
"age_s": age,
|
||
"stale": bool(stale),
|
||
"command": prev.get("command") or "",
|
||
"ts": prev_ts,
|
||
}
|
||
return {
|
||
"enabled": True,
|
||
"ok": False,
|
||
"light": "yellow",
|
||
"reason": "pay_never_run",
|
||
"age_s": None,
|
||
"stale": True,
|
||
"command": f"COUNT=1 ./taler-monitoring.sh -d {domain} zero-pay",
|
||
"ts": None,
|
||
}
|
||
result = _run_zero_pay(domain, timeout_s=PAY_TIMEOUT_S)
|
||
entry = {
|
||
"ts": now,
|
||
"ok": bool(result.get("ok")),
|
||
"light": result.get("light") or "orange",
|
||
"reason": result.get("reason") or "",
|
||
"command": result.get("command") or "",
|
||
"elapsed": result.get("elapsed"),
|
||
"rc": result.get("rc"),
|
||
}
|
||
state.setdefault("domains", {})[domain] = entry
|
||
save_payability_state(state)
|
||
return {
|
||
"enabled": True,
|
||
"ok": entry["ok"],
|
||
"light": entry["light"],
|
||
"reason": entry["reason"],
|
||
"age_s": 0.0,
|
||
"stale": False,
|
||
"command": entry["command"],
|
||
"ts": now,
|
||
}
|
||
|
||
|
||
def build_service_status(
|
||
row: dict[str, Any],
|
||
*,
|
||
prev_hosts: dict[str, dict[str, Any]],
|
||
now: float,
|
||
acute_s: float,
|
||
probe_fn: Callable[[str], dict[str, Any]],
|
||
osv_cache: dict[str, list],
|
||
published: dict[str, Any] | None = None,
|
||
) -> dict[str, Any]:
|
||
"""Probe a non-Taler company/service host (apex HTTPS)."""
|
||
host = row["host"]
|
||
url = f"https://{host}/"
|
||
raw = probe_fn(url)
|
||
classified = classify_probe(raw, prev=prev_hosts.get(host), now=now, acute_s=acute_s)
|
||
member = {
|
||
"role": "site",
|
||
"host": host,
|
||
**classified,
|
||
"endpoints": [],
|
||
"endpoints_summary": summarize_endpoints([]),
|
||
}
|
||
members = [member]
|
||
tls: dict[str, Any] = {}
|
||
if CERT_PROBES_ENABLED:
|
||
tls = probe_tls_cert(host)
|
||
if tls.get("light") and SEVERITY.get(str(tls["light"]), 0) > SEVERITY.get(
|
||
member.get("light") or "green", 0
|
||
):
|
||
members.append(
|
||
{
|
||
"role": "tls",
|
||
"host": host,
|
||
"ok": bool(tls.get("ok")),
|
||
"light": tls.get("light") or "yellow",
|
||
"reason": tls.get("reason") or "cert",
|
||
"down_since": None,
|
||
"code": 0,
|
||
"elapsed": 0.0,
|
||
"error": tls.get("error"),
|
||
"url": None,
|
||
"endpoints": [],
|
||
"endpoints_summary": summarize_endpoints([]),
|
||
}
|
||
)
|
||
label = str(row.get("service_label") or "")
|
||
git_probe: dict[str, Any] | None = None
|
||
if label.lower() == "git" or host.startswith("git."):
|
||
git_probe = collect_git_probe(host, now=now)
|
||
if git_probe.get("enabled"):
|
||
members.append(
|
||
{
|
||
"role": "git",
|
||
"host": host,
|
||
"ok": bool(git_probe.get("ok")),
|
||
"light": git_probe.get("light") or "yellow",
|
||
"reason": git_probe.get("reason") or "git",
|
||
"down_since": None,
|
||
"code": 0,
|
||
"elapsed": 0.0,
|
||
"error": None,
|
||
"url": None,
|
||
"endpoints": [],
|
||
"endpoints_summary": summarize_endpoints([]),
|
||
"command": ((git_probe.get("https") or {}).get("command") or ""),
|
||
}
|
||
)
|
||
light, intensity = aggregate_stack(members)
|
||
probe_cache = {url: raw}
|
||
vulns = assess_external_vulns(
|
||
probe_cache=probe_cache, members=members, osv_cache=osv_cache
|
||
)
|
||
light, intensity, members = apply_vuln_to_stack(light, intensity, members, vulns)
|
||
software = collect_software_from_probes(
|
||
probe_cache=probe_cache,
|
||
members=members,
|
||
vulns=vulns,
|
||
smtp=False,
|
||
published=published,
|
||
)
|
||
return {
|
||
"id": row.get("id") or host,
|
||
"currency": "",
|
||
"currency_info": None,
|
||
"kind": "service",
|
||
"service_label": row.get("service_label") or "service",
|
||
"commercial_prod": False,
|
||
"bank_backend": "",
|
||
"wire_methods": [],
|
||
"local": False,
|
||
"light": light,
|
||
"intensity": intensity,
|
||
"members": members,
|
||
"stats": None,
|
||
"software": software,
|
||
"git": git_probe or {},
|
||
"network": {
|
||
"zone": row.get("zone") or "public",
|
||
"label": row.get("zone") or "public",
|
||
"ssh": {"expected": False},
|
||
},
|
||
"vulns": {
|
||
"enabled": vulns.get("enabled"),
|
||
"worst": vulns.get("worst") or "",
|
||
"light": vulns.get("light"),
|
||
"hits": [
|
||
{
|
||
"id": h.get("id"),
|
||
"severity": h.get("severity"),
|
||
"package": h.get("package"),
|
||
"version": h.get("version"),
|
||
"host": h.get("host"),
|
||
"summary": h.get("summary") or "",
|
||
}
|
||
for h in (vulns.get("hits") or [])[:8]
|
||
],
|
||
"servers": vulns.get("servers") or [],
|
||
},
|
||
"breakdown": aggregate_breakdown(members),
|
||
"tls": {
|
||
"days_left": tls.get("days_left"),
|
||
"not_after": tls.get("not_after"),
|
||
"sans": tls.get("sans") or [],
|
||
"reason": tls.get("reason"),
|
||
}
|
||
if tls
|
||
else {},
|
||
"orphan": False,
|
||
}
|
||
|
||
|
||
def _host_covered_by_known(host: str, known_hosts: set[str]) -> bool:
|
||
"""True if host is known or a subdomain/parent of a monitored host (cert sibling)."""
|
||
h = _host_only(host).lower()
|
||
if not h or h in known_hosts:
|
||
return True
|
||
for k in known_hosts:
|
||
k = (k or "").lower()
|
||
if not k or k.upper() == "IBAN":
|
||
continue
|
||
if h.endswith("." + k) or k.endswith("." + h):
|
||
return True
|
||
return False
|
||
|
||
|
||
def collect_orphan_hosts(
|
||
*,
|
||
known_hosts: set[str],
|
||
tls_sans: list[str],
|
||
probe_fn: Callable[[str], dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
"""Flag SANs that look abandoned: DNS missing (default).
|
||
|
||
Skips wildcard expansions already covered as siblings of monitored hosts
|
||
(e.g. donations.demo.taler.net when demo.taler.net is known). HTTP probe
|
||
optional via TALER_MONITORD_ORPHAN_HTTP=1 (off by default — CDN noise).
|
||
"""
|
||
if not ORPHAN_PROBES_ENABLED:
|
||
return []
|
||
known = {_host_only(x).lower() for x in known_hosts if x}
|
||
candidates: list[str] = []
|
||
for raw_san in tls_sans:
|
||
san = str(raw_san or "").strip().lower()
|
||
if not san or san.startswith("*."):
|
||
# Wildcard SAN ≠ concrete orphan candidate
|
||
continue
|
||
h = _host_only(san)
|
||
if not h or _host_covered_by_known(h, known):
|
||
continue
|
||
if h not in candidates:
|
||
candidates.append(h)
|
||
orphans: list[dict[str, Any]] = []
|
||
for h in candidates[:25]:
|
||
addrs = dns_a_aaaa(h)
|
||
if not (addrs.get("a") or addrs.get("aaaa")):
|
||
orphans.append({"host": h, "reason": "dns_missing", "light": "orange"})
|
||
continue
|
||
if not ORPHAN_HTTP_ENABLED:
|
||
continue
|
||
raw = probe_fn(f"https://{h}/")
|
||
if not raw.get("ok") and int(raw.get("code") or 0) == 0:
|
||
orphans.append(
|
||
{
|
||
"host": h,
|
||
"reason": f"http_unreachable:{raw.get('error')}",
|
||
"light": "yellow",
|
||
}
|
||
)
|
||
return orphans
|
||
|
||
|
||
def scan_ports_list() -> list[int]:
|
||
raw = (os.environ.get("TALER_MONITORD_PORTSCAN_PORTS") or "").strip()
|
||
if raw:
|
||
out: list[int] = []
|
||
for part in re.split(r"[\s,]+", raw):
|
||
if not part:
|
||
continue
|
||
try:
|
||
out.append(int(part))
|
||
except ValueError:
|
||
continue
|
||
return sorted(set(out))
|
||
return list(_DEFAULT_SCAN_PORTS)
|
||
|
||
|
||
def parse_ports_whitelist(path: Path | None = None) -> list[dict[str, Any]]:
|
||
"""Whitelist lines: host_or_pattern port[,port…] [# note]
|
||
|
||
Patterns: exact host, ``*.example.com``, or ``*`` (any host).
|
||
"""
|
||
path = path or default_ports_whitelist_conf()
|
||
rows: list[dict[str, Any]] = []
|
||
if not path.is_file():
|
||
return rows
|
||
for raw in path.read_text(encoding="utf-8").splitlines():
|
||
line = raw.split("#", 1)[0].strip()
|
||
if not line:
|
||
continue
|
||
parts = line.split()
|
||
if len(parts) < 2:
|
||
continue
|
||
pattern = parts[0].lower().rstrip(".")
|
||
ports: list[int] = []
|
||
for tok in parts[1].split(","):
|
||
tok = tok.strip()
|
||
if not tok:
|
||
continue
|
||
try:
|
||
ports.append(int(tok))
|
||
except ValueError:
|
||
continue
|
||
if ports:
|
||
rows.append({"pattern": pattern, "ports": ports})
|
||
return rows
|
||
|
||
|
||
def whitelist_ports_for_host(host: str, rules: list[dict[str, Any]]) -> set[int]:
|
||
h = _host_only(host).lower()
|
||
allowed: set[int] = set()
|
||
for rule in rules or []:
|
||
pat = str(rule.get("pattern") or "").lower()
|
||
ports = set(int(p) for p in (rule.get("ports") or []))
|
||
if not pat or not ports:
|
||
continue
|
||
if pat == "*" or pat == h:
|
||
allowed |= ports
|
||
elif pat.startswith("*.") and (h.endswith(pat[1:]) or h == pat[2:]):
|
||
allowed |= ports
|
||
return allowed
|
||
|
||
|
||
def baseline_ports_for_host(host: str, *, mail_hosts: set[str] | None = None) -> set[int]:
|
||
"""Default expected open ports before whitelist."""
|
||
h = _host_only(host).lower()
|
||
allowed = set(_BASELINE_WEB_PORTS)
|
||
if mail_hosts and h in mail_hosts:
|
||
allowed |= set(_BASELINE_MAIL_PORTS)
|
||
return allowed
|
||
|
||
|
||
def tcp_scan_host(
|
||
host: str,
|
||
ports: list[int],
|
||
*,
|
||
timeout_s: float = PORT_SCAN_TIMEOUT_S,
|
||
workers: int = PORT_SCAN_WORKERS,
|
||
) -> list[int]:
|
||
"""Return sorted list of open TCP ports (banner-less connect)."""
|
||
h = _host_only(host)
|
||
if not h or h.upper() == "IBAN":
|
||
return []
|
||
open_ports: list[int] = []
|
||
|
||
def _one(p: int) -> int | None:
|
||
return p if tcp_port_open(h, p, timeout_s=timeout_s) else None
|
||
|
||
with ThreadPoolExecutor(max_workers=max(4, min(int(workers), len(ports) or 4))) as pool:
|
||
futs = [pool.submit(_one, p) for p in ports]
|
||
for fut in as_completed(futs):
|
||
try:
|
||
got = fut.result()
|
||
except Exception:
|
||
got = None
|
||
if got is not None:
|
||
open_ports.append(int(got))
|
||
return sorted(set(open_ports))
|
||
|
||
|
||
def assess_host_ports(
|
||
host: str,
|
||
*,
|
||
ports: list[int] | None = None,
|
||
whitelist_rules: list[dict[str, Any]] | None = None,
|
||
mail_hosts: set[str] | None = None,
|
||
open_ports: list[int] | None = None,
|
||
) -> dict[str, Any]:
|
||
"""Scan (or reuse) open ports and classify expected vs suspicious."""
|
||
h = _host_only(host)
|
||
ports = ports if ports is not None else scan_ports_list()
|
||
rules = whitelist_rules if whitelist_rules is not None else parse_ports_whitelist()
|
||
opened = list(open_ports) if open_ports is not None else tcp_scan_host(h, ports)
|
||
allowed = baseline_ports_for_host(h, mail_hosts=mail_hosts) | whitelist_ports_for_host(h, rules)
|
||
suspicious = [p for p in opened if p not in allowed]
|
||
expected_hit = [p for p in opened if p in allowed]
|
||
# hard = raise card colour; soft = list only (SSH/mail/http-alt often shared).
|
||
high = {21, 23, 135, 139, 445, 1433, 1521, 2049, 3306, 3389, 5432, 5900, 6379, 9200, 11211, 27017}
|
||
soft_set = {22, 25, 110, 143, 465, 587, 993, 995, 8080, 8443}
|
||
items: list[dict[str, Any]] = []
|
||
for p in suspicious:
|
||
if p in high:
|
||
sev = "hard"
|
||
pl = "red" if p in (23, 445, 3389, 27017) else "orange"
|
||
else:
|
||
sev, pl = "soft", "yellow"
|
||
if p not in soft_set:
|
||
sev, pl = "soft", "yellow"
|
||
items.append({"port": p, "label": _PORT_LABELS.get(p, str(p)), "severity": sev, "light": pl})
|
||
light = "green"
|
||
for it in items:
|
||
if SEVERITY.get(str(it["light"]), 0) > SEVERITY.get(light, 0):
|
||
light = str(it["light"])
|
||
return {
|
||
"host": h,
|
||
"scanned": list(ports),
|
||
"open": opened,
|
||
"allowed": sorted(allowed),
|
||
"expected_open": expected_hit,
|
||
"suspicious": items,
|
||
"light": light,
|
||
"ok": not any(it.get("severity") == "hard" for it in items),
|
||
}
|
||
|
||
|
||
def collect_port_scan(
|
||
hosts: list[str],
|
||
*,
|
||
mail_hosts: set[str] | None = None,
|
||
whitelist_rules: list[dict[str, Any]] | None = None,
|
||
) -> dict[str, Any]:
|
||
"""Scan unique hosts; return page-level findings."""
|
||
if not PORT_SCAN_ENABLED:
|
||
return {"enabled": False, "hosts": [], "suspicious": [], "light": None}
|
||
ports = scan_ports_list()
|
||
rules = whitelist_rules if whitelist_rules is not None else parse_ports_whitelist()
|
||
mail_hosts = { _host_only(x).lower() for x in (mail_hosts or set()) }
|
||
uniq: list[str] = []
|
||
seen: set[str] = set()
|
||
for h in hosts:
|
||
hh = _host_only(h).lower()
|
||
if not hh or hh.upper() == "IBAN" or hh in seen:
|
||
continue
|
||
# Skip pure IPs from rdns rows unless explicitly wanted
|
||
if re.match(r"^\d+\.\d+\.\d+\.\d+$", hh):
|
||
continue
|
||
seen.add(hh)
|
||
uniq.append(hh)
|
||
host_results: list[dict[str, Any]] = []
|
||
# Scan hosts concurrently (each host scans ports internally)
|
||
def _scan(h: str) -> dict[str, Any]:
|
||
return assess_host_ports(h, ports=ports, whitelist_rules=rules, mail_hosts=mail_hosts)
|
||
|
||
with ThreadPoolExecutor(max_workers=max(4, min(12, len(uniq) or 4))) as pool:
|
||
futs = {pool.submit(_scan, h): h for h in uniq}
|
||
for fut in as_completed(futs):
|
||
try:
|
||
host_results.append(fut.result())
|
||
except Exception as e:
|
||
host_results.append(
|
||
{
|
||
"host": futs[fut],
|
||
"scanned": ports,
|
||
"open": [],
|
||
"allowed": [],
|
||
"expected_open": [],
|
||
"suspicious": [],
|
||
"light": "yellow",
|
||
"ok": True,
|
||
"error": type(e).__name__,
|
||
}
|
||
)
|
||
host_results.sort(key=lambda r: r.get("host") or "")
|
||
suspicious_flat: list[dict[str, Any]] = []
|
||
worst = "green"
|
||
for r in host_results:
|
||
for s in r.get("suspicious") or []:
|
||
suspicious_flat.append(
|
||
{
|
||
"host": r.get("host"),
|
||
"port": s.get("port"),
|
||
"label": s.get("label"),
|
||
"severity": s.get("severity") or "soft",
|
||
"light": s.get("light") or "yellow",
|
||
}
|
||
)
|
||
Lgt = r.get("light") or "green"
|
||
if SEVERITY.get(str(Lgt), 0) > SEVERITY.get(worst, 0):
|
||
worst = str(Lgt)
|
||
hard = [s for s in suspicious_flat if s.get("severity") == "hard"]
|
||
if not suspicious_flat:
|
||
worst = "green"
|
||
elif not hard:
|
||
worst = "yellow"
|
||
return {
|
||
"enabled": True,
|
||
"ports": ports,
|
||
"hosts": host_results,
|
||
"suspicious": suspicious_flat,
|
||
"n_hosts": len(host_results),
|
||
"n_suspicious": len(suspicious_flat),
|
||
"n_hard": len(hard),
|
||
"light": worst if suspicious_flat else "green",
|
||
}
|
||
|
||
|
||
def build_stack_status(
|
||
stacks: list[dict[str, Any]],
|
||
*,
|
||
prev_doc: dict[str, Any] | None = None,
|
||
now: float | None = None,
|
||
acute_s: float = ACUTE_DOWN_S,
|
||
probe_fn: Callable[[str], dict[str, Any]] | None = None,
|
||
public_only: bool = True,
|
||
) -> dict[str, Any]:
|
||
"""Probe all stacks and build status document."""
|
||
now = time.time() if now is None else now
|
||
prev_doc = prev_doc or {}
|
||
prev_hosts: dict[str, dict[str, Any]] = {}
|
||
for s in prev_doc.get("stacks") or []:
|
||
for m in s.get("members") or []:
|
||
h = m.get("host")
|
||
if h:
|
||
prev_hosts[h] = m
|
||
|
||
probe = probe_fn or (lambda url: probe_http(url))
|
||
out_stacks: list[dict[str, Any]] = []
|
||
ssh_cache: dict[str, dict[str, Any]] = {}
|
||
osv_cache: dict[str, list] = {}
|
||
published_doc = fetch_published_taler_packages()
|
||
# Previous endpoint rows keyed by host+path for down_since continuity
|
||
prev_eps: dict[str, dict[str, Any]] = {}
|
||
for s in prev_doc.get("stacks") or []:
|
||
for m in s.get("members") or []:
|
||
h = m.get("host") or ""
|
||
for ep in m.get("endpoints") or []:
|
||
p = ep.get("path")
|
||
if h and p:
|
||
prev_eps[f"{h}{p}"] = ep
|
||
|
||
for st in stacks:
|
||
if public_only and st.get("local"):
|
||
continue
|
||
members_out: list[dict[str, Any]] = []
|
||
kind = st.get("kind") or ("taler" if st.get("currency") else "static_pending")
|
||
has_intro = stack_has_custom_intro(st)
|
||
legal_sev = legal_doc_severity(st)
|
||
bank_configured = st.get("bank_configured")
|
||
if bank_configured is None:
|
||
bank_configured = bank_host_configured(st.get("bank") or "")
|
||
bank_backend = "unknown"
|
||
wire_methods: set[str] = set()
|
||
currency_info: dict[str, Any] | None = None
|
||
keys_body_for_denoms: bytes | str | None = None
|
||
probe_cache: dict[str, dict[str, Any]] = {}
|
||
|
||
def cached_probe(url: str) -> dict[str, Any]:
|
||
if url not in probe_cache:
|
||
probe_cache[url] = probe(url)
|
||
return probe_cache[url]
|
||
|
||
if kind == "static_pending":
|
||
# Apex static site (prod Taler not up yet)
|
||
site_host = _host_only(st["id"])
|
||
site_url = f"https://{site_host}/"
|
||
site_raw = cached_probe(site_url)
|
||
site_c = classify_probe(site_raw, prev=prev_hosts.get(site_host), now=now, acute_s=acute_s)
|
||
if site_c.get("ok"):
|
||
site_c["light"] = "green"
|
||
site_c["reason"] = "static_ok"
|
||
site_member = {"role": "site", "host": site_host, **site_c}
|
||
site_prev = {k[len(site_host) :]: v for k, v in prev_eps.items() if k.startswith(site_host)}
|
||
site_eps = probe_role_endpoints(
|
||
"site",
|
||
site_host,
|
||
has_intro=False,
|
||
legal_severity=legal_sev,
|
||
probe_fn=cached_probe,
|
||
cache=probe_cache,
|
||
prev_by_path=site_prev,
|
||
now=now,
|
||
acute_s=acute_s,
|
||
)
|
||
members_out.append(merge_member_endpoint_light(site_member, site_eps))
|
||
|
||
# Probe exchange before bank so /keys payto://… can decide EBICS vs x-taler-bank.
|
||
role_hosts: list[tuple[str, str]] = [
|
||
("exchange", st["exchange"]),
|
||
("merchant", st["merchant"]),
|
||
]
|
||
if bank_configured:
|
||
role_hosts.insert(0, ("bank", st.get("bank") or ""))
|
||
|
||
classified_by_role: dict[str, dict[str, Any]] = {}
|
||
# Always resolve exchange (+ merchant) first for wire detection, then bank.
|
||
probe_order = ["exchange", "merchant", "bank"] if bank_configured else ["exchange", "merchant"]
|
||
host_by_role = {r: h for r, h in role_hosts}
|
||
|
||
for role in probe_order:
|
||
if role not in host_by_role:
|
||
continue
|
||
host = host_by_role[role]
|
||
if role == "bank":
|
||
# After exchange: skip public bank.* when wire is IBAN/EBICS-only.
|
||
bank_backend = classify_bank_backend(
|
||
wire_methods, bank_configured=bool(bank_configured)
|
||
)
|
||
if bank_backend == "ebics":
|
||
classified_by_role[role] = {
|
||
"ok": True,
|
||
"light": "blue",
|
||
"reason": "ebics_iban",
|
||
"down_since": None,
|
||
"code": 0,
|
||
"elapsed": 0.0,
|
||
"error": None,
|
||
"url": None,
|
||
"ts": now,
|
||
# No public bank.* — show wire kind in the host column
|
||
"host": "IBAN",
|
||
"role": role,
|
||
}
|
||
continue
|
||
if role == "exchange":
|
||
# Prefer /keys (accounts/payto); fall back to /config for liveness.
|
||
keys_url = f"https://{_host_only(host)}/keys"
|
||
raw = cached_probe(keys_url)
|
||
cfg_data: dict[str, Any] | None = None
|
||
if not raw.get("ok"):
|
||
alt = member_probe_url(role, host)
|
||
raw2 = cached_probe(alt)
|
||
if raw2.get("ok") or int(raw2.get("code") or 0) > int(raw.get("code") or 0):
|
||
raw = raw2
|
||
cfg_data = _json_body(raw)
|
||
else:
|
||
keys_body_for_denoms = raw.get("body")
|
||
wire_methods = payto_wire_methods(keys_body_for_denoms)
|
||
bank_backend = classify_bank_backend(
|
||
wire_methods, bank_configured=bool(bank_configured)
|
||
)
|
||
# Separate /config for currency_specification (small JSON).
|
||
cfg_raw = cached_probe(member_probe_url(role, host))
|
||
if cfg_raw.get("ok"):
|
||
cfg_data = _json_body(cfg_raw)
|
||
if cfg_data:
|
||
cs = cfg_data.get("currency_specification")
|
||
if not isinstance(cs, dict):
|
||
cs = {}
|
||
currency_info = summarize_currency_spec(
|
||
cs,
|
||
keys_body=keys_body_for_denoms,
|
||
fallback_currency=str(
|
||
cfg_data.get("currency") or st.get("currency") or ""
|
||
),
|
||
)
|
||
elif role == "bank":
|
||
url = member_probe_url(role, host)
|
||
raw = cached_probe(url)
|
||
if not raw.get("ok") and int(raw.get("code") or 0) in (0, 404):
|
||
alt = f"https://{_host_only(host)}/config"
|
||
raw2 = cached_probe(alt)
|
||
if raw2.get("ok") or int(raw2.get("code") or 0) > int(raw.get("code") or 0):
|
||
raw = raw2
|
||
else:
|
||
raw = cached_probe(member_probe_url(role, host))
|
||
|
||
classified = classify_probe(raw, prev=prev_hosts.get(host), now=now, acute_s=acute_s)
|
||
if kind == "static_pending" and not classified.get("ok"):
|
||
# Expected: no bank/exchange/merchant prod yet — not an outage.
|
||
classified = {
|
||
**classified,
|
||
"ok": True,
|
||
"light": "blue",
|
||
"reason": "no_prod_yet",
|
||
"down_since": None,
|
||
}
|
||
classified_by_role[role] = {"role": role, "host": host, **classified}
|
||
|
||
if not bank_configured:
|
||
bank_backend = "ebics"
|
||
classified_by_role["bank"] = {
|
||
"role": "bank",
|
||
"host": "IBAN",
|
||
"ok": True,
|
||
"light": "blue",
|
||
"reason": "ebics_iban",
|
||
"down_since": None,
|
||
"code": 0,
|
||
"elapsed": 0.0,
|
||
"error": None,
|
||
"url": None,
|
||
"ts": now,
|
||
}
|
||
|
||
# Stable member order: site (optional) already appended; then bank → exchange → merchant
|
||
for role in ("bank", "exchange", "merchant"):
|
||
if role not in classified_by_role:
|
||
continue
|
||
member = classified_by_role[role]
|
||
host = member.get("host") or ""
|
||
# EBICS placeholder — no public endpoint fan-out
|
||
if role == "bank" and (
|
||
bank_backend == "ebics" or (member.get("reason") or "") in _AGG_SKIP_REASONS
|
||
):
|
||
members_out.append(
|
||
{
|
||
**member,
|
||
"endpoints": [],
|
||
"endpoints_summary": summarize_endpoints([]),
|
||
}
|
||
)
|
||
continue
|
||
if kind == "static_pending" and (member.get("reason") or "") == "no_prod_yet":
|
||
members_out.append(
|
||
{
|
||
**member,
|
||
"endpoints": [],
|
||
"endpoints_summary": summarize_endpoints([]),
|
||
}
|
||
)
|
||
continue
|
||
host_prev = {k[len(host) :]: v for k, v in prev_eps.items() if host and k.startswith(host)}
|
||
eps = probe_role_endpoints(
|
||
role,
|
||
host,
|
||
has_intro=has_intro,
|
||
legal_severity=legal_sev,
|
||
probe_fn=cached_probe,
|
||
cache=probe_cache,
|
||
prev_by_path=host_prev,
|
||
now=now,
|
||
acute_s=acute_s,
|
||
)
|
||
members_out.append(merge_member_endpoint_light(member, eps))
|
||
|
||
light, intensity = aggregate_stack(members_out)
|
||
stats = fetch_stack_stats(
|
||
st, bank_backend=bank_backend, probe_fn=cached_probe, now=now, stale_s=STATS_STALE_S
|
||
)
|
||
vulns = assess_external_vulns(
|
||
probe_cache=probe_cache, members=members_out, osv_cache=osv_cache
|
||
)
|
||
light, intensity, members_out = apply_vuln_to_stack(light, intensity, members_out, vulns)
|
||
|
||
tls: dict[str, Any] = {}
|
||
primary_host = ""
|
||
for role in ("merchant", "exchange", "bank"):
|
||
m = classified_by_role.get(role) or {}
|
||
h = str(m.get("host") or "")
|
||
if h and h.upper() != "IBAN":
|
||
primary_host = h
|
||
break
|
||
if not primary_host:
|
||
primary_host = _host_only(str(st.get("id") or ""))
|
||
if CERT_PROBES_ENABLED and primary_host:
|
||
tls = probe_tls_cert(primary_host)
|
||
if tls.get("light") and SEVERITY.get(str(tls["light"]), 0) > SEVERITY.get(light, 0):
|
||
members_out.append(
|
||
{
|
||
"role": "tls",
|
||
"host": primary_host,
|
||
"ok": bool(tls.get("ok")),
|
||
"light": tls.get("light") or "yellow",
|
||
"reason": tls.get("reason") or "cert",
|
||
"down_since": None,
|
||
"code": 0,
|
||
"elapsed": 0.0,
|
||
"error": tls.get("error"),
|
||
"url": None,
|
||
"endpoints": [],
|
||
"endpoints_summary": summarize_endpoints([]),
|
||
}
|
||
)
|
||
light, intensity = aggregate_stack(members_out)
|
||
|
||
# Prefer live exchange currency; keep domains.conf as fallback label.
|
||
detected_currency = ""
|
||
if currency_info and currency_info.get("currency"):
|
||
detected_currency = str(currency_info["currency"])
|
||
elif st.get("currency"):
|
||
detected_currency = str(st.get("currency"))
|
||
software = collect_software_from_probes(
|
||
probe_cache=probe_cache,
|
||
members=members_out,
|
||
vulns=vulns,
|
||
smtp=False,
|
||
published=published_doc,
|
||
)
|
||
pay: dict[str, Any] = {}
|
||
zone = classify_network_zone(st)["zone"]
|
||
pay_id = str(st.get("id") or "")
|
||
if PAY_ENABLED and zone in ("tops-stage", "tops-prod", "goa-local", "fp-stage"):
|
||
pay = collect_payability(pay_id, now=now)
|
||
if pay.get("enabled") and pay.get("light") not in (None, "blue"):
|
||
members_out.append(
|
||
{
|
||
"role": "payability",
|
||
"host": pay_id,
|
||
"ok": bool(pay.get("ok")),
|
||
"light": pay.get("light") or "yellow",
|
||
"reason": pay.get("reason") or "pay",
|
||
"down_since": None,
|
||
"code": 0,
|
||
"elapsed": 0.0,
|
||
"error": None,
|
||
"url": None,
|
||
"endpoints": [],
|
||
"endpoints_summary": summarize_endpoints([]),
|
||
"command": pay.get("command") or "",
|
||
}
|
||
)
|
||
light, intensity = aggregate_stack(members_out)
|
||
out_stacks.append(
|
||
{
|
||
"id": st["id"],
|
||
"currency": detected_currency or (st.get("currency") or ""),
|
||
"currency_info": currency_info,
|
||
"kind": kind,
|
||
"commercial_prod": stack_is_commercial_prod(st),
|
||
"bank_backend": bank_backend,
|
||
"wire_methods": sorted(wire_methods),
|
||
"local": bool(st.get("local")),
|
||
"light": light,
|
||
"intensity": intensity,
|
||
"members": members_out,
|
||
"stats": stats,
|
||
"software": software,
|
||
"payability": pay,
|
||
"network": build_network_status(
|
||
st, members=members_out, ssh_cache=ssh_cache
|
||
),
|
||
"vulns": {
|
||
"enabled": vulns.get("enabled"),
|
||
"worst": vulns.get("worst") or "",
|
||
"light": vulns.get("light"),
|
||
"hits": [
|
||
{
|
||
"id": h.get("id"),
|
||
"severity": h.get("severity"),
|
||
"package": h.get("package"),
|
||
"version": h.get("version"),
|
||
"host": h.get("host"),
|
||
"summary": h.get("summary") or "",
|
||
}
|
||
for h in (vulns.get("hits") or [])[:8]
|
||
],
|
||
"servers": vulns.get("servers") or [],
|
||
},
|
||
"breakdown": aggregate_breakdown(members_out),
|
||
"tls": {
|
||
"days_left": tls.get("days_left"),
|
||
"not_after": tls.get("not_after"),
|
||
"sans": tls.get("sans") or [],
|
||
"reason": tls.get("reason"),
|
||
}
|
||
if tls
|
||
else {},
|
||
"orphan": False,
|
||
}
|
||
)
|
||
|
||
out_services: list[dict[str, Any]] = []
|
||
if SERVICES_ENABLED:
|
||
for row in parse_services_conf():
|
||
out_services.append(
|
||
build_service_status(
|
||
row,
|
||
prev_hosts=prev_hosts,
|
||
now=now,
|
||
acute_s=acute_s,
|
||
probe_fn=probe,
|
||
osv_cache=osv_cache,
|
||
published=published_doc,
|
||
)
|
||
)
|
||
|
||
out_mail: list[dict[str, Any]] = []
|
||
if MAIL_ENABLED:
|
||
mail_port_done: set[str] = set()
|
||
deliver_state = load_deliverability_state() if DELIVER_ENABLED else {"domains": {}}
|
||
for row in parse_mail_catalog():
|
||
out_mail.append(
|
||
assess_mail_domain(
|
||
row,
|
||
port_done=mail_port_done,
|
||
deliver_state=deliver_state,
|
||
now=now,
|
||
)
|
||
)
|
||
|
||
known_hosts: set[str] = set()
|
||
tls_sans: list[str] = []
|
||
scan_hosts: list[str] = []
|
||
for group in (out_stacks, out_services):
|
||
for st_out in group:
|
||
for m in st_out.get("members") or []:
|
||
h = _host_only(str(m.get("host") or ""))
|
||
if h and h.upper() != "IBAN":
|
||
known_hosts.add(h)
|
||
scan_hosts.append(h)
|
||
for san in (st_out.get("tls") or {}).get("sans") or []:
|
||
tls_sans.append(str(san))
|
||
known_hosts.add(_host_only(str(st_out.get("id") or "")))
|
||
scan_hosts.append(_host_only(str(st_out.get("id") or "")))
|
||
# Also scan env-mapped SSH targets — findings may be hidden hosts
|
||
for st_cfg in stacks:
|
||
if public_only and st_cfg.get("local"):
|
||
continue
|
||
for h in ssh_targets_for_stack(st_cfg).get("hosts") or []:
|
||
scan_hosts.append(h)
|
||
mail_hosts: set[str] = set()
|
||
for row in parse_mail_catalog() if MAIL_ENABLED else []:
|
||
for h in row.get("mail_hosts") or []:
|
||
mail_hosts.add(_host_only(h).lower())
|
||
scan_hosts.append(h)
|
||
orphans = collect_orphan_hosts(
|
||
known_hosts=known_hosts, tls_sans=tls_sans, probe_fn=probe
|
||
)
|
||
ports_doc = collect_port_scan(scan_hosts, mail_hosts=mail_hosts)
|
||
# Attach per-card port slice + optional synthetic member when suspicious
|
||
by_host = {str(r.get("host") or ""): r for r in (ports_doc.get("hosts") or [])}
|
||
for group in (out_stacks, out_services):
|
||
for st_out in group:
|
||
ssh_meta = st_out.get("network") or {}
|
||
ssh_block = ssh_meta.get("ssh") or {}
|
||
# UI clears candidates when hide_host; recover env-mapped hosts for port attach
|
||
hidden_ssh: set[str] = set()
|
||
if ssh_block.get("hide_host"):
|
||
zone = str(ssh_meta.get("zone") or "")
|
||
env_map = {
|
||
"goa-local": "TALER_MONITORD_SSH_GOA",
|
||
"fp-stage": "TALER_MONITORD_SSH_FP",
|
||
"tops-stage": "TALER_MONITORD_SSH_TOPS",
|
||
"tops-prod": "TALER_MONITORD_SSH_TOPS_PROD",
|
||
}
|
||
if zone in env_map:
|
||
hidden_ssh = {
|
||
_host_only(x).lower()
|
||
for x in _env_ssh_hosts(env_map[zone])
|
||
if x
|
||
}
|
||
card_hosts: list[str] = []
|
||
for m in st_out.get("members") or []:
|
||
h = _host_only(str(m.get("host") or ""))
|
||
if h and h.upper() != "IBAN":
|
||
card_hosts.append(h)
|
||
sid = _host_only(str(st_out.get("id") or ""))
|
||
if sid:
|
||
card_hosts.append(sid)
|
||
card_hosts.extend(hidden_ssh)
|
||
sus: list[dict[str, Any]] = []
|
||
opens: list[dict[str, Any]] = []
|
||
worst = "green"
|
||
for h in dict.fromkeys(_host_only(x).lower() for x in card_hosts if x):
|
||
r = by_host.get(h)
|
||
if not r:
|
||
continue
|
||
hide = h in hidden_ssh
|
||
for s in r.get("suspicious") or []:
|
||
sus.append(
|
||
{
|
||
"host": "" if hide else h,
|
||
"port": s.get("port"),
|
||
"label": s.get("label"),
|
||
"severity": s.get("severity") or "soft",
|
||
"light": s.get("light") or "yellow",
|
||
}
|
||
)
|
||
if r.get("open"):
|
||
opens.append(
|
||
{
|
||
"host": "" if hide else h,
|
||
"open": r.get("open"),
|
||
"suspicious": [s.get("port") for s in (r.get("suspicious") or [])],
|
||
}
|
||
)
|
||
for s in r.get("suspicious") or []:
|
||
if s.get("severity") != "hard":
|
||
continue
|
||
Lgt = s.get("light") or "orange"
|
||
if SEVERITY.get(str(Lgt), 0) > SEVERITY.get(worst, 0):
|
||
worst = str(Lgt)
|
||
hard_sus = [s for s in sus if s.get("severity") == "hard"]
|
||
st_out["ports"] = {
|
||
"suspicious": sus,
|
||
"hosts": opens,
|
||
"light": worst if hard_sus else ("yellow" if sus else "green"),
|
||
"n_suspicious": len(sus),
|
||
"n_hard": len(hard_sus),
|
||
}
|
||
# Only hard findings pull the card colour (soft stays informational).
|
||
if hard_sus:
|
||
members = list(st_out.get("members") or [])
|
||
members.append(
|
||
{
|
||
"role": "ports",
|
||
"host": ",".join(
|
||
f"{s.get('label') or s.get('port')}"
|
||
+ (f"@{s.get('host')}" if s.get("host") else "")
|
||
for s in hard_sus[:6]
|
||
),
|
||
"ok": False,
|
||
"light": worst,
|
||
"reason": f"suspicious_ports_hard:{len(hard_sus)}",
|
||
"down_since": None,
|
||
"code": 0,
|
||
"elapsed": 0.0,
|
||
"error": None,
|
||
"url": None,
|
||
"endpoints": [],
|
||
"endpoints_summary": summarize_endpoints([]),
|
||
}
|
||
)
|
||
st_out["members"] = members
|
||
light2, intensity2 = aggregate_stack(members)
|
||
st_out["light"] = light2
|
||
st_out["intensity"] = intensity2
|
||
st_out["breakdown"] = aggregate_breakdown(members)
|
||
return {
|
||
"schema": SCHEMA,
|
||
"ts": now,
|
||
"recorder": os.environ.get("TALER_MONITORD_RECORDER") or socket.gethostname(),
|
||
"acute_down_s": acute_s,
|
||
"legend": dict(LEGEND),
|
||
"stacks": out_stacks,
|
||
"services": out_services,
|
||
"mail": out_mail,
|
||
"orphans": orphans,
|
||
"ports": ports_doc,
|
||
"published": {
|
||
"enabled": published_doc.get("enabled"),
|
||
"suite": published_doc.get("suite"),
|
||
"error": published_doc.get("error"),
|
||
"url": published_doc.get("url"),
|
||
"packages": {
|
||
k: published_doc.get("packages", {}).get(k)
|
||
for k in (
|
||
"taler-exchange",
|
||
"taler-merchant",
|
||
"libeufin-bank",
|
||
"taler-merchant-webui",
|
||
"taler-wallet-cli",
|
||
)
|
||
if (published_doc.get("packages") or {}).get(k)
|
||
},
|
||
"apps": published_doc.get("apps") or [],
|
||
},
|
||
"score": compute_total_score(out_stacks, out_services, out_mail),
|
||
}
|
||
|
||
|
||
def read_status(path: Path | None = None) -> dict[str, Any]:
|
||
paths = [path] if path else status_json_paths()
|
||
for p in paths:
|
||
if p and p.is_file():
|
||
try:
|
||
return json.loads(p.read_text(encoding="utf-8"))
|
||
except (OSError, json.JSONDecodeError):
|
||
continue
|
||
return {}
|
||
|
||
|
||
def atomic_write_json(path: Path, doc: dict[str, Any]) -> None:
|
||
path = Path(path)
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
text = json.dumps(doc, ensure_ascii=False, indent=2) + "\n"
|
||
tmp = path.with_name(f".{path.name}.tmp.{os.getpid()}")
|
||
try:
|
||
tmp.write_text(text, encoding="utf-8")
|
||
os.replace(tmp, path)
|
||
finally:
|
||
if tmp.exists():
|
||
try:
|
||
tmp.unlink()
|
||
except OSError:
|
||
pass
|
||
|
||
|
||
def write_stack_status(doc: dict[str, Any], paths: list[Path] | None = None) -> Path:
|
||
paths = paths or status_json_paths()
|
||
primary = paths[0]
|
||
for p in paths:
|
||
atomic_write_json(p, doc)
|
||
return primary
|