From 6d5adeef9c2298e62fd509a26e66f42a7e9c42be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 6 Sep 2026 00:38:17 +0200 Subject: [PATCH 1/2] fix 1.26.2: merchant webui version.txt missing is INFO --- TESTS.md | 2 +- VERSION | 2 +- VERSIONS.md | 1 + check_auth401.sh | 2 +- check_urls.sh | 8 +++----- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/TESTS.md b/TESTS.md index f7b199e..bfe227f 100644 --- a/TESTS.md +++ b/TESTS.md @@ -56,7 +56,7 @@ Numbering follows **executed** checks (early skip may shift later NN inside the | **www.stats-** | `/intro/stats.json` **reachable**; **timestamps** (`generated_at_unix` + human/ISO); **freshness** vs wall clock (`STATS_STALE_SECS` WARN, `STATS_FAIL_SECS` ERROR); **display fields** (bank-shape withdraws/accounts, or exchange-db / merchant-db keys); optional performance block; shared-feed equal `gen_unix` when bank collector is published to all three | | **www.bank-** | `/config`, currency, alt_unit_names; integration/webui/intro; **auto-account.json** (required GOA/local; skip/soft off-GOA e.g. TESTPAYSAN); `/terms`, `/privacy` | | **www.merchant-** | `/config` currency + currencies alt_unit_names; listed exchanges alt; webui/intro; **`/terms`**, **`/privacy`** | -| **www.webui-** | SPA fingerprints: **`/webui/version.txt`**, **`/webui/version-overlay.txt`**, `index.html` / `index.js`; optional pin via `EXPECT_WEBUI_VERSION` / `EXPECT_WEBUI_OVERLAY` / `WEBUI_OVERLAY_DENY` | +| **www.webui-** | SPA fingerprints: **`/webui/version.txt`** (INFO if missing — earlier merchant SPA had it; bank still has; current merchant often omits), **`/webui/version-overlay.txt`**, `index.html` / `index.js`; optional pin via `EXPECT_WEBUI_VERSION` / `EXPECT_WEBUI_OVERLAY` / `WEBUI_OVERLAY_DENY` | | **www.paivana-** | local GOA paywall front (redirect to template) | | **www.landing-** | own-stack intro links; static assets (`qrcode.min.js` hard; `og-goa-shop.png` hard only GOA/local); **demo-withdraw.json** GOA-only; shop-pay soft; stage merchant shop: `shop-ui.js` + `shops.css` + `qrcode.min.js`; cross-links local | | **www.taler-uri-ports-** | **no default `:443`/`:80`** in public `taler://` URIs — bank/exchange/merchant `/intro/` HTML, `demo-withdraw.json`, `auto-account.json`, bank `stats.json` (ERROR on GOA/local). Unit: `tests/test_normalize_taler_uri.sh` | diff --git a/VERSION b/VERSION index dd43a14..c7c3f33 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.26.1 +1.26.2 diff --git a/VERSIONS.md b/VERSIONS.md index cb1174e..db12157 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| +| **v1.26.2** | 2026-09-06 | **Bugfix / urls:** missing merchant `/webui/version.txt` is **INFO** (earlier merchant SPA had stamp; current packages often omit; bank `/webui/version.txt` still present). | | **v1.26.1** | 2026-08-31 | **Bugfix:** public order settle accepts `StatusPaidResponse` (`refunded`/`refund_taken`); `paid`/`order_status` not on public GET. | | **v1.26.0** | 2026-08-30 | **Feature:** `--wallet-db PATH` / `--wallet-migrate` (taler-wallet-cli / taler-harness names) for e2e, ladder, zero-pay. TESTPAYSAN: one Quadlet `stage-testpaysan`; e2e token-pay via env `E2E_TOKEN_*` / `--choice-index`. Stack READMEs `README_testpaysan.md` / `README_mytops.md`. | | **v1.24.0** | 2026-08-30 | **Feature / zero-pay:** public-template pays as phase `zero-pay` (`check_zero_pay.sh`). Default merchant **stage.my.taler-ops.ch** (CHF:0 stickers). Wrapper `pay_talersticker_zero.sh` COUNT=100. TESTPAYSAN: empty POST `{}` + auto-confirm of bank wires (host `auto-confirm-withdrawals`). Not in default 4h PHASES (use COUNT=1). | diff --git a/check_auth401.sh b/check_auth401.sh index dfca606..a78cfda 100755 --- a/check_auth401.sh +++ b/check_auth401.sh @@ -314,7 +314,7 @@ if [ "${AUTH401_FULL}" = "1" ]; then if [ "$vcode" = "200" ] && [ -s "$tmp/spa-ver.txt" ]; then ok "webui version.txt" "$(tr -d '\n\r' <"$tmp/spa-ver.txt")" else - warn "webui version.txt" "HTTP ${vcode:-000}" + info "webui version.txt" "HTTP ${vcode:-000} · earlier merchant SPA versions had this stamp; current merchant packages often omit it · bank /webui/version.txt still present on current bank builds" fi # Download a slice of index.js — look for lowercasing mitigation (#11340) jcode=$(curl -skS -m 60 -o "$tmp/spa-index.js" -w '%{http_code}' "$BASE/webui/index.js" 2>/dev/null || echo 000) diff --git a/check_urls.sh b/check_urls.sh index f2557ff..0748940 100755 --- a/check_urls.sh +++ b/check_urls.sh @@ -879,11 +879,9 @@ PY info "merchant webui version" "${got_ver} (set EXPECT_WEBUI_VERSION= to pin)" fi else - if [ "${LOCAL_STACK:-1}" = "1" ]; then - fail "merchant /webui/version.txt" "HTTP ${vcode:-000} (SPA missing version stamp)" - else - warn "merchant /webui/version.txt" "HTTP ${vcode:-000}" - fi + # Newer merchant-webui packages omit version.txt (GOA / FP stage / prod). + # INFO only — earlier merchant SPA builds shipped this stamp; bank webui still has it. + info "merchant /webui/version.txt" "HTTP ${vcode:-000} · earlier merchant SPA versions had this stamp; current merchant packages often omit it · bank /webui/version.txt still present on current bank builds" fi ocode=$(http_body "$MERCHANT_PUBLIC/webui/version-overlay.txt" "$tmp/webui-overlay.txt") From fb68a1b011202d776646df9cb41ce9103052f5e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Tue, 8 Sep 2026 01:39:31 +0200 Subject: [PATCH 2/2] fix: update version.txt for 1.26.2 release --- TODO-OPEN-ngi0-landing.md | 18 + TODO-OPEN-stack-status.md | 26 ++ TODO-TASK114-catalog-pass.md | 14 + check_mail_deliverability.sh | 332 ++++++++++++++++++ host-agent/run-mail-deliv-probe.sh | 40 +++ .../taler-monitoring-mail-deliv.service | 17 + host-agent/taler-monitoring-mail-deliv.timer | 12 + mail_deliv_probe.py | 232 ++++++++++++ 8 files changed, 691 insertions(+) create mode 100644 TODO-OPEN-ngi0-landing.md create mode 100644 TODO-OPEN-stack-status.md create mode 100644 TODO-TASK114-catalog-pass.md create mode 100755 check_mail_deliverability.sh create mode 100755 host-agent/run-mail-deliv-probe.sh create mode 100644 host-agent/taler-monitoring-mail-deliv.service create mode 100644 host-agent/taler-monitoring-mail-deliv.timer create mode 100755 mail_deliv_probe.py diff --git a/TODO-OPEN-ngi0-landing.md b/TODO-OPEN-ngi0-landing.md new file mode 100644 index 0000000..2f5f7b9 --- /dev/null +++ b/TODO-OPEN-ngi0-landing.md @@ -0,0 +1,18 @@ +# OPEN — ngi0-landing bereinigen (**für mich**, nicht Agent-Queue) + +Stand: 2026-09-05 · **persönliche** Merkhilfe Hernâni · **unvollständiger Stand** + +## Ist (Teil) +Domains für Dossiers-Testseiten (ngi0-landing) angelegt: + +- `dossiers.ngi.hacktivism.ch` +- `dossiers.2.ngi.hacktivism.ch` +- `dossiers.3.ngi.hacktivism.ch` + +## Offen / unvollständig +- Caddy static vhosts + Docroots auf koopa (sudo) — Snippet besprochen, **nicht** eingespielt +- `ngi0-landing` Repo **bereinigen** +- Inhalt der Testseiten / Deploy-Pfad klären + +## Kanonisch +`PATH[llm-configs]/TODO-AGENT.md` → **HOLD-ngi0-landing** (außerhalb 1–11; Agents nur auf expliziten Auftrag). diff --git a/TODO-OPEN-stack-status.md b/TODO-OPEN-stack-status.md new file mode 100644 index 0000000..197c9e1 --- /dev/null +++ b/TODO-OPEN-stack-status.md @@ -0,0 +1,26 @@ +# OPEN — LLM Lightweight Monitoring / stack status + +Stand: 2026-09-05 · tip `e293ff4` · prefer `/compact` + +## Product +**LLM** = **Lightweight Monitoring**. Suite tree still `taler-monitoring`. + +## Live +- https://hacktivism.ch/stacks/ +- Wiki: `~/git/www-hacktivism-wiki` (4h) + +## IDs +**TASK106–114 stay fixed.** New work → next free id. + +## Done +- TASK107–111+113 · TASK114 catalog ok +- TASK112: trixie×4 + **ubuntu noble** apt-src on koopa · `ANSIBLE_SMOKE_REQUIRE_UBUNTU=1` → **OVERALL=PASS** +- Branch `llm-lightweight-monitoring` **pushed** · tag **v1.27.0** @ `e8f2820` · tip `e293ff4` +- Admin-log `2026-09-05--taler-apt-src-ubuntu.md` → Forgejo `a2300b6` + +## Next +**Keine zweite Liste.** Kanonisch: `PATH[llm-configs]/TODO-AGENT.md` § **Arbeitsliste 1–11**. + +**#8 TASK116 (2026-09-05):** Domain **monitoring.hacktivism.ch** (CNAME→taler→WAN). koopa `taler-monitord.service` **active**; local HTML at `/var/www/monitoring-sites/monitoring.hacktivism.ch/index.html`. **Blocker:** Caddy vhost+cert (sudo) — snippet `host-agent/caddy-monitoring.hacktivism.ch.snippet`. Clementine #5 still blocked. + +**HOLD-ngi0-landing (für mich, nicht Agent):** **unvollständiger Stand** — Domains da, Caddy/Repo-Bereinigung offen — `TODO-OPEN-ngi0-landing.md` / `TODO-AGENT.md`. diff --git a/TODO-TASK114-catalog-pass.md b/TODO-TASK114-catalog-pass.md new file mode 100644 index 0000000..0861f41 --- /dev/null +++ b/TODO-TASK114-catalog-pass.md @@ -0,0 +1,14 @@ +# TASK114 catalog pass + +Stand: 2026-09-05 · Grok + Osaurus `foundation` (hallucinated; discarded) · Jan GP hung 0 tools (killed) + +STATUS: ok (partial notes only) + +FINDING: +- services-catalog ↔ ports-whitelist ↔ mail-catalog: coherent for pixel/firefly/vecigate (git/mm/buildbot + WAN 212.51.151.254 present). +- domains.conf holds Taler stacks; services-catalog correctly stays non-Taler/company — no duplicate stacks. +- mail-catalog planes (firefly / mail.anastasis.lu / pixel) match ports whitelist mail hosts. +- Soft gap: no dedicated `tortuna` services row (only WAN DNAT ports on vecigate) — intentional per comments. +- Soft gap: apex `taler-ops.ch` is mail+domains; services has `www.taler-ops.ch` company web only. + +NEXT: no catalog edit required; optional later: document tortuna-as-DNAT-only in services header comment. diff --git a/check_mail_deliverability.sh b/check_mail_deliverability.sh new file mode 100755 index 0000000..eaaf015 --- /dev/null +++ b/check_mail_deliverability.sh @@ -0,0 +1,332 @@ +#!/usr/bin/env bash +# check_mail_deliverability.sh — Taler mail-plane deliverability (ops-checklist) +# +# SUT domains (NOT cyon): taler.net, gnunet.org, firefly.gnunet.org, +# taler-systems.com, anastasis.lu, taler-ops.ch (from mail-catalog.conf) +# +# Based on admin-log/firefly/ops-checklist.md (2026-03 MT/ED scores): +# dig SPF + DMARC + DKIM selector(s); SMTP RCPT smoke via monitord helper; +# optional IMAP sink poll (mon001–003@hacktivism.ch on cyon = sink only). +# +# Explicit phase (not in all/full): +# ./taler-monitoring.sh mail-deliverability +# ./taler-monitoring.sh maildeliv +# +# Env: +# MAIL_CATALOG default mail-catalog.conf +# MAIL_DKIM_SELECTORS default "mail-2026-3 mail" (space-separated) +# MAIL_DELIV_IMAP=1 enable sink IMAP login+INBOX check (needs pass) +# MAIL_DELIV_IMAP_USERS default mon001@hacktivism.ch mon002@… mon003@… +# MAIL_DELIV_IMAP_HOST default mail.cyon.ch (sink host — not SUT) +# MAIL_DELIV_IMAP_PASS shared password for mon* (prefer secrets.env) +# MAIL_DELIV_SEND=0 1 = authenticated send (Freigabe; needs SMTP_*) +# MAIL_DELIV_ALERT=0 1 = on ERROR, send alert via sink SMTP (Freigabe) +# MAIL_DELIV_ALERT_FROM default mon001@hacktivism.ch +# MAIL_DELIV_ALERT_TO default hernani+mon@taler-systems.com +# hernani+mon@vecirex.net vecirex+mon@icloud.com +# fdold+mon@taler.net +# MAIL_DELIV_ALERT_SMTP default mail.cyon.ch:587 (sink submission — not SUT) +# MAIL_DELIV_ALERT_PASS default MAIL_DELIV_IMAP_PASS +# MAIL_DELIV_ALERT_COOLDOWN_S default 21600 (6h); stamp under monitord state +# TALER_MONITORD_DELIVER_* see site-gen smtp_deliverability_probe +# +set -euo pipefail +ROOT=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=lib.sh +source "$ROOT/lib.sh" + +set_area maildeliv +section "maildeliv · Taler mail domains (SPF/DMARC/DKIM + SMTP RCPT; cyon=sink only)" + +CATALOG="${MAIL_CATALOG:-$ROOT/mail-catalog.conf}" +SELECTORS="${MAIL_DKIM_SELECTORS:-mail-2026-3 mail}" +: "${MAIL_DELIV_IMAP:=0}" +: "${MAIL_DELIV_SEND:=0}" +: "${MAIL_DELIV_ALERT:=0}" +: "${MAIL_DELIV_PROBE:=0}" +IMAP_HOST="${MAIL_DELIV_IMAP_HOST:-mail.cyon.ch}" +IMAP_USERS="${MAIL_DELIV_IMAP_USERS:-mon001@hacktivism.ch mon002@hacktivism.ch mon003@hacktivism.ch}" +ALERT_FROM="${MAIL_DELIV_ALERT_FROM:-mon001@hacktivism.ch}" +ALERT_TO="${MAIL_DELIV_ALERT_TO:-hernani+mon@taler-systems.com hernani+mon@vecirex.net vecirex+mon@icloud.com fdold+mon@taler.net}" +ALERT_SMTP="${MAIL_DELIV_ALERT_SMTP:-mail.cyon.ch:587}" +ALERT_PASS="${MAIL_DELIV_ALERT_PASS:-${MAIL_DELIV_IMAP_PASS:-}}" +ALERT_COOLDOWN_S="${MAIL_DELIV_ALERT_COOLDOWN_S:-21600}" +STATE_DIR="${TALER_MONITORD_STATE_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/taler-monitord}" +ALERT_STAMP="${STATE_DIR}/mail-deliv-alert.stamp" + +if [ ! -f "$CATALOG" ]; then + fail "catalog" "missing $CATALOG" + summary + exit 1 +fi + +info "scope" "SUT = catalog Taler domains · sink IMAP host=$IMAP_HOST (not under test)" + +dns_txt() { + local name="$1" + if command -v dig >/dev/null 2>&1; then + dig +short +time=3 +tries=2 TXT "$name" 2>/dev/null | tr -d '"' + else + host -t TXT "$name" 2>/dev/null | sed 's/.*"\(.*\)"/\1/' + fi +} + +dns_mx() { + local d="$1" + dig +short +time=3 +tries=2 MX "$d" 2>/dev/null | awk '{print tolower($2)}' | sed 's/\.$//' +} + +# --- per catalog domain --- +while IFS= read -r line || [ -n "$line" ]; do + line=${line%%#*} + line=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + [ -z "$line" ] && continue + domain=$(echo "$line" | awk '{print $1}') + expect_mx=$(echo "$line" | awk '{print $2}') + label=$(echo "$line" | awk '{print $6}') + [ -n "$domain" ] || continue + + set_group "${label:-$domain}" + + # SPF + spf=$(dns_txt "$domain" | tr '\n' ' ' | grep -oE 'v=spf1[^"]*' | head -1 || true) + if [ -n "$spf" ]; then + ok "spf" "$domain has SPF" "$spf" + else + fail "spf" "$domain missing SPF TXT" + fi + + # DMARC + dmarc=$(dns_txt "_dmarc.$domain" | tr '\n' ' ' | grep -oE 'v=DMARC1.*' | head -1 || true) + if [ -n "$dmarc" ]; then + case "$dmarc" in + *p=none*) + if [ "$domain" = "taler-ops.ch" ] || [ "$domain" = "firefly.gnunet.org" ]; then + warn "dmarc" "$domain DMARC p=none (known weak · ops-checklist)" "$dmarc" + else + warn "dmarc" "$domain DMARC p=none" "$dmarc" + fi + ;; + *) + ok "dmarc" "$domain DMARC present" "$dmarc" + ;; + esac + else + fail "dmarc" "$domain missing _dmarc TXT" + fi + + # DKIM selectors (ops-checklist / 2026-03) + dkim_ok=0 + dkim_hit="" + for sel in $SELECTORS; do + [ -n "$sel" ] || continue + txt=$(dns_txt "${sel}._domainkey.${domain}" | tr '\n' ' ') + if echo "$txt" | grep -qiE 'v=DKIM1|p='; then + dkim_ok=1 + dkim_hit="$sel" + break + fi + done + if [ "$dkim_ok" = "1" ]; then + ok "dkim" "$domain DKIM selector $dkim_hit" + else + warn "dkim" "$domain no DKIM TXT for selectors [$SELECTORS] (mail-tester often -1)" + fi + + # MX matches catalog expectation (first token) + mx_got=$(dns_mx "$domain" | head -1) + exp_first=${expect_mx%%,*} + exp_first=${exp_first%% *} + if [ -n "$mx_got" ]; then + if [ -z "$exp_first" ] || [ "$exp_first" = "-" ] || [ "$mx_got" = "$exp_first" ]; then + ok "mx" "$domain MX=$mx_got" + else + # allow alternate listed in expect_mx comma list + case ",${expect_mx}," in + *",${mx_got},"*) ok "mx" "$domain MX=$mx_got (listed)" ;; + *) warn "mx" "$domain MX=$mx_got want~$exp_first" ;; + esac + fi + else + fail "mx" "$domain no MX" + continue + fi + + # SMTP RCPT smoke (no DATA) — reuse site-gen helper + if [ -f "$ROOT/site-gen/stack_status_lib.py" ]; then + out=$( + ROOT="$ROOT" DOMAIN="$domain" MX="$mx_got" python3 - <<'PY' 2>/dev/null || echo "err=import" +import os, sys +sys.path.insert(0, os.path.join(os.environ["ROOT"], "site-gen")) +from stack_status_lib import smtp_deliverability_probe +r = smtp_deliverability_probe(os.environ["MX"], domain=os.environ["DOMAIN"]) +print(f"ok={r.get('ok')} light={r.get('light')} reason={r.get('reason')} rcpt={r.get('rcpt')}") +PY + ) + case "$out" in + ok=True*) + ok "smtp-rcpt" "$domain $out" + ;; + ok=False*light=yellow*|ok=False*rcpt_reject*|ok=False*reason=rcpt_reject*) + # Many MTAs reject postmaster@ from random clients; still useful signal (ops-checklist uses mail-tester for full score). + warn "smtp-rcpt" "$domain $out" + ;; + ok=False*) + fail "smtp-rcpt" "$domain $out" + ;; + *) + warn "smtp-rcpt" "$domain probe unavailable · $out" + ;; + esac + else + info "smtp-rcpt" "skipped (no site-gen/stack_status_lib.py)" + fi +done <"$CATALOG" + +# --- optional IMAP sinks (cyon hosts mon*; not SUT) --- +if [ "$MAIL_DELIV_IMAP" = "1" ]; then + set_group sink-imap + if [ -z "${MAIL_DELIV_IMAP_PASS:-}" ]; then + warn "sink-imap" "MAIL_DELIV_IMAP=1 but MAIL_DELIV_IMAP_PASS unset — skip" + else + info "sink-imap" "probing sinks on $IMAP_HOST (infrastructure only · not Taler MTA)" + for user in $IMAP_USERS; do + res=$( + MAIL_DELIV_IMAP_HOST="$IMAP_HOST" MAIL_DELIV_IMAP_USER="$user" \ + MAIL_DELIV_IMAP_PASS="$MAIL_DELIV_IMAP_PASS" python3 - <<'PY' 2>/dev/null || echo FAIL +import imaplib, os, ssl +host=os.environ["MAIL_DELIV_IMAP_HOST"] +user=os.environ["MAIL_DELIV_IMAP_USER"] +pw=os.environ["MAIL_DELIV_IMAP_PASS"] +ctx=ssl.create_default_context() +M=imaplib.IMAP4_SSL(host, 993, ssl_context=ctx) +M.login(user, pw) +typ, data=M.select("INBOX") +n=data[0].decode() if data and data[0] else "?" +# also see junk/spam exists +boxes=[] +t, bl=M.list() +for b in bl or []: + s=b.decode(errors="replace") + if "Junk" in s or "spam" in s.lower(): + boxes.append("junk/spam") +M.logout() +print(f"OK inbox={n} extra={','.join(boxes) or 'none'}") +PY + ) + case "$res" in + OK*) ok "sink" "$user · $res" ;; + *) fail "sink" "$user · $res" ;; + esac + done + fi +else + info "sink-imap" "skipped (MAIL_DELIV_IMAP=0) · sinks mon001–003 for Freigabe send tests" +fi + +if [ "$MAIL_DELIV_SEND" = "1" ]; then + warn "send" "MAIL_DELIV_SEND=1 deprecated here — use MAIL_DELIV_PROBE=1 (inbox vs spam)" +fi + +if [ "$MAIL_DELIV_PROBE" = "1" ]; then + set_group probe + if [ -z "${MAIL_DELIV_IMAP_PASS:-}${MAIL_DELIV_ALERT_PASS:-}" ]; then + warn "probe" "MAIL_DELIV_PROBE=1 but no IMAP pass" + else + info "probe" "mon* send + IMAP inbox-vs-spam (regular path · host-agent timer 6h)" + if out=$(python3 "$ROOT/mail_deliv_probe.py" 2>&1); then + ok "probe" "all sinks inbox" + info "probe-detail" "$(echo "$out" | tr '\n' ' ' | head -c 240)" + else + ec=$? + echo "$out" | while IFS= read -r line; do + case "$line" in + *result=spam*|*result=both*) + fail "probe-spam" "$line" + ;; + *result=missing*|*IMAP_FAIL*|*SEND_FAIL*) + fail "probe-miss" "$line" + ;; + SENT*|ALERT*) + info "probe" "$line" + ;; + *) + info "probe" "$line" + ;; + esac + done + [ "$ec" -eq 0 ] || true + fi + fi +else + info "probe" "skipped (MAIL_DELIV_PROBE=0) · enable for inbox-vs-spam; timer: taler-monitoring-mail-deliv.timer" +fi + +# --- alert on ERROR via sink SMTP (cyon submission; recipients = hernani+mon / icloud) --- +maildeliv_alert_if_needed() { + set_group alert + if [ "$MAIL_DELIV_ALERT" != "1" ]; then + info "alert" "skipped (MAIL_DELIV_ALERT=0) · would To: $ALERT_TO" + return 0 + fi + if [ "${#ERRORS[@]}" -eq 0 ]; then + ok "alert" "no ERRORs — no alert mail" + return 0 + fi + if [ -z "$ALERT_PASS" ]; then + warn "alert" "MAIL_DELIV_ALERT=1 but no MAIL_DELIV_ALERT_PASS / MAIL_DELIV_IMAP_PASS" + return 0 + fi + if ! command -v swaks >/dev/null 2>&1; then + warn "alert" "swaks missing — apt/brew install swaks" + return 0 + fi + mkdir -p "$STATE_DIR" 2>/dev/null || true + now=$(date +%s) + if [ -f "$ALERT_STAMP" ]; then + prev=$(cat "$ALERT_STAMP" 2>/dev/null || echo 0) + if [ "$prev" -gt 0 ] 2>/dev/null; then + age=$((now - prev)) + if [ "$age" -lt "$ALERT_COOLDOWN_S" ]; then + info "alert" "cooldown ${age}s < ${ALERT_COOLDOWN_S}s — skip send" + return 0 + fi + fi + fi + smtp_host=${ALERT_SMTP%%:*} + smtp_port=${ALERT_SMTP##*:} + [ "$smtp_port" = "$ALERT_SMTP" ] && smtp_port=587 + body=$(mktemp) + { + echo "taler-monitoring mail-deliverability ALERT" + echo "host=$(hostname 2>/dev/null || echo ?)" + echo "when=$(date -Iseconds 2>/dev/null || date)" + echo "errors=${#ERRORS[@]}" + echo "" + for e in "${ERRORS[@]}"; do + echo "- $e" + done + echo "" + echo "SUT = Taler mail domains (not cyon). From=$ALERT_FROM via $ALERT_SMTP" + } >"$body" + subj="[taler-mon maildeliv] ${#ERRORS[@]} ERROR(s)" + send_fail=0 + for to in $ALERT_TO; do + if swaks --server "$smtp_host" --port "$smtp_port" \ + --auth LOGIN --auth-user "$ALERT_FROM" --auth-password "$ALERT_PASS" \ + --tls --from "$ALERT_FROM" --to "$to" \ + --header "Subject: $subj" --body "$body" >/dev/null 2>&1; then + ok "alert" "sent → $to" + else + warn "alert" "send failed → $to" + send_fail=1 + fi + done + rm -f "$body" + if [ "$send_fail" = "0" ]; then + echo "$now" >"$ALERT_STAMP" 2>/dev/null || true + fi +} +maildeliv_alert_if_needed + +summary diff --git a/host-agent/run-mail-deliv-probe.sh b/host-agent/run-mail-deliv-probe.sh new file mode 100755 index 0000000..740ce49 --- /dev/null +++ b/host-agent/run-mail-deliv-probe.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Regular mon* inbox-vs-spam probe (Taler deliverability sinks). +# Timer: taler-monitoring-mail-deliv.timer (default every 6h). +# +# Loads secrets.env if present. Does NOT print passwords. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +if [[ -d /home/hernani/src/taler-monitoring ]]; then + ROOT=/home/hernani/src/taler-monitoring +fi +cd "$ROOT" + +# secrets (gitignored) +for f in \ + "${TALER_MONITORING_ENV:-}" \ + "$HOME/.config/taler-monitoring/env" \ + "$ROOT/secrets.env" \ + "$HOME/src/taler-monitoring-env/secrets.env" +do + [[ -n "$f" && -f "$f" ]] || continue + set -a + # shellcheck disable=SC1090 + source "$f" + set +a + break +done + +export MAIL_DELIV_PROBE_FROM="${MAIL_DELIV_PROBE_FROM:-mon001@hacktivism.ch}" +export MAIL_DELIV_PROBE_TO="${MAIL_DELIV_PROBE_TO:-mon001@hacktivism.ch mon002@hacktivism.ch mon003@hacktivism.ch}" +export MAIL_DELIV_PROBE_SMTP="${MAIL_DELIV_PROBE_SMTP:-mail.cyon.ch:587}" +export MAIL_DELIV_IMAP_HOST="${MAIL_DELIV_IMAP_HOST:-mail.cyon.ch}" +export MAIL_DELIV_PROBE_WAIT_S="${MAIL_DELIV_PROBE_WAIT_S:-45}" +export MAIL_DELIV_PROBE_ALERT_TO="${MAIL_DELIV_PROBE_ALERT_TO:-hernani+mon@taler-systems.com hernani+mon@vecirex.net vecirex+mon@icloud.com fdold+mon@taler.net}" + +if [[ -z "${MAIL_DELIV_IMAP_PASS:-}${MAIL_DELIV_ALERT_PASS:-}" ]]; then + echo "run-mail-deliv-probe: no MAIL_DELIV_IMAP_PASS in env/secrets.env" >&2 + exit 2 +fi + +exec python3 "$ROOT/mail_deliv_probe.py" diff --git a/host-agent/taler-monitoring-mail-deliv.service b/host-agent/taler-monitoring-mail-deliv.service new file mode 100644 index 0000000..b0f888f --- /dev/null +++ b/host-agent/taler-monitoring-mail-deliv.service @@ -0,0 +1,17 @@ +[Unit] +Description=taler-monitoring mon* inbox-vs-spam mail probe +Documentation=file:%h/src/taler-monitoring/check_mail_deliverability.sh +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=%h/src/taler-monitoring +Environment=PYTHONUNBUFFERED=1 +# Prefer secrets via EnvironmentFile if present (create on host; never commit) +EnvironmentFile=-%h/.config/taler-monitoring/env +EnvironmentFile=-%h/src/taler-monitoring/secrets.env +ExecStart=%h/src/taler-monitoring/host-agent/run-mail-deliv-probe.sh + +[Install] +WantedBy=default.target diff --git a/host-agent/taler-monitoring-mail-deliv.timer b/host-agent/taler-monitoring-mail-deliv.timer new file mode 100644 index 0000000..b5f67d7 --- /dev/null +++ b/host-agent/taler-monitoring-mail-deliv.timer @@ -0,0 +1,12 @@ +[Unit] +Description=taler-monitoring mon* mail probe every 6h +Documentation=file:%h/src/taler-monitoring/host-agent/run-mail-deliv-probe.sh + +[Timer] +OnBootSec=20min +OnUnitActiveSec=6h +Persistent=true +RandomizedDelaySec=10min + +[Install] +WantedBy=timers.target diff --git a/mail_deliv_probe.py b/mail_deliv_probe.py new file mode 100755 index 0000000..40cd409 --- /dev/null +++ b/mail_deliv_probe.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +"""Send a tagged probe mail via sink SMTP, then IMAP-search INBOX vs Junk/spam. + +SUT remains Taler mail domains elsewhere; this only exercises mon* sinks +(and optional g0v) for inbox-vs-spam visibility. + +Env: + MAIL_DELIV_IMAP_PASS / MAIL_DELIV_ALERT_PASS required (shared default) + MAIL_DELIV_PASS_mon001 / _mon002 / _mon003 optional per-mailbox override + MAIL_DELIV_PROBE_FROM default mon001@hacktivism.ch + MAIL_DELIV_PROBE_TO default mon001 mon002 mon003 (keep all — odd-looking addrs) + MAIL_DELIV_PROBE_SMTP default mail.cyon.ch:587 + MAIL_DELIV_IMAP_HOST default mail.cyon.ch + MAIL_DELIV_PROBE_WAIT_S default 25 + MAIL_DELIV_PROBE_ALERT_TO optional extra alert recipients on spam/miss +""" +from __future__ import annotations + +import imaplib +import os +import smtplib +import ssl +import sys +import time +import uuid +from datetime import datetime, timezone +from email.message import EmailMessage +from pathlib import Path + + +def _env(name: str, default: str = "") -> str: + return (os.environ.get(name) or default).strip() + + +def _pass_for(addr: str) -> str: + """Shared pass, or MAIL_DELIV_PASS_ (e.g. MAIL_DELIV_PASS_mon002).""" + local = addr.split("@", 1)[0].replace(".", "_").replace("+", "_") + specific = _env(f"MAIL_DELIV_PASS_{local}") or _env(f"MAIL_DELIV_IMAP_PASS_{local}") + if specific: + return specific + return _env("MAIL_DELIV_IMAP_PASS") or _env("MAIL_DELIV_ALERT_PASS") + + +def smtp_send(host: str, port: int, user: str, pw: str, to: str, subj: str, body: str, msgid: str) -> None: + msg = EmailMessage() + msg["From"] = user + msg["To"] = to + msg["Subject"] = subj + msg["Message-ID"] = msgid + msg["X-Taler-Mon-Probe"] = msgid.strip("<>") + msg.set_content(body) + ctx = ssl.create_default_context() + with smtplib.SMTP(host, port, timeout=45) as s: + s.ehlo() + s.starttls(context=ctx) + s.ehlo() + s.login(user, pw) + s.send_message(msg) + + +def imap_find(host: str, user: str, pw: str, token: str) -> dict: + """Return {folder: count} for folders where token appears in Message-ID/Subject/body.""" + ctx = ssl.create_default_context() + M = imaplib.IMAP4_SSL(host, 993, ssl_context=ctx) + M.login(user, pw) + typ, bl = M.list() + folders = [] + for raw in bl or []: + line = raw.decode(errors="replace") + # LIST (\HasNoChildren) "." INBOX.Junk + if '"/"' in line or ' "." ' in line or ' "."' in line: + name = line.split(' "." ')[-1].strip().strip('"') + elif ' "/" ' in line: + name = line.split(' "/" ')[-1].strip().strip('"') + else: + parts = line.rsplit(" ", 1) + name = parts[-1].strip().strip('"') if parts else "" + if name: + folders.append(name) + # ensure common names + for n in ("INBOX", "INBOX.Junk", "INBOX.spam", "Junk", "Spam"): + if n not in folders: + folders.append(n) + + hits: dict[str, int] = {} + needle = token.encode() + for folder in folders: + try: + typ, _ = M.select(folder, readonly=True) + if typ != "OK": + continue + except Exception: + continue + # SEARCH TEXT is broad; prefer HEADER Message-ID + found = 0 + for crit in (f'(HEADER Message-ID "{token}")', f'(HEADER X-Taler-Mon-Probe "{token}")', f'(SUBJECT "{token}")'): + try: + typ, data = M.search(None, crit) + if typ == "OK" and data and data[0]: + ids = data[0].split() + found = max(found, len(ids)) + except Exception: + continue + if found: + hits[folder] = found + M.logout() + return hits + + +def classify(hits: dict[str, int]) -> str: + if not hits: + return "missing" + # prefer INBOX without junk + junkish = [] + inboxish = [] + for f, n in hits.items(): + fl = f.lower() + if "junk" in fl or "spam" in fl: + junkish.append(f) + elif fl == "inbox" or fl.endswith(".inbox") or fl == "inbox.inbox": + inboxish.append(f) + elif "inbox" in fl and "junk" not in fl and "spam" not in fl and "trash" not in fl: + inboxish.append(f) + if inboxish and not junkish: + return "inbox" + if junkish and not inboxish: + return "spam" + if inboxish and junkish: + return "both" + return "other:" + ",".join(hits.keys()) + + +def main() -> int: + frm = _env("MAIL_DELIV_PROBE_FROM", "mon001@hacktivism.ch") + # Keep mon001–003 even if one address looks odd / one pass differs. + tos = _env( + "MAIL_DELIV_PROBE_TO", + "mon001@hacktivism.ch mon002@hacktivism.ch mon003@hacktivism.ch", + ).split() + pw_from = _pass_for(frm) + if not pw_from: + print("FAIL missing MAIL_DELIV_IMAP_PASS (or per-mailbox MAIL_DELIV_PASS_*)", file=sys.stderr) + return 2 + smtp = _env("MAIL_DELIV_PROBE_SMTP", "mail.cyon.ch:587") + imap_host = _env("MAIL_DELIV_IMAP_HOST", "mail.cyon.ch") + wait_s = float(_env("MAIL_DELIV_PROBE_WAIT_S", "25") or "25") + host, _, port_s = smtp.partition(":") + port = int(port_s or "587") + + token = uuid.uuid4().hex + msgid = f"<{token}@taler-monitoring.probe>" + now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + subj = f"[taler-mon probe] {token[:12]}" + body = ( + f"taler-monitoring regular mon* probe\n" + f"token={token}\n" + f"when={now}\n" + f"Hallo Hernâni, hallo Florian — Inbox-vs-Spam check.\n" + ) + + results = [] + sent_ok = [] + for to in tos: + try: + smtp_send(host, port, frm, pw_from, to, subj, body, msgid) + print(f"SENT to={to} token={token[:12]}") + sent_ok.append(to) + except Exception as e: + print(f"SEND_FAIL to={to} {type(e).__name__}: {e}") + results.append((to, "send_fail", {})) + + time.sleep(max(5.0, wait_s)) + + rc = 0 + for to in tos: + pw = _pass_for(to) + if not pw: + print(f"IMAP_SKIP to={to} no password (set MAIL_DELIV_PASS_{to.split('@')[0]})") + results.append((to, "no_pass", {})) + rc = 1 + continue + try: + hits = imap_find(imap_host, to, pw, token) + kind = classify(hits) + print(f"IMAP to={to} result={kind} hits={hits}") + results.append((to, kind, hits)) + if kind != "inbox": + rc = 1 + except Exception as e: + # Keep name in rotation; soft note — odd addr / wrong pass still listed + print(f"IMAP_FAIL to={to} {type(e).__name__}: {e}") + results.append((to, "imap_fail", {})) + rc = 1 + + # optional notify alert recipients if any non-inbox + alert_to = _env("MAIL_DELIV_PROBE_ALERT_TO") or _env("MAIL_DELIV_ALERT_TO") + bad = [r for r in results if r[1] not in ("inbox",)] + if alert_to and bad: + summary = "\n".join(f"- {t}: {k} {h}" for t, k, h in results) + for ato in alert_to.split(): + try: + smtp_send( + host, + port, + frm, + pw_from, + ato, + f"[taler-mon probe] spam/miss {token[:12]}", + f"Probe issues:\n{summary}\n", + f"<{uuid.uuid4().hex}@taler-monitoring.alert>", + ) + print(f"ALERT → {ato}") + except Exception as e: + print(f"ALERT_FAIL → {ato} {e}") + + # stamp state + state = Path( + _env( + "TALER_MONITORD_STATE_DIR", + str(Path.home() / ".local/state/taler-monitord"), + ) + ) + state.mkdir(parents=True, exist_ok=True) + (state / "mail-deliv-probe.last").write_text( + f"ts={now}\ntoken={token}\nrc={rc}\n" + "\n".join(f"{t}\t{k}\t{h}" for t, k, h in results) + "\n", + encoding="utf-8", + ) + return rc + + +if __name__ == "__main__": + raise SystemExit(main())