From a2afe690b7a0c2a62254087c8501f3bbecd6d300 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sat, 18 Jul 2026 13:55:49 +0200 Subject: [PATCH 001/101] taler-monitoring v1.0: standalone suite --- .gitignore | 9 + CLI-AUTOMATION-NOTES.md | 228 +++ DEPENDENCIES.md | 70 + README.md | 362 ++++ TESTS.md | 193 ++ VERSION | 1 + VERSIONS.md | 14 + android-test/.gitignore | 11 + android-test/GUI-AUTOMATION-NOTES.md | 362 ++++ android-test/README.md | 111 ++ android-test/lib_android_env.sh | 158 ++ android-test/lib_release_age.sh | 133 ++ android-test/lib_ui.py | 192 ++ android-test/run-android-build-and-smoke.sh | 167 ++ android-test/run-android-gui-smoke.sh | 201 ++ android-test/run-android-pay-smoke.sh | 308 +++ android-test/run-android-variant-matrix.sh | 322 ++++ android-test/run-goa-gui-chain.sh | 366 ++++ android-test/start-android-emulator.sh | 90 + check_apt_deploy.sh | 279 +++ check_auth401.sh | 704 +++++++ check_e2e.sh | 1711 +++++++++++++++++ check_goa_ladder.sh | 1364 +++++++++++++ check_inside.sh | 546 ++++++ check_qr_payloads.py | 294 +++ check_sanity.sh | 285 +++ check_server.sh | 158 ++ check_surface.sh | 692 +++++++ check_urls.sh | 1595 +++++++++++++++ check_versions.sh | 563 ++++++ domains.conf | 46 + harness-live.txt | 7 + harness-tests.txt | 41 + host-agent/README.md | 183 ++ host-agent/ROOT-APPLY-MONITORING.md | 57 + .../ensure-apt-deploy-test-containers.sh | 373 ++++ host-agent/env.example | 26 + host-agent/env/francpaysan.env.example | 24 + host-agent/env/hacktivism.env.example | 24 + host-agent/env/stagepaysan.env.example | 27 + host-agent/install-host-agent.sh | 107 ++ host-agent/run-aptdeploy-monitoring.sh | 43 + host-agent/run-aptdeploy.sh | 150 ++ host-agent/run-fp-prod-monitoring.sh | 20 + host-agent/run-fp-stage-monitoring.sh | 21 + host-agent/run-hacktivism-monitoring.sh | 23 + host-agent/run-host-report.sh | 358 ++++ host-agent/run-surface-monitoring.sh | 41 + host-agent/taler-monitoring-aptdeploy.service | 18 + host-agent/taler-monitoring-aptdeploy.timer | 14 + host-agent/taler-monitoring-hacktivism.path | 19 + .../taler-monitoring-hacktivism.service | 21 + host-agent/taler-monitoring-hacktivism.timer | 14 + host-agent/taler-monitoring-surface.service | 18 + host-agent/taler-monitoring-surface.timer | 14 + host-agent/touch-software-stamp.sh | 17 + host-agent/update-suite.sh | 45 + lib.sh | 1257 ++++++++++++ metrics.sh | 1431 ++++++++++++++ secrets.env.example | 87 + site-gen/README.md | 81 + site-gen/ROOT-ON-KOOPA.md | 92 + site-gen/caddy-monitoring-handles.snippet | 57 + ...om.hacktivism.taler-monitoring-sites.plist | 34 + site-gen/console_to_html.py | 935 +++++++++ site-gen/deploy-monitoring-sites.sh | 44 + site-gen/generate-monitoring-sites.sh | 273 +++ site-gen/install-firecuda-timer.sh | 79 + site-gen/pull-firecuda-to-koopa.sh | 15 + site-gen/run-on-firecuda.sh | 59 + site-gen/settings.conf.example | 55 + surface-catalog.conf | 52 + taler-monitoring.sh | 477 +++++ 73 files changed, 18268 insertions(+) create mode 100644 .gitignore create mode 100644 CLI-AUTOMATION-NOTES.md create mode 100644 DEPENDENCIES.md create mode 100644 README.md create mode 100644 TESTS.md create mode 100644 VERSION create mode 100644 VERSIONS.md create mode 100644 android-test/.gitignore create mode 100644 android-test/GUI-AUTOMATION-NOTES.md create mode 100644 android-test/README.md create mode 100644 android-test/lib_android_env.sh create mode 100644 android-test/lib_release_age.sh create mode 100755 android-test/lib_ui.py create mode 100755 android-test/run-android-build-and-smoke.sh create mode 100755 android-test/run-android-gui-smoke.sh create mode 100755 android-test/run-android-pay-smoke.sh create mode 100755 android-test/run-android-variant-matrix.sh create mode 100755 android-test/run-goa-gui-chain.sh create mode 100755 android-test/start-android-emulator.sh create mode 100755 check_apt_deploy.sh create mode 100755 check_auth401.sh create mode 100755 check_e2e.sh create mode 100755 check_goa_ladder.sh create mode 100755 check_inside.sh create mode 100755 check_qr_payloads.py create mode 100755 check_sanity.sh create mode 100755 check_server.sh create mode 100755 check_surface.sh create mode 100755 check_urls.sh create mode 100755 check_versions.sh create mode 100644 domains.conf create mode 100644 harness-live.txt create mode 100644 harness-tests.txt create mode 100644 host-agent/README.md create mode 100644 host-agent/ROOT-APPLY-MONITORING.md create mode 100755 host-agent/ensure-apt-deploy-test-containers.sh create mode 100644 host-agent/env.example create mode 100644 host-agent/env/francpaysan.env.example create mode 100644 host-agent/env/hacktivism.env.example create mode 100644 host-agent/env/stagepaysan.env.example create mode 100755 host-agent/install-host-agent.sh create mode 100755 host-agent/run-aptdeploy-monitoring.sh create mode 100755 host-agent/run-aptdeploy.sh create mode 100755 host-agent/run-fp-prod-monitoring.sh create mode 100755 host-agent/run-fp-stage-monitoring.sh create mode 100755 host-agent/run-hacktivism-monitoring.sh create mode 100755 host-agent/run-host-report.sh create mode 100755 host-agent/run-surface-monitoring.sh create mode 100644 host-agent/taler-monitoring-aptdeploy.service create mode 100644 host-agent/taler-monitoring-aptdeploy.timer create mode 100644 host-agent/taler-monitoring-hacktivism.path create mode 100644 host-agent/taler-monitoring-hacktivism.service create mode 100644 host-agent/taler-monitoring-hacktivism.timer create mode 100644 host-agent/taler-monitoring-surface.service create mode 100644 host-agent/taler-monitoring-surface.timer create mode 100755 host-agent/touch-software-stamp.sh create mode 100755 host-agent/update-suite.sh create mode 100755 lib.sh create mode 100644 metrics.sh create mode 100644 secrets.env.example create mode 100644 site-gen/README.md create mode 100644 site-gen/ROOT-ON-KOOPA.md create mode 100644 site-gen/caddy-monitoring-handles.snippet create mode 100644 site-gen/com.hacktivism.taler-monitoring-sites.plist create mode 100755 site-gen/console_to_html.py create mode 100755 site-gen/deploy-monitoring-sites.sh create mode 100755 site-gen/generate-monitoring-sites.sh create mode 100755 site-gen/install-firecuda-timer.sh create mode 100755 site-gen/pull-firecuda-to-koopa.sh create mode 100755 site-gen/run-on-firecuda.sh create mode 100644 site-gen/settings.conf.example create mode 100644 surface-catalog.conf create mode 100755 taler-monitoring.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..eb91f32 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +secrets.env +__pycache__/ +*.pyc +android-test/apks/ +android-test/out*/ +android-test/artifacts/ +site-gen/__pycache__/ +*.log +.DS_Store diff --git a/CLI-AUTOMATION-NOTES.md b/CLI-AUTOMATION-NOTES.md new file mode 100644 index 0000000..3cd311a --- /dev/null +++ b/CLI-AUTOMATION-NOTES.md @@ -0,0 +1,228 @@ +# CLI automation notes (`taler-wallet-cli` + monitoring) + +Living list of **recurring issues** seen while driving GOA (hacktivism) and +stage TESTPAYSAN via **CLI** (`taler-wallet-cli`, bank/merchant HTTP, monitoring +e2e/ladder). Ordered roughly by **how often they hurt automation** and how +useful a fix in **wallet-cli / wallet-core** (or clearer APIs) would be. + +Companion: Android GUI → [`android-test/GUI-AUTOMATION-NOTES.md`](./android-test/GUI-AUTOMATION-NOTES.md). + +Scripts: `check_e2e.sh`, `check_goa_ladder.sh`, `taler-monitoring.sh`. + +Legend: + +| Tag | Meaning | +|-----|---------| +| **cli** | wallet-cli UX / flags / hang behaviour | +| **core** | wallet-core protocol / state machine | +| **bank** | libeufin-bank / integration API | +| **ops** | secrets, paths, multi-host | +| **doc** | missing or wrong documentation | + +--- + +## 1. `run-until-done` is unusable for automation + +| | | +|--|--| +| **Seen** | e2e, ladder, Android notes, macOS comments: *hangs / banned in monitoring* | +| **Area** | **cli** / **core** | +| **Problem** | `taler-wallet-cli run-until-done` (or long shepherd runs) often **never returns** or blocks CI-length timeouts. Monitoring **forbids** it and polls balance + bank `transfer_done` instead. | +| **Workaround** | `advanced serve` + socket (`WSOCK`) or pure poll loops (`LADDER_SETTLE_*`, e2e settle wait). | +| **Wanted for wallet-cli** | Bounded wait: e.g. `run-until-done --timeout=Ns --exit-on=withdrawn|paid|idle`; stable non-zero exit on timeout; progress on stdout/JSON. | + +--- + +## 2. Withdraw completion is not observable from CLI alone + +| | | +|--|--| +| **Seen** | e2e “coins missing after withdraw”; ladder `OK_BANK` vs wallet avail | +| **Area** | **cli** / **core** / **bank** | +| **Problem** | After `accept-uri` + bank confirm, coins may lag (wirewatch). CLI has no single “withdraw settled” command with clear success/failure; automation reimplements status from bank integration JSON + balance polls. | +| **Workaround** | Poll `balance` + `…/withdrawal-operation/{id}` (`status`, `transfer_done`, `selected_reserve_pub`). | +| **Wanted** | `wallet-cli withdrawals wait --id=… --timeout=` or JSON event stream: `selected → confirmed → coins-available`. | + +--- + +## 3. `reserve_pub` / force-select is fragile (5114) + +| | | +|--|--| +| **Seen** | ladder force-select, e2e confirm path, Android spinner issues | +| **Area** | **cli** / **core** / **bank** | +| **Problem** | Cumulative wallets re-emit old reserves; binding the wrong `reserve_pub` yields **HTTP 409 / code 5114** (“already bound”). Automation scrapes accept/tx dumps for candidates. | +| **Workaround** | Score pubs by WID/amount; try several; treat 5114 as “stale reserve” not “out of money”. | +| **Wanted** | CLI returns **the** `reserve_pub` for the just-accepted withdraw in structured JSON; `force-select` / bank-side select API documented with idempotency. | + +--- + +## 4. Exchange not auto-known for bank deep links / withdraw URIs + +| | | +|--|--| +| **Seen** | Android endless spinner; GOA `bank.hacktivism.ch` withdraw; fixed in app branch `fix/bank-withdraw-auto-exchange` | +| **Area** | **core** / **cli** | +| **Problem** | Withdraw URI names an exchange the wallet has never added → stuck loading / empty exchange list. CLI requires explicit `exchanges add` + `update` + `accept-tos` every fresh DB. | +| **Workaround** | e2e always adds exchange before ATM ladder; Android needs ensureExchange-style fix. | +| **Wanted** | CLI/core: on `accept-uri` withdraw, **auto-add** exchange from URI/bank details (`allowCompletion`), then surface ToS if needed in one command. | + +--- + +## 5. Default ports in URIs (`:443` / `:80`) break round-trips + +| | | +|--|--| +| **Seen** | ladder/e2e strip ports; QR checks; mint URIs with `:443` | +| **Area** | **cli** / **bank** / **doc** | +| **Problem** | Some bank/wallet outputs include `taler://…host:443/…`. QR validators and some clients reject or double-normalize inconsistently. | +| **Workaround** | `sed 's/:443\//\//g'` everywhere in automation. | +| **Wanted** | Canonical form without default ports in **all** wallet-cli and bank integration outputs; document as invariant. | + +--- + +## 6. No first-class “demo withdraw” / explorer mint in wallet-cli + +| | | +|--|--| +| **Seen** | e2e uses bank admin + ATM create; ladder/gui-chain reimplement explorer mint in Python | +| **Area** | **cli** / **ops** | +| **Problem** | Automation always hand-rolls Basic auth → token → `POST …/withdrawals` → `taler_withdraw_uri`. Easy to get wrong (auth header, amount currency). | +| **Workaround** | Shared Python in `goa-chain` / ladder / e2e; secrets files. | +| **Wanted** | Optional helper: `wallet-cli testing mint-withdraw --bank=URL --user=explorer --password-file=… --amount=GOA:10` (or bank-side only tool in libeufin) emitting a clean URI. | + +--- + +## 7. Long-lived wallet process required for pay/withdraw progress + +| | | +|--|--| +| **Seen** | e2e `advanced serve` / shepherd socket | +| **Area** | **cli** | +| **Problem** | One-shot CLI invocations do not keep background work alive; without serve/shepherd, withdraw/pay may stall mid-flight. | +| **Workaround** | Start serve once per e2e run; route `wcli` through the socket. | +| **Wanted** | Documented, supported “session mode”: start/stop serve; or each mutating command optionally `--background-until=…` with timeout. | + +--- + +## 8. ToS accept is a separate step that fails silently or blocks + +| | | +|--|--| +| **Seen** | e2e `exchanges accept-tos`; ladder after add; Android GUI ToS screens | +| **Area** | **cli** / **core** | +| **Problem** | Fresh exchange → operations need ToS; CLI must call `accept-tos` explicitly. Failures are easy to miss in multi-step scripts. | +| **Workaround** | Always `accept-tos` after `update` in e2e/ladder. | +| **Wanted** | `accept-uri --accept-tos` or auto-prompt with non-interactive `--yes` that covers exchange ToS for that URI’s exchange. | + +--- + +## 9. Pay template / public order path is merchant-shaped, not CLI-shaped + +| | | +|--|--| +| **Seen** | e2e shop templates, stage farmer shops, ladder private orders | +| **Area** | **cli** / **doc** | +| **Problem** | Wallet-cli pays via `handle-uri` on `taler://pay/…`; creating the order is always custom curl (template POST or private order + token). No unified “pay this amount to instance” for public templates. | +| **Workaround** | Monitoring builds URI externally then `handle-uri --yes`. | +| **Wanted** | Documented recipe only, or `wallet-cli testing pay-template --base=… --instance=… --id=…` for demos. | + +--- + +## 10. Amount / currency parsing edge cases + +| | | +|--|--| +| **Seen** | ladder min denom 0.01 vs GOA 1e-6; zero withdraw rejected (HTTP 409 / amount too low); max wire ceilings | +| **Area** | **cli** / **bank** | +| **Problem** | `CURRENCY:0` and sub-min amounts fail differently per stack; CLI error strings are not machine-stable. | +| **Workaround** | Soft-skip zero rung; clamp ladder min to exchange denoms / bank max_wire. | +| **Wanted** | Structured errors (`AMOUNT_too_small`, `currency_unknown`) in JSON mode; `wallet-cli amount validate --exchange=`. | + +--- + +## 11. Finding the right `taler-wallet-cli` binary + +| | | +|--|--| +| **Seen** | `find_wallet_cli` in `lib.sh`; hardcoded laptop paths; wrapper vs `.mjs` | +| **Area** | **ops** / **cli** | +| **Problem** | Debian package is a shell wrapper; some tools need `node …/taler-wallet-cli.mjs`. Hardcoded `/Users/…` paths break other hosts. | +| **Workaround** | `find_wallet_cli` search list; `WALLET_CLI=` override. | +| **Wanted** | Single install story: `wallet-cli --version` JSON with path + libversion; no need to pass `.mjs` to node by hand. | + +--- + +## 12. Secrets and multi-stack confusion + +| | | +|--|--| +| **Seen** | explorer vs admin password; GOA secrets used on stage; ladder EXP_PW | +| **Area** | **ops** | +| **Problem** | CLI does not know “which stack”; wrong password → 401 mid-ladder. Not a wallet-cli bug, but every CLI automation hits it. | +| **Workaround** | `SECRETS_ROOT`, stage SSH, `STACK=` profiles. | +| **Wanted** | Optional `~/.config/taler/stacks.d/goa.env` convention documented next to wallet-cli; still no secrets in repo. | + +--- + +## 13. No stable machine-readable “step result” for scripts + +| | | +|--|--| +| **Seen** | e2e greps accept output; ladder scrapes JSON from mixed stdout | +| **Area** | **cli** | +| **Problem** | Human logs + occasional JSON blobs; hard to parse reliably. | +| **Workaround** | Python scrapers, temp files, `tee`. | +| **Wanted** | Global `--json` / `--ndjson` for all commands; one object per completed operation with `ok`, `op`, `ids`, `amounts`. | + +--- + +## 14. Pay settlement wait is symmetric to withdraw pain + +| | | +|--|--| +| **Seen** | e2e pay settle loops; ladder pay settle rounds | +| **Area** | **cli** / **core** | +| **Problem** | After `handle-uri` pay, success is “order paid” / balance drop / merchant order status — not one CLI wait. | +| **Workaround** | Short poll loops; never run-until-done. | +| **Wanted** | Same as §1–2: bounded wait on transaction id / order id. | + +--- + +## Priority shortlist for `taler-wallet-cli` / core + +If only a few changes land, these unlock the most automation: + +1. **Bounded `run-until-done` / wait-for-state** (§1, §2, §14) +2. **Structured JSON on every command** (§13) +3. **Auto-add exchange + ToS on accept-uri** (§4, §8) +4. **Canonical URIs without :443** (§5) +5. **Emit reserve_pub for last withdraw** (§3) + +--- + +## How we work around today (monitoring) + +| Issue | Monitoring behaviour | +|-------|----------------------| +| run-until-done | Disabled; balance + bank poll | +| serve | Optional long-lived socket in e2e | +| exchange | Explicit add/update/accept-tos | +| ladder explorer | `read_secret` / EXP_PW_FILE | +| stage maxima | bank `max_wire` + keys min denom | +| force-select | multi-rpub try; soft-skip confirm | + +--- + +## How to add an issue + +```markdown +### N. short title +| | | +|--|--| +| **Seen** | where / which stack | +| **Area** | cli / core / bank / ops / doc | +| **Problem** | … | +| **Workaround** | … | +| **Wanted** | concrete CLI/core behaviour | +``` diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md new file mode 100644 index 0000000..940dea3 --- /dev/null +++ b/DEPENDENCIES.md @@ -0,0 +1,70 @@ +# taler-monitoring — host package dependencies + +## koopa (openSUSE Tumbleweed, user `hernani`) + +### Required (host-agent / GOA primary: `urls inside versions`) + +| Binary | Package (zypper) | Role | +|--------|------------------|------| +| `bash` | (base) | runner | +| `curl` | `curl` | HTTPS/API probes | +| `python3` | `python313-base` (provides `/usr/bin/python3`) | JSON, HTML site-gen, metrics | +| `podman` | `podman` | **inside**: `podman exec` into taler-hacktivism* | +| `git` | `git` | pin commit SHA in monitoring HTML footer | +| `rsync` | `rsync` | optional: deploy/sync suite | +| `timeout` / `mktemp` / `getent` | coreutils / glibc | helpers | + +**Status on koopa (2026-07-18):** all of the above present; rootless podman owns GOA containers. + +### Recommended (full `urls` QR group) + +| Binary | Package | Role | +|--------|---------|------| +| `qrencode` | **`qrencode`** | encode taler:// / payto:// → PNG | +| `zbarimg` | **`zbar`** | decode PNG; exact payload match | + +Without these, QR checks **WARN** and skip encode/decode (`apt install` messages in code are Debian-oriented; on Tumbleweed use zypper). + +```bash +# as root on koopa +zypper refresh +zypper in qrencode zbar +# verify as hernani: +command -v qrencode zbarimg +``` + +### Optional (not needed for host-agent default) + +| Binary / tool | When | +|---------------|------| +| `jq` | nice-to-have; suite uses python3 for JSON | +| `dig` / `bind-utils` | debug DNS; not required for default phases | +| `ssh` | only for access mode **ssh** (laptop → `KOOPA_SSH` / stage `INSIDE_SSH`); not needed for **host-podman** | +| `node` / `pnpm` + wallet-cli | **e2e** / ladder only (+ secrets) | +| `secrets.env` / admin passwords | e2e only — never system packages | + +### systemd user (no extra packages) + +- `systemd` user session + **`loginctl enable-linger hernani`** (already **Linger=yes** on koopa) +- Units: `host-agent/*.service|path|timer` + +### Caddy HTML publish (separate from monitoring run) + +- Host Caddy already installed (root service). +- Writing `/var/www/monitoring-sites`: either root `rsync` or directory owned/writable by `hernani` (you set this up). + +--- + +## firecuda-external (macOS, outside-only timer) + +| Need | Notes | +|------|--------| +| `bash`, `curl`, `python3`, `rsync`, `git` (optional) | usually Homebrew / Xcode CLT | +| **No** `podman` | phases are `urls versions` only (`SKIP_SSH=1`) | +| `qrencode` / `zbarimg` | optional for QR; `brew install qrencode zbar` if desired | + +--- + +## Franc Paysan hosts (francpaysan / stagepaysan) + +Document package lists under **`francpaysan-admin-log`** when those host-agents are installed (same idea: curl, python3, podman if inside, qrencode/zbar for full urls). diff --git a/README.md b/README.md new file mode 100644 index 0000000..b8125f0 --- /dev/null +++ b/README.md @@ -0,0 +1,362 @@ +# taler-monitoring + +Standalone repository: https://git.hacktivism.ch/hernani/taler-monitoring (tag **v1.0**). + +Formerly: `koopa-admin-log/scripts/taler-monitoring/`. + +# taler-monitoring + +Host packages (koopa / runners): **[DEPENDENCIES.md](./DEPENDENCIES.md)**. +Host-agents (GOA + FP; shared report pipeline): **[host-agent/README.md](./host-agent/README.md)**. +Public HTML site-gen (same timeout/log/HTML defaults): **[site-gen/README.md](./site-gen/README.md)**. +FP install notes: `francpaysan-admin-log/docs/taler-monitoring-host-agents.md`. + +Report for the **GOA** stack with boxed severity badges and **grouped test IDs** +`area.group-NN` (e.g. `www.exchange-01`, `e2e.pay-03`): + +| Tag | Meaning | +|-----|---------| +| `┌ OK ┐` | check passed | +| `┌ INFO ┐` | status / note | +| `┌ WARN ┐` | degraded but maybe not fatal | +| `┌ ERROR ┐` | component problem | +| `┌ BLOCKER ┐` | **withdraw/pay path cannot complete** because of this | + +IDs reset per **group** inside an area — so “too many www tests” become +`www.exchange-*`, `www.bank-*`, `www.landing-*`, etc. +Catalog: **[TESTS.md](./TESTS.md)**. +Android / tooling: **[VERSIONS.md](./VERSIONS.md)**. +GUI Android notes: **[android-test/GUI-AUTOMATION-NOTES.md](./android-test/GUI-AUTOMATION-NOTES.md)**. +CLI / wallet-cli recurring issues (for upstream improvements): **[CLI-AUTOMATION-NOTES.md](./CLI-AUTOMATION-NOTES.md)**. + +```text +┌ OK ┐ www.exchange-01 exchange /config · HTTP 200 +┌ BLOCKER┐ e2e.prereq-02 merchant HTTP 502 +``` + +End of each phase: totals + list of **BLOCKERS** and **ERRORS** (ids included). + +## Secrets (local GOA e2e) + +**Do not commit passwords.** Canonical trees: `~/src/koopa/koopa-admin-log` + +sibling `~/src/koopa/koopa-admin-secrets`. + +### Local file (recommended on laptop) + +```bash +cd ~/src/koopa/src/taler-monitoring +cp secrets.env.example secrets.env # gitignored +$EDITOR secrets.env +./taler-monitoring.sh e2e +``` + +See **[secrets.env.example](./secrets.env.example)** for every variable that +may appear (admin pass, merchant token, SECRETS_ROOT, progress knobs). + +### Search order + +1. Already-exported env (`E2E_BANK_ADMIN_PASS`, …) +2. `secrets.env` next to this suite (`MONITORING_SECRETS_ENV=` override) +3. `SECRETS_ROOT` / `KOOPA_ADMIN_SECRETS` file tree +4. Sibling `../koopa-admin-secrets/koopa/host-root` +5. `$HOME/src/koopa/koopa-admin-secrets/koopa/host-root` +6. `$HOME/.config/taler-landing/bank-admin-password.txt` +7. SSH koopa `/root/…` or bank container + +### Line IDs + progress + +Each check prints **global** and **grouped** ids: + +```text +┌ OK ┐ #014 www.exchange-03 exchange /config · HTTP 200 +┌ WARN ┐ #015 www.stats-02 stats age soft · yellow badge (not red) +┌ 42% ┐ ████████░░░░░░░░░░░░░░░░░░░░ 14/55 +``` + +- `#014` — run-wide counter (whole `./taler-monitoring.sh` invocation) +- `www.exchange-03` — area.group-NN +- **Colours (left badge + label):** + - **OK** green · **INFO** blue/cyan · **WARN** yellow (never red) + - **ERROR** red · **BLOCKER** magenta +- Progress: badge shows **percent** (not “PROG”); bar fills **left → right**; + `done/total` on the right. Estimate auto-grows if short. Override: + `PROGRESS_TOTAL=N` / `PROGRESS_OFF=1` +- **QR (urls):** harvest `taler://` / `payto://` (+ app store `data-qr-url`) from landings and + `demo-withdraw.json` / `auto-account.json` → validate form → `qrencode` PNG → `zbarimg` + decode must match. Requires packages **`qrencode`** + **`zbar-tools`**. + +## Commands + +```bash +# Local GOA stack (may use SSH for inside/e2e) +./taler-monitoring.sh # urls + inside + versions + e2e +./taler-monitoring.sh inside # containers on koopa +./taler-monitoring.sh versions # deb.taler.net + package versions vs trixie +./taler-monitoring.sh sanity +./taler-monitoring.sh e2e + +# FrancPaysan STAGE (TESTPAYSAN) — like hacktivism breadth: +# default phases: urls + inside + versions + e2e +./taler-monitoring.sh -d stage.lefrancpaysan.ch +./taler-monitoring.sh -d stage.lefrancpaysan.ch inside # stagepaysan only +# inside: low-priv SSH francpaysan-stage-user (User stagepaysan) → podman +# containers stage-lfp-{bank,exchange-ansible,merchant} · ports 9032/9031/9030 +# stats: outside-in public …/intro/stats.json (age via generated_at_unix) +# Admin credit for e2e: E2E_BANK_ADMIN_PASS=… or francpaysan-host secrets +# (bank-admin-password.txt under /mnt/data/stagepaysan/bank/secrets/) +# Override: INSIDE_SSH=… INSIDE_PROFILE=stage-lfp + +# Other domains — public HTTPS only (pass e2e/inside explicitly) +./taler-monitoring.sh -d taler.net +./taler-monitoring.sh --domain taler-ops.ch +./taler-monitoring.sh -d my.taler-ops.ch urls +./taler-monitoring.sh -d demo.taler.net urls +./taler-monitoring.sh taler.net # bare domain = same as -d + +# Explicit endpoints (any mix; override profile or skip -d) +./taler-monitoring.sh --bank https://bank.demo.taler.net \ + --exchange https://exchange.demo.taler.net \ + --merchant https://backend.demo.taler.net --currency KUDOS urls + +# Landing stats.json freshness (outside-in; age via generated_at_unix) +STATS_STALE_SECS=900 STATS_FAIL_SECS=3600 ./taler-monitoring.sh -d stage.lefrancpaysan.ch urls + +# --- Full load (GOA / hacktivism) --- +# all = urls + inside + versions + sanity + e2e +# full = all + server + ladder + auth401 (long; needs secrets.env) +./taler-monitoring.sh -d hacktivism.ch full +./taler-monitoring.sh -d hacktivism.ch urls inside versions sanity server e2e ladder auth401 + +# SPA pin after selfbuild-webui.sh (container): +EXPECT_WEBUI_VERSION=1.6.11 EXPECT_WEBUI_OVERLAY=selfbuild-v1.6.11 \ + WEBUI_OVERLAY_DENY='c778af04|master-11340' \ + ./taler-monitoring.sh -d hacktivism.ch urls + +# Only 401 matrix (may create throwaway instance): +./taler-monitoring.sh -d hacktivism.ch auth401 + +# Stage FrancPaysan — durable mon401 account (password in francpaysan-secrets): +./taler-monitoring.sh -d stage.monnaie.lefrancpaysan.ch auth401 +# secrets: ~/francpaysan-secrets/stage/instances/mon401.password +# ~/francpaysan-secrets/stage/default-instance-token.txt +# force throwaway create instead: AUTH401_PREFER_DURABLE=0 AUTH401_CREATE=1 … +``` + +### Landing `stats.json` (www.stats) + +Public `GET …/intro/stats.json` must expose timestamps so monitoring can tell +whether the collector is still updating: + +| Field | Role | +|-------|------| +| `generated_at_unix` | epoch seconds — **preferred** for age = now − unix | +| `generated_at` | ISO-8601 (fallback if unix missing) | +| `generated_at_human` | UI “updated” / footer string | + +Also checks display payloads (bank-shape `withdraws` / accounts, or exchange-db / +merchant-db counters). Soft WARN if age > `STATS_STALE_SECS` (default 900); +ERROR if age > `STATS_FAIL_SECS` (default 3600; set `0` to disable). +Stage shared feed: same `gen_unix` on bank + exchange + monnaies. + +## Domain profiles (`domains.conf`) + +Each monitored stack must say **what is bank, exchange, and merchant-backend**. + +Edit **[domains.conf](./domains.conf)** (or set `TALER_DOMAINS_CONF=`): + +```text +# name bank exchange merchant-backend currency local [canonical] +taler-ops.ch bank.taler-ops.ch exchange.taler-ops.ch my.taler-ops.ch CHF 0 +``` + +| Field | Meaning | +|-------|---------| +| **name** | `-d NAME` / `TALER_DOMAIN` match (add alias lines as needed) | +| **bank** | libeufin/bank public host or `https://…` | +| **exchange** | exchange public host or URL | +| **merchant** | merchant **backend** (SPA/API), e.g. `my.taler-ops.ch` | +| **currency** | `GOA` / `KUDOS` / `CHF` / … | +| **local** | `1` = koopa SSH stack; `0` = public only | +| **landing** | `1` = check `/intro` landings + assets; `0` = skip (TOPS / mytops) | +| **canonical** | optional label after alias (e.g. `my.taler-ops.ch` → `taler-ops.ch`) | + +Then: + +```bash +./taler-monitoring.sh -d taler-ops.ch urls +``` + +Overrides (always win after profile load): + +```bash +./taler-monitoring.sh -d taler-ops.ch \ + --merchant https://my.taler-ops.ch \ + --exchange https://exchange.taler-ops.ch \ + --bank https://bank.taler-ops.ch urls +``` + +Helpers in `lib.sh`: `set_taler_stack`, `load_domain_profile`, `apply_taler_domain`. + +### Built-in profiles (see `domains.conf`) + +| Domain / alias | Bank | Exchange | Merchant | Currency | +|----------------|------|----------|----------|----------| +| `hacktivism.ch` / `koopa` | bank.hacktivism.ch | exchange.hacktivism.ch | taler.hacktivism.ch | GOA | +| `taler.net` / `demo.taler.net` | bank.demo.taler.net | exchange.demo.taler.net | backend.demo.taler.net | KUDOS | +| `test.taler.net` | bank.test… | exchange.test… | backend.test… | **TESTKUDOS** · no landings | +| `lefrancpaysan.ch` | bank.… | exchange.… | **monnaie.…** (merchant) | report-only · no landings | +| **`stage.lefrancpaysan.ch`** | stage.bank.… | stage.exchange.… | **stage.monnaie.…** | **TESTPAYSAN** · landings · **inside via stagepaysan** | +| `taler-ops.ch` / `my.taler-ops.ch` | bank.taler-ops.ch | exchange.taler-ops.ch | **my.taler-ops.ch** | CHF · **no landings** | +| `stage.taler-ops.ch` | bank.stage… | exchange.stage… | **my.stage…** | CHF · **no landings** | +| unknown | bank.DOMAIN | exchange.DOMAIN | my.DOMAIN then probe | any · no landings | + +**Landings:** GOA, stage LFP (`TESTPAYSAN`), and optionally demo use public `/intro` pages. **taler-ops.ch does not** — `CHECK_LANDING=0` skips intro crawl. + +**SSH:** +| Stack | Host alias | User | Scope | +|-------|------------|------|--------| +| koopa / hacktivism | `koopa` | hernani | full host + containers | +| stage LFP | `francpaysan-stage-user` | **stagepaysan** | own podman only (no sudo / no prod data) | + +Other domains: never SSH. Optional **e2e** aborts cleanly on login/KYC. + +### E2E amounts (variable) + +| | Local (koopa) | Remote | +|--|---------------|--------| +| **ATM withdraw** | 20 · 50 · 100 · 200 · **4200** (paivana) | 10 · 20 · 50 | +| **Pay ladder** | 0.01 … 10 | 0.01 … 1 | +| **Paivana** | HTTP paywall + template pay **GOA:4200** (`goa-shop` / `paivana`) | skipped | + +```bash +./taler-monitoring.sh e2e +./taler-monitoring.sh -d taler.net urls e2e +# Amount ladder (withdraw 0 → … → max, then pay): same script for GOA + stage +./taler-monitoring.sh ladder # GOA / current domain +./taler-monitoring.sh -d stage.lefrancpaysan.ch ladder # TESTPAYSAN maxima +# Stage auto-reads bank max_wire_transfer_amount (e.g. 2000) and min denom (0.01). +# Explorer password: francpaysan-stage-user …/bank-explorer-password.txt (or EXP_PW=). +# Override: LADDER_MAX_AMOUNT=100 LADDER_STEPS=7 LADDER_PAY=0 … +# customize e2e: +E2E_WITHDRAW_VALUES="20 50 100" E2E_PAY_VALUES="0.05 1 5" ./taler-monitoring.sh e2e +E2E_VARIABLE=0 WITHDRAW_AMT=GOA:50 PAY_AMT=GOA:1 ./taler-monitoring.sh e2e # single fixed +# GOA shop catalog (local hacktivism): full list in E2E_SHOP_PRODUCTS; each run +# shuffles and pays E2E_SHOP_PICK_N products (default 2). +# E2E_SHOP_PRODUCTS lines: id|Product name|GOA:amount +# E2E_SHOP_PICK_N=2 +# (landing QR = taler://pay-template/…/{id}; popup = live taler://pay after POST templates/{id}) +# Paivana paywall (local GOA only): +# E2E_PAIVANA=1 (default) PAIVANA_PUBLIC=https://paivana.hacktivism.ch +# E2E_PAIVANA_TEMPLATE=paivana E2E_PAIVANA_AMOUNT=GOA:4200 E2E_PAIVANA_INSTANCE=goa-shop +# E2E_PAIVANA=0 # skip +# remote secrets: +# E2E_BANK_ADMIN_PASS=… E2E_MERCHANT_TOKEN=… +``` + +## Phases + +| Phase | What | +|-------|------| +| **inside** | SSH koopa: processes, postgres, local /config,/keys, wirewatch, recent log ERRORs | +| **versions** | `deb.taler.net` reachable (InRelease/Packages/pool `.deb`); containers can reach it + have apt source; installed Taler packages vs **trixie** | +| **sanity** | bank · exchange · merchant sections (public + server) | +| **e2e** | account → credit → withdraw → wallet → confirm → order → pay | +| **ladder** | GOA withdraw: fixed **0** + low bands + **random high ranges** + fixed **max** | + +```bash +# package suite (default trixie on deb.taler.net) +TALER_APT_SUITE=trixie ./taler-monitoring.sh versions +TALER_PKG_BEHIND=error ./taler-monitoring.sh versions # any behind = ERROR +``` + +E2E maps failures to blockers, e.g.: + +- `bank-confirm HTTP 409` → wallet did not select exchange +- `no GOA balance` → wirewatch / transfer +- `create order failed` → merchant auth/instance +- `Alarm clock` during pay → usually missing `handle-uri --yes` or wrong pay URI (must be `…/instances/{inst}/{oid}/?c={token}` from merchant `taler_pay_uri`) +- `insufficient balance` → withdraw incomplete + +## Performance (urls phase) + +Outside-in HTTPS RTT for bank / exchange / merchant critical paths. Each probe prints +`HTTP … · N ms` on the `[OK]`/`[WARN]`/`[ERROR]` line; end of the block prints +`perf summary` (n / min / p50 / avg / max). + +| Env | Default | Meaning | +|-----|---------|---------| +| `PERF_WARN_MS` | `8000` | WARN if latency ≥ this | +| `PERF_FAIL_MS` | `20000` | ERROR if latency ≥ this | +| `PERF_CURL_TIMEOUT` | `25` | curl max seconds per probe | + +```bash +PERF_WARN_MS=3000 PERF_FAIL_MS=10000 ./taler-monitoring.sh urls +``` + +## Wallet coins (e2e · ladder) + +At every relevant withdraw/pay step the wallet is snapped via `advanced dump-coins`. +Amounts use exchange **`alt_unit_names`** (Kilo-GOA, Mega-GOA, …) with the base +**GOA value in parentheses**: + +```text +[INFO] coins after-ATM — amount_circ=2 Kilo-GOA (GOA:2000) +[INFO] coins after-ATM — denoms_in_circ 1 Kilo-GOA (GOA:1000)×2 (=2 Kilo-GOA (GOA:2000)) +== ladder · rung 12 GOA:1000000 · 1 Mega-GOA (GOA:1000000) == +``` + +Also: coin counts (in circulation / spent), status histogram, Δ vs previous snap. +History TSV: `$METRICS_DIR/coins-history.tsv`. Map loaded from +`${EXCHANGE_PUBLIC}/config` → `currency_specification.alt_unit_names`. + +### Final statistics dashboard (e2e · ladder) + +At the end of **e2e** and **ladder**, a boxed **FINAL STATISTICS** block prints: + +| Section | Content | +|---------|---------| +| **Coins** | in circulation / spent counts + amounts + denoms (alt names) | +| **Money flow** | withdrawn events/amount, spent (paid), net (wd − spent) | +| **Tendency** | coin/amount trend over snaps (↑↓→), slope, last history rows | +| **Performance** | min / p50 / avg / max per phase; first-half vs second-half latency | +| **Load** | host loadavg + mem before/after; per-container RSS/CPU; tendencies | + +Successful withdraw/pay steps append to `flow-withdrawn.txt` / `flow-spent.txt` (ladder also uses its TSVs). + +## Load / memory (inside · e2e · ladder) + +Host **loadavg**, **RAM used/avail**, and per-container **RSS / CPU / block I/O** +(plus postgres DB sizes and role RSS: libeufin, exchange, merchant, …) are +snapshotted via SSH whenever the stack is under load: + +| Phase | When | +|-------|------| +| **inside** | end of container status | +| **e2e** | before work · after ATM withdraws · after payments · after shop · e2e-end (+ delta) | +| **ladder** | before first rung · after last rung (+ delta + timing overall) | + +```bash +# LAN down? force WAN DNAT host +KOOPA_SSH=koopa-external ./taler-monitoring.sh inside +# or rely on automatic fallback (default KOOPA_SSH_FALLBACKS=koopa-external) + +METRICS_LOAD=0 ./taler-monitoring.sh e2e # skip host probes +LADDER_LOAD=0 ./taler-monitoring.sh ladder +``` + +| Env | Default | Meaning | +|-----|---------|---------| +| `KOOPA_SSH` | `koopa` | primary SSH host | +| `KOOPA_SSH_FALLBACKS` | `koopa-external` | tried if primary fails | +| `METRICS_LOAD` | `1` | `0` = skip all host/container load probes | +| `LADDER_LOAD` | `1` | `0` = skip load in ladder only | +| `LADDER_PAY` | `1` | `0` = skip payment ladder after withdraws | +| `LADDER_WITHDRAW_SCALE` | `1.5` | withdraw mids / pay mids ratio (funding headroom) | +| `LADDER_STEPS` | `23` | rungs for both withdraw and pay (0 + mids + max−1 + max) | +| `METRICS_LOAD_SSH_TIMEOUT` | `90` | seconds for remote load python | + +## Needs + +- SSH `koopa` or `koopa-external` (inside / load / sanity server bits) +- secrets under `koopa-admin-secrets/...` for e2e +- `taler-wallet-cli` for e2e diff --git a/TESTS.md b/TESTS.md new file mode 100644 index 0000000..a762f39 --- /dev/null +++ b/TESTS.md @@ -0,0 +1,193 @@ +# taler-monitoring — grouped test IDs + +Every check line has a **global** run number and a **grouped** id: + +```text +┌ OK ┐ #001 www.exchange-01 exchange /config · HTTP 200 +┌ WARN ┐ #002 www.stats-02 soft age · yellow (never red) +┌ ERROR ┐ #003 e2e.pay-03 order create failed · HTTP 502 +┌ BLOCKER┐ #004 e2e.prereq-02 merchant secret missing +┌ PROG ┐ ████░░░░ 25% 4/16 +``` + +- `#NNN` — monotonic for the whole monitoring run +- `area.group-NN` — logical block (resets per group, continues if re-entered) + +| Area | Phase script | Groups (examples) | +|------|--------------|-------------------| +| **www** | `check_urls.sh` | `exchange` `perf` `stats` `bank` `merchant` `paivana` `landing` `qr` | +| **inside** | `check_inside.sh` | `ssh` `bank` `exchange` `merchant` `caddy` `load` | +| **versions** | `check_versions.sh` | `outside` `inside` `compare` | +| **aptdeploy** | `check_apt_deploy.sh` | `trixie` `trixie-testing` (koopa podman apt-src merchant smoke) | +| **surface** | `check_surface.sh` | **remote-only** ecosystem / `-d` domain inventory (not in default/all/full) | +| **sanity** | `check_sanity.sh` | `bank` `exchange` `merchant` | +| **server** | `check_server.sh` | (flat `server-NN` or host groups) | +| **e2e** | `check_e2e.sh` | `prereq` `load` `bank` `wallet` `atm` `settle` `pay` `shop` `paivana` `dig` `report` | +| **ladder** | `check_goa_ladder.sh` | `plan` `load` `withdraw` `pay` `report` | + +**Why groups:** flat `www-001`…`www-080` was hard to talk about. +`www.bank-04` / `e2e.atm-02` pin the failure to a logical block. + +Usage in scripts: + +```bash +set_area www +set_group exchange # → www.exchange-01, www.exchange-02, … +set_group bank # counter resets → www.bank-01 +``` + +`set_group` also prints a small group chip (`┌ www.exchange ┐`) so logs show section boundaries. +Re-entering the same group later (e.g. exchange legal after perf) **continues** NN — `www.exchange-03`, not a second `-01`. +Without `set_group`, IDs stay flat: `area-01`, `area-02`, … + +Optional soft checks still consume a number when they WARN. +Numbering follows **executed** checks (early skip may shift later NN inside the same group). + +--- + +## www — public URLs (`./taler-monitoring.sh urls`) + +| Group | Checks (typical) | +|-------|------------------| +| **www.exchange-** | `/config`, currency, **alt_unit_names**; `/keys` (+ alt soft); `/intro/`, `/`; **`/terms`**, **`/privacy`**, `/terms/` | +| **www.perf-** | outside-in RTT: bank/exchange/merchant `/config`, keys, webui, intro — ms; WARN ≥ `PERF_WARN_MS` (8000); ERROR ≥ `PERF_FAIL_MS` (20000) | +| **www.stats-** | `/intro/stats.json` **reachable**; **timestamps** (`generated_at_unix` + human/ISO); **freshness** vs wall clock (`STATS_STALE_SECS` WARN, `STATS_FAIL_SECS` ERROR); **display fields** (bank-shape withdraws/accounts, or exchange-db / merchant-db keys); optional performance block; shared-feed equal `gen_unix` when bank collector is published to all three | +| **www.bank-** | `/config`, currency, alt_unit_names; integration/webui/intro; **auto-account.json** (required GOA/local; skip/soft off-GOA e.g. TESTPAYSAN); `/terms`, `/privacy` | +| **www.merchant-** | `/config` currency + currencies alt_unit_names; listed exchanges alt; webui/intro; **`/terms`**, **`/privacy`** | +| **www.webui-** | SPA fingerprints: **`/webui/version.txt`**, **`/webui/version-overlay.txt`**, `index.html` / `index.js`; optional pin via `EXPECT_WEBUI_VERSION` / `EXPECT_WEBUI_OVERLAY` / `WEBUI_OVERLAY_DENY` | +| **www.paivana-** | local GOA paywall front (redirect to template) | +| **www.landing-** | own-stack intro links; static assets (`qrcode.min.js` hard; `og-goa-shop.png` hard only GOA/local); **demo-withdraw.json** GOA-only; shop-pay soft; stage merchant shop: `shop-ui.js` + `shops.css` + `qrcode.min.js`; cross-links local | +| **www.qr-** | QR payloads: harvest `taler://` / `payto://` / app `data-qr-url` from landings + mint JSON; **form** check; **qrencode → zbarimg** exact roundtrip; optional static QR images. Needs `qrencode` + `zbar-tools`. Skip: `QR_CHECK=0` | + +**QR rule:** `taler://withdraw/HOST/taler-integration/UUID` (no default `:443`/`:80`); `taler://pay/` / `pay-template/`; `payto://…` shape OK; decoded PNG must equal payload. Form errors on withdraw/pay/payto are **ERROR** on local/GOA. + +**Legal docs rule:** HTTP 200, non-empty body, not plain `not configured`, not merchant API JSON `code:21`. Local stack may require content needle (terms/privacy/FADP/GOA…). + +**Performance rule:** Measured from the **monitoring runner** (public via Caddy), not container loopback. Latency (ms) on every perf line + **perf summary** (n / min / p50 / avg / max). + +**Landing links rule:** Own-stack must be HTTP 200 (or redirect→200). External stores/docs soft WARN. Auto-account wallet link must be `taler://withdraw/HOST/taler-integration/…` (default ports stripped), never payto. + +**alt_unit_names rule:** non-empty map including scale key `"0"`. Multi-currency merchant: follow each `exchanges[]` public `/config`. + +--- + +## inside — koopa SSH (`./taler-monitoring.sh inside`) + +| Group | Checks | +|-------|--------| +| **inside.host-** | **host-podman**: reachability via `podman exec` on this machine (GOA host / `INSIDE_PODMAN=1`) — not SSH | +| **inside.ssh-** | **ssh**: from laptop/remote — SSH to `KOOPA_SSH` / `INSIDE_SSH`, then podman there | +| **inside.bank-** | container, libeufin, postgres, local `/config`, nginx, DNS pins | +| **inside.exchange-** | container, httpd, wirewatch, aggregator, transfer, local keys, DNS | +| **inside.merchant-** | container, httpd, wirewatch, depositcheck, DNS | +| **inside.caddy-** | host reverse-proxy process | +| **inside.load-** | host loadavg + RAM; per-container RSS/CPU | +| **inside.disk-** / **aptdeploy.disk-** / **versions.disk-** / **server.disk-** | free space via `df`: **WARN** if use ≥ `DISK_WARN_USED_PCT` (85), **ERROR** if full or use ≥ `DISK_ERR_USED_PCT` (95); host + containers | + +Remote lines `E|comp|LEVEL|key|detail` each become one numbered result under that component group. + +SSH: `KOOPA_SSH` (default `koopa`), then `KOOPA_SSH_FALLBACKS` when LAN is unreachable. + +--- + +## e2e — withdraw + pay (`./taler-monitoring.sh e2e`) + +| Group | Checks | +|-------|--------| +| **e2e.prereq-** | wallet-cli, currency, budgets, secrets, public reachability | +| **e2e.load-** | host/container load snapshots (before / after ATM / after pay) | +| **e2e.bank-** | account, credit, withdraw ops | +| **e2e.wallet-** | exchange + ToS, wallet setup | +| **e2e.atm-** | ATM withdraw ladder rungs | +| **e2e.settle-** | wallet settlement wait / spendable balance | +| **e2e.pay-** | variable payments | +| **e2e.shop-** | GOA shop product catalog pays | +| **e2e.paivana-** | template paywall pay | +| **e2e.dig-** | coins-missing dig (inside + withdrawal-operation) | +| **e2e.report-** | final tallies | + +When filing an issue, quote the full id + label, e.g. +`e2e.atm-03 ATM withdraw 5 GOA · bank confirmed, wallet empty`. + +--- + +## versions / sanity / ladder / server / auth401 + +| Area.group | Meaning | +|------------|---------| +| **versions.outside-** | deb.taler.net suite index | +| **versions.host-** | host-podman: packages via local `podman exec` (`INSIDE_PODMAN=1`) | +| **versions.ssh-** | ssh: packages via `KOOPA_SSH` / `INSIDE_SSH` from laptop/remote | +| **versions.compare-** | installed vs suite | +| **aptdeploy.trixie-fresh-** | fresh rebuild container `…-apt-src-trixie` | +| **aptdeploy.trixie-testing-fresh-** | fresh rebuild `…-trixie-testing` | +| **aptdeploy.trixie-upgrade-** | upgrade-track `…-trixie-upgrade` (install once, then mytops-style upgrade) | +| **aptdeploy.trixie-testing-upgrade-** | upgrade-track `…-trixie-testing-upgrade` | + +Standalone: `host-agent/run-aptdeploy.sh` (on koopa local; else `ssh koopa-external`). + +### surface (remote-only public inventory) + +```bash +./taler-monitoring.sh surface # taler.net / gnunet / taler-systems / mattermost / … +./taler-monitoring.sh -d hacktivism.ch surface # all things under that domain +``` + +- Catalog: `surface-catalog.conf` (extend as hosts are found) +- DNS, optional ICMP, TCP ports, then **protocol** checks (HTTPS/HTTP/SSH) before ERROR +- TLS cert expiry; Server-header / `/config` version hints; else **WARN: version unknown** +- OSV CVE query when package+version known → **ERROR** on hits +- Never SSH / never podman on the *targets* +| **sanity.bank-** / **.exchange-** / **.merchant-** | public + optional server-side per component | +| **ladder.plan-** / **.load-** / **.withdraw-** / **.pay-** / **.report-** | amount ladder (GOA ceiling or stage TESTPAYSAN `max_wire`) | +| **server-** | SSH host ports / processes (flat unless grouped later) | +| **auth401.*** | merchant 401 / case matrix (WebUI APIs): **signup** `POST /instances` MIX id, **login** Basic user low vs MIX/UPPER, full **case** matrix, **password** bad/empty, **pwchange** `POST private/auth` + re-login, **admin-create** `POST /management/instances` (if admin token), **bearer**, **durable mon401**, **webui** SPA `toLowerCase` probe (`check_auth401.sh`) | + +### SPA pin (after selfbuild-webui) + +```bash +EXPECT_WEBUI_VERSION=1.6.11 \ +EXPECT_WEBUI_OVERLAY=selfbuild-v1.6.11 \ +WEBUI_OVERLAY_DENY='c778af04|master-11340' \ + ./taler-monitoring.sh -d hacktivism.ch urls +``` + +Skip SPA block: `CHECK_WEBUI_SPA=0`. + +### Full load (maximum phases) + +| Alias | Expands to (local GOA) | +|-------|-------------------------| +| **`all`** | `urls inside versions sanity e2e` | +| **`full`** | `urls inside versions sanity server e2e ladder auth401` | + +```bash +cd ~/src/koopa/src/taler-monitoring +# secrets.env with bank admin + merchant tokens (see secrets.env.example) +./taler-monitoring.sh -d hacktivism.ch full +# same, explicit: +./taler-monitoring.sh -d hacktivism.ch \ + urls inside versions sanity server e2e ladder auth401 +``` + +--- + +## Visual format (colour on) + +```text +╔══════════════════════════════╗ +║ www · public URLs · … ║ +╚══════════════════════════════╝ +┌ www.exchange ┐ +┌ OK ┐ www.exchange-01 exchange /config · HTTP 200 +┌ www.bank ┐ +┌ OK ┐ www.bank-01 bank /config · HTTP 200 +┌ ERROR ┐ www.bank-04 bank /intro/auto-account.json · invalid withdraw + +┌ ERRORS · failed checks ┐ + • www.bank-04 bank /intro/auto-account.json · invalid withdraw +totals: 40 OK, 1 ERROR, 2 WARN, 5 INFO +``` + +`NO_COLOR=1` or `CLICOLOR=0` disables boxes/colours (ASCII `[ OK ]` + `-- group --` headers). +Always use `printf --` friendly plain headers when colour is off (leading `---` is not a printf option). diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..afaf360 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.0.0 \ No newline at end of file diff --git a/VERSIONS.md b/VERSIONS.md new file mode 100644 index 0000000..baf7929 --- /dev/null +++ b/VERSIONS.md @@ -0,0 +1,14 @@ +# VERSIONS — taler-monitoring + +| Tag | Date (UTC) | Notes | +|-----|------------|--------| +| **v1.0** | 2026-07-18 | Initial standalone release. Split from `koopa-admin-log` at `scripts/taler-monitoring/` (tip after monitoring host-agent/site-gen split commits). | + +Source: https://git.hacktivism.ch/hernani/taler-monitoring + +```bash +git clone https://git.hacktivism.ch/hernani/taler-monitoring.git ~/src/taler-monitoring +cd ~/src/taler-monitoring && git checkout v1.0 # or stay on main +``` + +Default host-agent tracks `origin/main` (`SUITE_UPDATE_MODE=reset`). Pin with `SUITE_GIT_REF=v1.0` in `~/.config/taler-monitoring/env`. diff --git a/android-test/.gitignore b/android-test/.gitignore new file mode 100644 index 0000000..49b08e5 --- /dev/null +++ b/android-test/.gitignore @@ -0,0 +1,11 @@ +apks/*.apk +out/ +*.png +out-source/ +out-source-*/ +out-gui/ +out-source-gui/ +out-gui-chain/ +out-matrix-*/ +out-stack-run-*/ +out-survey-*/ diff --git a/android-test/GUI-AUTOMATION-NOTES.md b/android-test/GUI-AUTOMATION-NOTES.md new file mode 100644 index 0000000..18fe398 --- /dev/null +++ b/android-test/GUI-AUTOMATION-NOTES.md @@ -0,0 +1,362 @@ +# GUI automation notes (Android wallet) + +Canonical notes for **graphical** Android wallet tests against GOA / stage. + +--- + +## Minimal variant (implemented first) — status 2026-07 + +This is the **minimal vanilla** path we actually shipped; full dual-platform +GUI is planned via flags (below), not required to use these scripts today. + +| Piece | Status | Notes | +|-------|--------|--------| +| F-Droid published APK install | **done** | `net.taler.wallet.fdroid` 1.6.1 / 854 | +| Source build + same smoke | **done** | `:wallet:assembleFdroidDebug` → `net.taler.wallet.fdroid.debug` | +| Default source branch | **done** | `dev/hernani-inference/fix-bank-withdraw-auto-exchange` (minimal GOA exchange auto-add) | +| Deep-link entry (≡ QR) | **done** | `adb` `VIEW` **withdraw then pay** (`taler://withdraw/…` → `taler://pay…`) | +| Explorer API mint (chain) | **done** | `run-goa-gui-chain.sh` (GOA + stage) | +| UI taps via uiautomator | **done** | best-effort Confirm/ToS/Pay; ANR → prefer **Wait** | +| Screenshots / logcat artifacts | **done** | under `out*`, `out-gui-chain/` (gitignored) | +| Host CLI e2e settlement | **separate** | `taler-monitoring.sh e2e` / `ladder` (not Android UI) | +| Reliable unattended Confirm on 4 GiB Linux host | **limited** | System UI ANR; use phone or more RAM | +| iOS GUI | **not in this tree** | see taler-ios `dev/hernani-inference/gui-workflows` | +| macOS full dual AVD helpers | **upstream** | taler-android `gui-workflows` (Homebrew); ported ideas here for Linux | + +**Ground rule:** bring it to run; app diffs only if needed and **minimal**; prefer +extending existing `dev/hernani-inference/*` branches over new trees. + +### Minimalvariante: was geht / was nicht + +Einschränkungen der **aktuell implementierten** Minimalvariante (deep-link + +uiautomator best-effort). Kein voller Dual-Platform-GUI-Stack, kein Ersatz für +Host-CLI-Settlement. + +#### Geht (grün) + +| Fähigkeit | Wie / Hinweis | +|-----------|----------------| +| APK installieren | F-Droid `net.taler.wallet.fdroid` **oder** Source `assembleFdroidDebug` (`.debug`) | +| GOA-Exchange ohne manuelles Add | Source-Build von `fix-bank-withdraw-auto-exchange` (Default in `run-android-build-and-smoke.sh`) | +| **Abheben + Bezahlen** | Beide Beine: erst Withdraw-URI + Taps, dann Pay-URI + Taps (`DO_WITHDRAW`/`DO_PAY`) | +| Light UI-Drive | Confirm / Accept / Pay-Taps über Label-Suche (`lib_ui.py`, gui-smoke, chain) | +| Multi-Runden-Taps | `run-android-gui-smoke.sh` / `run-goa-gui-chain.sh` (best-effort) | +| Explorer-Mint (Kette) | `run-goa-gui-chain.sh` + `EXP_PW_FILE` (GOA + stage) | +| Stack-Wahl | `STACK=goa` / `stage` (Bank/Merchant-URLs) | +| Artefakte | Screenshots, UI-XML, logcat unter `out*` / `out-gui-chain/` (gitignored) | +| Hybrid-Smoke | Install + Intent + leichte Taps: `run-android-pay-smoke.sh` | +| Settlement **nachweisen** | **Host-seitig**: `taler-monitoring.sh e2e` / `ladder` (nicht Android-UI) | +| Flags abschalten | `AUTO_ANDROID=0` skip; `AUTO_GUI=0` → nur deep-link/pay-smoke | + +#### Geht **nicht** oder nur eingeschränkt (rot / gelb) + +| Beschränkung | Folge / Workaround | +|--------------|--------------------| +| **Unattended Confirm → settled im Emulator (~4 GiB Host)** | Häufig **System UI ANR**; Taps bleiben hängen. Prefer **Wait** auf ANR-Dialog, echtes Gerät, oder Host mit mehr RAM. Kein zuverlässiges grünes E2E nur über GUI auf dem schwachen Emulator. | +| **Vollständige Wallet-Settlement-Assertion in der GUI** | Minimalvariante prüft **nicht** „Balance final / transfer_done“ in der App-Oberfläche. Proof bleibt CLI/e2e. | +| **Kamera / physischer QR-Scan** | Absichtlich nicht automatisiert; deep-link ist der Shortcut. Landing-QR-Pfade sind **HTTP**-Thema, nicht Android-GUI. | +| **Browser-Shop-Checkout-UI** | Kein Chromium/WebView-Drive; Pay-URI kommt per Template-`POST` + Intent. | +| **Volles ToS-Scroll / alle Dialoge** | Best-effort Accept-Tap; lange ToS oder unerwartete Sheets können hängen bleiben. | +| **Published F-Droid allein auf GOA** | Kann Exchange-Add / Spinner-Probleme zeigen → **fix-branch-APK** bauen; bei Spinner-Hang optional `fix-withdraw-spinner-fallback`. | +| **iOS-GUI** | Nicht in diesem Tree; siehe taler-ios `gui-workflows`. | +| **macOS dual AVD / Homebrew-Helpers** | Upstream `gui-workflows`; hier nur Linux-taugliche Port-Ideen. | +| **`AUTO_*` in jedem Script erzwungen** | Design + teilweise verdrahtet; Wrapper-Checkliste noch offen. | +| **`run-until-done` / CLI-Wallet in GUI-Skripten** | Gehört zu CLI-Automation, nicht GUI; siehe `CLI-AUTOMATION-NOTES.md`. | +| **CI grün = „User hat bezahlt“** | Smoke = Intent + best-effort UI. Wirtschaftlicher Erfolg = Host-e2e / ladder / Explorer. | + +#### Erwartetes Ergebnis pro Schicht + +| Schicht | Erfolgskriterium Minimalvariante | +|---------|----------------------------------| +| A Install | APK installed, App startet | +| B Deep-link | Intent delivered, Wallet öffnet Withdraw/Pay-Flow | +| C GUI taps | Dump+Tap ohne Crash; ANR ggf. mit Wait; **kein** Garant für „Paid“ | +| D Evidence | Artefakte geschrieben | +| Settlement | **Außerhalb** GUI: Host e2e/ladder | + +#### Wann Minimalvariante reicht + +- Smoke nach APK-/Branch-Build (Intent kommt an, App crasht nicht). +- Manuelle Nacharbeit am Gerät nach vorbereiteten URIs. +- GOA/stage **Erreichbarkeit** von Landing/Template + Wallet-Einstieg. + +#### Wann nicht reicht → nächster Schritt + +- Unattended grünes Pay-E2E auf dem Emulator → mehr RAM / physisches Gerät, ggf. `fix-withdraw-spinner-fallback`. +- Beweis „Münzen da / Merchant paid“ → CLI e2e/ladder, nicht GUI-smoke. +- iOS / Dual-Platform → andere Repos/Flags, nicht diese Skripte. + +```bash +cd scripts/taler-monitoring/android-test + +# Minimal hybrid (install + deep-link + light taps) +STACK=goa ./run-android-pay-smoke.sh + +# GUI multi-round taps +STACK=goa ./run-android-gui-smoke.sh +STACK=stage ./run-android-gui-smoke.sh + +# Explorer mint + multi withdraw/pay chain (from gui-workflows) +EXP_PW_FILE=$HOME/src/koopa/koopa-admin-secrets/koopa/host-root/taler-bank/bank-explorer-password.txt \ + STACK=goa PKG=net.taler.wallet.fdroid.debug \ + ./run-goa-gui-chain.sh + +# Build fix-branch APK then smoke (GUI=1 for gui smoke) +./run-android-build-and-smoke.sh +GUI=1 ./run-android-build-and-smoke.sh +``` + +--- + +## Platform capability flags (planned + defaults) + +Automation can run on hosts that support **Android only**, **GUI tooling only** +(conceptual), or **both**. Flags keep that explicit for CI and laptops. + +### Proposed env flags + +| Variable | Values | Meaning | +|----------|--------|---------| +| `AUTO_ANDROID` | `0` / `1` | Run Android wallet automation (adb, APK, emulator/device) | +| `AUTO_GUI` | `0` / `1` | Drive **graphical** UI (uiautomator taps, multi-round). If `0`, deep-link/CLI-only install smoke. | +| `AUTO_PLATFORM` | `auto` / `linux` / `macos` / `ios` | Host family (optional override) | +| `EMULATOR_HEADLESS` | **`1` (default)** / `0` | No host window (`-no-window`). Server/CI-safe. | +| `WINDOWED` | `0` / `1` | Shortcut: `WINDOWED=1` ⇒ show emulator window (`EMULATOR_HEADLESS=0`) | +| `EMULATOR_GPU` | default **`swiftshader_indirect`** | Guest GLES still runs headless (layout/taps/screenshots). Override `host` only if windowed + real GPU. | +| `AUTO_START_EMULATOR` | **`1` (default)** / `0` | If no adb device, start AVD via `start-android-emulator.sh --wait` | + +--- + +## Headless mode (default) + +**Default for all GUI / pay smokes:** no window on the host. Opening an emulator +GUI on servers (or headless CI) is wrong; these scripts therefore start the AVD +with **`-no-window`** and still keep **graphics logic** in the guest: + +| Piece | Headless default | +|-------|------------------| +| Host window | **off** (`-no-window`, `QT_QPA_PLATFORM=offscreen`, no `DISPLAY`) | +| Guest GLES | **on** via **SwiftShader** (`-gpu swiftshader_indirect`) | +| UI automation | **on** — `uiautomator` dump/tap + `screencap` (no X11 needed) | +| Audio / boot anim | off (`-no-audio -no-boot-anim`) | + +```bash +# explicit (same as default) +./start-android-emulator.sh --wait + +# or let smoke auto-start headless AVD when no phone is plugged in +STACK=goa ./run-android-gui-smoke.sh + +# laptop: show the emulator window +WINDOWED=1 ./start-android-emulator.sh --wait +# or: EMULATOR_HEADLESS=0 EMULATOR_GPU=host ./start-android-emulator.sh --wait +``` + +Shared code: `lib_android_env.sh` + `start-android-emulator.sh`. + +**Semantics:** + +| `AUTO_ANDROID` | `AUTO_GUI` | Behaviour | +|----------------|------------|-----------| +| `1` | `0` | Android install + deep-link smoke only (no multi-round taps) | +| `1` | `1` | Android + graphical drive (vanilla GUI / chain) | +| `0` | `1` | Reserved (e.g. future desktop/web GUI); currently **no-op** with a clear message | +| `0` | `0` | Skip mobile automation | + +### Defaults by host (when flags unset) + +| Host (`uname -s`) | Default `AUTO_ANDROID` | Default `AUTO_GUI` | Rationale | +|-------------------|------------------------|--------------------|-----------| +| **Linux** | **`1`** | **`1`** if device/emulator present, else scripts exit 3 | This repo’s day-to-day path; **Android only** (no iOS here) | +| **Darwin (macOS)** | `1` | `1` | Can run Android emulators **and** (separately) taler-ios GUI helpers; both flags on for Android scripts; iOS is out of tree | +| **Other** | `0` | `0` | Fail closed | + +**Linux default = Android** (no second mobile platform in this suite). +**macOS** may enable both ecosystems in the wider monorepo sense; for *these* +scripts only Android is implemented — set `AUTO_ANDROID=1` (default) and keep +iOS under `taler-ios`. + +### Resolution helper (convention for future wrappers) + +```bash +# Example for a future run-all-mobile.sh +os=$(uname -s) +: "${AUTO_PLATFORM:=auto}" +case "$AUTO_PLATFORM" in + auto) case "$os" in Linux) AUTO_PLATFORM=linux ;; Darwin) AUTO_PLATFORM=macos ;; *) AUTO_PLATFORM=other ;; esac ;; +esac +case "$AUTO_PLATFORM" in + linux) + : "${AUTO_ANDROID:=1}" + : "${AUTO_GUI:=1}" + # no iOS + ;; + macos) + : "${AUTO_ANDROID:=1}" + : "${AUTO_GUI:=1}" + # optional later: AUTO_IOS=1 for taler-ios scripts + ;; + *) + : "${AUTO_ANDROID:=0}" + : "${AUTO_GUI:=0}" + ;; +esac +``` + +Scripts **today** implement Android only; they should honour: + +- `AUTO_ANDROID=0` → exit 0 with “skipped (AUTO_ANDROID=0)” +- `AUTO_GUI=0` → call deep-link smoke without multi-round GUI (or set `GUI_ROUNDS=0`) + +--- + +## Vanilla layers (detail) + +| Layer | What | Tooling | +|-------|------|---------| +| **A — Install** | F-Droid or from-source APK | `adb install` | +| **B — Entry (shortcut)** | `taler://withdraw/…` / `taler://pay…` | `adb am start -a VIEW` | +| **C — Graphical UI** | Confirm / ToS / Pay taps; ANR Wait | `uiautomator` + `lib_ui.py` / chain | +| **D — Evidence** | Screenshots, XML, logcat, URIs | `out*/` | + +### Legitimate shortcuts + +| Shortcut | Replaces | +|----------|----------| +| Deep-link withdraw/pay | Camera QR / opening paywall | +| Host `POST` template → pay URI | Browser shop checkout UI | +| Label-list taps | Human reading button text | +| Skip full ToS scroll | Tap Accept if shown | + +### Scripts + +| Script | Role | +|--------|------| +| `run-android-pay-smoke.sh` | Hybrid: install + deep-link + light taps | +| `run-android-gui-smoke.sh` | Multi-round GUI drive | +| `run-goa-gui-chain.sh` | Multi mint/withdraw/pay (gui-workflows port) | +| `run-android-build-and-smoke.sh` | Build git ref (branch/tag) + smoke (`GUI=1` optional) | +| `run-android-variant-matrix.sh` | **published + stable-self + master** (compare builds) | +| `start-android-emulator.sh` | Start AVD (**headless default**, SwiftShader) | +| `lib_android_env.sh` | Flags, PATH, `android_ensure_device` | +| `lib_ui.py` | Dump/tap/ANR helpers | + +### Build variants (anti–“weird local build”) + +| Variant | Source | Severity | Purpose | +|---------|--------|----------|---------| +| **stable-self** | **only** rebuild of current stable tag | WARN | Self-build vs published for **one** release | +| **published** | F-Droid current APK | **≤14d → BLOCKER**, else WARN | What users install | +| **older / milestones** | F-Droid APK ~**3 / 6 / 9 / 12 mo** | **always WARN** | No self-build of old tags | +| **master** | `origin/master` rebuild | WARN | Tip — never hard-blocks | +| **fix** (optional) | inference branch | soft | GOA automation fixes | + +**Self-build:** current stable only (plus master/fix if requested) — not older releases. +**Blocker window:** releases ≤ 14 days. +**Milestones:** nearest tags to 90/180/270/365d with a still-hosted F-Droid APK. + +--- + +## Stacks under test + +| Stack | Withdraw | Pay | +|-------|----------|-----| +| **GOA** | explorer mint / `demo-withdraw.json` @ bank.hacktivism.ch | goa-shop templates / Paivana pay-template | +| **stage** | explorer / demo-withdraw @ stage.bank… | fermes / jardin public templates | + +### Lageübersicht GOA + stage (as-is survey · 2026-07-17 ~22:05–22:18) + +**Device:** headless `emulator-5554` · host ~3.7 GiB RAM (often <300 MiB free) · timeout per leg ≤420 s. +**Legs:** Abheben + Bezahlen (`DO_WITHDRAW=1 DO_PAY=1`). Artefakte: `out-survey-20260717-220527/` (gitignored). + +#### Varianten + +| Variante | Quelle | Bemerkung | +|----------|--------|-----------| +| **stable (published)** | F-Droid `net.taler.wallet.fdroid` **1.6.1 / 854** | as-is | +| **master as-is** | rebuild `origin/master` @ `f65a976c` → `.fdroid.debug` | as-is, no patches | +| **ältere Releases** (3/6/9/12 mon) | F-Droid APKs für `1.5.0`/`1.3.0`/`1.1.0`/`1.0.8` | **nicht auffindbar** (HTTP miss) → **SKIP** (kein Selbstbau älterer Tags) | +| **master + inference** (extra) | local branch `local/master-with-inference` @ `3ebe8b09` | merges `fix-bank-withdraw-auto-exchange` + `fix-withdraw-spinner-fallback` · **not pushed** | + +#### Ergebnis-Matrix + +| Variant | Stack | Install / deep-link | Withdraw URI | Pay URI | wallet-core Abheben | wallet-core Pay | ANR | Exit | +|---------|-------|---------------------|--------------|---------|---------------------|-----------------|-----|------| +| stable published | **goa** (hacktivism) | OK | OK bank.hacktivism.ch | OK paivana template | **NO** | soft/YES once* | **YES** | **10** | +| stable published | **stage** (*.lefrancpaysan) | OK | OK stage.bank… | OK fermes order | **NO** | **NO** | **YES** | **10** | +| master as-is | **goa** | OK | OK | OK paivana | **NO** | **NO** | **YES** | **10** | +| master as-is | **stage** | OK | OK | OK fermes | **NO** | **NO** | **YES** | **10** | +| master+inference | **goa** | OK | OK | OK | **NO** | **NO** | **YES** | **10** | +| master+inference | **stage** | OK | OK | OK | **NO** | **NO** | **YES** | **10** | + +\*stable-goa console once reported pay wallet-core YES mid-run; final score still ANR-dominated / not reliable settlement. + +#### Was pro Stack **steht** (Infra) + +| Check | **goa / hacktivism** | **stage / \*.lefrancpaysan** | +|-------|----------------------|------------------------------| +| Bank `…/intro/demo-withdraw.json` → `taler://withdraw/…` | **OK** | **OK** | +| Pay entry | paivana `pay-template` **OK** | fermes `panier-legumes` POST → `taler://pay/…` **OK** | +| adb `VIEW` Intent delivery | **OK** | **OK** | +| Unattended Confirm/Abheben/Pay GUI | **FAIL (ANR)** | **FAIL (ANR)** | +| wallet-core prepare/accept withdraw in logcat | **not seen** | **not seen** | + +**Fazit as-is:** Beide Stacks liefern die **Eingänge** (Withdraw- + Pay-URIs). Die **GUI/wallet-core-Strecke** ist auf diesem Emulator für **alle** getesteten App-Varianten (stable, master, inference) **gleich tot** — System UI / Android System ANR, keine brauchbaren Taps. Das ist **kein Stack-Unterschied** GOA vs stage und **kein** belastbarer App-Versions-Vergleich. + +#### Inference-Patch-Versuch (lokal, **kein Push**) + +Weil **nichts** der as-is-Varianten wallet-core-grün war, wurde zusätzlich gebaut: + +```text +taler-android branch: local/master-with-inference # NOT pushed +base: origin/master +merged: origin/dev/hernani-inference/fix-bank-withdraw-auto-exchange + origin/dev/hernani-inference/fix-withdraw-spinner-fallback +commit: 3ebe8b091379 +APK: apks/wallet-master-inference-3ebe8b091379.apk (local) +``` + +| Stack | Inference smoke | Besser als master/stable? | +|-------|-----------------|---------------------------| +| goa | exit 10, ANR, no withdraw core | **Nein** | +| stage | exit 10, ANR, no withdraw core | **Nein** | + +**Kein grünes Inference-Signal** auf diesem Host → wir können **nicht** belegen, dass die Patches Withdraw/Pay „retten“. +Ebenso können wir **nicht** belegen, dass master ohne Patches „schuld“ ist: die Umgebung erreicht wallet-core gar nicht. + +> **Wenn später auf Gerät/mehr RAM die Inference-APK plötzlich grün und master/stable rot ist:** +> das ist der **klare Fix-Hinweis** — Patches aus `dev/hernani-inference/*` upstreamen / in master holen. +> **Heute:** noch **nicht** der Fall; primärer Blocker = **Emulator/RAM/ANR**. + +#### Nächste sinnvolle Schritte (später) + +1. Physisches Gerät oder Host ≥6–8 GiB free RAM +2. Nur **eine** Wallet-Package-ID installiert (kein Open-with) +3. Survey wiederholen: stable · master · `local/master-with-inference` auf **goa + stage** +4. Ältere F-Droid-APKs nur wenn wieder gehostet; kein Selbstbau alter Tags + +--- + +## Upstream branches (`taler-android` `dev/hernani-inference/*`) + +| Branch | Role | +|--------|------| +| `gui-workflows` | macOS emulator helpers + original `goa-chain-emu.sh` | +| `fix/bank-withdraw-auto-exchange` | **Minimal** GOA exchange auto-add (default build branch) | +| `fix/withdraw-spinner-fallback` | Spinner + OIM; use if spinner still hangs | +| `experimental-oim*` | Optional cash UI | + +--- + +## Checklist + +- [x] Minimal variant documented +- [x] Minimalvariante: was geht / was nicht (Beschränkungen) +- [x] Two stacks (goa / stage) +- [x] Dual-stack run documented (2026-07-17: both FAIL ANR, landings OK) +- [x] Flags design (`AUTO_ANDROID` / `AUTO_GUI` / platform defaults) +- [x] **Headless emulator default** (`EMULATOR_HEADLESS=1`, SwiftShader) +- [x] Linux default = Android suite only +- [x] Inference-branch policy +- [ ] Wrapper enforces flags in every entry script (incremental) +- [ ] Reliable unattended Confirm on low-RAM Linux emulator diff --git a/android-test/README.md b/android-test/README.md new file mode 100644 index 0000000..8ed2fb0 --- /dev/null +++ b/android-test/README.md @@ -0,0 +1,111 @@ +# Android wallet smoke (Abheben + Bezahlen) + +Smokes always cover **withdraw first**, then **pay** (not pay-only). + +| Leg | Default | What | +|-----|---------|------| +| **1 WITHDRAW** | on | `taler://withdraw/…` + Confirm/Abheben taps | +| **2 PAY** | on | `taler://pay…` / pay-template + Pay taps | + +```bash +# needs adb device/emulator +./run-android-pay-smoke.sh +STACK=goa ./run-android-pay-smoke.sh +DO_PAY=0 ./run-android-pay-smoke.sh # withdraw only +DO_WITHDRAW=0 DO_PAY=1 … # pay only (unusual) +``` + +APK: F-Droid **1.6.1** (`net.taler.wallet.fdroid`). + +## 2) Source build + same tests + +Builds `taler-android` at a git **ref** (`:wallet:assembleFdroidDebug`), then +runs the **same** smoke (install + withdraw/pay deep-links). + +```bash +# default: inference fix branch (GOA withdraw reliability) +./run-android-build-and-smoke.sh +# tip of master +BRANCH=master ./run-android-build-and-smoke.sh +# self-build of published stable **tag** (same release as F-Droid 1.6.1) +BRANCH=wallet-1.6.1 LABEL=stable-self ./run-android-build-and-smoke.sh +SKIP_PULL=1 BRANCH=master ./run-android-build-and-smoke.sh +TALER_ANDROID_SRC=/path/to/taler-android ./run-android-build-and-smoke.sh +``` + +Built package id: `net.taler.wallet.fdroid.debug`. +Needs `ANDROID_HOME` (e.g. `$HOME/Android/Sdk`), Java 17+, network for Maven. + +## 2b) Variant matrix (published + stable self-build + master) + +Runs several smokes so a single odd developer build cannot be the only sample: + +| Variant | What | +|---------|------| +| Order | Variant | Severity | What | +|------|---------|----------|------| +| **1** | **stable-self** | WARN | **Only** self-build: current stable tag | +| **2** | **published** | age ≤14d → **BLOCKER**, else WARN | Current F-Droid APK | +| **3** | **older / milestones** | **always WARN** | F-Droid APKs nearest **3 / 6 / 9 / 12 months** (if still hosted) — **no self-build** | +| **4** | **master** | WARN | Rebuild `origin/master` | + +**Self-build policy:** only **current published stable** (`stable-self`) + optional **master**. +Historical versions: published F-Droid APK only; skip if APK gone. +**Milestones:** `MILESTONE_DAYS="90 180 270 365"` (±50d), warn only. +**Young releases** (≤ 14 days): hard blocker on fail. + +```bash +./run-android-variant-matrix.sh +STACK=goa ./run-android-variant-matrix.sh +INCLUDE_OLDER=0 VARIANTS=stable-self,published,master ./run-android-variant-matrix.sh +# custom milestones (days): +MILESTONE_DAYS="90 180 270 365" ./run-android-variant-matrix.sh +``` + +Artifacts + `SUMMARY.txt` under `out-matrix-/`. + +## 3) Graphical UI (vanilla) + +Deep-link entry (≡ QR) **plus** multi-round uiautomator taps: + +```bash +STACK=goa ./run-android-gui-smoke.sh +STACK=stage ./run-android-gui-smoke.sh +# after source build: +GUI=1 ./run-android-build-and-smoke.sh +``` + +## 4) Full GOA/stage chain (from `dev/hernani-inference/gui-workflows`) + +Bank API mint → withdraw deep-link → taps → template pay → taps: + +```bash +# needs explorer password + device +EXP_PW_FILE=$HOME/src/koopa/koopa-admin-secrets/koopa/host-root/taler-bank/bank-explorer-password.txt \ + STACK=goa PKG=net.taler.wallet.fdroid.debug \ + ./run-goa-gui-chain.sh + +STACK=stage ./run-goa-gui-chain.sh +``` + +Canonical notes (minimal variant, **flags**, Linux defaults): +**[GUI-AUTOMATION-NOTES.md](./GUI-AUTOMATION-NOTES.md)**. + +| Flag | Linux default | Meaning | +|------|---------------|---------| +| `AUTO_ANDROID` | `1` | Run Android adb/APK automation | +| `AUTO_GUI` | `1` | Graphical uiautomator drive (else deep-link only) | +| `AUTO_PLATFORM` | `auto` → `linux` | Host family; macOS can later pair with iOS out of tree | +| **`EMULATOR_HEADLESS`** | **`1`** | **No host window** (`-no-window`). Server-safe. | +| `EMULATOR_GPU` | `swiftshader_indirect` | Guest GLES still active for UI/taps/screenshots | +| `AUTO_START_EMULATOR` | `1` | Auto-start headless AVD if no adb device | +| `WINDOWED` | `0` | `WINDOWED=1` → show emulator window | + +```bash +# headless AVD only (default path for servers) +./start-android-emulator.sh --wait +# windowed (laptops) +WINDOWED=1 ./start-android-emulator.sh --wait +``` + +See also parent [VERSIONS.md](../VERSIONS.md). diff --git a/android-test/lib_android_env.sh b/android-test/lib_android_env.sh new file mode 100644 index 0000000..670d796 --- /dev/null +++ b/android-test/lib_android_env.sh @@ -0,0 +1,158 @@ +# shellcheck shell=bash +# Shared Android env for GUI / adb smokes. +# Source: . "$ROOT/lib_android_env.sh" +# +# Defaults favour **servers / CI**: no host window (headless emulator), while the +# guest still runs GLES via SwiftShader so UI layout / uiautomator work. +# +# See GUI-AUTOMATION-NOTES.md → Headless mode + +# --- SDK / PATH --------------------------------------------------------------- +export ANDROID_HOME="${ANDROID_HOME:-${ANDROID_SDK_ROOT:-$HOME/Android/Sdk}}" +export ANDROID_SDK_ROOT="${ANDROID_SDK_ROOT:-$ANDROID_HOME}" +for d in \ + "$ANDROID_HOME/platform-tools" \ + "$ANDROID_HOME/emulator" \ + "$ANDROID_HOME/cmdline-tools/latest/bin" +do + [ -d "$d" ] && case ":$PATH:" in *":$d:"*) ;; *) PATH="$d:$PATH" ;; esac +done +export PATH + +# AVD path (some hosts keep AVDs under ~/.config/.android) +if [ -z "${ANDROID_AVD_HOME:-}" ]; then + if [ -d "$HOME/.android/avd" ]; then + export ANDROID_AVD_HOME="$HOME/.android/avd" + elif [ -d "$HOME/.config/.android/avd" ]; then + export ANDROID_AVD_HOME="$HOME/.config/.android/avd" + # emulator also looks at ~/.android/avd — soft-link if missing + if [ ! -e "$HOME/.android/avd" ]; then + mkdir -p "$HOME/.android" + ln -sfn "$HOME/.config/.android/avd" "$HOME/.android/avd" 2>/dev/null || true + fi + fi +fi + +# --- Platform / feature flags ------------------------------------------------- +os=$(uname -s) +case "${AUTO_PLATFORM:-auto}" in + auto) + case "$os" in + Linux) : "${AUTO_ANDROID:=1}"; : "${AUTO_GUI:=1}" ;; + Darwin) : "${AUTO_ANDROID:=1}"; : "${AUTO_GUI:=1}" ;; + *) : "${AUTO_ANDROID:=0}"; : "${AUTO_GUI:=0}" ;; + esac + ;; + linux|macos) : "${AUTO_ANDROID:=1}"; : "${AUTO_GUI:=1}" ;; + *) : "${AUTO_ANDROID:=0}"; : "${AUTO_GUI:=0}" ;; +esac + +# Headless = default (no X11/Wayland window on host). GUI *logic* still runs in +# the guest via software GPU. Set EMULATOR_HEADLESS=0 or WINDOWED=1 for a window. +: "${EMULATOR_HEADLESS:=1}" +if [ "${WINDOWED:-0}" = "1" ]; then + EMULATOR_HEADLESS=0 +fi +export EMULATOR_HEADLESS + +# Soft GLES for headless; windowed may override to host/auto if set by user. +if [ "${EMULATOR_HEADLESS}" = "1" ]; then + : "${EMULATOR_GPU:=swiftshader_indirect}" +else + : "${EMULATOR_GPU:=swiftshader_indirect}" # still safe default; host GPU: EMULATOR_GPU=host +fi +export EMULATOR_GPU + +: "${EMULATOR_AVD:=TalerWallet34}" +: "${EMULATOR_MEMORY:=1536}" +: "${EMULATOR_CORES:=2}" +# If no adb device: start headless AVD (0 = require phone/manual emulator) +: "${AUTO_START_EMULATOR:=1}" +: "${EMULATOR_BOOT_TIMEOUT:=360}" +: "${EMULATOR_LOG:=/tmp/taler-android-emulator.log}" +: "${EMULATOR_NO_SNAPSHOT:=1}" + +android_adb_serial() { + adb devices 2>/dev/null | awk '/\tdevice$/{print $1; exit}' +} + +android_has_device() { + [ -n "$(android_adb_serial)" ] +} + +# Build emulator argv for current headless/windowed mode. +# Prints args; caller exec/nohup. +android_emulator_args() { + local avd="${1:-$EMULATOR_AVD}" + local args=( -avd "$avd" -gpu "$EMULATOR_GPU" -no-audio -no-boot-anim ) + if [ "${EMULATOR_HEADLESS}" = "1" ]; then + args+=( -no-window ) + # Avoid Qt trying to open a display on servers + export QT_QPA_PLATFORM="${QT_QPA_PLATFORM:-offscreen}" + fi + # Broken default_boot snapshots leave adb offline forever — cold boot by default. + if [ "${EMULATOR_NO_SNAPSHOT:-1}" = "1" ]; then + args+=( -no-snapshot -no-snapshot-load -no-snapshot-save ) + fi + if [ -n "${EMULATOR_MEMORY:-}" ]; then + args+=( -memory "$EMULATOR_MEMORY" ) + fi + if [ -n "${EMULATOR_CORES:-}" ]; then + args+=( -cores "$EMULATOR_CORES" ) + fi + # extra user flags + # shellcheck disable=SC2206 + if [ -n "${EMULATOR_EXTRA_ARGS:-}" ]; then + # intentional word-split for extra flags + args+=( ${EMULATOR_EXTRA_ARGS} ) + fi + printf '%s\n' "${args[@]}" +} + +# Ensure an adb "device" is online. Optionally start headless emulator. +# Sets SERIAL / ANDROID_SERIAL on success. Returns 0 ok, 3 no device. +android_ensure_device() { + local serial + serial="${SERIAL:-${ANDROID_SERIAL:-}}" + if [ -z "$serial" ]; then + serial=$(android_adb_serial || true) + fi + if [ -n "$serial" ]; then + SERIAL="$serial" + export ANDROID_SERIAL="$serial" + return 0 + fi + + if [ "${AUTO_START_EMULATOR}" != "1" ]; then + echo "no adb device (AUTO_START_EMULATOR=0 — plug phone or start AVD manually)" >&2 + return 3 + fi + + if ! command -v emulator >/dev/null 2>&1; then + echo "no adb device and emulator binary missing under $ANDROID_HOME" >&2 + return 3 + fi + + echo "no adb device → starting AVD ${EMULATOR_AVD} (headless=${EMULATOR_HEADLESS} gpu=${EMULATOR_GPU})" + local starter + starter="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/start-android-emulator.sh" + if [ ! -x "$starter" ]; then + echo "missing $starter" >&2 + return 3 + fi + EMULATOR_HEADLESS="$EMULATOR_HEADLESS" EMULATOR_GPU="$EMULATOR_GPU" \ + EMULATOR_AVD="$EMULATOR_AVD" EMULATOR_MEMORY="$EMULATOR_MEMORY" \ + EMULATOR_CORES="$EMULATOR_CORES" EMULATOR_LOG="$EMULATOR_LOG" \ + EMULATOR_BOOT_TIMEOUT="$EMULATOR_BOOT_TIMEOUT" \ + "$starter" --wait || return 3 + + serial=$(android_adb_serial || true) + if [ -z "$serial" ]; then + echo "emulator started but no adb device yet — see $EMULATOR_LOG" >&2 + return 3 + fi + SERIAL="$serial" + export ANDROID_SERIAL="$serial" + echo "device: $SERIAL (headless=${EMULATOR_HEADLESS})" + return 0 +} diff --git a/android-test/lib_release_age.sh b/android-test/lib_release_age.sh new file mode 100644 index 0000000..7be935f --- /dev/null +++ b/android-test/lib_release_age.sh @@ -0,0 +1,133 @@ +# shellcheck shell=bash +# Release age helpers for Android wallet matrix. +# Source after ROOT/lib_android_env.sh (optional). +# +# BLOCKER only when a *released* app is within BLOCKER_MAX_AGE_DAYS (default 14). +# Older releases: try them, but failures are WARN only. + +: "${BLOCKER_MAX_AGE_DAYS:=14}" +: "${TALER_ANDROID_SRC:=$HOME/taler/taler-android}" + +# Days since git tag (tagger/committer date). Empty if unknown. +# Usage: release_age_days wallet-1.6.1 → e.g. 28 +release_age_days() { + local tag="$1" + local src="${2:-$TALER_ANDROID_SRC}" + [ -d "$src/.git" ] || return 1 + local epoch + epoch=$(git -C "$src" log -1 --format=%ct "refs/tags/${tag}" 2>/dev/null || true) + [ -n "$epoch" ] || epoch=$(git -C "$src" log -1 --format=%ct "$tag" 2>/dev/null || true) + [ -n "$epoch" ] || return 1 + local now + now=$(date +%s) + echo $(( (now - epoch) / 86400 )) +} + +# severity from age: blocker if age_days <= BLOCKER_MAX_AGE_DAYS, else warn +# Unknown age → warn (never accidental hard fail on mystery builds) +severity_from_age_days() { + local days="${1:-}" + if [ -z "$days" ] || ! [[ "$days" =~ ^[0-9]+$ ]]; then + echo warn + return + fi + if [ "$days" -le "${BLOCKER_MAX_AGE_DAYS}" ]; then + echo blocker + else + echo warn + fi +} + +severity_for_tag() { + local tag="$1" + local days + days=$(release_age_days "$tag" 2>/dev/null || true) + severity_from_age_days "$days" +} + +# List wallet-* release tags newest first (exclude rc/dev). +# Args: max count (default 5). Pass 0 or "all" for no limit. +list_wallet_release_tags() { + local n="${1:-5}" + local src="${TALER_ANDROID_SRC:-$HOME/taler/taler-android}" + [ -d "$src/.git" ] || return 1 + git -C "$src" fetch --tags origin 2>/dev/null || true + local cmd=(git -C "$src" for-each-ref --sort=-creatordate --format='%(refname:short)' refs/tags) + if [ "$n" = "all" ] || [ "$n" = "0" ]; then + "${cmd[@]}" | grep -E '^wallet-[0-9]+\.[0-9]+(\.[0-9]+)?$' + else + "${cmd[@]}" | grep -E '^wallet-[0-9]+\.[0-9]+(\.[0-9]+)?$' | head -n "$n" + fi +} + +# Pick one tag closest to each age milestone (days). Always intended as WARN samples. +# Default milestones: 3,6,9,12 months ≈ 90,180,270,365 days. +# Env: MILESTONE_DAYS="90 180 270 365" MILESTONE_TOLERANCE_DAYS=50 +# Prints lines: TAG AGE_DAYS TARGET_DAYS (only when a tag within tolerance exists) +select_milestone_tags() { + local src="${TALER_ANDROID_SRC:-$HOME/taler/taler-android}" + local milestones="${MILESTONE_DAYS:-90 180 270 365}" + local tol="${MILESTONE_TOLERANCE_DAYS:-50}" + local skip="${1:-}" # optional tag to exclude (e.g. current STABLE_TAG) + local -a tags=() + mapfile -t tags < <(list_wallet_release_tags all 2>/dev/null || true) + [ "${#tags[@]}" -gt 0 ] || return 0 + + local target best_tag best_age best_delta age t delta + local -A used=() + for target in $milestones; do + best_tag="" + best_age="" + best_delta=99999 + for t in "${tags[@]}"; do + [ -n "$t" ] || continue + [ "$t" = "$skip" ] && continue + [ -n "${used[$t]:-}" ] && continue + age=$(release_age_days "$t" "$src" 2>/dev/null || true) + [[ "$age" =~ ^[0-9]+$ ]] || continue + delta=$(( age > target ? age - target : target - age )) + if [ "$delta" -le "$tol" ] && [ "$delta" -lt "$best_delta" ]; then + best_delta=$delta + best_tag=$t + best_age=$age + fi + done + if [ -n "$best_tag" ]; then + used[$best_tag]=1 + echo "$best_tag $best_age $target" + fi + done +} + +# Guess F-Droid APK URL for a wallet version name (e.g. 1.6.1). +# Uses versionCode from tag's build.gradle if possible, else STABLE mapping. +# Prints: versionCode apk_url or returns 1 +fdroid_apk_for_version_name() { + local vn="$1" # 1.6.1 + local src="${TALER_ANDROID_SRC:-$HOME/taler/taler-android}" + local tag="wallet-${vn}" + local vc="" + if git -C "$src" cat-file -e "refs/tags/${tag}:wallet/build.gradle" 2>/dev/null; then + vc=$(git -C "$src" show "refs/tags/${tag}:wallet/build.gradle" 2>/dev/null \ + | sed -n 's/^[[:space:]]*versionCode[[:space:]]\+\([0-9]\+\).*/\1/p' | head -1) + fi + # ABI x86_64 → +4 in override scheme 10*vc+abi (common F-Droid multi-apk) + # Prefer universal-style code used by F-Droid: for 1.6.1 → 854 = 10*85+4 + local apk_vc="" + if [ -n "$vc" ]; then + apk_vc=$((10 * vc + 4)) + fi + # Known overrides / suggested + case "$vn" in + 1.6.1) apk_vc=854 ;; + esac + [ -n "$apk_vc" ] || return 1 + echo "${apk_vc}|https://f-droid.org/repo/net.taler.wallet.fdroid_${apk_vc}.apk" +} + +# Check HTTP HEAD/GET that APK exists (0 = ok) +fdroid_apk_exists() { + local url="$1" + curl -sfS -m 15 -o /dev/null -r 0-0 "$url" 2>/dev/null \ + || curl -sfS -m 15 -o /dev/null -I "$url" 2>/dev/null +} diff --git a/android-test/lib_ui.py b/android-test/lib_ui.py new file mode 100755 index 0000000..a7cdad5 --- /dev/null +++ b/android-test/lib_ui.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +"""Minimal adb + uiautomator helpers for Taler wallet GUI smoke (no root). + +Does not replace Maestro/Appium; uses only adb shell + uiautomator dump. +""" +from __future__ import annotations + +import re +import subprocess +import time +from pathlib import Path +from typing import Iterable, List, Optional, Sequence, Tuple + +Bounds = Tuple[int, int, int, int] + + +def adb(serial: str, *args: str, check: bool = False, timeout: Optional[float] = 60) -> subprocess.CompletedProcess: + cmd = ["adb", "-s", serial, *args] + return subprocess.run(cmd, check=check, capture_output=True, text=True, timeout=timeout) + + +def dump_ui(serial: str, dest: Path) -> str: + adb(serial, "shell", "uiautomator", "dump", "/sdcard/ui.xml", timeout=90) + adb(serial, "pull", "/sdcard/ui.xml", str(dest), timeout=30) + return dest.read_text(errors="replace") if dest.is_file() else "" + + +def screencap(serial: str, dest: Path) -> None: + p = subprocess.run( + ["adb", "-s", serial, "exec-out", "screencap", "-p"], + capture_output=True, + timeout=60, + ) + if p.returncode == 0 and p.stdout: + dest.write_bytes(p.stdout) + + +def texts(xml: str) -> List[str]: + return [t for t in re.findall(r'text="([^"]*)"', xml) if t.strip()] + + +def find_bounds(xml: str, label: str) -> List[Bounds]: + """Return list of (x1,y1,x2,y2) for nodes whose text or content-desc matches label (exact).""" + out: List[Bounds] = [] + esc = re.escape(label) + patterns = [ + rf'(?:text|content-desc)="{esc}"[^>]*bounds="\[(\d+),(\d+)\]\[(\d+),(\d+)\]"', + rf'bounds="\[(\d+),(\d+)\]\[(\d+),(\d+)\]"[^>]*(?:text|content-desc)="{esc}"', + ] + for pat in patterns: + for m in re.finditer(pat, xml, flags=re.I): + out.append(tuple(map(int, m.groups()))) # type: ignore[arg-type] + return out + + +def find_bounds_re(xml: str, pattern: str) -> List[Tuple[str, Bounds]]: + out: List[Tuple[str, Bounds]] = [] + for m in re.finditer( + rf'(?:text|content-desc)="([^"]*{pattern}[^"]*)"[^>]*bounds="\[(\d+),(\d+)\]\[(\d+),(\d+)\]"', + xml, + flags=re.I, + ): + label = m.group(1) + b = tuple(map(int, m.group(2, 3, 4, 5))) + out.append((label, b)) # type: ignore[arg-type] + return out + + +def center(b: Bounds) -> Tuple[int, int]: + x1, y1, x2, y2 = b + return (x1 + x2) // 2, (y1 + y2) // 2 + + +def tap(serial: str, x: int, y: int) -> None: + adb(serial, "shell", "input", "tap", str(x), str(y)) + + +def tap_bounds(serial: str, b: Bounds) -> None: + x, y = center(b) + tap(serial, x, y) + + +def dismiss_anr(serial: str, xml: str) -> bool: + """If System UI isn't responding, prefer Wait over Close.""" + if "isn't responding" not in xml and "reagiert nicht" not in xml.lower(): + return False + for label in ("Wait", "Warten", "OK"): + bs = find_bounds(xml, label) + if bs: + tap_bounds(serial, bs[0]) + return True + # fallback: right-ish button often Wait + adb(serial, "shell", "input", "keyevent", "22") + adb(serial, "shell", "input", "keyevent", "66") + return True + + +# Labels seen on wallet UI (EN/DE/FR) — expand as needed +CONFIRM_LABELS = [ + "Confirm", + "Confirm withdrawal", + "Withdraw", + "Accept", + "I accept", + "Agree", + "Continue", + "Next", + "OK", + "Pay", + "Pay now", + "Bestätigen", + "Abheben", + "Akzeptieren", + "Zustimmen", + "Weiter", + "Bezahlen", + "Confirmer", + "Retirer", + "Accepter", + "Payer", +] + +TOS_LABELS = [ + "I accept", + "Accept", + "Agree", + "Akzeptieren", + "Zustimmen", + "Accepter", +] + + +def try_tap_any(serial: str, xml: str, labels: Sequence[str]) -> Optional[str]: + for label in labels: + bs = find_bounds(xml, label) + if bs: + tap_bounds(serial, bs[0]) + return label + return None + + +def gui_drive( + serial: str, + out_dir: Path, + *, + rounds: int = 8, + sleep_s: float = 3.0, + phase: str = "gui", +) -> dict: + """Poll UI, dismiss ANR, tap confirm/ToS-like buttons. Returns status dict.""" + out_dir.mkdir(parents=True, exist_ok=True) + seen_texts: List[str] = [] + taps: List[str] = [] + anr = 0 + for i in range(1, rounds + 1): + xml_path = out_dir / f"{phase}-{i:02d}-ui.xml" + png_path = out_dir / f"{phase}-{i:02d}.png" + try: + xml = dump_ui(serial, xml_path) + except Exception as e: + taps.append(f"dump-fail:{e}") + time.sleep(sleep_s) + continue + try: + screencap(serial, png_path) + except Exception: + pass + ts = texts(xml) + seen_texts.extend(ts[:20]) + if dismiss_anr(serial, xml): + anr += 1 + time.sleep(sleep_s) + continue + hit = try_tap_any(serial, xml, CONFIRM_LABELS) + if hit: + taps.append(hit) + time.sleep(sleep_s) + continue + # partial regex for amount confirm rows + for label, b in find_bounds_re(xml, r"Confirm|Withdraw|Pay|Accept|Bestätig|Abheben|Payer"): + tap_bounds(serial, b) + taps.append(f"re:{label}") + time.sleep(sleep_s) + break + else: + time.sleep(sleep_s) + return { + "taps": taps, + "anr_dismissals": anr, + "sample_texts": seen_texts[-40:], + "rounds": rounds, + } diff --git a/android-test/run-android-build-and-smoke.sh b/android-test/run-android-build-and-smoke.sh new file mode 100755 index 0000000..659b446 --- /dev/null +++ b/android-test/run-android-build-and-smoke.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +# Build GNU Taler Android wallet from source at a git **ref**, then run the same +# adb deep-link (or GUI) smoke as run-android-pay-smoke.sh. +# +# Source (default): $HOME/taler/taler-android +# Build: ./gradlew :wallet:assembleFdroidDebug +# Package id: net.taler.wallet.fdroid.debug +# +# Typical refs: +# BRANCH=master # tip of master +# BRANCH=wallet-1.6.1 # published stable **tag** (self-build) +# BRANCH=dev/hernani-inference/fix-… # GOA automation fix branch +# +# For a multi-variant matrix (master + stable self-build + published APK): +# ./run-android-variant-matrix.sh +# +# Usage: +# ./run-android-build-and-smoke.sh +# BRANCH=master STACK=goa ./run-android-build-and-smoke.sh +# BRANCH=wallet-1.6.1 LABEL=stable-self ./run-android-build-and-smoke.sh +# SKIP_PULL=1 BRANCH=master ./run-android-build-and-smoke.sh +# +set -euo pipefail +ROOT=$(cd "$(dirname "$0")" && pwd) +SMOKE="$ROOT/run-android-pay-smoke.sh" +SRC="${TALER_ANDROID_SRC:-$HOME/taler/taler-android}" +SKIP_PULL="${SKIP_PULL:-0}" +# Prefer existing inference fix branch for GOA withdraw (spinner/exchange resolve). +# Matrix uses BRANCH=master and BRANCH=$STABLE_TAG explicitly. +BRANCH="${BRANCH:-dev/hernani-inference/fix-bank-withdraw-auto-exchange}" +# Optional human label for logs / artifact names (default = sanitized BRANCH) +LABEL="${LABEL:-}" +VARIANT="${VARIANT:-fdroidDebug}" +export ANDROID_HOME="${ANDROID_HOME:-$HOME/Android/Sdk}" +export ANDROID_SDK_ROOT="${ANDROID_SDK_ROOT:-$ANDROID_HOME}" + +if [ ! -x "$SRC/gradlew" ]; then + echo "taler-android not found at $SRC (set TALER_ANDROID_SRC=…)" >&2 + exit 2 +fi +if [ ! -d "$ANDROID_HOME" ]; then + echo "ANDROID_HOME missing: $ANDROID_HOME" >&2 + exit 2 +fi + +printf 'sdk.dir=%s\n' "$ANDROID_HOME" >"$SRC/local.properties" + +# --- checkout REF (branch, origin/branch, or tag) -------------------------------- +cd "$SRC" +if [ -d .git ]; then + git fetch origin --tags 2>/dev/null || git fetch origin 2>/dev/null || true + REF="$BRANCH" + checked=0 + if [ -n "$REF" ]; then + if git rev-parse --verify "refs/tags/${REF}" >/dev/null 2>&1; then + echo "checkout tag ${REF} (stable / release self-build)" + git checkout -f "refs/tags/${REF}" + checked=1 + SKIP_PULL=1 + elif git rev-parse --verify "origin/${REF}" >/dev/null 2>&1; then + echo "checkout origin/${REF}" + git checkout -B "$REF" "origin/${REF}" 2>/dev/null || git checkout "$REF" 2>/dev/null || true + checked=1 + elif git rev-parse --verify "${REF}" >/dev/null 2>&1; then + echo "checkout ${REF}" + git checkout -f "$REF" + checked=1 + else + echo "WARN: ref '${REF}' not found locally after fetch — staying on current HEAD" >&2 + fi + fi + if [ "$SKIP_PULL" != "1" ] && [ "$checked" = "1" ]; then + # only pull real branches, never tags + if git symbolic-ref -q HEAD >/dev/null 2>&1; then + br=$(git rev-parse --abbrev-ref HEAD) + git pull --ff-only origin "$br" 2>/dev/null || true + fi + fi +fi + +COMMIT=$(git rev-parse --short=12 HEAD 2>/dev/null || echo unknown) +HEAD_DESC=$(git describe --tags --always 2>/dev/null || echo "$COMMIT") +BR_NOW=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo detached) +if [ -z "$LABEL" ]; then + LABEL=$(printf '%s' "$BRANCH" | tr '/:' '--' | tr -c 'A-Za-z0-9._-' '-' | cut -c1-48) +fi +echo "source: $SRC @ $COMMIT ($HEAD_DESC) ref=$BRANCH label=$LABEL head=$BR_NOW" +echo "variant: $VARIANT" + +# Map VARIANT → gradle task +case "$VARIANT" in + fdroidDebug|FdroidDebug) task=assembleFdroidDebug ;; + fdroidRelease|FdroidRelease) task=assembleFdroidRelease ;; + nightlyDebug|NightlyDebug) task=assembleNightlyDebug ;; + googleDebug|GoogleDebug) task=assembleGoogleDebug ;; + *) + task="assemble$(printf '%s' "$VARIANT" | python3 -c 'import sys; s=sys.stdin.read().strip(); print(s[:1].upper()+s[1:] if s else "FdroidDebug")')" + ;; +esac + +# Low-RAM hosts: cap JVM +export GRADLE_OPTS="${GRADLE_OPTS:--Xmx1280m -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1}" +echo "gradle :wallet:$task …" +./gradlew ":wallet:$task" --no-daemon --max-workers="${GRADLE_MAX_WORKERS:-1}" \ + -Dorg.gradle.jvmargs="${GRADLE_JVMARGS:--Xmx1280m}" + +# Prefer fdroid debug APK if multiple exist +APK=$(find "$SRC/wallet/build/outputs/apk" -path '*fdroid*debug*.apk' -type f 2>/dev/null | sort | tail -1) +if [ -z "$APK" ] || [ ! -f "$APK" ]; then + APK=$(find "$SRC/wallet/build/outputs/apk" -name '*.apk' -type f 2>/dev/null | sort | tail -1) +fi +if [ -z "$APK" ] || [ ! -f "$APK" ]; then + echo "no APK under wallet/build/outputs/apk" >&2 + exit 4 +fi +echo "built apk: $APK ($(wc -c <"$APK") bytes)" + +PKG="${PKG:-}" +if [ -z "$PKG" ] && command -v aapt >/dev/null 2>&1; then + PKG=$(aapt dump badging "$APK" 2>/dev/null | sed -n "s/^package: name='\([^']*\)'.*/\1/p" | head -1) +fi +if [ -z "$PKG" ] && command -v aapt2 >/dev/null 2>&1; then + PKG=$(aapt2 dump badging "$APK" 2>/dev/null | sed -n "s/^package: name='\([^']*\)'.*/\1/p" | head -1) +fi +: "${PKG:=net.taler.wallet.fdroid.debug}" + +mkdir -p "$ROOT/apks" "$ROOT/out" +STAMP_APK="$ROOT/apks/wallet-${LABEL}-${COMMIT}.apk" +cp -f "$APK" "$STAMP_APK" +echo "copied → $STAMP_APK" +{ + echo "label=$LABEL" + echo "ref=$BRANCH" + echo "commit=$COMMIT" + echo "describe=$HEAD_DESC" + echo "apk=$STAMP_APK" + echo "pkg=$PKG" + echo "task=$task" +} | tee "$ROOT/out/build-${LABEL}.txt" +echo "$COMMIT" >"$ROOT/out/build-commit.txt" +echo "$APK" >"$ROOT/out/build-apk-path.txt" + +export APK_PATH="$STAMP_APK" +export APK_NAME +APK_NAME="$(basename "$STAMP_APK")" +export PKG +export OUT_DIR="${OUT_DIR:-$ROOT/out-source-$LABEL}" +mkdir -p "$OUT_DIR" +cp -f "$ROOT/out/build-${LABEL}.txt" "$OUT_DIR/build-meta.txt" 2>/dev/null || true + +SMOKE_CMD="$SMOKE" +if [ "${GUI:-0}" = "1" ]; then + SMOKE_CMD="$ROOT/run-android-gui-smoke.sh" + case "$OUT_DIR" in + *-gui) ;; + *) OUT_DIR="${OUT_DIR}-gui"; export OUT_DIR; mkdir -p "$OUT_DIR" ;; + esac +fi +echo "=== smoke label=$LABEL PKG=$PKG GUI=${GUI:-0} OUT_DIR=$OUT_DIR ===" +# Do not exec: allow matrix wrapper to run multiple variants +set +e +env APK_DIR="$ROOT/apks" APK_NAME="$APK_NAME" APK_PATH="$APK_PATH" PKG="$PKG" OUT_DIR="$OUT_DIR" \ + "$SMOKE_CMD" +rc=$? +set -e +echo "=== smoke label=$LABEL exit=$rc ===" +exit "$rc" diff --git a/android-test/run-android-gui-smoke.sh b/android-test/run-android-gui-smoke.sh new file mode 100755 index 0000000..a22f1b8 --- /dev/null +++ b/android-test/run-android-gui-smoke.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +# GUI-oriented Android smoke: **withdraw (Abheben)** then **pay**, each with +# deep-link entry + multi-round uiautomator taps. Not pay-only. +# +# 1) taler://withdraw/… + Confirm/Abheben/ToS taps +# 2) taler://pay… + Pay/Confirm taps +# +# See GUI-AUTOMATION-NOTES.md +# +set -euo pipefail +ROOT=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=lib_android_env.sh +. "$ROOT/lib_android_env.sh" + +[ "${AUTO_ANDROID}" = "1" ] || { echo "skipped: AUTO_ANDROID=${AUTO_ANDROID}"; exit 0; } +if [ "${AUTO_GUI}" != "1" ]; then + echo "AUTO_GUI=0 → deep-link smoke only" + exec env AUTO_ANDROID=1 "$ROOT/run-android-pay-smoke.sh" +fi + +export OUT_DIR="${OUT_DIR:-$ROOT/out-gui}" +export GUI_ROUNDS="${GUI_ROUNDS:-10}" +export GUI_SLEEP="${GUI_SLEEP:-3}" + +# Install + deliver intents via existing smoke, but skip its weak single-tap +# by setting GUI_MODE=1 which we implement here more fully. +# Simpler: do install/URI here by sourcing patterns from pay-smoke via env + python. + +# Delegate install + URI start to pay-smoke with GUI_AFTER=1 if we patch it; +# instead run a self-contained flow calling pay-smoke pieces. + +APK_DIR="${APK_DIR:-$ROOT/apks}" +APK_URL="${APK_URL:-https://f-droid.org/repo/net.taler.wallet.fdroid_854.apk}" +APK_NAME="${APK_NAME:-net.taler.wallet.fdroid_854.apk}" +APK_PATH="${APK_PATH:-}" +PKG="${PKG:-net.taler.wallet.fdroid}" +STACK="${STACK:-auto}" +SERIAL="${SERIAL:-}" +mkdir -p "$OUT_DIR" + +command -v adb >/dev/null || { echo "adb missing" >&2; exit 2; } + +# Headless default: no host window; guest GLES via SwiftShader (uiautomator). +android_ensure_device || { echo "no adb device (try ./start-android-emulator.sh --wait)" >&2; exit 3; } +ADB=(adb -s "$SERIAL") +echo "device: $SERIAL mode: GUI (uiautomator) headless=${EMULATOR_HEADLESS} gpu=${EMULATOR_GPU}" + +case "$STACK" in + auto) + if curl -sfS -m 5 -o /dev/null https://bank.hacktivism.ch/config; then STACK=goa; else STACK=stage; fi + ;; +esac +case "$STACK" in + goa) + BANK=https://bank.hacktivism.ch + MERCHANT=https://taler.hacktivism.ch + WITHDRAW_JSON="$BANK/intro/demo-withdraw.json" + ;; + stage) + BANK=https://stage.bank.lefrancpaysan.ch + MERCHANT=https://stage.monnaie.lefrancpaysan.ch + WITHDRAW_JSON="$BANK/intro/demo-withdraw.json" + ;; + *) echo "unknown STACK" >&2; exit 2 ;; +esac +echo "stack: $STACK" + +if [ -n "$APK_PATH" ] && [ -f "$APK_PATH" ]; then + : +elif [ -f "$APK_DIR/$APK_NAME" ]; then + APK_PATH="$APK_DIR/$APK_NAME" +else + APK_PATH="$APK_DIR/$APK_NAME" + mkdir -p "$APK_DIR" + curl -fL --retry 3 -o "$APK_PATH" "$APK_URL" +fi +echo "apk: $APK_PATH pkg: $PKG" + +"${ADB[@]}" wait-for-device +"${ADB[@]}" install -r "$APK_PATH" +"${ADB[@]}" shell settings put global window_animation_scale 0 || true +"${ADB[@]}" shell settings put global transition_animation_scale 0 || true +"${ADB[@]}" shell settings put global animator_duration_scale 0 || true + +: "${DO_WITHDRAW:=1}" +: "${DO_PAY:=1}" +: "${SMOKE_STRICT:=0}" + +WURI="" +PAY_URI="" +WITHDRAW_OK=0 +PAY_OK=0 + +"${ADB[@]}" shell am force-stop "$PKG" 2>/dev/null || true +"${ADB[@]}" logcat -c 2>/dev/null || true + +if [ "$DO_WITHDRAW" = "1" ]; then + echo + echo "======== 1 WITHDRAW (Abheben) ========" + WURI=$(curl -sfS -m 25 "$WITHDRAW_JSON" | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("taler_withdraw_uri") or d.get("qr_payload") or "")') + [ -n "$WURI" ] || { echo "no withdraw uri" >&2; exit 4; } + echo "withdraw: $WURI" + echo "$WURI" >"$OUT_DIR/last-withdraw.uri" + "${ADB[@]}" shell am start -a android.intent.action.VIEW -d "$WURI" | tee "$OUT_DIR/start-withdraw.txt" + sleep 5 + python3 - "$ROOT/lib_ui.py" "$SERIAL" "$OUT_DIR" "$GUI_ROUNDS" "$GUI_SLEEP" <<'PY' +import json, sys, importlib.util +from pathlib import Path +lib_path, serial, out, rounds, sleep_s = ( + Path(sys.argv[1]), sys.argv[2], Path(sys.argv[3]), int(sys.argv[4]), float(sys.argv[5]) +) +spec = importlib.util.spec_from_file_location("lib_ui", lib_path) +lib = importlib.util.module_from_spec(spec) +spec.loader.exec_module(lib) +st = lib.gui_drive(serial, out / "gui-withdraw", rounds=rounds, sleep_s=sleep_s, phase="wd") +(out / "gui-withdraw-status.json").write_text(json.dumps(st, indent=2)) +print("gui-withdraw taps:", st.get("taps")) +print("gui-withdraw anr:", st.get("anr_dismissals")) +PY +else + echo "======== 1 WITHDRAW skipped (DO_WITHDRAW=0) ========" +fi + +if [ "$DO_PAY" = "1" ]; then + echo + echo "======== 2 PAY (Bezahlen) ========" + if [ "$STACK" = "stage" ]; then + if RESP=$(curl -sfS -m 20 -X POST -H 'Content-Type: application/json' -d '{}' \ + "$MERCHANT/instances/fermes-des-collines/templates/panier-legumes" 2>/dev/null); then + PAY_URI=$(printf '%s' "$RESP" | python3 -c 'import json,sys;d=json.load(sys.stdin);oid=d.get("order_id")or"";tok=d.get("token")or""; +print("taler://pay/stage.monnaie.lefrancpaysan.ch/instances/fermes-des-collines/%s/?c=%s"%(oid,tok) if oid and tok else "")') + fi + else + PAY_URI="taler://pay-template/taler.hacktivism.ch/instances/goa-shop/paivana" + fi + if [ -n "$PAY_URI" ]; then + echo "pay: $PAY_URI" + echo "$PAY_URI" >"$OUT_DIR/last-pay.uri" + "${ADB[@]}" shell am start -a android.intent.action.VIEW -d "$PAY_URI" | tee "$OUT_DIR/start-pay.txt" + sleep 5 + python3 - "$ROOT/lib_ui.py" "$SERIAL" "$OUT_DIR" "$GUI_ROUNDS" "$GUI_SLEEP" <<'PY' +import json, sys, importlib.util +from pathlib import Path +lib_path, serial, out, rounds, sleep_s = ( + Path(sys.argv[1]), sys.argv[2], Path(sys.argv[3]), int(sys.argv[4]), float(sys.argv[5]) +) +spec = importlib.util.spec_from_file_location("lib_ui", lib_path) +lib = importlib.util.module_from_spec(spec) +spec.loader.exec_module(lib) +st = lib.gui_drive(serial, out / "gui-pay", rounds=rounds, sleep_s=sleep_s, phase="pay") +(out / "gui-pay-status.json").write_text(json.dumps(st, indent=2)) +print("gui-pay taps:", st.get("taps")) +print("gui-pay anr:", st.get("anr_dismissals")) +PY + else + echo "pay: (no URI)" + fi +else + echo "======== 2 PAY skipped (DO_PAY=0) ========" +fi + +"${ADB[@]}" logcat -d -t 500 >"$OUT_DIR/logcat.txt" || true +grep -iE 'taler-wallet|prepareBank|acceptWithdrawal|confirmWithdrawal|preparePay|confirmPay|Error|success|withdraw|pay' \ + "$OUT_DIR/logcat.txt" | tail -100 >"$OUT_DIR/logcat-taler.txt" || true + +if [ "$DO_WITHDRAW" = "1" ] && grep -qE 'prepareBankIntegratedWithdrawal|acceptWithdrawal|confirmWithdrawal' \ + "$OUT_DIR/logcat.txt" 2>/dev/null; then + WITHDRAW_OK=1 +fi +if [ "$DO_PAY" = "1" ] && [ -n "$PAY_URI" ] && \ + grep -qiE 'preparePay|confirmPay|preparePurchase|checkPay|pay-template|PayForTemplate' \ + "$OUT_DIR/logcat.txt" 2>/dev/null; then + PAY_OK=1 +fi + +rc=0 +echo +echo "=== GUI smoke summary ($STACK) — withdraw + pay ===" +echo "package: $PKG DO_WITHDRAW=$DO_WITHDRAW DO_PAY=$DO_PAY STRICT=$SMOKE_STRICT" +if [ "$DO_WITHDRAW" = "1" ]; then + echo "withdraw URI: delivered → $OUT_DIR/gui-withdraw-status.json" + echo "withdraw wallet-core: $([ "$WITHDRAW_OK" = 1 ] && echo YES || echo NO)" + if [ "$SMOKE_STRICT" = "1" ] && [ "$WITHDRAW_OK" != "1" ]; then + echo "FAIL: strict withdraw missing"; rc=11 + fi +fi +if [ "$DO_PAY" = "1" ]; then + echo "pay URI: ${PAY_URI:-none} → $OUT_DIR/gui-pay-status.json" + echo "pay wallet-core: $([ "$PAY_OK" = 1 ] && echo YES || echo NO)" + if [ "$SMOKE_STRICT" = "1" ] && [ "$PAY_OK" != "1" ]; then + echo "FAIL: strict pay missing"; [ "$rc" -eq 0 ] && rc=13 + fi +fi +if grep -qiE "isn't responding|not responding" "$OUT_DIR/logcat.txt" 2>/dev/null \ + || grep -rqiE "isn't responding" "$OUT_DIR/gui-withdraw" "$OUT_DIR/gui-pay" 2>/dev/null; then + echo "FAIL: ANR"; rc=10 +fi +echo "artifacts: $OUT_DIR/" +ls -la "$OUT_DIR" | sed 's/^/ /' +echo "Documented shortcuts → GUI-AUTOMATION-NOTES.md" +exit "$rc" diff --git a/android-test/run-android-pay-smoke.sh b/android-test/run-android-pay-smoke.sh new file mode 100755 index 0000000..48ec49f --- /dev/null +++ b/android-test/run-android-pay-smoke.sh @@ -0,0 +1,308 @@ +#!/usr/bin/env bash +# Android wallet smoke: install APK, then **withdraw (Abheben)** and **pay** via +# deep-link + best-effort UI taps. Both legs are first-class (not pay-only). +# +# Flow (default both on): +# 1) WITHDRAW taler://withdraw/… from bank demo-withdraw.json + confirm taps +# 2) PAY taler://pay… / pay-template + optional taps +# +# Env: +# DO_WITHDRAW=1 (default) DO_PAY=1 (default) +# SMOKE_STRICT=1 → require wallet-core withdraw (+ pay if DO_PAY) + no ANR +# WITHDRAW_GUI_ROUNDS multi-round Abheben taps (default 6) +# +# Usage: +# ./android-test/run-android-pay-smoke.sh +# STACK=goa ./android-test/run-android-pay-smoke.sh +# DO_PAY=0 ./android-test/run-android-pay-smoke.sh # withdraw-only +# +set -euo pipefail +ROOT=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=lib_android_env.sh +. "$ROOT/lib_android_env.sh" + +if [ "${AUTO_ANDROID}" != "1" ]; then + echo "skipped: AUTO_ANDROID=${AUTO_ANDROID} (platform flags — see GUI-AUTOMATION-NOTES.md)" + exit 0 +fi + +APK_DIR="${APK_DIR:-$ROOT/apks}" +APK_URL="${APK_URL:-https://f-droid.org/repo/net.taler.wallet.fdroid_854.apk}" +APK_NAME="${APK_NAME:-net.taler.wallet.fdroid_854.apk}" +# Optional override: full path to an already-built APK (from-source) +APK_PATH="${APK_PATH:-}" +PKG="${PKG:-net.taler.wallet.fdroid}" +STACK="${STACK:-auto}" # auto | goa | stage +SERIAL="${SERIAL:-}" +OUT_DIR="${OUT_DIR:-$ROOT/out}" +mkdir -p "$APK_DIR" "$OUT_DIR" + +if ! command -v adb >/dev/null 2>&1; then + echo "adb missing — install android-tools-adb or user SDK platform-tools" >&2 + exit 2 +fi + +# Default: headless AVD (no host window). WINDOWED=1 / EMULATOR_HEADLESS=0 for UI. +if ! android_ensure_device; then + cat >&2 </dev/null; then + STACK=goa + else + STACK=stage + fi + ;; +esac +case "$STACK" in + goa) + BANK=https://bank.hacktivism.ch + MERCHANT=https://taler.hacktivism.ch + WITHDRAW_JSON="$BANK/intro/demo-withdraw.json" + PAY_HINT="paivana template / goa-shop" + ;; + stage) + BANK=https://stage.bank.lefrancpaysan.ch + MERCHANT=https://stage.monnaie.lefrancpaysan.ch + WITHDRAW_JSON="$BANK/intro/demo-withdraw.json" + PAY_HINT="farmer shop templates" + ;; + *) echo "unknown STACK=$STACK" >&2; exit 2 ;; +esac +echo "stack: $STACK bank: $BANK" + +# APK: explicit path, or download published F-Droid into APK_DIR +if [ -n "$APK_PATH" ] && [ -f "$APK_PATH" ]; then + : +elif [ -f "$APK_DIR/$APK_NAME" ]; then + APK_PATH="$APK_DIR/$APK_NAME" +else + APK_PATH="$APK_DIR/$APK_NAME" + echo "downloading published wallet APK…" + curl -fL --retry 3 -o "$APK_PATH" "$APK_URL" +fi +echo "apk: $APK_PATH ($(wc -c <"$APK_PATH") bytes)" +echo "pkg: $PKG" + +: "${DO_WITHDRAW:=1}" +: "${DO_PAY:=1}" +: "${WITHDRAW_GUI_ROUNDS:=6}" +: "${PAY_GUI_ROUNDS:=4}" +: "${SMOKE_STRICT:=0}" + +"${ADB[@]}" wait-for-device +"${ADB[@]}" install -r "$APK_PATH" +"${ADB[@]}" shell settings put global window_animation_scale 0 || true +"${ADB[@]}" shell settings put global transition_animation_scale 0 || true +"${ADB[@]}" shell settings put global animator_duration_scale 0 || true + +"${ADB[@]}" shell am force-stop "$PKG" 2>/dev/null || true +"${ADB[@]}" logcat -c 2>/dev/null || true + +WURI="" +PAY_URI="" +WITHDRAW_OK=0 +PAY_OK=0 + +# ═══════════════════════════════════════════════════════════════ +# 1) WITHDRAW / Abheben (required by default — not pay-only) +# ═══════════════════════════════════════════════════════════════ +if [ "$DO_WITHDRAW" = "1" ]; then + echo + echo "======== 1 WITHDRAW (Abheben) ========" + WURI=$(curl -sfS -m 25 "$WITHDRAW_JSON" | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("taler_withdraw_uri") or d.get("qr_payload") or "")') + if [ -z "$WURI" ]; then + echo "FAIL: no taler_withdraw_uri from $WITHDRAW_JSON" >&2 + exit 4 + fi + echo "withdraw URI: $WURI" + echo "$WURI" >"$OUT_DIR/last-withdraw.uri" + + "${ADB[@]}" shell am start -a android.intent.action.VIEW -d "$WURI" 2>&1 | tee "$OUT_DIR/start-withdraw.txt" + sleep 6 + "${ADB[@]}" exec-out screencap -p >"$OUT_DIR/01-after-withdraw-uri.png" || true + "${ADB[@]}" shell uiautomator dump /sdcard/ui.xml 2>/dev/null || true + "${ADB[@]}" pull /sdcard/ui.xml "$OUT_DIR/01-ui.xml" 2>/dev/null || true + + # Multi-round confirm / Abheben / ToS (same idea as GUI smoke) + if [ -f "$ROOT/lib_ui.py" ]; then + python3 - "$ROOT/lib_ui.py" "$SERIAL" "$OUT_DIR" "$WITHDRAW_GUI_ROUNDS" 2 <<'PY' || true +import json, sys, importlib.util +from pathlib import Path +lib_path, serial, out, rounds, sleep_s = ( + Path(sys.argv[1]), sys.argv[2], Path(sys.argv[3]), int(sys.argv[4]), float(sys.argv[5]) +) +spec = importlib.util.spec_from_file_location("lib_ui", lib_path) +lib = importlib.util.module_from_spec(spec) +spec.loader.exec_module(lib) +st = lib.gui_drive(serial, out / "gui-withdraw", rounds=rounds, sleep_s=sleep_s, phase="wd") +(out / "gui-withdraw-status.json").write_text(json.dumps(st, indent=2)) +print("withdraw GUI taps:", st.get("taps"), "anr:", st.get("anr_dismissals")) +PY + else + python3 - "$SERIAL" "$OUT_DIR/01-ui.xml" <<'PY' || true +import re, subprocess, sys +serial, path = sys.argv[1], sys.argv[2] +try: + xml = open(path, errors="replace").read() +except FileNotFoundError: + raise SystemExit(0) +labels = [ + "Confirm", "Confirm withdrawal", "Withdraw", "Accept", "Continue", "OK", + "I accept", "Agree", "Bestätigen", "Abheben", "Akzeptieren", "Retirer", +] +for label in labels: + for pat in ( + r'text="%s"[^>]*bounds="\[(\d+),(\d+)\]\[(\d+),(\d+)\]"' % re.escape(label), + r'bounds="\[(\d+),(\d+)\]\[(\d+),(\d+)\]"[^>]*text="%s"' % re.escape(label), + ): + for m in re.finditer(pat, xml, flags=re.I): + x1, y1, x2, y2 = map(int, m.groups()) + x, y = (x1 + x2) // 2, (y1 + y2) // 2 + print(f"tap {label} @ {x},{y}") + subprocess.run(["adb", "-s", serial, "shell", "input", "tap", str(x), str(y)], check=False) +PY + sleep 8 + fi + "${ADB[@]}" exec-out screencap -p >"$OUT_DIR/02-after-withdraw-taps.png" || true + # Mid-flow logcat: withdraw must register before pay + "${ADB[@]}" logcat -d -t 400 >"$OUT_DIR/logcat-after-withdraw.txt" || true + if grep -qE 'prepareBankIntegratedWithdrawal|acceptWithdrawal|confirmWithdrawal' \ + "$OUT_DIR/logcat-after-withdraw.txt" 2>/dev/null; then + WITHDRAW_OK=1 + echo "withdraw: wallet-core activity YES" + else + echo "withdraw: wallet-core activity NOT seen yet (onboarding/ANR/slow?)" + fi +else + echo "======== 1 WITHDRAW skipped (DO_WITHDRAW=0) ========" +fi + +# ═══════════════════════════════════════════════════════════════ +# 2) PAY (after withdraw; still first-class, but second leg) +# ═══════════════════════════════════════════════════════════════ +if [ "$DO_PAY" = "1" ]; then + echo + echo "======== 2 PAY (Bezahlen) ========" + if [ "$STACK" = "stage" ]; then + if RESP=$(curl -sfS -m 20 -X POST -H 'Content-Type: application/json' -d '{}' \ + "$MERCHANT/instances/fermes-des-collines/templates/panier-legumes" 2>/dev/null); then + PAY_URI=$(printf '%s' "$RESP" | python3 -c ' +import json,sys +d=json.load(sys.stdin) +oid=d.get("order_id") or "" +tok=d.get("token") or "" +if oid and tok: + print("taler://pay/stage.monnaie.lefrancpaysan.ch/instances/fermes-des-collines/%s/?c=%s"%(oid,tok)) +' 2>/dev/null || true) + fi + elif [ "$STACK" = "goa" ]; then + PAY_URI="taler://pay-template/taler.hacktivism.ch/instances/goa-shop/paivana" + fi + + if [ -n "$PAY_URI" ]; then + echo "pay URI: $PAY_URI ($PAY_HINT)" + echo "$PAY_URI" >"$OUT_DIR/last-pay.uri" + "${ADB[@]}" shell am start -a android.intent.action.VIEW -d "$PAY_URI" 2>&1 | tee "$OUT_DIR/start-pay.txt" + sleep 6 + if [ -f "$ROOT/lib_ui.py" ]; then + python3 - "$ROOT/lib_ui.py" "$SERIAL" "$OUT_DIR" "$PAY_GUI_ROUNDS" 2 <<'PY' || true +import json, sys, importlib.util +from pathlib import Path +lib_path, serial, out, rounds, sleep_s = ( + Path(sys.argv[1]), sys.argv[2], Path(sys.argv[3]), int(sys.argv[4]), float(sys.argv[5]) +) +spec = importlib.util.spec_from_file_location("lib_ui", lib_path) +lib = importlib.util.module_from_spec(spec) +spec.loader.exec_module(lib) +st = lib.gui_drive(serial, out / "gui-pay", rounds=rounds, sleep_s=sleep_s, phase="pay") +(out / "gui-pay-status.json").write_text(json.dumps(st, indent=2)) +print("pay GUI taps:", st.get("taps"), "anr:", st.get("anr_dismissals")) +PY + fi + "${ADB[@]}" exec-out screencap -p >"$OUT_DIR/03-after-pay-uri.png" || true + else + echo "pay: (no public template URI for stack=$STACK)" + fi +else + echo "======== 2 PAY skipped (DO_PAY=0) ========" +fi + +# Final evidence +"${ADB[@]}" logcat -d -t 500 >"$OUT_DIR/logcat.txt" || true +grep -iE 'taler-wallet|prepareBank|acceptWithdrawal|confirmWithdrawal|preparePay|confirmPay|error|Error|success|withdraw|pay' \ + "$OUT_DIR/logcat.txt" | tail -100 >"$OUT_DIR/logcat-taler.txt" || true + +# Re-evaluate withdraw/pay from full logcat +if [ "$DO_WITHDRAW" = "1" ]; then + if grep -qE 'prepareBankIntegratedWithdrawal|acceptWithdrawal|confirmWithdrawal' \ + "$OUT_DIR/logcat.txt" 2>/dev/null; then + WITHDRAW_OK=1 + fi +fi +if [ "$DO_PAY" = "1" ] && [ -n "$PAY_URI" ]; then + if grep -qiE 'preparePay|confirmPay|preparePurchase|checkPay|pay-template|PayForTemplate' \ + "$OUT_DIR/logcat.txt" 2>/dev/null; then + PAY_OK=1 + fi +fi + +rc=0 +echo +echo "=== summary (withdraw + pay) ===" +echo "package: $PKG stack: $STACK" +echo "DO_WITHDRAW=$DO_WITHDRAW DO_PAY=$DO_PAY SMOKE_STRICT=$SMOKE_STRICT" +if [ "$DO_WITHDRAW" = "1" ]; then + echo "withdraw URI: ${WURI:-(none)}" + if [ "$WITHDRAW_OK" = "1" ]; then + echo "withdraw wallet-core: YES" + else + echo "withdraw wallet-core: NO" + if [ "$SMOKE_STRICT" = "1" ]; then + echo "FAIL: SMOKE_STRICT requires withdraw (prepareBank/accept/confirm) in logcat" + rc=11 + fi + fi +fi +if [ "$DO_PAY" = "1" ]; then + echo "pay URI: ${PAY_URI:-(none)}" + if [ -z "$PAY_URI" ]; then + echo "pay: no URI (template/API)" + if [ "$SMOKE_STRICT" = "1" ]; then + echo "FAIL: SMOKE_STRICT requires a pay URI when DO_PAY=1" + rc=12 + fi + elif [ "$PAY_OK" = "1" ]; then + echo "pay wallet-core: YES" + else + echo "pay wallet-core: NO (intent may still have been delivered)" + if [ "$SMOKE_STRICT" = "1" ]; then + echo "FAIL: SMOKE_STRICT requires pay path activity in logcat" + [ "$rc" -eq 0 ] && rc=13 + fi + fi +fi +if grep -qiE "isn't responding|isn.t responding|System UI isn" "$OUT_DIR/01-ui.xml" 2>/dev/null \ + || grep -qiE "isn't responding|not responding" "$OUT_DIR/logcat.txt" 2>/dev/null \ + || grep -rqiE "isn't responding" "$OUT_DIR/gui-withdraw" 2>/dev/null; then + echo "FAIL: System UI / app ANR" + rc=10 +fi +echo "artifacts: $OUT_DIR/" +ls -la "$OUT_DIR" | sed 's/^/ /' +echo +echo "NOTE: Full unattended Abheben+Bezahlen needs responsive device (≥6–8 GiB host RAM)." +echo "Smoke legs: withdraw=$( [ "$DO_WITHDRAW" = 1 ] && echo on || echo off) pay=$( [ "$DO_PAY" = 1 ] && echo on || echo off) rc=$rc" +exit "$rc" diff --git a/android-test/run-android-variant-matrix.sh b/android-test/run-android-variant-matrix.sh new file mode 100755 index 0000000..54fe5dc --- /dev/null +++ b/android-test/run-android-variant-matrix.sh @@ -0,0 +1,322 @@ +#!/usr/bin/env bash +# Run Android wallet smoke against several build / published variants. +# +# Default order: +# 1) stable-self — **only** self-build of current published stable (WARN) +# 2) published — current F-Droid APK +# 3) older — F-Droid APKs ~3/6/9/12 mo if still hosted (WARN, no self-build) +# 4) master — tip self-build (WARN) +# +# Self-build policy: current stable + master/fix only — never rebuild old releases. +# +# Severity: age ≤14d → BLOCKER; milestones/older/self-builds → WARN +# +# Env: +# STABLE_TAG, STABLE_APK_URL, STABLE_APK_NAME +# MASTER_BRANCH, FIX_BRANCH +# VARIANTS default: stable-self,published,older,master +# INCLUDE_OLDER=1 expand token "older" / "milestones" (default 1) +# MILESTONE_DAYS default "90 180 270 365" (≈ 3,6,9,12 months) +# MILESTONE_TOLERANCE_DAYS default 50 — max distance to accept a tag +# BLOCKER_MAX_AGE_DAYS default 14 +# PUBLISHED_FORCE_BLOCKER=1 always treat current published as blocker +# FAIL_FAST=1, GUI=0, STACK=… +# +set -euo pipefail +ROOT=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=lib_android_env.sh +. "$ROOT/lib_android_env.sh" +# shellcheck source=lib_release_age.sh +. "$ROOT/lib_release_age.sh" + +: "${STABLE_TAG:=wallet-1.6.1}" +: "${STABLE_APK_URL:=https://f-droid.org/repo/net.taler.wallet.fdroid_854.apk}" +: "${STABLE_APK_NAME:=net.taler.wallet.fdroid_854.apk}" +: "${MASTER_BRANCH:=master}" +: "${FIX_BRANCH:=dev/hernani-inference/fix-bank-withdraw-auto-exchange}" +: "${VARIANTS:=stable-self,published,older,master}" +: "${INCLUDE_OLDER:=1}" +: "${MILESTONE_DAYS:=90 180 270 365}" +: "${MILESTONE_TOLERANCE_DAYS:=50}" +: "${BLOCKER_MAX_AGE_DAYS:=14}" +: "${PUBLISHED_FORCE_BLOCKER:=0}" +: "${FAIL_FAST:=1}" +: "${GUI:=0}" +: "${STACK:=auto}" +: "${TALER_ANDROID_SRC:=$HOME/taler/taler-android}" +export TALER_ANDROID_SRC BLOCKER_MAX_AGE_DAYS MILESTONE_DAYS MILESTONE_TOLERANCE_DAYS + +TS=$(date +%Y%m%d-%H%M%S) +MATRIX_OUT="${MATRIX_OUT:-$ROOT/out-matrix-$TS}" +mkdir -p "$MATRIX_OUT" +SUMMARY="$MATRIX_OUT/SUMMARY.txt" +: >"$SUMMARY" + +echo "=== Android variant matrix ===" | tee -a "$SUMMARY" +echo "STACK=$STACK GUI=$GUI headless=${EMULATOR_HEADLESS} variants=$VARIANTS" | tee -a "$SUMMARY" +echo "STABLE_TAG=$STABLE_TAG BLOCKER_MAX_AGE_DAYS=$BLOCKER_MAX_AGE_DAYS" | tee -a "$SUMMARY" +echo "milestones(days)=$MILESTONE_DAYS tolerance=$MILESTONE_TOLERANCE_DAYS (always WARN)" | tee -a "$SUMMARY" +echo "out: $MATRIX_OUT" | tee -a "$SUMMARY" + +android_ensure_device || { + echo "no adb device — start ./start-android-emulator.sh --wait" >&2 + exit 3 +} +export SERIAL ANDROID_SERIAL +echo "device: $SERIAL" | tee -a "$SUMMARY" + +# --- expand VARIANTS (token "older" → concrete older-pub / older-self entries) --- +# Append one older release as WARN-only — **F-Droid APK only** (no self-build). +# Returns 0 if queued, 1 if skipped (no published APK). +append_older_warn() { + local t="$1" age="$2" target="${3:-}" + local vn meta apk_vc apk_url + vn="${t#wallet-}" + if [ -n "$target" ]; then + echo " milestone ~${target}d → $t (age=${age}d) severity=WARN" | tee -a "$SUMMARY" + else + echo " older $t (age=${age}d) severity=WARN" | tee -a "$SUMMARY" + fi + meta=$(fdroid_apk_for_version_name "$vn" 2>/dev/null || true) + if [ -n "$meta" ]; then + apk_vc="${meta%%|*}" + apk_url="${meta#*|}" + if fdroid_apk_exists "$apk_url"; then + out+=("older-pub:${vn}:${apk_vc}:warn") + return 0 + fi + echo " SKIP $t — no F-Droid APK at ${apk_url:-?} (self-build only for current stable)" | tee -a "$SUMMARY" + else + echo " SKIP $t — cannot map F-Droid versionCode (self-build only for current stable)" | tee -a "$SUMMARY" + fi + return 1 +} + +expand_variants() { + local raw="$1" + local -a out=() + local v tag vn age sev line t target + + IFS=',' read -r -a toks <<<"$raw" + for v in "${toks[@]}"; do + v=$(echo "$v" | tr -d '[:space:]') + [ -n "$v" ] || continue + if [ "$v" = "older" ] || [ "$v" = "older-releases" ] || [ "$v" = "milestones" ]; then + [ "$INCLUDE_OLDER" = "1" ] || continue + # 3/6/9/12-month samples: published APK only, always WARN. + local found=0 queued=0 + while read -r t age target; do + [ -n "$t" ] || continue + [ "$t" = "$STABLE_TAG" ] && continue + found=1 + if append_older_warn "$t" "$age" "$target"; then + queued=$((queued + 1)) + fi + done < <(select_milestone_tags "$STABLE_TAG" 2>/dev/null || true) + if [ "$found" = "0" ]; then + echo " (no milestone tags within ±${MILESTONE_TOLERANCE_DAYS}d of $MILESTONE_DAYS)" | tee -a "$SUMMARY" + elif [ "$queued" = "0" ]; then + echo " (milestone tags found but none still on F-Droid — no older APK smokes)" | tee -a "$SUMMARY" + fi + else + out+=("$v") + fi + done + printf '%s\n' "${out[@]}" +} + +mapfile -t VLIST < <(expand_variants "$VARIANTS") +echo "expanded variants: ${VLIST[*]}" | tee -a "$SUMMARY" + +declare -a RESULTS=() +declare -a WARNINGS=() +overall=0 + +severity_for_published_current() { + if [ "${PUBLISHED_FORCE_BLOCKER}" = "1" ]; then + echo blocker + return + fi + local days + days=$(release_age_days "$STABLE_TAG" 2>/dev/null || true) + severity_from_age_days "$days" +} + +run_published_apk() { + local label="$1" + local apk_url="$2" + local apk_name="$3" + local sev="$4" + local out="$MATRIX_OUT/$label" + mkdir -p "$out" + echo "" | tee -a "$SUMMARY" + echo "-------- variant: $label (F-Droid APK) [$(echo "$sev" | tr 'a-z' 'A-Z')] --------" | tee -a "$SUMMARY" + echo " url=$apk_url" | tee -a "$SUMMARY" + local smoke="$ROOT/run-android-pay-smoke.sh" + [ "$GUI" = "1" ] && smoke="$ROOT/run-android-gui-smoke.sh" + local strict=1 + [ "$sev" = "warn" ] && strict="${SMOKE_STRICT_WARN:-0}" + [ "$sev" = "blocker" ] && strict="${SMOKE_STRICT:-1}" + set +e + env STACK="$STACK" \ + APK_URL="$apk_url" \ + APK_NAME="$apk_name" \ + APK_PATH="" \ + PKG=net.taler.wallet.fdroid \ + OUT_DIR="$out" \ + SERIAL="$SERIAL" \ + AUTO_START_EMULATOR=0 \ + SMOKE_STRICT="$strict" \ + "$smoke" + local rc=$? + set -e + echo "$label exit=$rc severity=$sev" | tee -a "$SUMMARY" + echo "$label $rc $sev $apk_url" >>"$MATRIX_OUT/results.tsv" + return "$rc" +} + +run_selfbuild() { + local label="$1" + local ref="$2" + local sev="${3:-warn}" + local out="$MATRIX_OUT/$label" + mkdir -p "$out" + echo "" | tee -a "$SUMMARY" + echo "-------- variant: $label (self-build ref=$ref) [$(echo "$sev" | tr 'a-z' 'A-Z')] --------" | tee -a "$SUMMARY" + local strict="${SMOKE_STRICT:-1}" + [ "$sev" = "warn" ] && strict="${SMOKE_STRICT_SELF:-0}" + [ "$sev" = "blocker" ] && strict="${SMOKE_STRICT:-1}" + set +e + env STACK="$STACK" \ + BRANCH="$ref" \ + LABEL="$label" \ + GUI="$GUI" \ + OUT_DIR="$out" \ + SERIAL="$SERIAL" \ + ANDROID_SERIAL="$SERIAL" \ + AUTO_START_EMULATOR=0 \ + PKG=net.taler.wallet.fdroid.debug \ + SMOKE_STRICT="$strict" \ + TALER_ANDROID_SRC="$TALER_ANDROID_SRC" \ + "$ROOT/run-android-build-and-smoke.sh" + local rc=$? + set -e + echo "$label (ref=$ref) exit=$rc severity=$sev" | tee -a "$SUMMARY" + echo "$label $rc $ref $sev" >>"$MATRIX_OUT/results.tsv" + if [ -f "$out/build-meta.txt" ]; then + echo "build-meta:" | tee -a "$SUMMARY" + cat "$out/build-meta.txt" | tee -a "$SUMMARY" + fi + return "$rc" +} + +record_result() { + local v="$1" rc="$2" sev="$3" + if [ "$rc" -eq 0 ]; then + RESULTS+=("$v:OK($sev)") + return + fi + case "$sev" in + warn) + echo "WARN: $v failed rc=$rc — not a blocker; continuing" | tee -a "$SUMMARY" + WARNINGS+=("$v:$rc") + RESULTS+=("$v:WARN($rc)") + ;; + blocker) + echo "BLOCKER: $v failed rc=$rc (release within ${BLOCKER_MAX_AGE_DAYS}d window)" | tee -a "$SUMMARY" + RESULTS+=("$v:BLOCKER($rc)") + overall=1 + if [ "$FAIL_FAST" = "1" ]; then + echo "FAIL_FAST: stopping after blocker $v" | tee -a "$SUMMARY" + return 2 + fi + ;; + soft) + echo "FAIL: $v rc=$rc" | tee -a "$SUMMARY" + RESULTS+=("$v:FAIL($rc)") + overall=1 + if [ "$FAIL_FAST" = "1" ]; then + echo "FAIL_FAST: stopping after $v" | tee -a "$SUMMARY" + return 2 + fi + ;; + esac + return 0 +} + +for raw in "${VLIST[@]}"; do + v=$(echo "$raw" | tr -d '[:space:]') + [ -n "$v" ] || continue + rc=0 + sev=warn + stop=0 + + case "$v" in + published|fdroid|stable-published) + sev=$(severity_for_published_current) + age=$(release_age_days "$STABLE_TAG" 2>/dev/null || echo "?") + echo "published current: tag=$STABLE_TAG age_days=$age → severity=$sev" | tee -a "$SUMMARY" + run_published_apk "published" "$STABLE_APK_URL" "$STABLE_APK_NAME" "$sev" || rc=$? + ;; + stable-self|stable|self-stable|release-self) + # rebuild of stable: always warn (toolchain), age does not make self-build a blocker + sev=warn + run_selfbuild "stable-self" "$STABLE_TAG" warn || rc=$? + ;; + master|main) + sev=warn + run_selfbuild "master" "$MASTER_BRANCH" warn || rc=$? + ;; + fix|fix-branch|inference) + sev=soft + run_selfbuild "fix" "$FIX_BRANCH" soft || rc=$? + ;; + older-pub:*) + # older-pub:VERSION:VC:SEV + IFS=':' read -r _ vn apk_vc sev <<<"$v" + : "${sev:=warn}" + label="older-pub-${vn}" + url="https://f-droid.org/repo/net.taler.wallet.fdroid_${apk_vc}.apk" + name="net.taler.wallet.fdroid_${apk_vc}.apk" + age=$(release_age_days "wallet-${vn}" 2>/dev/null || echo "?") + echo "older published $vn age_days=$age severity=$sev" | tee -a "$SUMMARY" + run_published_apk "$label" "$url" "$name" "$sev" || rc=$? + v="$label" + ;; + older-self:*) + # Explicit older-self is disabled by policy (self-build = current stable only). + echo "SKIP $v — self-build only for current stable ($STABLE_TAG); use older-pub or published" | tee -a "$SUMMARY" + RESULTS+=("$v:SKIP(no-selfbuild-older)") + continue + ;; + *) + # raw git ref: only master-like / fix via named tokens; bare wallet-* tags → no self-build + if [[ "$v" =~ ^wallet-[0-9] ]]; then + echo "SKIP self-build $v — only current stable self-build ($STABLE_TAG) is enabled" | tee -a "$SUMMARY" + RESULTS+=("$v:SKIP(no-selfbuild-older)") + continue + fi + sev=warn + run_selfbuild "ref-${v//\//-}" "$v" warn || rc=$? + ;; + esac + + record_result "$v" "$rc" "$sev" || stop=$? + [ "$stop" = "2" ] && break +done + +echo "" | tee -a "$SUMMARY" +echo "=== matrix done (overall=$overall) ===" | tee -a "$SUMMARY" +echo "rule: age ≤ ${BLOCKER_MAX_AGE_DAYS}d → BLOCKER; 3/6/9/12mo milestones + older → WARN only" | tee -a "$SUMMARY" +for r in "${RESULTS[@]}"; do + echo " $r" | tee -a "$SUMMARY" +done +if [ "${#WARNINGS[@]}" -gt 0 ]; then + echo "warnings (non-blocking):" | tee -a "$SUMMARY" + for w in "${WARNINGS[@]}"; do + echo " WARN $w" | tee -a "$SUMMARY" + done +fi +echo "" +echo "Artifacts: $MATRIX_OUT" +exit "$overall" diff --git a/android-test/run-goa-gui-chain.sh b/android-test/run-goa-gui-chain.sh new file mode 100755 index 0000000..1c20a63 --- /dev/null +++ b/android-test/run-goa-gui-chain.sh @@ -0,0 +1,366 @@ +#!/usr/bin/env bash +# GOA (and optionally stage) GUI chain on Android — adapted from +# taler-android branch **dev/hernani-inference/gui-workflows** +# (`scripts/goa-chain-emu.sh`, docs/gui-workflows.md). +# +# Linux + macOS: uses adb + uiautomator only (no Homebrew hard dependency). +# Upstream scripts are documented as macOS-only for SDK install paths. +# +# Flow: +# 1) API mint withdraw (explorer) → VIEW taler://withdraw → GUI taps +# 2) Merchant template POST → VIEW taler://pay → GUI taps +# +# Usage: +# ./run-goa-gui-chain.sh +# STACK=stage ./run-goa-gui-chain.sh +# WITHDRAW_AMOUNTS='GOA:10' PAY_LIMIT=2 ./run-goa-gui-chain.sh +# PKG=net.taler.wallet.fdroid.debug ./run-goa-gui-chain.sh --no-clear +# +# Env: ANDROID_SERIAL, PKG, EXP_PW_FILE, BANK, MERCHANT, INSTANCE, +# WITHDRAW_AMOUNTS, PRODUCTS, PAY_LIMIT, SHOTDIR, PAUSE +# +# See GUI-AUTOMATION-NOTES.md +# +set -euo pipefail +ROOT=$(cd "$(dirname "$0")" && pwd) + +CLEAR=1 +for a in "$@"; do + case "$a" in + --no-clear) CLEAR=0 ;; + -h|--help) sed -n '2,28p' "$0"; exit 0 ;; + esac +done + +die() { echo "ERROR: $*" >&2; exit 1; } +step() { echo; echo "======== $* ========"; } +info() { echo " $*"; } +pause() { sleep "${1:-${PAUSE:-0.8}}"; } + +# Platform flags + headless emulator defaults (see GUI-AUTOMATION-NOTES.md) +# shellcheck source=lib_android_env.sh +. "$ROOT/lib_android_env.sh" +[ "${AUTO_ANDROID}" = "1" ] || { echo "skipped: AUTO_ANDROID=${AUTO_ANDROID}"; exit 0; } +[ "${AUTO_GUI}" = "1" ] || { echo "skipped: AUTO_GUI=${AUTO_GUI} (chain is GUI)"; exit 0; } + +command -v adb >/dev/null || die "adb not found" +command -v python3 >/dev/null || die "python3 required" + +STACK="${STACK:-goa}" +case "$STACK" in + goa|hacktivism) + PKG="${PKG:-net.taler.wallet.fdroid.debug}" + BANK="${BANK:-https://bank.hacktivism.ch}" + MERCHANT="${MERCHANT:-https://taler.hacktivism.ch}" + INSTANCE=$(printf '%s' "${INSTANCE:-goa-shop}" | tr '[:upper:]' '[:lower:]') + DONATE_INSTANCE=$(printf '%s' "${DONATE_INSTANCE:-goa-demo-cp4zqk}" | tr '[:upper:]' '[:lower:]') + DONATE_TEMPLATE="${DONATE_TEMPLATE:-goa-free}" + DEFAULT_AMOUNTS=("GOA:10" "GOA:20") + DEFAULT_PAYS=("product:orbit-sticker" "product:nebula-coffee" "product:voidwave-playlist" "donate:GOA:12") + CUR_HINT=GOA + ;; + stage|testpaysan) + PKG="${PKG:-net.taler.wallet.fdroid.debug}" + BANK="${BANK:-https://stage.bank.lefrancpaysan.ch}" + MERCHANT="${MERCHANT:-https://stage.monnaie.lefrancpaysan.ch}" + INSTANCE=$(printf '%s' "${INSTANCE:-fermes-des-collines}" | tr '[:upper:]' '[:lower:]') + DONATE_INSTANCE=$(printf '%s' "${DONATE_INSTANCE:-fermes-des-collines}" | tr '[:upper:]' '[:lower:]') + DONATE_TEMPLATE="${DONATE_TEMPLATE:-don-panier-libre}" + DEFAULT_AMOUNTS=("TESTPAYSAN:10" "TESTPAYSAN:20") + DEFAULT_PAYS=("product:panier-legumes" "product:fromage-chevre" "product:oeufs-6") + CUR_HINT=TESTPAYSAN + ;; + *) die "STACK must be goa|stage (got $STACK)" ;; +esac + +ACT="${ACT:-$PKG/net.taler.wallet.main.MainActivity}" +EXP_USER="${EXP_USER:-explorer}" +PAY_LIMIT="${PAY_LIMIT:-4}" +PAUSE="${PAUSE:-0.8}" +SHOTDIR="${SHOTDIR:-$ROOT/out-gui-chain/$(date +%Y%m%d-%H%M%S)-$STACK}" +mkdir -p "$SHOTDIR" + +# Explorer password (same search as taler-monitoring ladder) +if [[ -z "${EXP_PW_FILE:-}" ]]; then + for cand in \ + "${HOME}/src/koopa/koopa-admin-secrets/koopa/host-root/taler-bank/bank-explorer-password.txt" \ + "${HOME}/.config/taler-landing/bank-explorer-password.txt" + do + [[ -f "$cand" ]] && EXP_PW_FILE="$cand" && break + done +fi +# stage: try stagepaysan secret via ssh if missing +if [[ ! -f "${EXP_PW_FILE:-}" && "$STACK" = "stage" ]]; then + for host in francpaysan-stage-user francpaysan-host; do + tmp=$(mktemp) + if ssh -o BatchMode=yes -o ConnectTimeout=10 "$host" \ + 'tr -d "\n\r" "$tmp" 2>/dev/null && [[ -s "$tmp" ]]; then + EXP_PW_FILE="$tmp" + info "explorer password via $host" + break + fi + rm -f "$tmp" + done +fi +[[ -n "${EXP_PW_FILE:-}" && -f "$EXP_PW_FILE" ]] || die "set EXP_PW_FILE (explorer password)" + +# Headless AVD by default (no host window). WINDOWED=1 for a visible emulator. +SERIAL="${SERIAL:-${ANDROID_SERIAL:-}}" +android_ensure_device || die "no adb device (./start-android-emulator.sh --wait)" +export ANDROID_SERIAL="$SERIAL" +info "ANDROID_SERIAL=$ANDROID_SERIAL STACK=$STACK PKG=$PKG SHOTDIR=$SHOTDIR headless=$EMULATOR_HEADLESS gpu=$EMULATOR_GPU" +ADB=(adb -s "$ANDROID_SERIAL") + +adb_sh() { "${ADB[@]}" shell "$@"; } + +shot() { + local n="$1" + adb_sh screencap -p "/sdcard/demo-$n.png" 2>/dev/null || true + "${ADB[@]}" pull "/sdcard/demo-$n.png" "$SHOTDIR/$n.png" >/dev/null 2>&1 || true + info "[shot] $SHOTDIR/$n.png" +} + +tap_text() { + local label="$1" + adb_sh uiautomator dump /sdcard/ui.xml >/dev/null 2>&1 || return 1 + "${ADB[@]}" pull /sdcard/ui.xml "$SHOTDIR/ui.xml" >/dev/null 2>&1 || return 1 + SERIAL="$ANDROID_SERIAL" python3 - "$label" "$SHOTDIR/ui.xml" <<'PY' +import os, re, subprocess, sys +from pathlib import Path +label, path = sys.argv[1], sys.argv[2] +serial = os.environ.get("SERIAL") or os.environ.get("ANDROID_SERIAL") or "" +xml = Path(path).read_text(errors="replace") +# ANR dismiss +if "isn't responding" in xml or "reagiert nicht" in xml.lower(): + for lab in ("Wait", "Warten"): + for m in re.finditer(rf'text="{lab}"[^>]*bounds="\[(\d+),(\d+)\]\[(\d+),(\d+)\]"', xml): + x = (int(m.group(1))+int(m.group(3)))//2 + y = (int(m.group(2))+int(m.group(4)))//2 + subprocess.run(["adb","-s",serial,"shell","input","tap",str(x),str(y)], check=False) + print(f"anr-wait @{x},{y}") + sys.exit(0) +for pat in [ + rf'text="{re.escape(label)}"[^>]*bounds="\[(\d+),(\d+)\]\[(\d+),(\d+)\]"', + rf'text="[^"]*{re.escape(label)}[^"]*"[^>]*bounds="\[(\d+),(\d+)\]\[(\d+),(\d+)\]"', + rf'content-desc="[^"]*{re.escape(label)}[^"]*"[^>]*bounds="\[(\d+),(\d+)\]\[(\d+),(\d+)\]"', +]: + for m in re.finditer(pat, xml, re.I): + chunk = xml[max(0, m.start()-120):m.end()] + if 'enabled="false"' in chunk: + continue + x = (int(m.group(1))+int(m.group(3)))//2 + y = (int(m.group(2))+int(m.group(4)))//2 + print(f"tap {label!r} @{x},{y}") + cmd = ["adb","shell","input","tap",str(x),str(y)] + if serial: + cmd = ["adb","-s",serial,"shell","input","tap",str(x),str(y)] + subprocess.run(cmd, check=False) + sys.exit(0) +print("miss", label) +sys.exit(1) +PY +} + +mint_withdraw() { + local amount="$1" + EXP_USER="$EXP_USER" EXP_PW_FILE="$EXP_PW_FILE" BANK="$BANK" \ + python3 - "$amount" <<'PY' +import base64, json, os, ssl, sys, urllib.request +from pathlib import Path +amount = sys.argv[1] +bank = os.environ["BANK"].rstrip("/") +user = os.environ["EXP_USER"] +pw = Path(os.environ["EXP_PW_FILE"]).read_text().strip() +auth = "Basic " + base64.b64encode(f"{user}:{pw}".encode()).decode() +ctx = ssl.create_default_context() + +def req(method, url, data=None, headers=None): + h = dict(headers or {}) + body = None + if data is not None: + body = json.dumps(data).encode() + h["Content-Type"] = "application/json" + r = urllib.request.Request(url, data=body, headers=h, method=method) + with urllib.request.urlopen(r, context=ctx, timeout=25) as resp: + return json.load(resp) + +tok = req( + "POST", + f"{bank}/accounts/{user}/token", + {"scope": "readwrite", "duration": {"d_us": 3600_000_000}}, + {"Authorization": auth}, +)["access_token"] +wd = req( + "POST", + f"{bank}/accounts/{user}/withdrawals", + {"amount": amount}, + {"Authorization": f"Bearer {tok}"}, +) +print(wd.get("taler_withdraw_uri") or wd.get("talerWithdrawUri") or "") +PY +} + +mint_pay() { + local mode="$1" + MERCHANT="$MERCHANT" INSTANCE="$INSTANCE" \ + DONATE_INSTANCE="$DONATE_INSTANCE" DONATE_TEMPLATE="$DONATE_TEMPLATE" \ + python3 - "$mode" <<'PY' +import json, os, ssl, sys, urllib.error, urllib.request +mode = sys.argv[1] +merchant = os.environ["MERCHANT"].rstrip("/") +inst = os.environ["INSTANCE"] +d_inst = os.environ["DONATE_INSTANCE"] +d_tmpl = os.environ["DONATE_TEMPLATE"] +ctx = ssl.create_default_context() + +def post(url, data): + body = json.dumps(data).encode() + r = urllib.request.Request(url, data=body, method="POST", + headers={"Content-Type": "application/json"}) + try: + with urllib.request.urlopen(r, context=ctx, timeout=25) as resp: + return resp.status, json.load(resp) + except urllib.error.HTTPError as e: + raw = e.read() if hasattr(e, "read") else b"" + try: + return getattr(e, "code", None), json.loads(raw) + except Exception as err: + raise SystemExit(f"POST {url} failed: {e} body={raw[:200]!r}") from err + +if mode.startswith("product:"): + pid = mode.split(":", 1)[1] + st, d = post(f"{merchant}/instances/{inst}/templates/{pid}", {}) + use_inst = inst +elif mode.startswith("donate:"): + amt = mode.split(":", 1)[1] + st, d = post(f"{merchant}/instances/{d_inst}/templates/{d_tmpl}", {"amount": amt}) + use_inst = d_inst +else: + raise SystemExit(f"bad mode {mode!r}") + +if "taler_pay_uri" in d: + print(d["taler_pay_uri"]); raise SystemExit(0) +oid, tok = d.get("order_id"), d.get("token") +if not oid or not tok: + raise SystemExit(f"no order: {d!r}"[:200]) +host = merchant.replace("https://","").replace("http://","").split("/")[0] +print(f"taler://pay/{host}/instances/{use_inst}/{oid}/?c={tok}") +PY +} + +click_through() { + local labels=("$@") + local i lab misses=0 + # Prefer dismissing ANR before hunting Confirm (low-RAM hosts thrash System UI) + for i in 1 2 3 4 5 6; do + if tap_text "Wait" 2>/dev/null || tap_text "Warten" 2>/dev/null; then + info "dismissed ANR (Wait)" + pause 2.0 + misses=0 + continue + fi + local hit=0 + for lab in "${labels[@]}"; do + if tap_text "$lab" 2>/dev/null; then + hit=1 + misses=0 + pause 0.9 + break + fi + done + if [[ "$hit" -eq 0 ]]; then + misses=$((misses + 1)) + # stop early if hierarchy has nothing useful (avoid burning order deadlines) + [[ "$misses" -ge 3 ]] && break + fi + pause 0.5 + done +} + +if [[ -n "${WITHDRAW_AMOUNTS:-}" ]]; then + # shellcheck disable=SC2206 + AMOUNTS=($WITHDRAW_AMOUNTS) +else + AMOUNTS=("${DEFAULT_AMOUNTS[@]}") +fi +if [[ -n "${PRODUCTS:-}" ]]; then + PAYS=() + for p in $PRODUCTS; do PAYS+=("product:$p"); done +else + PAYS=("${DEFAULT_PAYS[@]}") +fi + +############################ +step "0 device + wallet ($STACK / $CUR_HINT)" +adb_sh input keyevent KEYCODE_WAKEUP 2>/dev/null || true +adb_sh wm dismiss-keyguard 2>/dev/null || true +if [[ "$CLEAR" -eq 1 ]]; then + info "pm clear $PKG" + adb_sh pm clear "$PKG" >/dev/null 2>&1 || true +fi +adb_sh am force-stop "$PKG" 2>/dev/null || true +pause 0.4 +# Prefer explicit MainActivity (from gui-workflows); fall back to VIEW launcher +if ! adb_sh am start -n "$ACT" >/dev/null 2>&1; then + info "MainActivity path failed — monkey launch" + adb_sh monkey -p "$PKG" -c android.intent.category.LAUNCHER 1 >/dev/null 2>&1 || true +fi +pause 2.0 +shot "00-start" + +############################ +step "2 withdraw chain" +idx=0 +for AMT in "${AMOUNTS[@]}"; do + idx=$((idx + 1)) + step "2.$idx withdraw $AMT" + URI="$(mint_withdraw "$AMT" | head -1)" + [[ -n "$URI" ]] || die "mint failed for $AMT" + # wallet / monitoring QR rules: strip default ports + URI=$(printf '%s' "$URI" | sed 's/:443\//\//g; s/:443?/?/g; s/:80\//\//g') + info "URI=$URI" + echo "$URI" >>"$SHOTDIR/withdraw-uris.txt" + adb_sh am start -a android.intent.action.VIEW -d "$URI" "$PKG" >/dev/null + pause 1.5 + shot "w${idx}-open" + click_through "Accept" "I accept" "Accept terms" "Confirm" "Continue" "Withdraw" "Next" "OK" "Agree" "Bestätigen" "Abheben" "Akzeptieren" + pause 3.0 + shot "w${idx}-after" + adb_sh am start -n "$ACT" >/dev/null 2>&1 || true + pause 0.6 + tap_text "Assets" 2>/dev/null || true + pause 0.4 + shot "w${idx}-assets" +done + +############################ +step "3 pay chain (limit=$PAY_LIMIT)" +pidx=0 +for PAY in "${PAYS[@]}"; do + pidx=$((pidx + 1)) + [[ "$pidx" -gt "$PAY_LIMIT" ]] && break + step "3.$pidx pay $PAY" + if ! PAYURI="$(mint_pay "$PAY" | head -1)"; then + info "mint_pay failed for $PAY — skip" + continue + fi + [[ -n "$PAYURI" ]] || { info "empty pay uri — skip"; continue; } + PAYURI=$(printf '%s' "$PAYURI" | sed 's/:443\//\//g; s/:443?/?/g; s/:80\//\//g') + info "PAYURI=$PAYURI" + echo "$PAYURI" >>"$SHOTDIR/pay-uris.txt" + adb_sh am start -a android.intent.action.VIEW -d "$PAYURI" "$PKG" >/dev/null + pause 1.5 + shot "p${pidx}-open" + click_through "Pay" "Confirm" "Accept" "Continue" "OK" "Next" "Bezahlen" "Bestätigen" "Payer" + pause 1.5 + shot "p${pidx}-after" + adb_sh am start -n "$ACT" >/dev/null 2>&1 || true + pause 0.5 +done + +step "4 done → $SHOTDIR" +ls -la "$SHOTDIR" | tail -40 +adb_sh dumpsys window 2>/dev/null | grep mCurrentFocus | head -1 || true +info "PKG=$PKG amounts=${AMOUNTS[*]} pays(limit)=$PAY_LIMIT" +echo "See GUI-AUTOMATION-NOTES.md (ported from taler-android dev/hernani-inference/gui-workflows)" diff --git a/android-test/start-android-emulator.sh b/android-test/start-android-emulator.sh new file mode 100755 index 0000000..e62b5f4 --- /dev/null +++ b/android-test/start-android-emulator.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Start the Taler Android AVD. **Default = headless** (no host window). +# Guest GLES still runs via SwiftShader so uiautomator / screenshots work. +# +# Usage: +# ./start-android-emulator.sh # headless, background +# ./start-android-emulator.sh --wait # headless + wait for boot +# ./start-android-emulator.sh --windowed # show emulator window +# EMULATOR_HEADLESS=0 ./start-android-emulator.sh +# EMULATOR_GPU=host WINDOWED=1 ./start-android-emulator.sh +# +# Env: EMULATOR_AVD EMULATOR_MEMORY EMULATOR_CORES EMULATOR_GPU +# EMULATOR_HEADLESS EMULATOR_LOG EMULATOR_BOOT_TIMEOUT EMULATOR_EXTRA_ARGS +# +set -euo pipefail +ROOT=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=lib_android_env.sh +. "$ROOT/lib_android_env.sh" + +WAIT=0 +for a in "$@"; do + case "$a" in + --wait) WAIT=1 ;; + --windowed| --gui) EMULATOR_HEADLESS=0; export EMULATOR_HEADLESS ;; + --headless) EMULATOR_HEADLESS=1; export EMULATOR_HEADLESS ;; + -h|--help) + sed -n '2,18p' "$0" + exit 0 + ;; + esac +done + +command -v emulator >/dev/null || { + echo "emulator not found (ANDROID_HOME=$ANDROID_HOME)" >&2 + exit 2 +} +command -v adb >/dev/null || { + echo "adb not found" >&2 + exit 2 +} + +# Already booted? +if serial=$(android_adb_serial); [ -n "$serial" ]; then + boot=$(adb -s "$serial" shell getprop sys.boot_completed 2>/dev/null | tr -d '\r' || true) + if [ "$boot" = "1" ]; then + echo "already running: $serial (boot_completed=1) — headless default still applies to *new* starts" + exit 0 + fi +fi + +# Avoid pkill patterns that match this wrapper's argv (self-kill). +mapfile -t EMU_ARGS < <(android_emulator_args "$EMULATOR_AVD") +echo "starting: emulator ${EMU_ARGS[*]}" +echo " headless=$EMULATOR_HEADLESS gpu=$EMULATOR_GPU log=$EMULATOR_LOG" + +if [ "${EMULATOR_HEADLESS}" = "1" ]; then + export QT_QPA_PLATFORM="${QT_QPA_PLATFORM:-offscreen}" + # Do not require DISPLAY + unset DISPLAY || true +fi + +mkdir -p "$(dirname "$EMULATOR_LOG")" +# Background; do not use pkill -f emulator from callers matching full cmdline +nohup emulator "${EMU_ARGS[@]}" >"$EMULATOR_LOG" 2>&1 & +echo "emulator pid=$! log=$EMULATOR_LOG" + +if [ "$WAIT" != "1" ]; then + echo "not waiting (pass --wait for boot). adb devices when ready." + exit 0 +fi + +echo "waiting for boot (timeout ${EMULATOR_BOOT_TIMEOUT}s)…" +deadline=$((SECONDS + EMULATOR_BOOT_TIMEOUT)) +serial="" +while (( SECONDS < deadline )); do + adb wait-for-device 2>/dev/null || true + serial=$(android_adb_serial || true) + if [ -n "$serial" ]; then + boot=$(adb -s "$serial" shell getprop sys.boot_completed 2>/dev/null | tr -d '\r' || true) + if [ "$boot" = "1" ]; then + echo "BOOT_OK serial=$serial headless=$EMULATOR_HEADLESS gpu=$EMULATOR_GPU" + exit 0 + fi + fi + sleep 2 +done + +echo "boot timeout — last log:" >&2 +tail -30 "$EMULATOR_LOG" >&2 || true +exit 3 diff --git a/check_apt_deploy.sh b/check_apt_deploy.sh new file mode 100755 index 0000000..5240e82 --- /dev/null +++ b/check_apt_deploy.sh @@ -0,0 +1,279 @@ +#!/usr/bin/env bash +# check_apt_deploy.sh — podman apt-src merchant deploy tests on koopa. +# +# Containers (default): +# Fresh (rebuild when repo versions change): +# koopa-taler-deploy-test-apt-src-trixie +# koopa-taler-deploy-test-apt-src-trixie-testing +# Upgrade track (initial install, then mytops-style apt upgrade): +# koopa-taler-deploy-test-apt-src-trixie-upgrade +# koopa-taler-deploy-test-apt-src-trixie-testing-upgrade +# +# Env: +# APT_DEPLOY_CONTAINERS "name:suite:mode …" mode=fresh|upgrade (optional, default fresh) +# APT_DEPLOY_SKIP=1 +# APT_DEPLOY_PODMAN=podman +# +set -euo pipefail +ROOT=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=lib.sh +source "$ROOT/lib.sh" + +if [ "${APT_DEPLOY_SKIP:-0}" = "1" ]; then + echo "[INFO] skip phase aptdeploy (APT_DEPLOY_SKIP=1)" + exit 0 +fi + +set_area aptdeploy +section "aptdeploy · podman apt-src merchant (fresh + upgrade tracks)" + +PODMAN_BIN="${APT_DEPLOY_PODMAN:-podman}" +DEFAULT_LIST="\ +koopa-taler-deploy-test-apt-src-trixie:trixie:fresh \ +koopa-taler-deploy-test-apt-src-trixie-testing:trixie-testing:fresh \ +koopa-taler-deploy-test-apt-src-trixie-upgrade:trixie:upgrade \ +koopa-taler-deploy-test-apt-src-trixie-testing-upgrade:trixie-testing:upgrade" +LIST="${APT_DEPLOY_CONTAINERS:-$DEFAULT_LIST}" + +if ! command -v "$PODMAN_BIN" >/dev/null 2>&1; then + err "podman" "podman binary missing ($PODMAN_BIN)" + summary + exit 1 +fi +ok "podman" "$("$PODMAN_BIN" --version 2>/dev/null | head -1)" +info "host" "$(hostname 2>/dev/null || echo unknown) · whoami=$(whoami)" + +fail_any=0 + +print_pkg_table() { + local name=$1 + local rows + rows=$("$PODMAN_BIN" exec "$name" bash -lc ' + echo "| package | version |" + echo "|---------|---------|" + for p in taler-merchant libtalermerchant libtalerexchange libdonau libgnunet \ + taler-merchant-webui taler-merchant-typst taler-terms-generator; do + v=$(dpkg-query -W -f="\${Version}" "$p" 2>/dev/null || echo missing) + echo "| $p | $v |" + done + ' 2>/dev/null || echo "| (query failed) | |") + info "pkg-table" "$name" + # emit as plain lines so HTML log keeps the table + while IFS= read -r line; do + [ -n "$line" ] || continue + info "pkg" "$line" + done <<<"$rows" +} + +check_ldd_deep() { + local name=$1 + local out + out=$("$PODMAN_BIN" exec "$name" bash -lc ' + set +e + bin=$(command -v taler-merchant-httpd) + echo "=== ldd taler-merchant-httpd ===" + ldd "$bin" 2>&1 | grep -E "libtalerutil|libdonau|libgnunet|not found" || true + echo "=== ldd libdonau (if present) ===" + for f in /usr/lib/*/libdonau.so* /usr/lib/*/libdonauutil.so*; do + [ -e "$f" ] || continue + echo "-- $f --" + ldd "$f" 2>&1 | grep -E "libtalerutil|not found" || true + done + echo "=== libtalerutil files ===" + ls -la /usr/lib/*/libtalerutil* 2>/dev/null || echo "(none)" + ' 2>/dev/null || echo "ldd probe failed") + while IFS= read -r line; do + [ -n "$line" ] || continue + info "ldd" "$line" + done <<<"$out" + if echo "$out" | grep -q 'not found'; then + err "ldd" "shared library missing in $name" "see ldd lines above" + return 1 + fi + return 0 +} + +check_systemd() { + local name=$1 + local out active_target active_httpd enabled + out=$("$PODMAN_BIN" exec "$name" bash -lc ' + set +e + echo "system: $(systemctl is-system-running 2>&1)" + echo "target_enabled: $(systemctl is-enabled taler-merchant.target 2>&1)" + echo "target_active: $(systemctl is-active taler-merchant.target 2>&1)" + echo "httpd_active: $(systemctl is-active taler-merchant-httpd.service 2>&1)" + echo "httpd_enabled: $(systemctl is-enabled taler-merchant-httpd.service 2>&1)" + systemctl start taler-merchant.target 2>&1 | tail -5 + sleep 2 + echo "after_start_target: $(systemctl is-active taler-merchant.target 2>&1)" + echo "after_start_httpd: $(systemctl is-active taler-merchant-httpd.service 2>&1)" + systemctl --failed --no-legend 2>/dev/null | grep -i taler || echo "failed_taler: none" + ' 2>/dev/null || echo "systemd probe failed") + while IFS= read -r line; do + [ -n "$line" ] || continue + info "systemd" "$line" + done <<<"$out" + + active_httpd=$(echo "$out" | sed -n 's/^after_start_httpd: //p' | tail -1) + if [ "$active_httpd" = "active" ]; then + ok "systemd httpd" "taler-merchant-httpd active after start taler-merchant.target" + return 0 + fi + # soft if target not shipped as enabled in container — still ERROR for deploy smoke + err "systemd httpd" "taler-merchant-httpd not active" "after_start_httpd=${active_httpd:-?} · see systemd lines" + return 1 +} + +check_merchant_basics() { + local name=$1 + local out code + # unix socket or curl localhost if httpd listens + out=$("$PODMAN_BIN" exec "$name" bash -lc ' + set +e + echo "=== binaries ===" + command -v taler-merchant-httpd taler-merchant-dbinit taler-config 2>/dev/null + echo "=== --version ===" + taler-merchant-httpd --version 2>&1 + echo "=== config probe ===" + # try common paths without TLS + for u in \ + http://127.0.0.1:9966/config \ + http://127.0.0.1:8081/config \ + http://127.0.0.1/config + do + code=$(curl -sS -m 2 -o /tmp/mcfg -w "%{http_code}" "$u" 2>/dev/null || echo 000) + echo "curl $u -> $code" + [ "$code" = "200" ] && head -c 120 /tmp/mcfg && echo + done + sock=$(ls /run/taler-merchant/httpd/*.sock 2>/dev/null | head -1) + if [ -n "$sock" ]; then + echo "socket: $sock" + code=$(curl -sS -m 2 --unix-socket "$sock" -o /tmp/mcfg -w "%{http_code}" http://localhost/config 2>/dev/null || echo 000) + echo "curl --unix-socket -> $code" + [ "$code" = "200" ] && head -c 160 /tmp/mcfg && echo + else + echo "socket: (none under /run/taler-merchant/httpd/)" + fi + ' 2>/dev/null || echo "basics probe failed") + + while IFS= read -r line; do + [ -n "$line" ] || continue + info "basics" "$line" + done <<<"$out" + + if echo "$out" | grep -qi 'error while loading shared libraries'; then + err "httpd" "shared library error on --version" "see basics/ldd" + return 1 + fi + if ! echo "$out" | grep -q 'taler-merchant-httpd --version\|v\.\|taler-merchant-httpd'; then + # version line varies; success if exit was ok — check via separate exec + : + fi + + set +e + "$PODMAN_BIN" exec "$name" taler-merchant-httpd --version >/dev/null 2>&1 + local ec=$? + set -e + if [ "$ec" -eq 0 ]; then + ok "httpd --version" "exit 0" + else + err "httpd --version" "exit $ec" + return 1 + fi + + # /config is nice-to-have if stack is fully configured + if echo "$out" | grep -qE 'curl .* -> 200'; then + ok "merchant /config" "HTTP 200 (local)" + else + warn "merchant /config" "no local HTTP 200 (unit may need BASE_URL/db — still report version/ldd)" + fi + return 0 +} + +check_one() { + local name="$1" expect_suite="$2" mode="${3:-fresh}" + local st suite_line pkgs + + # group ids: aptdeploy.trixie-01 / aptdeploy.trixie-testing-upgrade-01 + set_group "${expect_suite}${mode:+-$mode}" + section "aptdeploy · $name (suite=$expect_suite mode=$mode)" + + if ! "$PODMAN_BIN" container exists "$name" 2>/dev/null; then + err "container" "$name missing" "run ensure-apt-deploy-test-containers.sh" + fail_any=1 + return + fi + st=$("$PODMAN_BIN" inspect -f '{{.State.Status}}' "$name" 2>/dev/null || echo unknown) + if [ "$st" != "running" ]; then + err "container" "$name not running" "status=$st" + fail_any=1 + return + fi + ok "container" "$name running (mode=$mode)" + + suite_line=$("$PODMAN_BIN" exec "$name" bash -lc \ + 'grep -h "^Suites:" /etc/apt/sources.list.d/*taler* 2>/dev/null | head -1' 2>/dev/null || true) + if echo "$suite_line" | grep -qE "Suites:[[:space:]]*${expect_suite}([[:space:]]|$)"; then + ok "apt suite" "$expect_suite" + else + err "apt suite" "expected $expect_suite" "got: ${suite_line:-empty}" + fail_any=1 + fi + + print_pkg_table "$name" + + if ! check_merchant_basics "$name"; then + fail_any=1 + check_ldd_deep "$name" || fail_any=1 + else + # still show ldd summary as info when ok + ldd_out=$("$PODMAN_BIN" exec "$name" bash -lc \ + 'ldd "$(command -v taler-merchant-httpd)" 2>&1 | grep -E "libtalerutil|not found" || true' 2>/dev/null || true) + if echo "$ldd_out" | grep -q 'not found'; then + err "libtalerutil" "not found" "$ldd_out" + check_ldd_deep "$name" || true + fail_any=1 + else + info "libtalerutil" "$(echo "$ldd_out" | tr '\n' ' ')" + fi + fi + + if ! check_systemd "$name"; then + fail_any=1 + check_ldd_deep "$name" || true + fi + + # Disk free inside this deploy-test container (+ host once, first container only) + if [ "${_APT_DEPLOY_HOST_DISK_DONE:-0}" != "1" ]; then + set_group disk + mon_disk_check_host "host" || fail_any=1 + _APT_DEPLOY_HOST_DISK_DONE=1 + fi + set_group disk + if ! mon_disk_check_podman "$name" "$PODMAN_BIN"; then + fail_any=1 + fi +} + +for entry in $LIST; do + # name:suite or name:suite:mode + cname=${entry%%:*} + rest=${entry#*:} + case "$rest" in + *:*) + suite=${rest%%:*} + mode=${rest#*:} + ;; + *) + suite=$rest + mode=fresh + ;; + esac + check_one "$cname" "$suite" "$mode" +done + +summary +if [ "$fail_any" -ne 0 ]; then + exit 1 +fi +exit 0 diff --git a/check_auth401.sh b/check_auth401.sh new file mode 100755 index 0000000..dfca606 --- /dev/null +++ b/check_auth401.sh @@ -0,0 +1,704 @@ +#!/usr/bin/env bash +# auth401 — merchant HTTP 401 / case-sensitivity matrix (WebUI-shaped flows) +# +# Exercises the same APIs the merchant SPA uses for: +# • self-provision signup (POST /instances) — mixed-case id +# • login / createAccessToken (POST …/private/token) — Basic user case +# • password change (POST …/private/auth) +# • admin-create (POST /management/instances) when admin bearer available +# • access-token + private/ with Bearer +# +# Backend fact (#11340): Basic-auth *username* is case-sensitive; URL path is +# usually not. SPA must always lowercase the username (createAccessToken, signup +# id, admin-create, pw change re-login). These tests assert the *raw API* +# behaviour so a missing SPA lowercasing shows up as 401 where 200 is wanted +# only after lowercasing. +# +# ./taler-monitoring.sh -d stage.monnaie.lefrancpaysan.ch auth401 +# ./taler-monitoring.sh -d hacktivism.ch auth401 +# +# Continue through all groups / collect every ERROR (optional): +# AUTH401_CONTINUE=1 ./taler-monitoring.sh -d stage… auth401 +# CONTINUE_ON_ERROR=1 ./taler-monitoring.sh -d stage… auth401 # same +# +# Env: see secrets.env.example (AUTH401_*). +# +# NOTE: we intentionally do NOT use `set -e`. A single failed curl/grep under +# set -e would abort with no ERROR summary — the opposite of what monitoring +# needs. Optional CONTINUE_ON_ERROR still controls hard prereq early-exit. +set -uo pipefail +ROOT=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=lib.sh +source "$ROOT/lib.sh" +load_monitoring_secrets_env 2>/dev/null || true + +tmp=$(mktemp -d) +AUTH401_FINISHED=0 +_auth401_finish() { + [ "${AUTH401_FINISHED}" = "1" ] && return 0 + AUTH401_FINISHED=1 + # summary() returns non-zero when FAIL_N>0 — never let that skip the print + summary || true + rm -rf "${tmp:-}" + if [ "${FAIL_N:-0}" -gt 0 ]; then + exit 1 + fi + exit 0 +} +trap '_auth401_finish' EXIT + +# --------------------------------------------------------------------------- +# Parameters +# --------------------------------------------------------------------------- +: "${AUTH401_TIMEOUT:=${TIMEOUT:-20}}" +: "${AUTH401_CREATE:=1}" +: "${AUTH401_INSTANCE:=}" +: "${AUTH401_PASSWORD:=${MERCHANT_INSTANCE_PASSWORD:-${AUTH401_PASS:-}}}" +: "${AUTH401_ID_PREFIX:=mon401}" +: "${AUTH401_SKIP_IF_MFA:=1}" +: "${AUTH401_BEARER:=${E2E_MERCHANT_TOKEN:-${MERCHANT_TOKEN:-}}}" +: "${AUTH401_CREATE_PASSWORD:=${AUTH401_PASSWORD:-Mon401-Test-Pass!}}" +: "${AUTH401_DURABLE_ID:=mon401}" +# Prefer durable alone (skip throwaway) — default 0 so full matrix always runs +: "${AUTH401_PREFER_DURABLE:=0}" +# Run expanded groups (signup/login/pwchange/admin/webui). Default 1. +: "${AUTH401_FULL:=1}" +# Admin bearer for POST /management/instances (optional) +: "${AUTH401_ADMIN_TOKEN:=${MERCHANT_ADMIN_TOKEN:-}}" +: "${AUTH401_ADMIN_PASSWORD:=}" +: "${AUTH401_ADMIN_USER:=admin}" +# 1 = do not abort on hard prereq fail; run remaining groups (default 0) +# Alias: CONTINUE_ON_ERROR=1 +: "${AUTH401_CONTINUE:=${CONTINUE_ON_ERROR:-0}}" +# Never rotate password on durable account (default 1 — keep mon401.password valid) +: "${AUTH401_PWCHANGE_THROWAY_ONLY:=1}" + +TIMEOUT="$AUTH401_TIMEOUT" +export TIMEOUT + +_auth401_abort_or_continue() { + local why="${1:-prereq failed}" + if [ "${AUTH401_CONTINUE}" = "1" ]; then + warn "continue" "${why} (AUTH401_CONTINUE=1 / CONTINUE_ON_ERROR=1)" + return 0 + fi + # hard stop — EXIT trap prints summary + all ERRORS collected so far + exit 1 +} + +BASE="${MERCHANT_PUBLIC%/}" +if [ -z "$BASE" ]; then + echo "MERCHANT_PUBLIC empty — set -d domain or --merchant" >&2 + exit 2 +fi + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- +expect_http() { + local label="$1" want="$2" got="$3" detail="${4:-}" + case ",$want," in + *",$got,"*) ok "$label" "HTTP $got${detail:+ · $detail}" ;; + *) fail "$label" "HTTP $got (want $want)${detail:+ · $detail}" ;; + esac +} + +to_lower() { printf '%s' "$1" | tr '[:upper:]' '[:lower:]'; } +to_upper() { printf '%s' "$1" | tr '[:lower:]' '[:upper:]'; } + +# First char upper, rest lower → synthetic MIX for Basic user (always ≠ low if len>0) +synth_mix() { + python3 -c "s='''$1'''; print((s[:1].upper()+s[1:].lower()) if s else s)" +} + +# First 3 upper (partial mix) +synth_partmix() { + python3 -c "s='''$1'''; print(s[:3].upper()+s[3:] if len(s)>=3 else s.upper())" +} + +# Normalize curl status: "200", "000", never "200000" / "000000" +_http_norm() { + local raw="$1" dig + dig=$(printf '%s' "$raw" | tr -cd '0-9') + if [ -z "$dig" ]; then + printf '000' + else + # first 3 digits = real code (curl 200 then || echo 000 → 200000) + printf '%s' "$dig" | head -c 3 + fi +} + +token_http() { + local path_id="$1" user="$2" password="$3" + local b64 raw + b64=$(printf '%s' "${user}:${password}" | base64 -w0 2>/dev/null \ + || printf '%s' "${user}:${password}" | base64) + raw=$(curl -skS --max-redirs 0 -m "${TIMEOUT}" \ + -o "$tmp/last_token.json" -w '%{http_code}' \ + -X POST "${BASE}/instances/${path_id}/private/token" \ + -H "Authorization: Basic ${b64}" \ + -H 'Content-Type: application/json' \ + -d '{"scope":"write","duration":{"d_us":3600000000},"refreshable":true}' \ + 2>/dev/null || true) + _http_norm "${raw:-}" +} + +json_code_hint() { + python3 - "$tmp/last_token.json" <<'PY' 2>/dev/null || true +import json, sys +try: + d = json.load(open(sys.argv[1])) +except Exception: + print("") + raise SystemExit +c = d.get("code", "") +h = (d.get("hint") or d.get("detail") or "")[:70] +print(f"code={c} {h}".strip()) +PY +} + +extract_token() { + python3 - "$tmp/last_token.json" <<'PY' 2>/dev/null || true +import json, sys +try: + d = json.load(open(sys.argv[1])) +except Exception: + print("") + raise SystemExit(0) +print(d.get("access_token") or d.get("token") or "") +PY +} + +instance_body() { + local id="$1" pass="$2" name="${3:-auth401 probe}" + python3 - "$id" "$pass" "$name" <<'PY' +import json, sys +print(json.dumps({ + "address": {}, + "auth": {"method": "token", "password": sys.argv[2]}, + "default_pay_delay": {"d_us": 60000000}, + "default_wire_transfer_delay": {"d_us": 30000000}, + "default_refund_delay": {"d_us": 30000000}, + "id": sys.argv[1], + "jurisdiction": {}, + "name": sys.argv[3], + "use_stefan": True, +})) +PY +} + +auth_change_http() { + local path_id="$1" bearer="$2" new_pass="$3" + local tok="$bearer" body + case "$tok" in secret-token:*) ;; *) tok="secret-token:${tok}" ;; esac + body=$(NEW_PASS="$new_pass" python3 -c 'import json,os; print(json.dumps({"method":"token","password":os.environ["NEW_PASS"]}))') + curl -skS --max-redirs 0 -m "${TIMEOUT}" \ + -o "$tmp/auth_change.json" -w '%{http_code}' \ + -X POST "${BASE}/instances/${path_id}/private/auth" \ + -H "Authorization: Bearer ${tok}" \ + -H 'Content-Type: application/json' \ + -d "$body" \ + 2>/dev/null || echo 000 +} + +# --------------------------------------------------------------------------- +# Stage secrets (quiet) +# --------------------------------------------------------------------------- +_auth401_stage_secret_roots() { + printf '%s\n' \ + "${FRANCPAYSAN_SECRETS:-}" \ + "${HOME}/francpaysan-secrets" \ + "${HOME}/src/francpaysan-secrets" \ + "${HOME}/taler/src/francpaysan-secrets" +} + +_auth401_resolve_secrets_quiet() { + local root inst_id pass + for root in $(_auth401_stage_secret_roots); do + [ -n "$root" ] && [ -d "$root" ] || continue + # durable ids: prefer mon401fix when mon401 was rotated by old tests + if [ -z "${DURABLE_PASS:-}" ]; then + for inst_id in "${AUTH401_DURABLE_ID}" mon401fix mon401 "${MERCHANT_INSTANCE:-}"; do + [ -n "$inst_id" ] || continue + pass="" + if [ -f "$root/stage/instances/${inst_id}.password" ]; then + pass=$(tr -d '\n\r' <"$root/stage/instances/${inst_id}.password") + fi + # skip empty password files + if [ -n "$pass" ]; then + # verify file is for this id (mon401.password may be stale copy of mon401fix) + DURABLE_PASS="$pass" + DURABLE_ID=$(to_lower "$inst_id") + AUTH401_SECRET_NOTE="durable ${inst_id} from ${root}/stage/instances/" + # only set AUTH401_PASSWORD if caller did not pin an instance + if [ -z "${AUTH401_PASSWORD}" ] && [ -z "${AUTH401_INSTANCE}" ]; then + AUTH401_PASSWORD="$pass" + fi + break + fi + done + fi + if [ -z "${AUTH401_BEARER}" ] && [ -f "$root/stage/default-instance-token.txt" ]; then + AUTH401_BEARER=$(tr -d '\n\r' <"$root/stage/default-instance-token.txt") + AUTH401_BEARER_NOTE="default-instance-token.txt" + fi + if [ -z "${AUTH401_ADMIN_TOKEN}" ] && [ -f "$root/stage/admin-instance-token.txt" ]; then + AUTH401_ADMIN_TOKEN=$(tr -d '\n\r' <"$root/stage/admin-instance-token.txt") + AUTH401_ADMIN_NOTE="admin-instance-token.txt" + fi + [ -n "${DURABLE_PASS:-}" ] && break + done + # also keep AUTH401_INSTANCE for durable-only mode + if [ "${AUTH401_PREFER_DURABLE}" = "1" ] && [ -n "${DURABLE_ID:-}" ] && [ -n "${DURABLE_PASS:-}" ]; then + AUTH401_INSTANCE="${AUTH401_INSTANCE:-$DURABLE_ID}" + AUTH401_PASSWORD="${AUTH401_PASSWORD:-$DURABLE_PASS}" + AUTH401_CREATE=0 + AUTH401_MODE_NOTE="PREFER_DURABLE=1 → only durable ${AUTH401_INSTANCE}" + fi +} + +_auth401_resolve_secrets_quiet +: "${AUTH401_CREATE_PASSWORD:=${AUTH401_PASSWORD:-Mon401-Test-Pass!}}" +export AUTH401_INSTANCE AUTH401_PASSWORD AUTH401_BEARER AUTH401_CREATE AUTH401_ADMIN_TOKEN + +# --------------------------------------------------------------------------- +set_area auth401 +section "auth401 · merchant 401 / case matrix (WebUI APIs) · ${BASE}" + +if [ -n "${AUTH401_SECRET_NOTE:-}" ]; then info "stage secret" "$AUTH401_SECRET_NOTE"; fi +if [ -n "${AUTH401_BEARER_NOTE:-}" ]; then info "stage secret" "AUTH401_BEARER from ${AUTH401_BEARER_NOTE}"; fi +if [ -n "${AUTH401_ADMIN_NOTE:-}" ]; then info "stage secret" "AUTH401_ADMIN_TOKEN from ${AUTH401_ADMIN_NOTE}"; fi +if [ -n "${AUTH401_MODE_NOTE:-}" ]; then info "auth401 mode" "$AUTH401_MODE_NOTE"; fi +info "auth401 scope" "FULL=${AUTH401_FULL} CREATE=${AUTH401_CREATE} PREFER_DURABLE=${AUTH401_PREFER_DURABLE} CONTINUE=${AUTH401_CONTINUE}" +if [ "${AUTH401_CONTINUE}" = "1" ]; then + info "continue mode" "CONTINUE_ON_ERROR/AUTH401_CONTINUE=1 — hard prereqs soft; always full ERROR list via EXIT trap" +fi + +# =========================================================================== +set_group config +# =========================================================================== +cfg_code=$(http_body "$BASE/config" "$tmp/mer-config.json") +expect_http "merchant /config" 200 "$cfg_code" "$BASE/config" + +HAVE_SP=0 +MANDATORY_TAN=0 +if [ "$cfg_code" = "200" ]; then + eval "$(python3 - "$tmp/mer-config.json" <<'PY' +import json, sys +d = json.load(open(sys.argv[1])) +sp = 1 if d.get("have_self_provisioning") else 0 +tans = d.get("mandatory_tan_channels") or [] +print(f"HAVE_SP={sp}") +print(f"MANDATORY_TAN={1 if tans else 0}") +print(f"TAN_CH={','.join(tans) if tans else ''}") +print(f"API_VER={d.get('version','')}") +PY +)" + info "self-provision" "have_self_provisioning=${HAVE_SP} mandatory_tan=${MANDATORY_TAN}${TAN_CH:+ ($TAN_CH)} api=${API_VER:-?}" +else + fail "merchant config required for auth401" "HTTP $cfg_code" + _auth401_abort_or_continue "no /config" +fi + +# =========================================================================== +set_group webui +# SPA surface: /webui/ loads + version stamp; optional toLowerCase in bundle +# =========================================================================== +if [ "${AUTH401_FULL}" = "1" ]; then + wcode=$(http_code "$BASE/webui/") + case "$wcode" in + 200|301|302) ok "webui /" "HTTP $wcode" ;; + *) warn "webui /" "HTTP $wcode" ;; + esac + vcode=$(http_body "$BASE/webui/version.txt" "$tmp/spa-ver.txt" 2>/dev/null || echo 000) + if [ "$vcode" = "200" ] && [ -s "$tmp/spa-ver.txt" ]; then + ok "webui version.txt" "$(tr -d '\n\r' <"$tmp/spa-ver.txt")" + else + warn "webui version.txt" "HTTP ${vcode:-000}" + fi + # Download a slice of index.js — look for lowercasing mitigation (#11340) + jcode=$(curl -skS -m 60 -o "$tmp/spa-index.js" -w '%{http_code}' "$BASE/webui/index.js" 2>/dev/null || echo 000) + jcode=$(printf '%s' "$jcode" | tr -cd '0-9' | head -c 3) + if [ "$jcode" = "200" ] && [ -s "$tmp/spa-index.js" ]; then + ok "webui index.js" "HTTP 200 · $(wc -c <"$tmp/spa-index.js" | tr -d ' ') bytes" + if grep -q 'toLowerCase' "$tmp/spa-index.js" 2>/dev/null; then + ok "webui bundle toLowerCase" "present (SPA may mitigate #11340)" + else + warn "webui bundle toLowerCase" "not found in index.js — login MIX may 401 without lowercasing" + fi + else + warn "webui index.js" "HTTP ${jcode:-000}" + fi +fi + +# =========================================================================== +set_group signup +# WebUI self-provision: POST /instances with mixed-case id (as typed) +# =========================================================================== +CREATED=0 +USE_ID="" +USE_PASS="" +ID_MIX="" +ID_LOW="" +CREATE_PASS="$AUTH401_CREATE_PASSWORD" +stamp=$(date +%H%M%S) +# Mixed-case id like the SPA would send before lowercasing (or if it forgot) +ID_MIX="$(printf '%s' "${AUTH401_ID_PREFIX}" | awk '{print toupper(substr($0,1,1)) substr($0,2)}')X${stamp}" +ID_LOW=$(to_lower "$ID_MIX") + +if [ "${AUTH401_FULL}" = "1" ] && [ "$AUTH401_CREATE" = "1" ] && [ "$HAVE_SP" = "1" ]; then + if [ "$MANDATORY_TAN" = "1" ] && [ "$AUTH401_SKIP_IF_MFA" = "1" ]; then + warn "signup skipped" "mandatory_tan_channels set — set AUTH401_INSTANCE+PASSWORD" + else + body=$(instance_body "$ID_MIX" "$CREATE_PASS" "auth401 signup MIX") + ccode=$(_http_norm "$(curl -skS --max-redirs 0 -m "${TIMEOUT}" \ + -o "$tmp/create.json" -w '%{http_code}' \ + -X POST "${BASE}/instances" \ + -H 'Content-Type: application/json' \ + -d "$body" 2>/dev/null || true)") + case "$ccode" in + 200|204) + ok "signup POST /instances id=MIX" "HTTP $ccode id=$ID_MIX (SPA self-provision)" + CREATED=1 + USE_ID="$ID_MIX" + USE_PASS="$CREATE_PASS" + # Immediate login as SPA after signup: must use *lowercase* Basic user + code=$(token_http "$ID_LOW" "$ID_LOW" "$CREATE_PASS") + expect_http "signup→login SPA-sim path=low user=low → 200" 200 "$code" "$(json_code_hint)" + # Bug path: SPA forgot toLowerCase on login after signup with MIX id typed + code=$(token_http "$ID_LOW" "$ID_MIX" "$CREATE_PASS") + expect_http "signup→login RAW user=MIX (no lower) → 401" 401 "$code" "$(json_code_hint)" + ;; + 202) + warn "signup needs MFA" "HTTP 202 — use AUTH401_INSTANCE" + ;; + *) + fail "signup POST /instances" "HTTP $ccode $(head -c 120 "$tmp/create.json" 2>/dev/null | tr '\n' ' ')" + ;; + esac + fi +elif [ -n "${AUTH401_INSTANCE:-}" ] && [ -n "${AUTH401_PASSWORD:-}" ]; then + USE_ID=$(to_lower "$AUTH401_INSTANCE") + USE_PASS="$AUTH401_PASSWORD" + ID_LOW=$(to_lower "$USE_ID") + ID_MIX=$(synth_mix "$ID_LOW") + info "signup" "using existing AUTH401_INSTANCE=$USE_ID (no POST /instances)" + ok "using existing instance" "$USE_ID" +else + warn "signup skipped" "CREATE=${AUTH401_CREATE} HAVE_SP=${HAVE_SP} (need instance for matrix)" +fi + +# Fallback durable +if [ -z "$USE_ID" ] && [ -n "${DURABLE_ID:-}" ] && [ -n "${DURABLE_PASS:-}" ]; then + USE_ID="$DURABLE_ID" + USE_PASS="$DURABLE_PASS" + ID_LOW=$(to_lower "$USE_ID") + ID_MIX=$(synth_mix "$ID_LOW") + info "fallback durable" "$USE_ID" + ok "using durable instance" "$USE_ID" +fi + +if [ -z "$USE_ID" ]; then + fail "no instance under test" "need signup OK or AUTH401_INSTANCE+PASSWORD or mon401.password" + _auth401_abort_or_continue "no instance under test" + # CONTINUE: synthesize dummy ids so later groups still run (expect many 404/401) + ID_LOW_USE="auth401-missing-instance" + ID_USER_MIX=$(synth_mix "$ID_LOW_USE") + ID_PATH_MIX="$ID_USER_MIX" + ID_UPPER=$(to_upper "$ID_LOW_USE") + ID_PARTMIX=$(synth_partmix "$ID_LOW_USE") + PASS_OK="invalid" + PASS_BAD='auth401-wrong-password-NOT-VALID' + USE_ID="$ID_LOW_USE" + USE_PASS="$PASS_OK" +fi + +ID_LOW_USE=$(to_lower "$USE_ID") +# Always synthetic case variants (even for mon401) so MIX ≠ low +ID_USER_MIX=$(synth_mix "$ID_LOW_USE") +ID_PATH_MIX="$ID_USER_MIX" +ID_UPPER=$(to_upper "$ID_LOW_USE") +ID_PARTMIX=$(synth_partmix "$ID_LOW_USE") +PASS_OK="$USE_PASS" +PASS_BAD='auth401-wrong-password-NOT-VALID' + +info "probe ids" "low=${ID_LOW_USE} MIX_user=${ID_USER_MIX} UPPER=${ID_UPPER} created=${CREATED} pass_len=${#PASS_OK}" + +# =========================================================================== +set_group login +# WebUI login = POST …/private/token with Basic (username, password) +# =========================================================================== +if [ "${AUTH401_FULL}" = "1" ]; then + code=$(token_http "$ID_LOW_USE" "$ID_LOW_USE" "$PASS_OK") + expect_http "login SPA-sim user=low pwd=OK → 200" 200 "$code" "$(json_code_hint)" + code=$(token_http "$ID_LOW_USE" "$ID_USER_MIX" "$PASS_OK") + expect_http "login RAW user=MIX pwd=OK → 401 (#11340)" 401 "$code" "$(json_code_hint)" + code=$(token_http "$ID_LOW_USE" "$ID_UPPER" "$PASS_OK") + expect_http "login RAW user=UPPER pwd=OK → 401" 401 "$code" "$(json_code_hint)" + code=$(token_http "$ID_PATH_MIX" "$ID_LOW_USE" "$PASS_OK") + expect_http "login path=MIX user=low pwd=OK → 200 (path tolerant)" 200 "$code" "$(json_code_hint)" + code=$(token_http "$ID_UPPER" "$ID_LOW_USE" "$PASS_OK") + expect_http "login path=UPPER user=low pwd=OK → 200" 200 "$code" "$(json_code_hint)" +fi + +# =========================================================================== +set_group case +# Full Basic case matrix (always hard expects — synthetic MIX) +# =========================================================================== +code=$(token_http "$ID_PATH_MIX" "$ID_USER_MIX" "$PASS_OK") +expect_http "path=MIX user=MIX pwd=OK → 401 (case)" 401 "$code" "$(json_code_hint)" + +code=$(token_http "$ID_PATH_MIX" "$ID_LOW_USE" "$PASS_OK") +expect_http "path=MIX user=low pwd=OK → 200" 200 "$code" "$(json_code_hint)" + +code=$(token_http "$ID_LOW_USE" "$ID_USER_MIX" "$PASS_OK") +expect_http "path=low user=MIX pwd=OK → 401 (case)" 401 "$code" "$(json_code_hint)" + +code=$(token_http "$ID_LOW_USE" "$ID_LOW_USE" "$PASS_OK") +expect_http "path=low user=low pwd=OK → 200" 200 "$code" "$(json_code_hint)" + +code=$(token_http "$ID_UPPER" "$ID_LOW_USE" "$PASS_OK") +expect_http "path=UPPER user=low pwd=OK → 200" 200 "$code" "$(json_code_hint)" + +code=$(token_http "$ID_LOW_USE" "$ID_UPPER" "$PASS_OK") +expect_http "path=low user=UPPER pwd=OK → 401 (case)" 401 "$code" "$(json_code_hint)" + +code=$(token_http "$ID_LOW_USE" "$ID_PARTMIX" "$PASS_OK") +expect_http "path=low user=MiX pwd=OK → 401 (case)" 401 "$code" "$(json_code_hint)" + +# =========================================================================== +set_group password +# =========================================================================== +code=$(token_http "$ID_LOW_USE" "$ID_LOW_USE" "$PASS_BAD") +expect_http "pwd=BAD → 401" 401 "$code" "$(json_code_hint)" + +code=$(token_http "$ID_LOW_USE" "$ID_LOW_USE" "") +expect_http "pwd=EMPTY → 401" 401 "$code" "$(json_code_hint)" + +code=$(token_http "$ID_PATH_MIX" "$ID_USER_MIX" "$PASS_BAD") +expect_http "path=MIX user=MIX pwd=BAD → 401" 401 "$code" "$(json_code_hint)" + +# =========================================================================== +set_group pwchange +# WebUI password change: POST private/auth then re-login +# Only on throwaway signup instances — never rotate durable mon401.password +# =========================================================================== +if [ "${AUTH401_FULL}" = "1" ]; then + if [ "${AUTH401_PWCHANGE_THROWAY_ONLY}" = "1" ] && [ "${CREATED:-0}" != "1" ]; then + info "pwchange skipped" "not a throwaway instance (CREATED=0) — refuse to rotate durable password" + else + code=$(token_http "$ID_LOW_USE" "$ID_LOW_USE" "$PASS_OK") + if [ "$code" != "200" ]; then + fail "pwchange: login for token" "HTTP $code — skip pwchange" + else + TOK=$(extract_token) + if [ -z "$TOK" ]; then + fail "pwchange: parse token" "empty access_token" + else + ok "pwchange: got bearer" "len=${#TOK}" + NEW_PASS="Auth401-New-${stamp}!" + ac=$(_http_norm "$(auth_change_http "$ID_LOW_USE" "$TOK" "$NEW_PASS")") + case "$ac" in + 200|204) + ok "pwchange POST private/auth" "HTTP $ac (SPA password form)" + code=$(token_http "$ID_LOW_USE" "$ID_LOW_USE" "$PASS_OK") + expect_http "pwchange: old pwd → 401" 401 "$code" "$(json_code_hint)" + code=$(token_http "$ID_LOW_USE" "$ID_LOW_USE" "$NEW_PASS") + expect_http "pwchange: new pwd user=low → 200" 200 "$code" "$(json_code_hint)" + code=$(token_http "$ID_LOW_USE" "$ID_USER_MIX" "$NEW_PASS") + expect_http "pwchange: new pwd user=MIX → 401" 401 "$code" "$(json_code_hint)" + PASS_OK="$NEW_PASS" + USE_PASS="$NEW_PASS" + ;; + 202) + warn "pwchange needs MFA" "HTTP 202" + ;; + *) + fail "pwchange POST private/auth" "HTTP $ac $(head -c 100 "$tmp/auth_change.json" 2>/dev/null | tr '\n' ' ')" + ;; + esac + fi + fi + fi +fi + +# =========================================================================== +set_group admin +# Admin UI: POST /management/instances (needs admin bearer) +# =========================================================================== +if [ "${AUTH401_FULL}" = "1" ]; then + ADMIN_TOK="${AUTH401_ADMIN_TOKEN:-}" + # Optional: login as admin user with password + if [ -z "$ADMIN_TOK" ] && [ -n "${AUTH401_ADMIN_PASSWORD}" ]; then + code=$(token_http "$AUTH401_ADMIN_USER" "$AUTH401_ADMIN_USER" "$AUTH401_ADMIN_PASSWORD") + if [ "$code" = "200" ]; then + ADMIN_TOK=$(extract_token) + ok "admin login user=${AUTH401_ADMIN_USER}" "got token" + else + # try lower only + au=$(to_lower "$AUTH401_ADMIN_USER") + code=$(token_http "$au" "$au" "$AUTH401_ADMIN_PASSWORD") + if [ "$code" = "200" ]; then + ADMIN_TOK=$(extract_token) + ok "admin login user=${au}" "got token" + else + warn "admin login" "HTTP $code — set AUTH401_ADMIN_TOKEN" + fi + fi + fi + + if [ -z "$ADMIN_TOK" ]; then + info "admin-create skipped" "no AUTH401_ADMIN_TOKEN / AUTH401_ADMIN_PASSWORD (stage default is not admin)" + else + case "$ADMIN_TOK" in secret-token:*) ;; *) ADMIN_TOK="secret-token:${ADMIN_TOK}" ;; esac + AID_MIX="Adm401X${stamp}" + AID_LOW=$(to_lower "$AID_MIX") + APASS="Adm401-Pass-${stamp}!" + body=$(instance_body "$AID_MIX" "$APASS" "auth401 admin-create MIX") + acode=$(curl -skS --max-redirs 0 -m "${TIMEOUT}" \ + -o "$tmp/admin_create.json" -w '%{http_code}' \ + -X POST "${BASE}/management/instances" \ + -H "Authorization: Bearer ${ADMIN_TOK}" \ + -H 'Content-Type: application/json' \ + -d "$body" 2>/dev/null || echo 000) + case "$acode" in + 200|204) + ok "admin-create POST /management/instances id=MIX" "HTTP $acode id=$AID_MIX" + code=$(token_http "$AID_LOW" "$AID_LOW" "$APASS") + expect_http "admin-create→login user=low → 200" 200 "$code" "$(json_code_hint)" + code=$(token_http "$AID_LOW" "$AID_MIX" "$APASS") + expect_http "admin-create→login user=MIX → 401" 401 "$code" "$(json_code_hint)" + # admin-set auth on new instance + code=$(token_http "$AID_LOW" "$AID_LOW" "$APASS") + if [ "$code" = "200" ]; then + AT=$(extract_token) + NEW_A="Adm401-New-${stamp}!" + ch=$(auth_change_http "$AID_LOW" "$AT" "$NEW_A") + case "$ch" in + 200|204) + ok "admin-created instance pwchange" "HTTP $ch" + code=$(token_http "$AID_LOW" "$AID_LOW" "$NEW_A") + expect_http "admin-created re-login new pwd → 200" 200 "$code" "$(json_code_hint)" + ;; + *) warn "admin-created pwchange" "HTTP $ch" ;; + esac + fi + ;; + 401|403) + warn "admin-create unauthorized" "HTTP $acode — token not admin (expected on stage default)" + ;; + 202) + warn "admin-create MFA" "HTTP 202" + ;; + *) + fail "admin-create" "HTTP $acode $(head -c 120 "$tmp/admin_create.json" 2>/dev/null | tr '\n' ' ')" + ;; + esac + fi +fi + +# =========================================================================== +set_group missing +# =========================================================================== +code=$(curl -skS --max-redirs 0 -m "${TIMEOUT}" \ + -o "$tmp/last_token.json" -w '%{http_code}' \ + -X POST "${BASE}/instances/${ID_LOW_USE}/private/token" \ + -H 'Content-Type: application/json' \ + -d '{"scope":"write","duration":{"d_us":3600000000},"refreshable":true}' \ + 2>/dev/null || echo 000) +expect_http "private/token NO_AUTH → 401" 401 "$code" "$(json_code_hint)" + +code=$(token_http "$ID_LOW_USE" "auth401-not-a-real-user" "$PASS_OK") +expect_http "user=OTHER pwd=OK → 401" 401 "$code" "$(json_code_hint)" + +code=$(token_http "auth401-does-not-exist-${stamp}" "$ID_LOW_USE" "$PASS_OK") +# Must not look like auth failure (401). Prefer 404; 502 from proxy is soft. +case "$code" in + 404) ok "path=missing → 404 (not 401)" "HTTP 404 · $(json_code_hint)" ;; + 401) fail "path=missing must not be 401" "HTTP 401 · $(json_code_hint)" ;; + 502|503) warn "path=missing" "HTTP $code (want 404; proxy flake, not auth)" ;; + *) fail "path=missing → 404 (not 401)" "HTTP $code (want 404) · $(json_code_hint)" ;; +esac + +# =========================================================================== +set_group bearer +# =========================================================================== +code=$(token_http "$ID_LOW_USE" "$ID_LOW_USE" "$PASS_OK") +if [ "$code" = "200" ]; then + TOKEN=$(extract_token) + if [ -n "$TOKEN" ]; then + ok "got access_token" "len=${#TOKEN}" + code=$(curl -skS --max-redirs 0 -m "${TIMEOUT}" \ + -o "$tmp/priv.json" -w '%{http_code}' \ + "${BASE}/instances/${ID_LOW_USE}/private/" \ + -H "Authorization: Bearer ${TOKEN}" 2>/dev/null || echo 000) + expect_http "GET private/ Bearer OK → 200" 200 "$code" + + code=$(curl -skS --max-redirs 0 -m "${TIMEOUT}" \ + -o "$tmp/priv.json" -w '%{http_code}' \ + "${BASE}/instances/${ID_LOW_USE}/private/" \ + -H "Authorization: Bearer secret-token:AUTH401INVALID000" 2>/dev/null || echo 000) + expect_http "GET private/ Bearer BAD → 401" 401 "$code" + + code=$(curl -skS --max-redirs 0 -m "${TIMEOUT}" \ + -o "$tmp/priv.json" -w '%{http_code}' \ + "${BASE}/instances/${ID_LOW_USE}/private/" 2>/dev/null || echo 000) + expect_http "GET private/ NO_AUTH → 401" 401 "$code" + else + fail "parse access_token" "HTTP $code but no token field" + fi +else + fail "token for bearer tests" "HTTP $code" +fi + +code=$(curl -skS --max-redirs 0 -m "${TIMEOUT}" \ + -o "$tmp/mgmt.json" -w '%{http_code}' \ + "${BASE}/management/instances" 2>/dev/null || echo 000) +expect_http "GET /management/instances NO_AUTH → 401" 401 "$code" + +if [ -n "${AUTH401_BEARER:-}" ]; then + if [ -z "${AUTH401_BEARER_INSTANCE:-}" ]; then + case "${AUTH401_BEARER_NOTE:-}" in + *default-instance-token*) AUTH401_BEARER_INSTANCE=default ;; + *) AUTH401_BEARER_INSTANCE="${DURABLE_ID:-default}" ;; + esac + fi + inst=$(to_lower "$AUTH401_BEARER_INSTANCE") + tok="$AUTH401_BEARER" + case "$tok" in secret-token:*) ;; *) tok="secret-token:${tok}" ;; esac + code=$(curl -skS --max-redirs 0 -m "${TIMEOUT}" \ + -o "$tmp/priv2.json" -w '%{http_code}' \ + "${BASE}/instances/${inst}/private/" \ + -H "Authorization: Bearer ${tok}" 2>/dev/null || echo 000) + case "$code" in + 200) ok "GET private/ secrets Bearer" "HTTP 200 instance=$inst" ;; + 401) warn "secrets Bearer rejected" "HTTP 401 on $inst" ;; + *) warn "secrets Bearer" "HTTP $code on $inst" ;; + esac +fi + +# =========================================================================== +set_group durable +# Extra: durable mon401 still works after throwaway tests (if present) +# =========================================================================== +if [ -n "${DURABLE_ID:-}" ] && [ -n "${DURABLE_PASS:-}" ]; then + dlow=$(to_lower "$DURABLE_ID") + dmix=$(synth_mix "$dlow") + code=$(token_http "$dlow" "$dlow" "$DURABLE_PASS") + expect_http "durable ${dlow} login user=low → 200" 200 "$code" "$(json_code_hint)" + code=$(token_http "$dlow" "$dmix" "$DURABLE_PASS") + expect_http "durable ${dlow} login user=MIX → 401" 401 "$code" "$(json_code_hint)" +else + info "durable" "no mon401.password — skipped" +fi + +# =========================================================================== +set_group report +# =========================================================================== +info "summary matrix" "401: Basic user case≠low | bad/empty pwd | no auth | bad bearer | wrong user · 200: user=low + path any case · 404: missing instance · SPA must toLowerCase username on login/signup/admin/pw-relogin" +if [ "$CREATED" = "1" ]; then + info "throwaway left" "id=${ID_MIX:-?} low=${ID_LOW_USE} (password rotated if pwchange ran)" +fi + +# fall through → EXIT trap runs _auth401_finish (summary + full ERROR list) +: diff --git a/check_e2e.sh b/check_e2e.sh new file mode 100755 index 0000000..05cb5f7 --- /dev/null +++ b/check_e2e.sh @@ -0,0 +1,1711 @@ +#!/usr/bin/env bash +# Full withdraw + pay cycle; clear BLOCKERS when the path cannot finish. +# Skips remaining steps if overall budget exceeded (E2E_TIMEOUT, default 90s). +set -euo pipefail +ROOT=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=lib.sh +source "$ROOT/lib.sh" +# shellcheck source=metrics.sh +source "$ROOT/metrics.sh" + +: "${E2E_TIMEOUT:=180}" # ATM ladder + wirewatch lag needs room (was 55 — too tight) +: "${E2E_PAY_SECS:=22}" # hard reserve for handle-uri + settle (do not starve pay) +: "${E2E_SETTLE_ROUNDS:=18}" +: "${E2E_SETTLE_SLEEP:=3}" +E2E_START=$(date +%s) +BAL_BEFORE="(not captured)" +BAL_AFTER="(not captured)" +E2E_REPORTED=0 +e2e_left() { echo $(( E2E_TIMEOUT - ($(date +%s) - E2E_START) )); } +e2e_over() { [ "$(e2e_left)" -le 0 ]; } + +# Fixed short timeout — used even when e2e budget is exhausted / on EXIT +# stdout only for JSON; logs go to stderr file so balance parse stays clean +wcli_bal_snap() { + local out="$1" + if [ -z "${WALLET_CLI:-}" ]; then + echo "(no wallet)" >"$out" + return 1 + fi + # one-shot needs DB file; serve mode needs socket + if [ -z "${WSERVE_PID:-}" ] || [ ! -S "${WSOCK:-}" ]; then + if [ ! -f "${WDB:-}" ]; then + echo "(no wallet)" >"$out" + return 1 + fi + fi + local err="${out}.err" + if command -v perl >/dev/null 2>&1; then + if [ -n "${WSERVE_PID:-}" ] && [ -S "${WSOCK:-}" ]; then + perl -e 'alarm shift; exec @ARGV' 10 \ + node "$WALLET_CLI" --wallet-connection="$WSOCK" --no-throttle balance \ + >"$out" 2>"$err" || true + else + perl -e 'alarm shift; exec @ARGV' 10 \ + node "$WALLET_CLI" --wallet-db="$WDB" --no-throttle --skip-defaults balance \ + >"$out" 2>"$err" || true + fi + else + if [ -n "${WSERVE_PID:-}" ] && [ -S "${WSOCK:-}" ]; then + node "$WALLET_CLI" --wallet-connection="$WSOCK" --no-throttle balance \ + >"$out" 2>"$err" || true + else + node "$WALLET_CLI" --wallet-db="$WDB" --no-throttle --skip-defaults balance \ + >"$out" 2>"$err" || true + fi + fi + [ -s "$out" ] +} + +fmt_bal() { + python3 -c ' +import json,re,sys +raw=open(sys.argv[1]).read() if sys.argv[1] else "" +if not raw.strip() or raw.strip().startswith("(no"): + print(raw.strip() or "(empty)"); sys.exit(0) +# extract first JSON object with "balances" +d=None +for m in re.finditer(r"\{", raw): + try: + d=json.loads(raw[m.start():]) + if isinstance(d, dict) and "balances" in d: + break + except Exception: + d=None +if not isinstance(d, dict): + print(re.sub(r"\s+"," ", raw)[:200]); sys.exit(0) +parts=[] +for b in d.get("balances") or []: + cur=(b.get("scopeInfo") or {}).get("currency") or "?" + parts.append("%s avail=%s pendingIn=%s" % (cur, b.get("available"), b.get("pendingIncoming"))) +print("; ".join(parts) if parts else "(no balances)") +' "${1:-/dev/null}" 2>/dev/null || echo "(unreadable)" +} + +# Numeric available for currency CUR (from balance file or fresh wallet snap) +wallet_avail_num() { + local f="${1:-}" + if [ -z "$f" ] || [ ! -s "$f" ]; then + f="$SCRATCH/bal-live.out" + wcli_bal_snap "$f" || wcli balance >"$f" 2>&1 || true + fi + python3 -c ' +import json,re,sys +cur=sys.argv[2] +raw=open(sys.argv[1]).read() if sys.argv[1] else "" +d=None +for m in re.finditer(r"\{", raw): + try: + d=json.loads(raw[m.start():]) + if isinstance(d, dict) and "balances" in d: break + except Exception: + d=None +if isinstance(d, dict): + for b in d.get("balances") or []: + c=(b.get("scopeInfo") or {}).get("currency") or "" + av=b.get("available") or "" + if c==cur or av.startswith(cur+":"): + try: + print(float(av.split(":",1)[1])); sys.exit(0) + except Exception: + pass +# text fallback +m=re.search(r"%s:([0-9]+(?:\.[0-9]+)?)" % re.escape(cur), raw) +print(float(m.group(1)) if m else 0.0) +' "${f:-/dev/null}" "${CUR:-GOA}" 2>/dev/null || echo 0 +} + +# Wait for spendable balance (settlement / wirewatch lag). Returns 0 if avail > min. +# Emphasizes timing, not hard failure, while waiting. +wait_wallet_balance() { + local min_n="${1:-0}" + local rounds="${2:-15}" + local sleep_s="${3:-3}" + local r av + # Poll balance only — never run-until-done (hangs; banned in monitoring). + info "wallet settle wait" "up to ${rounds}×${sleep_s}s for avail>${min_n} ${CUR} (poll balance only; no run-until-done)" + for r in $(seq 1 "$rounds"); do + wcli_bal_snap "$SCRATCH/bal-live.out" || wcli balance >"$SCRATCH/bal-live.out" 2>&1 || true + av=$(wallet_avail_num "$SCRATCH/bal-live.out") + if python3 -c "import sys; sys.exit(0 if float(sys.argv[1]) > float(sys.argv[2]) else 1)" "$av" "$min_n" 2>/dev/null; then + ok "wallet balance ready" "avail=${CUR}:${av} after ~$((r * sleep_s))s settle wait" + info "balance" "$(fmt_bal "$SCRATCH/bal-live.out")" + return 0 + fi + if [ "$r" = "1" ] || [ "$((r % 3))" = "0" ]; then + info "settlement lag" "round $r/${rounds}: avail=${CUR}:${av} (still waiting — not an error yet)" + fi + # gentle wirewatch nudge mid-wait (local only) + if [ "$r" = "4" ] || [ "$r" = "8" ]; then + if [ "${E2E_REMOTE:-0}" != "1" ] && koopa_ssh_ok; then + koopa_ssh_run 8 \ + 'podman exec taler-hacktivism-exchange-ansible systemctl restart taler-exchange-wirewatch 2>/dev/null || true' \ + >/dev/null 2>&1 || true + fi + fi + sleep "$sleep_s" + done + av=$(wallet_avail_num "$SCRATCH/bal-live.out") + warn "settlement lag" "after ${rounds}×${sleep_s}s still avail=${CUR}:${av} — coins may still be in flight" + return 1 +} + +print_balances() { + section "e2e · wallet balances" + info "BALANCE before" "$BAL_BEFORE" + # refresh after if we can + if [ -n "${SCRATCH:-}" ]; then + wcli_bal_snap "$SCRATCH/bal-end.out" || true + if [ -s "$SCRATCH/bal-end.out" ]; then + BAL_AFTER=$(fmt_bal "$SCRATCH/bal-end.out") + fi + fi + info "BALANCE after " "$BAL_AFTER" +} + +e2e_finish() { + local code="${1:-1}" + if [ "$E2E_REPORTED" = "1" ]; then + return "$code" + fi + E2E_REPORTED=1 + print_balances + # Final coin inventory + host/container load + statistics dashboard + if [ -n "${METRICS_DIR:-}" ]; then + section "metrics · e2e coins final" + metrics_report_coins "e2e-end" || true + if [ "${METRICS_LOAD:-1}" != "0" ]; then + metrics_report_load "${METRICS_DIR}/load-after.json" "e2e-end" || true + fi + metrics_print_overall "e2e final" || true + fi + summary || true + return "$code" +} + +e2e_skip_rest() { + local why="$1" + warn "e2e timeout" "budget ${E2E_TIMEOUT}s exceeded — $why" + blocker "e2e-timeout" "skipped remaining steps after ${E2E_TIMEOUT}s ($why)" + section "e2e · report" + info "WITHDRAW" "incomplete (timeout)" + info "PAY" "SKIPPED (timeout)" + # If we already started a withdrawal, dig inside for missing coins + if [ -n "${WID:-}" ]; then + dig_when_no_coins 2>/dev/null || true + fi + e2e_finish 1 + exit 1 +} + +# Area e2e-### — withdraw + pay cycle +set_area e2e +set_group prereq +section "e2e · prerequisites" +info "e2e budget" "${E2E_TIMEOUT}s (set E2E_TIMEOUT= to change)" + +# Remote domains (not koopa): public path only, tiny amounts, abort on login/KYC +: "${E2E_REMOTE:=0}" +if [ "${LOCAL_STACK:-1}" != "1" ]; then + E2E_REMOTE=1 + SKIP_SSH=1 + E2E_FAKE_INCOMING=0 +fi + +# Currency for amounts / balance checks +CUR="${EXPECT_CURRENCY:-}" +if [ -z "$CUR" ]; then + CUR=$(curl -skS -m 8 "$BANK_PUBLIC/config" 2>/dev/null | python3 -c 'import sys,json +try: print(json.load(sys.stdin).get("currency") or "") +except Exception: print("")' 2>/dev/null || true) +fi +[ -z "$CUR" ] && CUR=$(curl -skS -m 8 "$EXCHANGE_PUBLIC/config" 2>/dev/null | python3 -c 'import sys,json +try: print(json.load(sys.stdin).get("currency") or "") +except Exception: print("")' 2>/dev/null || true) +[ -z "$CUR" ] && CUR="GOA" + +# --------------------------------------------------------------------------- +# Variable amounts +# Withdraw = typical ATM notes only (Bankomat): 20 / 50 / 100 / 200 … +# Pay = several smaller purchase amounts +# Env: +# E2E_WITHDRAW_VALUES="20 50 100" # bare numbers → prefixed with CUR +# E2E_PAY_VALUES="0.01 0.05 0.5 1 2 5" +# E2E_VARIABLE=0 # single fixed WITHDRAW_AMT / PAY_AMT +# E2E_ATM_MAX=N # try at most N ATM withdraws (budget) +# --------------------------------------------------------------------------- +: "${E2E_VARIABLE:=1}" +: "${E2E_ATM_MAX:=5}" +if [ "$E2E_REMOTE" = "1" ]; then + # remote: still ATM-shaped; TESTPAYSAN needs larger notes for shop pays + if [ "$CUR" = "TESTPAYSAN" ]; then + : "${E2E_WITHDRAW_VALUES:=20 50}" + # farmer shop template face values (oeufs-6=5, fromage=8.5, jus=6) + : "${E2E_PAY_VALUES:=5 8.5 6}" + else + : "${E2E_WITHDRAW_VALUES:=10 20 50}" + : "${E2E_PAY_VALUES:=0.01 0.05 0.1 1}" + fi +else + # local GOA: classic ATM notes + 4200 for paivana paywall template + : "${E2E_WITHDRAW_VALUES:=20 50 100 200 4200}" + : "${E2E_PAY_VALUES:=0.01 0.05 0.1 0.5 1 2 5 10}" +fi +# Public template pays (landing/shop style) — default on for TESTPAYSAN stage +if [ -z "${E2E_USE_TEMPLATES:-}" ]; then + if [ "$CUR" = "TESTPAYSAN" ]; then E2E_USE_TEMPLATES=1; else E2E_USE_TEMPLATES=0; fi +fi +: "${E2E_USE_TEMPLATES:=0}" + +# Build CUR:amount lists +build_amt_list() { + local cur="$1"; shift + local v out="" + for v in "$@"; do + case "$v" in + *:*) out="${out}${out:+ }$v" ;; + *) out="${out}${out:+ }${cur}:${v}" ;; + esac + done + printf '%s' "$out" +} +# shellcheck disable=SC2086 +WITHDRAW_LIST=$(build_amt_list "$CUR" $E2E_WITHDRAW_VALUES) +# shellcheck disable=SC2086 +PAY_LIST=$(build_amt_list "$CUR" $E2E_PAY_VALUES) + +if [ "$E2E_VARIABLE" != "1" ]; then + if [ "$E2E_REMOTE" = "1" ]; then + WITHDRAW_LIST="${WITHDRAW_AMT:-${CUR}:20}" + PAY_LIST="${PAY_AMT:-${CUR}:0.01}" + else + WITHDRAW_LIST="${WITHDRAW_AMT:-${CUR}:20}" + PAY_LIST="${PAY_AMT:-${CUR}:0.01}" + fi +fi + +# Credit = sum of planned ATM withdraws + small buffer for fees +CREDIT_AMT=$(python3 -c ' +import sys +cur=sys.argv[1] +vals=sys.argv[2].split() +s=0.0 +for a in vals: + try: s += float(a.split(":",1)[-1]) + except Exception: pass +buf = max(10.0, s * 0.05) +print("%s:%g" % (cur, s + buf)) +' "$CUR" "$WITHDRAW_LIST" 2>/dev/null || echo "${CUR}:100") + +# First withdraw/pay for legacy single-step labels +WITHDRAW_AMT=$(printf '%s' "$WITHDRAW_LIST" | awk '{print $1}') +PAY_AMT=$(printf '%s' "$PAY_LIST" | awk '{print $1}') + +# More wall time when running ATM ladder +if [ "$E2E_VARIABLE" = "1" ]; then + n_w=$(printf '%s' "$WITHDRAW_LIST" | wc -w | tr -d ' ') + n_p=$(printf '%s' "$PAY_LIST" | wc -w | tr -d ' ') + need=$(( 50 + n_w * 40 + n_p * 20 + 60 )) # + room for paivana pay + if [ "${E2E_TIMEOUT}" -lt "$need" ]; then + E2E_TIMEOUT=$need + fi + # cap ATM attempts but always keep CUR:4200 if listed (paivana) + WITHDRAW_LIST=$( + printf '%s' "$WITHDRAW_LIST" | tr ' ' '\n' | awk -v max="$E2E_ATM_MAX" -v big="${CUR}:4200" ' + NF==0 { next } + $0==big { has=1; next } + { small[++n]=$0 } + END { + keep = max - (has ? 1 : 0) + if (keep < 0) keep = 0 + for (i = 1; i <= n && i <= keep; i++) printf "%s ", small[i] + if (has) printf "%s", big + print "" + }' | sed 's/ *$//' + ) +fi + +BANK_HOST=$(python3 -c 'from urllib.parse import urlparse; print(urlparse("'"$BANK_PUBLIC"'").hostname or "bank")' 2>/dev/null || echo bank) + +SCRATCH=$(mktemp -d) +WDB="$SCRATCH/wallet.sqlite3" +WSOCK="$SCRATCH/wallet.sock" +WSERVE_PID="" +# Client/server wallet (advanced serve): long-lived shepherd without run-until-done. +# Default on — wallet-cli only completes withdraw/pay while a serve process is up. +: "${E2E_WALLET_SERVE:=1}" +METRICS_DIR="${METRICS_DIR:-$SCRATCH/metrics}" +mkdir -p "$METRICS_DIR" +export METRICS_DIR CUR="${CUR:-${EXPECT_CURRENCY:-GOA}}" WDB +export PATH="${HOME}/.local/bin:${HOME}/taler/opt/taler-wallet-cli/usr/bin:/opt/homebrew/bin:/usr/local/bin:$PATH" + +stop_wallet_serve() { + if [ -n "${WSERVE_PID:-}" ] && kill -0 "$WSERVE_PID" 2>/dev/null; then + kill "$WSERVE_PID" 2>/dev/null || true + wait "$WSERVE_PID" 2>/dev/null || true + fi + WSERVE_PID="" + rm -f "${WSOCK:-}" 2>/dev/null || true +} + +# Always dump balances on any exit (timeout, blocker, signal, success) +trap 'ec=$?; e2e_finish "$ec"; stop_wallet_serve; rm -rf "$SCRATCH"; exit "$ec"' EXIT + +# Abort cleanly on login / KYC / registration barriers (esp. remote domains) +e2e_abort_auth() { + local step="$1" msg="$2" + warn "$step" "$msg" + blocker "$step" "abort (login/KYC): $msg" + section "e2e · report" + info "ABORTED" "login or KYC blocked further e2e — not retrying" + info "hint" "remote e2e needs open bank registration + credit path; set E2E_BANK_ADMIN_PASS / E2E_MERCHANT_TOKEN if you have them" + e2e_finish 1 + exit 1 +} + +is_auth_kyc_body() { + # stdin or file arg: true if looks like login/KYC barrier + local f="${1:-}" + local t + if [ -n "$f" ] && [ -f "$f" ]; then t=$(head -c 800 "$f" 2>/dev/null || true) + else t=$(cat 2>/dev/null || true); fi + printf '%s' "$t" | grep -qiE 'kyc|legitim|login required|unauthorized|forbidden|captcha|challenge|tan_|not.?allowed|registration.?disabled|admin.?only|permission.?denied|401|403' +} + +WALLET_CLI=$(find_wallet_cli) || { + blocker "prereq" "taler-wallet-cli not found (set WALLET_CLI=)" + exit 1 +} +# Implementation version (--version) + wallet-core API ranges (version command) +WALLET_IMPL_VER=$( + node "$WALLET_CLI" --version 2>/dev/null | head -1 | tr -d '\r' || true +) +WALLET_CORE_VER=$( + _wver_db=$(mktemp "${TMPDIR:-/tmp}/wver.XXXXXX.sqlite3" 2>/dev/null || echo "${TMPDIR:-/tmp}/wver-$$.sqlite3") + node "$WALLET_CLI" --wallet-db="$_wver_db" --no-throttle --skip-defaults version 2>/dev/null \ + | python3 -c ' +import json,re,sys +raw=sys.stdin.read() +d=None +for m in re.finditer(r"\{", raw): + try: + d=json.loads(raw[m.start():]) + if isinstance(d, dict) and "version" in d: + break + except Exception: + d=None +if not isinstance(d, dict): + sys.exit(0) +parts=[] +# wallet-core libversion + protocol ranges used against exchange/merchant/bank +for k in ("version","implementationSemver","exchange","merchant","bank","bankIntegrationApiRange","corebankApiRange"): + if k in d and d[k] is not None: + parts.append("%s=%s" % (k, d[k])) +print(" ".join(parts)) +' 2>/dev/null || true + rm -f "$_wver_db" "$_wver_db"-* 2>/dev/null || true +) +ok "wallet-cli" "${WALLET_IMPL_VER:-?} · ${WALLET_CLI}" +if [ -n "${WALLET_CORE_VER:-}" ]; then + info "wallet-core" "$WALLET_CORE_VER" +else + info "wallet-core" "version details unavailable (impl ${WALLET_IMPL_VER:-?})" +fi +info "e2e mode" "$([ "$E2E_REMOTE" = "1" ] && echo "remote/public domain (no SSH)" || echo "local koopa stack")" +info "currency" "$CUR" +info "ATM withdraw ladder" "$WITHDRAW_LIST (credit $CREDIT_AMT)" +info "pay ladder" "$PAY_LIST" +info "e2e budget" "${E2E_TIMEOUT}s" + +# alt_unit_names for human amounts (Kilo-GOA … + base GOA in parentheses) +ALT_UNITS_FILE="${METRICS_DIR}/alt_unit_names.json" +export ALT_UNITS_FILE +if metrics_load_alt_units "${EXCHANGE_PUBLIC}/config"; then + info "alt_unit_names" "from ${EXCHANGE_PUBLIC}/config" +else + warn "alt_unit_names" "using built-in SI fallback" +fi + +# Host + bank/exchange/merchant RAM/CPU/load (SSH koopa or koopa-external) +set_group load +section "e2e · load snapshot (before withdraw/pay)" +metrics_report_load "${METRICS_DIR}/load-before.json" "e2e-start" || true + +# Local stack: koopa secrets. Remote: env first; TESTPAYSAN stage may load francpaysan host secrets. +set_group prereq +if [ "$E2E_REMOTE" = "1" ]; then + ADMIN_PASS="${E2E_BANK_ADMIN_PASS:-${BANK_ADMIN_PASS:-}}" + MPW="${E2E_MERCHANT_TOKEN:-${MERCHANT_TOKEN:-}}" + # Stage FrancPaysan: allow admin credit without committing passwords. + # Order: already-set env → local secrets tree → SSH francpaysan-host (ops laptop). + if [ -z "$ADMIN_PASS" ] && [ "${CUR:-${EXPECT_CURRENCY:-}}" = "TESTPAYSAN" ]; then + _fp_try="" + for _fp_try in \ + "${FRANCPAYSAN_SECRETS:-}/stage/bank-admin-password.txt" \ + "${HOME}/francpaysan-secrets/stage/bank-admin-password.txt" \ + "${HOME}/src/francpaysan-secrets/stage/bank-admin-password.txt" \ + "${HOME}/taler/src/francpaysan-secrets/stage/bank-admin-password.txt" + do + if [ -n "$_fp_try" ] && [ -f "$_fp_try" ]; then + ADMIN_PASS=$(tr -d '\n\r' <"$_fp_try") + info "bank admin secret" "from $_fp_try" + break + fi + done + if [ -z "$ADMIN_PASS" ] && command -v ssh >/dev/null 2>&1; then + for _fp_host in "${FRANCPAYSAN_SSH:-francpaysan-host}" francpaysan-host; do + ADMIN_PASS=$(ssh -o BatchMode=yes -o ConnectTimeout=8 "$_fp_host" \ + 'sudo cat /mnt/data/stagepaysan/bank/secrets/bank-admin-password.txt 2>/dev/null' \ + 2>/dev/null | tr -d '\n\r' || true) + if [ -n "$ADMIN_PASS" ]; then + info "bank admin secret" "from ${_fp_host}:/mnt/data/stagepaysan/bank/secrets/" + break + fi + done + fi + unset _fp_try _fp_host + fi + # Stage merchant: public templates need no token; optional default-instance token + if [ -z "$MPW" ] && [ "${CUR:-${EXPECT_CURRENCY:-}}" = "TESTPAYSAN" ]; then + for _fp_try in \ + "${FRANCPAYSAN_SECRETS:-}/stage/default-instance-token.txt" \ + "${HOME}/francpaysan-secrets/stage/default-instance-token.txt" + do + if [ -n "$_fp_try" ] && [ -f "$_fp_try" ]; then + MPW=$(tr -d '\n\r' <"$_fp_try") + break + fi + done + if [ -z "$MPW" ] && command -v ssh >/dev/null 2>&1; then + MPW=$(ssh -o BatchMode=yes -o ConnectTimeout=8 "${FRANCPAYSAN_SSH:-francpaysan-host}" \ + 'sudo cat /mnt/data/stagepaysan/merchant/secrets/default-instance-token.txt 2>/dev/null' \ + 2>/dev/null | tr -d '\n\r' || true) + fi + unset _fp_try + fi +else + ADMIN_PASS="${E2E_BANK_ADMIN_PASS:-$(read_secret "taler-bank/bank-admin-password.txt" || true)}" + MPW="${E2E_MERCHANT_TOKEN:-${MERCHANT_TOKEN:-$(read_secret "taler-merchant/merchant-goa-demo-cp4zqk-password.txt" || true)}}" +fi +if [ -n "$ADMIN_PASS" ]; then + ok "bank admin secret" "${SECRETS_ROOT:+SECRETS_ROOT=${SECRETS_ROOT}}" +elif [ "$E2E_REMOTE" = "1" ]; then + warn "bank admin secret" "missing — will try public registration only" +else + blocker "prereq" "bank admin password missing (SECRETS_ROOT or koopa /root)" + info "secrets" "SECRETS_ROOT=${SECRETS_ROOT:-'(unset)'}" + if type secrets_hint >/dev/null 2>&1; then + while IFS= read -r _line; do info "secrets-hint" "$_line"; done < <(secrets_hint) + fi + exit 1 +fi +if [ -n "$MPW" ]; then + # Secrets often store full "secret-token:…"; strip before re-prefixing Bearer. + MPW=$(printf '%s' "$MPW" | tr -d '\n\r') + case "$MPW" in + secret-token:*) MPW_TOKEN="$MPW" ;; + *) MPW_TOKEN="secret-token:${MPW}" ;; + esac + ok "merchant secret (instance ${MERCHANT_INSTANCE})" +elif [ "$E2E_REMOTE" = "1" ]; then + MPW_TOKEN="" + warn "merchant secret" "missing — private orders fail unless E2E_USE_TEMPLATES=1 (set E2E_MERCHANT_TOKEN)" +else + blocker "prereq" "merchant instance password missing" + info "secrets" "SECRETS_ROOT=${SECRETS_ROOT:-'(unset)'} — need taler-merchant/merchant-*-password.txt" + if type secrets_hint >/dev/null 2>&1; then + while IFS= read -r _line; do info "secrets-hint" "$_line"; done < <(secrets_hint) + fi + exit 1 +fi +# Authorization header for merchant private API (never double secret-token:) +merchant_auth_header() { + local t="${1:-${MPW_TOKEN:-}}" + [ -n "$t" ] || return 1 + case "$t" in + secret-token:*) printf 'Authorization: Bearer %s' "$t" ;; + *) printf 'Authorization: Bearer secret-token:%s' "$t" ;; + esac +} + +# Public reachability gates (remote: soft-skip if bank/merchant absent) +for pair in \ + "bank|$BANK_PUBLIC/config" \ + "exchange|$EXCHANGE_PUBLIC/config" \ + "exchange-keys|$EXCHANGE_PUBLIC/keys" \ + "bank-integration|$BANK_PUBLIC/taler-integration/config" \ + "merchant|$MERCHANT_PUBLIC/config" +do + IFS='|' read -r name url <<<"$pair" + code=$(http_code "$url") + if [ "$code" = "200" ]; then + ok "reachable $name" + else + if [ "$E2E_REMOTE" = "1" ] && [[ "$name" == bank* || "$name" == merchant ]]; then + warn "reachable $name" "HTTP $code — e2e may abort" + else + blocker "prereq" "$name HTTP $code ($url) — cannot start payment cycle" + fi + fi +done +# Remote: need at least bank+exchange+merchant for a full cycle +if [ "$E2E_REMOTE" = "1" ]; then + bc=$(http_code "$BANK_PUBLIC/config") + ec=$(http_code "$EXCHANGE_PUBLIC/config") + mc=$(http_code "$MERCHANT_PUBLIC/config") + if [ "$bc" != "200" ] || [ "$ec" != "200" ]; then + e2e_abort_auth "prereq" "bank/exchange not publicly usable (bank=$bc exchange=$ec) — skip e2e" + fi + if [ "$mc" != "200" ]; then + warn "prereq" "merchant HTTP $mc — will try withdraw only if possible" + fi +fi +if [ "${#BLOCKERS[@]}" -gt 0 ]; then + exit 1 +fi + +USER="mon$(date +%s | tail -c 6)" +USER_PW=$(python3 -c 'import secrets;print(secrets.token_urlsafe(10))') +info "e2e user" "$USER withdraw=$WITHDRAW_AMT pay=$PAY_AMT" +BANK="$BANK_PUBLIC" +INST="$MERCHANT_INSTANCE" + +start_wallet_serve() { + [ "${E2E_WALLET_SERVE}" = "1" ] || return 0 + [ -n "${WALLET_CLI:-}" ] && [ -f "${WALLET_CLI}" ] || return 1 + stop_wallet_serve + rm -f "$WSOCK" + # serve holds the DB open; clients use --wallet-connection only + node "$WALLET_CLI" --wallet-db="$WDB" --no-throttle --skip-defaults \ + advanced serve --unix-path "$WSOCK" \ + >"$SCRATCH/wallet-serve.log" 2>&1 & + WSERVE_PID=$! + local i + for i in $(seq 1 40); do + if [ -S "$WSOCK" ] && kill -0 "$WSERVE_PID" 2>/dev/null; then + ok "wallet serve" "pid=$WSERVE_PID sock=$WSOCK (no run-until-done)" + return 0 + fi + sleep 0.15 + done + warn "wallet serve" "socket not ready — falling back to one-shot CLI (coins may stay pending)" + stop_wallet_serve + return 1 +} + +# $1 = max seconds for this call (optional); rest = wallet-cli args +# Prefer live serve socket so the shepherd stays up (alternative to run-until-done). +wcli() { + local maxc=12 + if [[ "${1:-}" =~ ^[0-9]+$ ]]; then + maxc=$1 + shift + fi + e2e_over && return 124 + local cap + cap=$(e2e_left) + [ "$cap" -gt "$maxc" ] && cap=$maxc + [ "$cap" -lt 3 ] && return 124 + if [ -n "${WSERVE_PID:-}" ] && [ -S "${WSOCK:-}" ]; then + if command -v perl >/dev/null 2>&1; then + perl -e 'alarm shift; exec @ARGV' "$cap" \ + node "$WALLET_CLI" --wallet-connection="$WSOCK" --no-throttle "$@" + else + node "$WALLET_CLI" --wallet-connection="$WSOCK" --no-throttle "$@" + fi + else + if command -v perl >/dev/null 2>&1; then + perl -e 'alarm shift; exec @ARGV' "$cap" \ + node "$WALLET_CLI" --wallet-db="$WDB" --no-throttle --skip-defaults "$@" + else + node "$WALLET_CLI" --wallet-db="$WDB" --no-throttle --skip-defaults "$@" + fi + fi +} + +# Pay must not be starved: use reserved seconds even if global budget is tight +wcli_pay() { + local cap=$E2E_PAY_SECS + local left + left=$(e2e_left) + if [ "$left" -gt "$cap" ]; then + cap=$E2E_PAY_SECS + elif [ "$left" -ge 8 ]; then + cap=$left + else + # still try once with floor 12s so Alarm clock is not the blocker + cap=12 + fi + info "pay wallet cap" "${cap}s" + if [ -n "${WSERVE_PID:-}" ] && [ -S "${WSOCK:-}" ]; then + if command -v perl >/dev/null 2>&1; then + perl -e 'alarm shift; exec @ARGV' "$cap" \ + node "$WALLET_CLI" --wallet-connection="$WSOCK" --no-throttle "$@" + else + node "$WALLET_CLI" --wallet-connection="$WSOCK" --no-throttle "$@" + fi + else + if command -v perl >/dev/null 2>&1; then + perl -e 'alarm shift; exec @ARGV' "$cap" \ + node "$WALLET_CLI" --wallet-db="$WDB" --no-throttle --skip-defaults "$@" + else + node "$WALLET_CLI" --wallet-db="$WDB" --no-throttle --skip-defaults "$@" + fi + fi +} + +# When coins never become available: run inside + targeted bank/exchange dig +dig_when_no_coins() { + set_group dig + section "e2e · coins missing — inside dig" + info "reason" "wallet empty after withdraw path (often wirewatch timing) — probing bank + exchange" + if [ -n "${WID:-}" ]; then + curl -sS -m 6 -o "$SCRATCH/wd-dig.json" \ + "$BANK_PUBLIC/taler-integration/withdrawal-operation/${WID}" 2>/dev/null || true + info "backend withdrawal-operation" "$(python3 -c ' +import json +try: + d=json.load(open("'"$SCRATCH"'/wd-dig.json")) + print("status=%s transfer_done=%s selection_done=%s amount=%s reserve=%s" % ( + d.get("status"), d.get("transfer_done"), d.get("selection_done"), + d.get("amount"), (d.get("selected_reserve_pub") or "-")[:24])) +except Exception as e: + print("unreadable", e) +' 2>/dev/null)" + fi + # Full component inside report (capped SSH; ignore its exit) + if [ "${SKIP_SSH}" != "1" ] && [ -x "$ROOT/check_inside.sh" ]; then + info "inside" "running check_inside.sh …" + # subshell so its PASS_N/FAIL_N/summary don't poison ours; still print to stdout + ( + # shellcheck disable=SC2030 + "$ROOT/check_inside.sh" || true + ) || true + else + warn "inside" "skipped (SKIP_SSH=1 or check_inside.sh missing)" + fi + # Extra targeted: wirewatch + recent journal + bank→exchange path + if koopa_ssh_ok; then + section "e2e · coins missing — exchange/bank focus" + DIG=$(koopa_ssh_bash 15 <<'REMOTE' || true +set +e +emit() { printf 'D|%s|%s\n' "$1" "$(printf '%s' "$2" | tr '\n' ' ' | head -c 220)"; } +EX=$(podman ps --format '{{.Names}}' | grep -i exchange | head -1) +BANK=$(podman ps --format '{{.Names}}' | grep -iE 'hacktivism-bank|taler-bank' | head -1) +[ -z "$BANK" ] && BANK=$(podman ps --format '{{.Names}}' | grep -i bank | head -1) +if [ -n "$EX" ]; then + podman exec "$EX" pgrep -af taler-exchange-wirewatch 2>/dev/null | head -1 | \ + { read -r l; [ -n "$l" ] && emit OK "wirewatch: $l" || emit ERROR "wirewatch not running"; } + podman exec "$EX" pgrep -af taler-exchange-transfer 2>/dev/null | head -1 | \ + { read -r l; [ -n "$l" ] && emit OK "transfer: $l" || emit WARN "transfer not running"; } + podman exec "$EX" pgrep -af taler-exchange-aggregator 2>/dev/null | head -1 | \ + { read -r l; [ -n "$l" ] && emit OK "aggregator: $l" || emit WARN "aggregator not running"; } + # last wirewatch log lines if journal available + j=$(podman exec "$EX" bash -c 'journalctl -u "taler-exchange-wirewatch*" -n 8 --no-pager 2>/dev/null | tail -5' 2>/dev/null | tr '\n' ';' | head -c 280) + [ -n "$j" ] && emit INFO "wirewatch journal: $j" + # reserve lookup via exchange if tools exist (best-effort) + c=$(curl -sS -m 3 -o /dev/null -w '%{http_code}' http://127.0.0.1:9011/keys 2>/dev/null || echo 000) + emit INFO "exchange /keys HTTP $c" +else + emit ERROR "no exchange container" +fi +if [ -n "$BANK" ]; then + podman exec "$BANK" pgrep -af 'MainKt serve|libeufin-bank serve' 2>/dev/null | head -1 | \ + { read -r l; [ -n "$l" ] && emit OK "libeufin: $l" || emit ERROR "libeufin not running"; } + c=$(curl -sS -m 3 -o /dev/null -w '%{http_code}' http://127.0.0.1:9012/taler-integration/config 2>/dev/null || echo 000) + emit INFO "bank integration /config HTTP $c" +else + emit ERROR "no bank container" +fi +REMOTE +) + while IFS= read -r line; do + case "$line" in + D\|*) + IFS="|" read -r _ lvl msg <<<"$line" + case "$lvl" in + OK) ok "dig $msg" ;; + ERROR) err "dig" "$msg" ;; + WARN) warn "dig" "$msg" ;; + INFO) info "dig" "$msg" ;; + esac + ;; + esac + done <<<"$DIG" + fi +} + +# Baseline wallet balance (empty DB) — always shown even if we abort later +wcli_bal_snap "$SCRATCH/bal-before.out" || true +BAL_BEFORE=$(fmt_bal "$SCRATCH/bal-before.out") +info "BALANCE before" "$BAL_BEFORE" + +# --------------------------------------------------------------------------- +set_group bank +section "e2e · bank (account · credit · withdraw)" +# --------------------------------------------------------------------------- +AT="" +if [ -n "${ADMIN_PASS:-}" ]; then + AT=$(curl -sS -m 15 -u "admin:${ADMIN_PASS}" -H 'Content-Type: application/json' \ + -d '{"scope":"readwrite"}' \ + "$BANK/accounts/admin/token" 2>"$SCRATCH/at.err" | python3 -c 'import sys,json;print(json.load(sys.stdin).get("access_token",""))' 2>/dev/null || true) +fi +if [ -n "$AT" ]; then + ok "admin token" +elif [ "$E2E_REMOTE" = "1" ]; then + warn "admin token" "unavailable — trying open registration" +else + if is_auth_kyc_body "$SCRATCH/at.err"; then + e2e_abort_auth "bank-auth" "admin login blocked (KYC/auth) — $(head -c 80 "$SCRATCH/at.err" 2>/dev/null)" + fi + blocker "bank-auth" "admin token failed — check admin password / bank up ($(head -c 80 "$SCRATCH/at.err" 2>/dev/null))" + exit 1 +fi + +# Create account: with admin bearer if we have it, else unauthenticated registration (demo banks) +if [ -n "$AT" ]; then + code=$(curl -sS -m 15 -o "$SCRATCH/acc.json" -w '%{http_code}' -X POST \ + -H "Authorization: Bearer $AT" -H 'Content-Type: application/json' \ + -d "{\"username\":\"${USER}\",\"password\":\"${USER_PW}\",\"name\":\"Monitoring\",\"is_public\":false,\"is_taler_exchange\":false,\"debit_threshold\":\"${CUR}:1000\"}" \ + "$BANK/accounts") +else + code=$(curl -sS -m 15 -o "$SCRATCH/acc.json" -w '%{http_code}' -X POST \ + -H 'Content-Type: application/json' \ + -d "{\"username\":\"${USER}\",\"password\":\"${USER_PW}\",\"name\":\"Monitoring\",\"is_public\":false,\"is_taler_exchange\":false,\"debit_threshold\":\"${CUR}:1000\"}" \ + "$BANK/accounts") +fi +if [ "$code" = "200" ] || [ "$code" = "201" ]; then + ok "create account $USER" +elif [ "$code" = "401" ] || [ "$code" = "403" ] || is_auth_kyc_body "$SCRATCH/acc.json"; then + e2e_abort_auth "bank-account" "registration/login blocked HTTP $code — $(head -c 100 "$SCRATCH/acc.json" | tr '\n' ' ')" +else + if [ "$E2E_REMOTE" = "1" ]; then + e2e_abort_auth "bank-account" "POST /accounts HTTP $code — cannot open account on this domain" + fi + blocker "bank-account" "POST /accounts HTTP $code — $(head -c 100 "$SCRATCH/acc.json" | tr '\n' ' ')" + exit 1 +fi + +if [ -n "$AT" ]; then + export USER_NAME="$USER" CREDIT_AMT="$CREDIT_AMT" OUT="$SCRATCH/credit.json" BANK_HOST="$BANK_HOST" + python3 - <<'PY' +import json, os +ALPH = "0123456789ABCDEFGHJKMNPQRSTVWXYZ" +def crock32(b): + bits = val = 0; o = [] + for byte in b: + val = (val << 8) | byte; bits += 8 + while bits >= 5: + bits -= 5; o.append(ALPH[(val >> bits) & 31]) + if bits: o.append(ALPH[(val << (5 - bits)) & 31]) + return "".join(o) +uid = crock32(os.urandom(32)) +user = os.environ["USER_NAME"] +host = os.environ.get("BANK_HOST") or "bank" +json.dump({ + "payto_uri": f"payto://x-taler-bank/{host}/{user}?receiver-name={user}&message=mon", + "amount": os.environ["CREDIT_AMT"], + "request_uid": uid, +}, open(os.environ["OUT"], "w")) +PY + curl -sS -m 15 -o "$SCRATCH/credit.out" -H "Authorization: Bearer $AT" \ + -H 'Content-Type: application/json' -d @"$SCRATCH/credit.json" \ + "$BANK/accounts/admin/transactions" >/dev/null || true + if grep -q row_id "$SCRATCH/credit.out" 2>/dev/null; then + ok "credit $CREDIT_AMT → $USER" + else + if is_auth_kyc_body "$SCRATCH/credit.out"; then + e2e_abort_auth "bank-credit" "admin credit blocked (KYC/auth)" + fi + if [ "$E2E_REMOTE" = "1" ]; then + e2e_abort_auth "bank-credit" "admin transfer failed — no credit path on remote ($(head -c 80 "$SCRATCH/credit.out" | tr '\n' ' '))" + fi + blocker "bank-credit" "admin transfer failed — $(head -c 100 "$SCRATCH/credit.out" | tr '\n' ' ')" + exit 1 + fi +else + # No admin: remote demo may start funded or require cash-in — try withdraw later; warn + warn "bank-credit" "skipped (no admin) — withdraw may fail without balance" +fi + +UT=$(curl -sS -m 15 -u "${USER}:${USER_PW}" -H 'Content-Type: application/json' \ + -d '{"scope":"readwrite"}' \ + "$BANK/accounts/${USER}/token" 2>"$SCRATCH/ut.err" | python3 -c 'import sys,json;print(json.load(sys.stdin).get("access_token",""))' 2>/dev/null || true) +if [ -n "$UT" ]; then + ok "user token" +elif is_auth_kyc_body "$SCRATCH/ut.err" || is_auth_kyc_body /dev/null; then + e2e_abort_auth "bank-auth" "user token failed (login/KYC) for $USER" +else + e2e_abort_auth "bank-auth" "user token failed for $USER — $(head -c 80 "$SCRATCH/ut.err" 2>/dev/null)" +fi + +# --------------------------------------------------------------------------- +set_group wallet +section "e2e · wallet setup (exchange + ToS once)" +# --------------------------------------------------------------------------- +# Long-lived serve = client/server wallet (shepherd keeps running; no run-until-done). +start_wallet_serve || true +if ! wcli exchanges add "${EXCHANGE_PUBLIC}/" >"$SCRATCH/ex-add.out" 2>&1; then + if ! grep -qiE 'already|ok' "$SCRATCH/ex-add.out"; then + blocker "wallet-exchange" "exchanges add failed — $(tail -c 160 "$SCRATCH/ex-add.out" | tr '\n' ' ')" + else + ok "wallet exchange known" + fi +else + ok "wallet add exchange" +fi +wcli exchanges update "${EXCHANGE_PUBLIC}/" >"$SCRATCH/ex-up.out" 2>&1 || true +if wcli exchanges accept-tos "${EXCHANGE_PUBLIC}/" >"$SCRATCH/ex-tos.out" 2>&1; then + ok "wallet accept ToS" +else + if grep -qiE 'pending|tos|terms' "$SCRATCH/ex-tos.out"; then + blocker "wallet-tos" "accept-tos failed — wallet may refuse withdraw ($(tail -c 120 "$SCRATCH/ex-tos.out" | tr '\n' ' '))" + else + warn "wallet accept ToS" "$(tail -c 80 "$SCRATCH/ex-tos.out" | tr '\n' ' ')" + fi +fi +# Empty wallet baseline before ATM / pay load +metrics_report_coins "wallet-ready" || true + +wd_status() { + curl -sS -m 8 -o "$SCRATCH/wd-st.json" -w '' \ + "$BANK/taler-integration/withdrawal-operation/${WID}" 2>/dev/null || true + python3 -c 'import json;d=json.load(open("'"$SCRATCH"'/wd-st.json"));print(d.get("status",""))' 2>/dev/null || true +} +wd_status_detail() { + python3 -c 'import json;d=json.load(open("'"$SCRATCH"'/wd-st.json")) +print("status=%s reserve=%s exchange=%s" % ( + d.get("status"), + (d.get("selected_reserve_pub") or "-")[:20], + (d.get("selected_exchange") or d.get("selected_exchange_account") or "-")[:60], +))' 2>/dev/null || echo "(no status body)" +} + +fake_incoming_speedup() { + [ "${E2E_FAKE_INCOMING}" = "1" ] || return 0 + st=$(wd_status) + RPUB=$(python3 -c 'import json;d=json.load(open("'"$SCRATCH"'/wd-st.json"));print(d.get("selected_reserve_pub") or "")' 2>/dev/null || true) + AMT=$(python3 -c 'import json;d=json.load(open("'"$SCRATCH"'/wd-st.json"));print(d.get("amount") or "'"$WITHDRAW_AMT"'")' 2>/dev/null || echo "$WITHDRAW_AMT") + DEBIT=$(python3 -c 'import json;d=json.load(open("'"$SCRATCH"'/wd-st.json"));print(d.get("sender_wire") or "")' 2>/dev/null || true) + [ -z "$DEBIT" ] && DEBIT="payto://x-taler-bank/${BANK_HOST}/${USER}?receiver-name=${USER}" + [ -z "$RPUB" ] && return 0 + [ -z "${AT:-}" ] && return 0 + WG="${BANK}/accounts/exchange/taler-wire-gateway/admin/add-incoming" + curl -sS -m 10 -o "$SCRATCH/fake-in.json" -w '%{http_code}' -X POST \ + -H "Authorization: Bearer $AT" -H 'Content-Type: application/json' \ + -d "{\"amount\":\"${AMT}\",\"reserve_pub\":\"${RPUB}\",\"debit_account\":\"${DEBIT}\"}" \ + "$WG" >/dev/null || true + if [ "$E2E_REMOTE" != "1" ] && koopa_ssh_ok; then + koopa_ssh_run 22 \ + 'podman exec taler-hacktivism-exchange-ansible bash -c "runuser -u taler-exchange-wire -- timeout 15 taler-exchange-wirewatch -c /etc/taler-exchange/taler-exchange.conf -t -L INFO 2>&1 | tail -5"' \ + >/dev/null 2>&1 || true + fi +} + +# One ATM-style withdraw: create → accept → select → confirm → settle +# returns 0 on wallet balance increase, 1 on soft fail (ladder continues) +e2e_one_withdraw() { + WITHDRAW_AMT="$1" + local tag + tag=$(printf '%s' "$WITHDRAW_AMT" | tr '.:' '__') + section "e2e · ATM withdraw $WITHDRAW_AMT · $(format_amount_alt "$WITHDRAW_AMT")" + e2e_over && { warn "ATM withdraw" "budget exhausted — skip $WITHDRAW_AMT"; return 1; } + metrics_report_coins "before-ATM-${tag}" || true + + curl -sS -m 15 -o "$SCRATCH/wd-$tag.json" -H "Authorization: Bearer $UT" \ + -H 'Content-Type: application/json' \ + -d "{\"amount\":\"${WITHDRAW_AMT}\",\"exchange_url\":\"${EXCHANGE_PUBLIC}/\"}" \ + "$BANK/accounts/${USER}/withdrawals" + WID=$(python3 -c 'import json;d=json.load(open("'"$SCRATCH"'/wd-'"$tag"'.json"));print(d.get("withdrawal_id") or d.get("id") or "")' 2>/dev/null || true) + URI=$(python3 -c 'import json;d=json.load(open("'"$SCRATCH"'/wd-'"$tag"'.json"));print(d.get("taler_withdraw_uri") or "")' 2>/dev/null || true) + URI_CLEAN=$(python3 -c 'import json;u=json.load(open("'"$SCRATCH"'/wd-'"$tag"'.json")).get("taler_withdraw_uri") or "";print(u.replace(":443/","/"))' 2>/dev/null || true) + if [ -z "$WID" ] || [ -z "$URI_CLEAN" ]; then + warn "ATM withdraw $WITHDRAW_AMT" "create failed — $(head -c 80 "$SCRATCH/wd-$tag.json" | tr '\n' ' ')" + return 1 + fi + ok "ATM withdrawal created $WITHDRAW_AMT ($WID)" + cp "$SCRATCH/wd-$tag.json" "$SCRATCH/wd.json" + + USE_URI="$URI" + [ -z "$USE_URI" ] && USE_URI="$URI_CLEAN" + if wcli withdraw accept-uri --exchange "${EXCHANGE_PUBLIC}/" "$USE_URI" >"$SCRATCH/accept-$tag.out" 2>&1; then + ok "wallet accept $WITHDRAW_AMT" + elif [ "$USE_URI" != "$URI_CLEAN" ] && [ -n "$URI_CLEAN" ] \ + && wcli withdraw accept-uri --exchange "${EXCHANGE_PUBLIC}/" "$URI_CLEAN" >"$SCRATCH/accept-$tag.out" 2>&1; then + ok "wallet accept $WITHDRAW_AMT (no :443)" + else + warn "ATM withdraw $WITHDRAW_AMT" "accept-uri failed — $(tail -c 100 "$SCRATCH/accept-$tag.out" | tr '\n' ' ')" + return 1 + fi + cp "$SCRATCH/accept-$tag.out" "$SCRATCH/accept.out" + + # Poll bank withdrawal status only (no run-until-done). force-select if stuck pending. + st="" + for i in 1 2 3 4 5 6 8 10; do + e2e_over && break + st=$(wd_status) + case "$st" in selected|confirmed|aborted) break ;; esac + sleep 0.4 + done + if [ "$st" != "selected" ] && [ "$st" != "confirmed" ]; then + wcli transactions >"$SCRATCH/tx-pre.json" 2>&1 || true + RPUB=$(python3 -c ' +import re +t=open("'"$SCRATCH"'/accept.out").read()+open("'"$SCRATCH"'/tx-pre.json").read() +m=re.search(r"reserve[_ ]?pub[\"=: ]+([A-Z0-9]{40,})", t, re.I) +if not m: m=re.search(r"\"reservePub\"\s*:\s*\"([^\"]+)\"", t) +print(m.group(1) if m else "") +' 2>/dev/null || true) + EPAYTO=$(curl -sS -m 12 "$EXCHANGE_PUBLIC/keys" 2>/dev/null | python3 -c ' +import json,sys +d=json.load(sys.stdin) +acc=d.get("accounts") or [] +for a in acc: + p=a.get("payto_uri") or a.get("payto_address") or "" + if "x-taler-bank" in p or "exchange" in p: + print(p); break +else: + if acc: print(acc[0].get("payto_uri") or "") +' 2>/dev/null || true) + if [ -n "$RPUB" ] && [ -n "$EPAYTO" ]; then + curl -sS -m 12 -o "$SCRATCH/sel.json" -X POST \ + -H 'Content-Type: application/json' \ + -d "{\"reserve_pub\":\"$RPUB\",\"selected_exchange\":\"$EPAYTO\"}" \ + "$BANK/taler-integration/withdrawal-operation/${WID}" >/dev/null || true + st=$(wd_status) + fi + fi + if [ "$st" != "selected" ] && [ "$st" != "confirmed" ]; then + warn "ATM withdraw $WITHDRAW_AMT" "not selected (status=${st:-?})" + return 1 + fi + if [ "$st" != "confirmed" ]; then + code=$(curl -sS -m 15 -o "$SCRATCH/conf-$tag.json" -w '%{http_code}' -X POST \ + -H "Authorization: Bearer $UT" -H 'Content-Type: application/json' -d '{}' \ + "$BANK/accounts/${USER}/withdrawals/${WID}/confirm") + case "$code" in + 200|204) ok "bank confirm $WITHDRAW_AMT" ;; + *) warn "ATM withdraw $WITHDRAW_AMT" "confirm HTTP $code"; return 1 ;; + esac + fi + fake_incoming_speedup + + # Short per-ATM settle: poll balance + bank transfer_done only (never run-until-done) + local ok_bal=0 r av xfer + for r in 1 2 3 4 5 6 8 10; do + wcli_bal_snap "$SCRATCH/bal-$tag.out" || wcli balance >"$SCRATCH/bal-$tag.out" 2>&1 || true + av=$(wallet_avail_num "$SCRATCH/bal-$tag.out") + if python3 -c "import sys; sys.exit(0 if float(sys.argv[1]) > 0 else 1)" "$av" 2>/dev/null; then + ok_bal=1 + ok "wallet funded after ATM $WITHDRAW_AMT" "avail=${CUR}:${av}" + info "balance" "$(fmt_bal "$SCRATCH/bal-$tag.out")" + break + fi + xfer=$(curl -sS -m 8 "$BANK/taler-integration/withdrawal-operation/${WID}" 2>/dev/null \ + | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("transfer_done"), d.get("status"))' 2>/dev/null || echo "?") + if echo "$xfer" | grep -qi True; then + ok "ATM $WITHDRAW_AMT bank transfer_done" "wallet avail=${CUR}:${av} ($xfer) — no run-until-done" + ok_bal=1 + break + fi + sleep 2 + done + metrics_report_coins "after-ATM-${tag}" || true + if [ "$ok_bal" = "1" ]; then + metrics_record_flow withdrawn "$WITHDRAW_AMT" || true + return 0 + fi + # Bank side often already confirmed — treat as timing lag, not hard fail + st=$(wd_status) + if [ "$st" = "confirmed" ]; then + warn "ATM withdraw $WITHDRAW_AMT" "bank confirmed; wallet not funded yet (settlement timing — will recheck later)" + return 2 + fi + warn "ATM withdraw $WITHDRAW_AMT" "no wallet balance yet (status=${st:-?})" + return 1 +} + +# Map CUR:amount → public template id. +# GOA default: e2e-* on goa-demo. TESTPAYSAN: farmer shop templates (set in apply_taler_domain). +# Override with E2E_TEMPLATE_MAP="5=oeufs-6 8.5=fromage-chevre" +e2e_template_id_for_amount() { + local amt="$1" + local val map_entry k v + val=$(printf '%s' "$amt" | awk -F: '{print $NF}') + if [ -z "${E2E_TEMPLATE_MAP:-}" ]; then + if [ "${CUR:-}" = "TESTPAYSAN" ]; then + E2E_TEMPLATE_MAP="5=oeufs-6 4.5=pain-seigle 6=jus-pomme 8.5=fromage-chevre 12=miel-printemps 25=panier-legumes" + else + E2E_TEMPLATE_MAP="0.01=e2e-001 0.05=e2e-005 1=e2e-1 1.0=e2e-1" + fi + fi + for map_entry in $E2E_TEMPLATE_MAP; do + k="${map_entry%%=*}" + v="${map_entry#*=}" + if [ "$k" = "$val" ]; then + printf '%s' "$v" + return 0 + fi + done + return 1 +} + +# Template id → merchant instance (stage farmer shops / GOA goa-shop) +e2e_template_instance_for_id() { + local pid="$1" + case "$pid" in + oeufs-6|pain-seigle|jus-pomme) printf '%s' "jardin-du-creux" ;; + panier-legumes|fromage-chevre|miel-printemps) printf '%s' "fermes-des-collines" ;; + e2e-001|e2e-005|e2e-1|orbit-sticker|nebula-coffee|voidwave-playlist|comet-cap|shuttle-pass|beacon-badge|relay-pin|eclipse-shades|star-chart|rainbow-pill|blue-or-red-pill) + printf '%s' "${E2E_SHOP_INSTANCE:-goa-shop}" + ;; + *) printf '%s' "${MERCHANT_INSTANCE:-default}" ;; + esac +} + +# One payment: e2e_one_pay AMOUNT [SUMMARY] [TAG] +# SUMMARY defaults to "monitoring pay $AMOUNT"; TAG defaults from amount. +# When E2E_USE_TEMPLATES=1 and amount maps to a public template, uses +# POST /instances/{inst}/templates/{id} (same as landings/shops). +e2e_one_pay() { + PAY_AMT="$1" + local PAY_SUM="${2:-monitoring pay ${PAY_AMT}}" + local tag="${3:-}" + if [ -z "$tag" ]; then + tag=$(printf '%s' "$PAY_AMT" | tr '.:' '__') + fi + # shell-safe tag for files + tag=$(printf '%s' "$tag" | tr -c 'A-Za-z0-9._-' '_') + section "e2e · pay $PAY_AMT · $(format_amount_alt "$PAY_AMT") · $PAY_SUM" + e2e_over && { warn "pay" "budget exhausted — skip $PAY_AMT ($PAY_SUM)"; return 1; } + metrics_report_coins "before-pay-${tag}" || true + + # Prefer public templates when enabled (stage TESTPAYSAN / shop-style orders) + local tpl_id="" + if [ "${E2E_USE_TEMPLATES:-0}" = "1" ]; then + tpl_id=$(e2e_template_id_for_amount "$PAY_AMT" 2>/dev/null || true) + if [ -n "$tpl_id" ]; then + info "pay $PAY_AMT" "via public template $tpl_id (E2E_USE_TEMPLATES=1)" + e2e_one_pay_public_template "$tpl_id" "$PAY_SUM" "$PAY_AMT" + return $? + fi + warn "pay $PAY_AMT" "no template map for amount — fall back to private order" + fi + + if [ -z "${MPW:-}" ] && [ -z "${MPW_TOKEN:-}" ]; then + warn "pay $PAY_AMT" "no merchant token" + return 1 + fi + local AUTH + AUTH=$(merchant_auth_header) || { + warn "pay $PAY_AMT" "no merchant token" + return 1 + } + # JSON-escape summary for curl -d + local SUM_JSON + SUM_JSON=$(python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "$PAY_SUM" 2>/dev/null || printf '"%s"' "$PAY_SUM") + curl -skS -m 15 -o "$SCRATCH/ord-$tag.json" -X POST \ + -H "$AUTH" -H 'Content-Type: application/json' \ + -d "{\"order\":{\"summary\":${SUM_JSON},\"amount\":\"${PAY_AMT}\",\"fulfillment_message\":\"ok\"},\"create_token\":true}" \ + "${MERCHANT_PUBLIC}/instances/${INST}/private/orders" 2>"$SCRATCH/ord-$tag.err" || true + if ! grep -q order_id "$SCRATCH/ord-$tag.json" 2>/dev/null; then + if [ "$E2E_REMOTE" != "1" ] && koopa_ssh_ok; then + local mpw_raw="${MPW_TOKEN:-secret-token:${MPW}}" + koopa_ssh_run 20 \ + "curl -skS -m 12 -X POST -H 'Authorization: Bearer ${mpw_raw}' -H 'Content-Type: application/json' -d '{\"order\":{\"summary\":${SUM_JSON},\"amount\":\"${PAY_AMT}\",\"fulfillment_message\":\"ok\"},\"create_token\":true}' 'https://127.0.0.1:9010/instances/${INST}/private/orders'" \ + >"$SCRATCH/ord-$tag.json" 2>"$SCRATCH/ord-$tag.err" || true + fi + fi + OID=$(python3 -c 'import json,re,sys;t=open(sys.argv[1]).read() +try: print(json.loads(t).get("order_id") or "") +except Exception: + m=re.search(r"\"order_id\"\s*:\s*\"([^\"]+)\"",t); print(m.group(1) if m else "") +' "$SCRATCH/ord-$tag.json" 2>/dev/null || true) + OTOK=$(python3 -c 'import json,re,sys;t=open(sys.argv[1]).read() +try: print(json.loads(t).get("token") or "") +except Exception: + m=re.search(r"\"token\"\s*:\s*\"([^\"]+)\"",t); print(m.group(1) if m else "") +' "$SCRATCH/ord-$tag.json" 2>/dev/null || true) + if [ -z "$OID" ]; then + if is_auth_kyc_body "$SCRATCH/ord-$tag.json"; then + e2e_abort_auth "merchant-order" "merchant auth/KYC on pay $PAY_AMT ($PAY_SUM)" + fi + warn "pay $PAY_AMT ($PAY_SUM)" "order create failed — $(head -c 80 "$SCRATCH/ord-$tag.json" | tr '\n' ' ')" + return 1 + fi + ok "merchant order $OID ($PAY_AMT · $PAY_SUM)" + curl -skS -m 12 -o "$SCRATCH/ord-det-$tag.json" -H "$AUTH" \ + "${MERCHANT_PUBLIC}/instances/${INST}/private/orders/${OID}" 2>/dev/null || true + PAYURI=$(python3 -c 'import json,sys +try: print(json.load(open(sys.argv[1])).get("taler_pay_uri") or "") +except Exception: print("") +' "$SCRATCH/ord-det-$tag.json" 2>/dev/null || true) + if [ -z "$PAYURI" ] && [ -n "$OTOK" ]; then + MH=$(python3 -c 'from urllib.parse import urlparse; print(urlparse("'"$MERCHANT_PUBLIC"'").hostname or "taler.hacktivism.ch")' 2>/dev/null || echo "taler.hacktivism.ch") + PAYURI="taler://pay/${MH}/instances/${INST}/${OID}/?c=${OTOK}" + fi + # normalize default HTTPS port (wallets accept both) + PAYURI=$(printf '%s' "$PAYURI" | sed 's/:443\//\//g; s/:443?/?/g') + if [ -z "$PAYURI" ]; then + warn "pay $PAY_AMT ($PAY_SUM)" "no pay URI for $OID" + return 1 + fi + ok "taler_pay_uri ready ($tag)" + if ! wcli_pay handle-uri --yes "$PAYURI" >"$SCRATCH/pay-$tag.out" 2>&1; then + warn "pay $PAY_AMT ($PAY_SUM)" "handle-uri failed — $(tail -c 100 "$SCRATCH/pay-$tag.out" | tr '\n' ' ')" + return 1 + fi + ok "wallet handle pay $PAY_AMT ($PAY_SUM)" + # Settle: poll merchant order + wallet transactions only (never run-until-done) + local r + for r in 1 2 3 4 5 6; do + wcli_pay transactions >"$SCRATCH/tx-$tag.out" 2>&1 || true + if [ -n "${AUTH:-}" ]; then + curl -skS -m 8 -o "$SCRATCH/ord-paid-$tag.json" -H "$AUTH" \ + "${MERCHANT_PUBLIC}/instances/${INST}/private/orders/${OID}" 2>/dev/null || true + fi + if [ -n "${OTOK:-}" ]; then + curl -skS -m 8 -o "$SCRATCH/ord-pub-$tag.json" \ + "${MERCHANT_PUBLIC}/instances/${INST}/orders/$(python3 -c 'import urllib.parse,sys; print(urllib.parse.quote(sys.argv[1], safe=""))' "$OID")?token=$(python3 -c 'import urllib.parse,sys; print(urllib.parse.quote(sys.argv[1], safe=""))' "$OTOK")" \ + 2>/dev/null || true + fi + if grep -qiE 'payment|paid|Payment' "$SCRATCH/tx-$tag.out" 2>/dev/null \ + || grep -qiE 'done|paid|success|Payment' "$SCRATCH/pay-$tag.out" 2>/dev/null \ + || python3 -c 'import json,sys +for p in sys.argv[1:]: + try: + d=json.load(open(p)) + if d.get("paid") is True or str(d.get("order_status","")).lower()=="paid": + sys.exit(0) + except Exception: + pass +sys.exit(1) +' "$SCRATCH/ord-paid-$tag.json" "$SCRATCH/ord-pub-$tag.json" 2>/dev/null; then + ok "payment settled $PAY_AMT ($PAY_SUM · order $OID)" + metrics_report_coins "after-pay-${tag}" || true + metrics_record_flow spent "$PAY_AMT" || true + return 0 + fi + sleep 1 + done + metrics_report_coins "after-pay-fail-${tag}" || true + warn "pay $PAY_AMT ($PAY_SUM)" "not settled for order $OID" + return 1 +} + +# GOA shop catalog (full list) — keep in sync with configs/merchant-landing/index.html +# id|product_name|amount +# Landing QR = taler://pay-template/…/{id}; popup = public POST templates/{id}. +# E2E does not pay every product every run: shuffle + pick E2E_SHOP_PICK_N (default 2). +: "${E2E_SHOP_PRODUCTS:=orbit-sticker|Orbit sticker pack|GOA:2 +nebula-coffee|Nebula coffee|GOA:5 +voidwave-playlist|Voidwave playlist|GOA:8 +comet-cap|Comet cap|GOA:15 +shuttle-pass|Shuttle day pass|GOA:25 +beacon-badge|Beacon badge|GOA:3 +relay-pin|Relay pin|GOA:4 +eclipse-shades|Eclipse shades|GOA:12 +star-chart|Star chart print|GOA:7 +rainbow-pill|Rainbow pill (sample/joke)|GOA:4.2 +blue-or-red-pill|Blue or red pill (sample/joke)|GOA:1.5}" +: "${E2E_SHOP_PICK_N:=2}" +# Public templates live on goa-shop (landing shop-pay.js); not the e2e default demo instance. +: "${E2E_SHOP_INSTANCE:=goa-shop}" + +# Settlement payto shown on landing shop popup (static) +: "${E2E_SHOP_PAYTO:=payto://x-taler-bank/bank.hacktivism.ch/goa-shop?receiver-name=GOA%20Shop}" + +# Pay one shop product the way the public landing does: +# POST /instances/{inst}/templates/{id} body {} → order_id+token → taler_pay_uri +# Also checks pay-template URI shape (what qrencode PNGs encode). +# Usage: e2e_one_pay_public_template PRODUCT_ID PRODUCT_NAME AMOUNT +e2e_one_pay_public_template() { + local pid="$1" + local pname="${2:-$1}" + local pamt="$3" + local inst="${4:-}" + local tag + tag=$(printf 'shop_%s' "$pid" | tr -c 'A-Za-z0-9._-' '_') + section "e2e · shop product · $pname ($pid) · $pamt · $(format_amount_alt "$pamt")" + e2e_over && { warn "shop $pname" "budget exhausted — skip"; return 1; } + metrics_report_coins "before-shop-${tag}" || true + + # Resolve instance: explicit arg → template map → global INST + if [ -z "$inst" ]; then + inst=$(e2e_template_instance_for_id "$pid") + fi + [ -n "$inst" ] || inst="${INST:-${MERCHANT_INSTANCE:-default}}" + + local MH + MH=$(python3 -c 'from urllib.parse import urlparse; print(urlparse("'"$MERCHANT_PUBLIC"'").hostname or "taler.hacktivism.ch")' 2>/dev/null || echo "taler.hacktivism.ch") + local TPL_URI="taler://pay-template/${MH}/instances/${inst}/${pid}" + local TPL_HTTPS="${MERCHANT_PUBLIC}/instances/${inst}/templates/$(python3 -c 'import urllib.parse,sys; print(urllib.parse.quote(sys.argv[1], safe=""))' "$pid")" + info "shop $pname" "instance $inst · pay-template URI $TPL_URI" + info "shop $pname" "public template POST $TPL_HTTPS" + # Stage settlement payto: instance bank account; GOA uses E2E_SHOP_PAYTO + local payto_show="${E2E_SHOP_PAYTO:-}" + if [ "${CUR:-}" = "TESTPAYSAN" ]; then + payto_show="payto://x-taler-bank/stage.bank.lefrancpaysan.ch/${inst}" + fi + info "shop $pname" "settlement payto ${payto_show:-(none)}" + + # Public create (no merchant secret) — same as shop-pay.js + curl -skS -m 15 -o "$SCRATCH/tpl-$tag.json" -X POST \ + -H 'Content-Type: application/json' \ + -d '{}' \ + "$TPL_HTTPS" 2>"$SCRATCH/tpl-$tag.err" || true + if ! grep -q order_id "$SCRATCH/tpl-$tag.json" 2>/dev/null; then + if [ "$E2E_REMOTE" != "1" ] && koopa_ssh_ok; then + koopa_ssh_run 20 \ + "curl -skS -m 12 -X POST -H 'Content-Type: application/json' -d '{}' 'https://127.0.0.1:9010/instances/${INST}/templates/${pid}'" \ + >"$SCRATCH/tpl-$tag.json" 2>"$SCRATCH/tpl-$tag.err" || true + fi + fi + OID=$(python3 -c 'import json,re,sys;t=open(sys.argv[1]).read() +try: print(json.loads(t).get("order_id") or "") +except Exception: + m=re.search(r"\"order_id\"\s*:\s*\"([^\"]+)\"",t); print(m.group(1) if m else "") +' "$SCRATCH/tpl-$tag.json" 2>/dev/null || true) + OTOK=$(python3 -c 'import json,re,sys;t=open(sys.argv[1]).read() +try: print(json.loads(t).get("token") or "") +except Exception: + m=re.search(r"\"token\"\s*:\s*\"([^\"]+)\"",t); print(m.group(1) if m else "") +' "$SCRATCH/tpl-$tag.json" 2>/dev/null || true) + if [ -z "$OID" ] || [ -z "$OTOK" ]; then + warn "shop $pname ($pid)" "public template POST failed — $(head -c 100 "$SCRATCH/tpl-$tag.json" 2>/dev/null | tr '\n' ' ')" + return 1 + fi + ok "shop $pname" "public order $OID (template $pid)" + + local statusUrl="${MERCHANT_PUBLIC}/instances/${INST}/orders/$(python3 -c 'import urllib.parse,sys; print(urllib.parse.quote(sys.argv[1], safe=""))' "$OID")?token=$(python3 -c 'import urllib.parse,sys; print(urllib.parse.quote(sys.argv[1], safe=""))' "$OTOK")" + curl -skS -m 12 -o "$SCRATCH/tpl-st-$tag.json" "$statusUrl" 2>/dev/null || true + PAYURI=$(python3 -c 'import json,sys +try: + d=json.load(open(sys.argv[1])) + print(d.get("taler_pay_uri") or "") +except Exception: print("") +' "$SCRATCH/tpl-st-$tag.json" 2>/dev/null || true) + if [ -z "$PAYURI" ]; then + PAYURI="taler://pay/${MH}/instances/${INST}/${OID}/?c=${OTOK}" + fi + PAYURI=$(printf '%s' "$PAYURI" | sed 's/:443\//\//g; s/:443?/?/g') + if ! printf '%s' "$PAYURI" | grep -q '^taler://pay/'; then + warn "shop $pname ($pid)" "bad pay URI: $PAYURI" + return 1 + fi + ok "shop $pname" "taler_pay_uri $PAYURI" + + if ! wcli_pay handle-uri --yes "$PAYURI" >"$SCRATCH/pay-$tag.out" 2>&1; then + warn "shop $pname ($pid)" "handle-uri failed — $(tail -c 120 "$SCRATCH/pay-$tag.out" | tr '\n' ' ')" + return 1 + fi + ok "shop $pname" "wallet accepted pay URI" + + # Settle: public order status + transactions only (never run-until-done) + local r AUTH="" + AUTH=$(merchant_auth_header 2>/dev/null) || AUTH="" + for r in 1 2 3 4 5 6; do + wcli_pay transactions >"$SCRATCH/tx-$tag.out" 2>&1 || true + if [ -n "$AUTH" ]; then + curl -skS -m 8 -o "$SCRATCH/ord-paid-$tag.json" -H "$AUTH" \ + "${MERCHANT_PUBLIC}/instances/${INST}/private/orders/${OID}" 2>/dev/null || true + fi + curl -skS -m 8 -o "$SCRATCH/ord-pub-$tag.json" "$statusUrl" 2>/dev/null || true + if python3 -c 'import json,sys +for p in sys.argv[1:]: + try: + d=json.load(open(p)) + if d.get("paid") is True or str(d.get("order_status","")).lower()=="paid": + sys.exit(0) + except Exception: + pass +sys.exit(1) +' "$SCRATCH/ord-paid-$tag.json" "$SCRATCH/ord-pub-$tag.json" 2>/dev/null \ + || grep -qiE 'payment|paid|Payment' "$SCRATCH/tx-$tag.out" 2>/dev/null \ + || grep -qiE 'done|paid|success|Payment' "$SCRATCH/pay-$tag.out" 2>/dev/null; then + ok "shop $pname" "payment settled ($pamt · order $OID)" + metrics_report_coins "after-shop-${tag}" || true + metrics_record_flow spent "$pamt" || true + return 0 + fi + sleep 1 + done + metrics_report_coins "after-shop-fail-${tag}" || true + warn "shop $pname ($pid)" "not settled for order $OID" + return 1 +} + +# --------------------------------------------------------------------------- +set_group atm +section "e2e · ATM withdraw ladder" +# --------------------------------------------------------------------------- +WITHDRAW_OK=0 +WITHDRAW_OK_N=0 +WITHDRAW_LAG_N=0 +WITHDRAW_FAIL_N=0 +WITHDRAW_REPORT="" +for WITHDRAW_AMT in $WITHDRAW_LIST; do + e2e_over && { warn "ATM ladder" "time budget low — stopping more ATM withdraws (not a protocol error)"; break; } + set +e + e2e_one_withdraw "$WITHDRAW_AMT" + wc=$? + set -e + case "$wc" in + 0) + WITHDRAW_OK=1 + WITHDRAW_OK_N=$((WITHDRAW_OK_N + 1)) + WITHDRAW_REPORT="${WITHDRAW_REPORT}${WITHDRAW_REPORT:+ }${WITHDRAW_AMT}=OK" + ;; + 2) + WITHDRAW_LAG_N=$((WITHDRAW_LAG_N + 1)) + WITHDRAW_REPORT="${WITHDRAW_REPORT}${WITHDRAW_REPORT:+ }${WITHDRAW_AMT}=LAG" + ;; + *) + WITHDRAW_FAIL_N=$((WITHDRAW_FAIL_N + 1)) + WITHDRAW_REPORT="${WITHDRAW_REPORT}${WITHDRAW_REPORT:+ }${WITHDRAW_AMT}=FAIL" + ;; + esac +done +info "ATM withdraw summary" "$WITHDRAW_REPORT (ok=$WITHDRAW_OK_N lag=$WITHDRAW_LAG_N fail=$WITHDRAW_FAIL_N)" +set_group load +section "e2e · load snapshot (after ATM withdraws)" +metrics_report_load "${METRICS_DIR}/load-after-withdraw.json" "after-withdraw" || true +cp -f "${METRICS_DIR}/load-after-withdraw.json" "${METRICS_DIR}/load-after.json" 2>/dev/null || true +metrics_report_coins "after-ATM-ladder" || true + +# Settlement catch-up: bank may have confirmed while wallet was still empty +set_group settle +section "e2e · wallet settlement (timing)" +if ! wait_wallet_balance 0 "${E2E_SETTLE_ROUNDS}" "${E2E_SETTLE_SLEEP}"; then + if [ "$WITHDRAW_LAG_N" -gt 0 ] || [ "$WITHDRAW_OK_N" -gt 0 ]; then + warn "settlement timing" "ATM path reached bank confirm (lag=$WITHDRAW_LAG_N ok=$WITHDRAW_OK_N) but wallet empty after wait — TIME lag, not protocol error" + fi +fi +av_now=$(wallet_avail_num) +if python3 -c "import sys; sys.exit(0 if float(sys.argv[1]) > 0 else 1)" "$av_now" 2>/dev/null; then + WITHDRAW_OK=1 + if [ "$WITHDRAW_OK_N" = "0" ]; then + warn "settlement timing" "coins arrived after ATM ladder (${CUR}:${av_now}) — earlier 'FAIL' was timing lag" + WITHDRAW_REPORT="${WITHDRAW_REPORT} → late-OK avail=${CUR}:${av_now}" + fi + ok "spendable balance for payments" "${CUR}:${av_now}" + metrics_report_coins "after-settle" || true +else + if [ "$WITHDRAW_OK" != "1" ]; then + warn "withdraw" "still no spendable ${CUR} after settle wait" + # diagnostic dig only — do not treat as hard stop if we can still see bank state + if [ "$E2E_REMOTE" != "1" ]; then + dig_when_no_coins || true + fi + # one last balance after dig (dig takes wall time — often enough for wirewatch) + wait_wallet_balance 0 8 3 || true + av_now=$(wallet_avail_num) + if python3 -c "import sys; sys.exit(0 if float(sys.argv[1]) > 0 else 1)" "$av_now" 2>/dev/null; then + WITHDRAW_OK=1 + warn "settlement timing" "coins present after dig wait (${CUR}:${av_now}) — timing, not blocker" + else + blocker "withdraw-settle" "no spendable ${CUR} after ATM ladder + settle wait ($WITHDRAW_LIST)" + section "e2e · report" + info "WITHDRAW" "$WITHDRAW_REPORT — no coins after extended wait" + info "PAY" "SKIPPED (no spendable balance after settle wait)" + exit 1 + fi + fi +fi + +# --------------------------------------------------------------------------- +set_group pay +section "e2e · variable payments (if balance allows)" +# --------------------------------------------------------------------------- +PAY_OK=0 +PAY_OK_N=0 +PAY_FAIL_N=0 +PAY_SKIP_N=0 +PAY_REPORT="" +if [ -z "${MPW:-}" ] && [ "${E2E_USE_TEMPLATES:-0}" != "1" ]; then + if [ "$E2E_REMOTE" = "1" ]; then + e2e_abort_auth "merchant-order" "no merchant token — set E2E_MERCHANT_TOKEN (or E2E_USE_TEMPLATES=1)" + fi + blocker "merchant-order" "no merchant token" + exit 1 +fi +if [ -z "${MPW:-}" ] && [ "${E2E_USE_TEMPLATES:-0}" = "1" ]; then + info "pay" "no merchant token — public templates only (E2E_USE_TEMPLATES=1)" +fi + +for PAY_AMT in $PAY_LIST; do + e2e_over && { warn "pay ladder" "time budget low — stopping more payments"; break; } + av_now=$(wallet_avail_num) + pay_n=$(python3 -c 'import sys; print(float(sys.argv[1].split(":",1)[-1]))' "$PAY_AMT" 2>/dev/null || echo 0) + if ! python3 -c "import sys; sys.exit(0 if float(sys.argv[1]) + 1e-12 >= float(sys.argv[2]) else 1)" "$av_now" "$pay_n" 2>/dev/null; then + warn "pay $PAY_AMT" "skip — insufficient avail ${CUR}:${av_now} (need ${pay_n})" + PAY_SKIP_N=$((PAY_SKIP_N + 1)) + PAY_REPORT="${PAY_REPORT}${PAY_REPORT:+ }${PAY_AMT}=SKIP(bal)" + continue + fi + set +e + e2e_one_pay "$PAY_AMT" + pc=$? + set -e + if [ "$pc" = "0" ]; then + PAY_OK=1 + PAY_OK_N=$((PAY_OK_N + 1)) + PAY_REPORT="${PAY_REPORT}${PAY_REPORT:+ }${PAY_AMT}=OK" + else + PAY_FAIL_N=$((PAY_FAIL_N + 1)) + PAY_REPORT="${PAY_REPORT}${PAY_REPORT:+ }${PAY_AMT}=FAIL" + fi +done +info "pay summary" "$PAY_REPORT (ok=$PAY_OK_N fail=$PAY_FAIL_N skip=$PAY_SKIP_N)" +set_group load +section "e2e · load snapshot (after payments)" +metrics_report_load "${METRICS_DIR}/load-after-pay.json" "after-pay" || true +cp -f "${METRICS_DIR}/load-after-pay.json" "${METRICS_DIR}/load-after.json" 2>/dev/null || true +metrics_report_coins "after-pay-ladder" || true +if [ "$PAY_OK" != "1" ]; then + av_now=$(wallet_avail_num) + if python3 -c "import sys; sys.exit(0 if float(sys.argv[1]) > 0 else 1)" "$av_now" 2>/dev/null; then + warn "pay" "balance ${CUR}:${av_now} but no pay amount completed — check merchant/timing (not necessarily empty wallet)" + else + blocker "pay-settle" "no payment succeeded and no spendable balance ($PAY_LIST)" + fi +fi + +# --------------------------------------------------------------------------- +set_group shop +section "e2e · shop products (public templates)" +# --------------------------------------------------------------------------- +# Public template POST + wallet pay — catalog random pick of N products. +# GOA: goa-shop on hacktivism. TESTPAYSAN: farmer shops (jardin / fermes). +SHOP_OK=0 +SHOP_OK_N=0 +SHOP_FAIL_N=0 +SHOP_SKIP_N=0 +SHOP_REPORT="" +# Auto catalog for stage TESTPAYSAN when not overridden +if [ "${CUR:-}" = "TESTPAYSAN" ] && [ -z "${E2E_SHOP_PRODUCTS_SET:-}" ] && [ -z "${E2E_SHOP_FORCE_GOA:-}" ]; then + # id|name|amount|instance — force stage catalog (GOA default is set earlier) + E2E_SHOP_PRODUCTS="oeufs-6|Œufs plein air × 6|TESTPAYSAN:5|jardin-du-creux +pain-seigle|Pain de seigle 800 g|TESTPAYSAN:4.5|jardin-du-creux +jus-pomme|Jus de pomme 1 L|TESTPAYSAN:6|jardin-du-creux +panier-legumes|Panier légumes|TESTPAYSAN:25|fermes-des-collines +fromage-chevre|Fromage de chèvre 200 g|TESTPAYSAN:8.5|fermes-des-collines +miel-printemps|Miel de printemps 250 g|TESTPAYSAN:12|fermes-des-collines" + # Force stage defaults (GOA goa-shop is already set earlier via :=) + if [ -z "${E2E_SHOP_INSTANCE_SET:-}" ]; then + E2E_SHOP_INSTANCE=jardin-du-creux + fi + if [ -z "${E2E_SHOP_PAYTO_SET:-}" ]; then + E2E_SHOP_PAYTO="payto://x-taler-bank/stage.bank.lefrancpaysan.ch/jardin-du-creux" + fi + E2E_SHOP_PICK_N="${E2E_SHOP_PICK_N:-2}" + E2E_SHOP_ENABLE=1 +fi +# GOA local only by default; remote non-GOA shops need E2E_SHOP_ENABLE=1 or TESTPAYSAN auto above +if [ -z "${E2E_SHOP_ENABLE:-}" ]; then + if [ "$E2E_REMOTE" = "1" ] || [ "${CUR:-}" != "GOA" ]; then + E2E_SHOP_ENABLE=0 + else + E2E_SHOP_ENABLE=1 + fi +fi +if [ "${E2E_SHOP_ENABLE}" != "1" ]; then + info "shop" "SKIPPED (set E2E_SHOP_ENABLE=1 or use TESTPAYSAN stage catalog)" +else + # Build catalog array from E2E_SHOP_PRODUCTS (id|name|amount[|instance] lines) + SHOP_CATALOG=() + while IFS= read -r line; do + [ -z "$line" ] && continue + case "$line" in \#*) continue ;; esac + SHOP_CATALOG+=("$line") + done </dev/null || echo 0) + if ! python3 -c "import sys; sys.exit(0 if float(sys.argv[1]) + 1e-12 >= float(sys.argv[2]) else 1)" "$av_now" "$pay_n" 2>/dev/null; then + warn "shop $pname ($pid)" "skip — insufficient avail ${CUR}:${av_now} (need ${pay_n})" + SHOP_SKIP_N=$((SHOP_SKIP_N + 1)) + SHOP_REPORT="${SHOP_REPORT}${SHOP_REPORT:+ }${pname}=SKIP(bal)" + continue + fi + set +e + e2e_one_pay_public_template "$pid" "$pname" "$pamt" + pc=$? + set -e + if [ "$pc" = "0" ]; then + SHOP_OK=1 + SHOP_OK_N=$((SHOP_OK_N + 1)) + PAY_OK=1 + PAY_OK_N=$((PAY_OK_N + 1)) + SHOP_REPORT="${SHOP_REPORT}${SHOP_REPORT:+ }${pname}=OK" + else + SHOP_FAIL_N=$((SHOP_FAIL_N + 1)) + SHOP_REPORT="${SHOP_REPORT}${SHOP_REPORT:+ }${pname}=FAIL" + fi + done </dev/null || true + fi +fi + +# --------------------------------------------------------------------------- +set_group paivana +section "e2e · paivana paywall (template GOA:4200)" +# --------------------------------------------------------------------------- +# Public paywall: https://paivana.hacktivism.ch → 302 to merchant template "paivana". +# Requires spendable ≥ 4200 GOA (ATM ladder includes GOA:4200 by default). +: "${PAIVANA_PUBLIC:=https://paivana.hacktivism.ch}" +: "${E2E_PAIVANA:=1}" +: "${E2E_PAIVANA_TEMPLATE:=paivana}" +: "${E2E_PAIVANA_AMOUNT:=GOA:4200}" +: "${E2E_PAIVANA_INSTANCE:=goa-shop}" +PAIVANA_OK=0 +PAIVANA_REPORT="" +if [ "$E2E_REMOTE" = "1" ] || [ "${CUR:-}" != "GOA" ] || [ "${E2E_PAIVANA}" = "0" ]; then + info "paivana" "SKIPPED (remote, non-GOA, or E2E_PAIVANA=0)" + PAIVANA_REPORT="SKIP" +else + PAIVANA_PUBLIC="${PAIVANA_PUBLIC%/}" + # HTTP: paywall front must redirect into template flow (or serve well-known) + hdr=$(curl -skS -m 12 -D - -o /dev/null "$PAIVANA_PUBLIC/" 2>/dev/null || true) + pcode=$(printf '%s' "$hdr" | awk 'BEGIN{c="000"} /^HTTP/{c=$2} END{print c}') + loc=$(printf '%s' "$hdr" | awk 'BEGIN{IGNORECASE=1} /^location:/{sub(/\r$/,""); sub(/^location:[[:space:]]*/,""); print; exit}') + case "$pcode" in + 301|302|303|307|308) + if printf '%s' "$loc" | grep -qiE 'paivana|templates|well-known'; then + ok "paivana HTTP" "HTTP $pcode → ${loc:0:120}" + else + ok "paivana HTTP" "HTTP $pcode redirect (Location: ${loc:0:80})" + fi + ;; + 200) + warn "paivana HTTP" "HTTP 200 without paywall redirect — unexpected for locked site" + ;; + *) + fail "paivana HTTP" "HTTP ${pcode:-000} on $PAIVANA_PUBLIC/ (want 302 to template)" + PAIVANA_REPORT="HTTP_FAIL" + ;; + esac + + # Ensure wallet can cover 4200 (extra ATM if ladder did not fund enough) + pay_need=$(python3 -c 'import sys; print(float(sys.argv[1].split(":",1)[-1]))' "$E2E_PAIVANA_AMOUNT" 2>/dev/null || echo 4200) + av_now=$(wallet_avail_num) + if ! python3 -c "import sys; sys.exit(0 if float(sys.argv[1]) + 1e-9 >= float(sys.argv[2]) else 1)" "$av_now" "$pay_need" 2>/dev/null; then + info "paivana" "avail ${CUR}:${av_now} < ${pay_need} — ATM withdraw ${E2E_PAIVANA_AMOUNT}" + e2e_over && warn "paivana" "budget low before 4200 withdraw" + set +e + e2e_one_withdraw "$E2E_PAIVANA_AMOUNT" + wc=$? + set -e + if [ "$wc" = "0" ]; then + WITHDRAW_OK=1 + WITHDRAW_OK_N=$((WITHDRAW_OK_N + 1)) + WITHDRAW_REPORT="${WITHDRAW_REPORT}${WITHDRAW_REPORT:+ }${E2E_PAIVANA_AMOUNT}=OK(paivana)" + elif [ "$wc" = "2" ]; then + WITHDRAW_LAG_N=$((WITHDRAW_LAG_N + 1)) + WITHDRAW_REPORT="${WITHDRAW_REPORT}${WITHDRAW_REPORT:+ }${E2E_PAIVANA_AMOUNT}=LAG(paivana)" + else + WITHDRAW_FAIL_N=$((WITHDRAW_FAIL_N + 1)) + WITHDRAW_REPORT="${WITHDRAW_REPORT}${WITHDRAW_REPORT:+ }${E2E_PAIVANA_AMOUNT}=FAIL(paivana)" + fi + wait_wallet_balance "$(python3 -c "print(max(0, $pay_need - 1))" 2>/dev/null || echo 4199)" 12 3 || true + av_now=$(wallet_avail_num) + fi + + if ! python3 -c "import sys; sys.exit(0 if float(sys.argv[1]) + 1e-9 >= float(sys.argv[2]) else 1)" "$av_now" "$pay_need" 2>/dev/null; then + warn "paivana" "skip pay — insufficient avail ${CUR}:${av_now} (need ${pay_need})" + PAIVANA_REPORT="${PAIVANA_REPORT:+$PAIVANA_REPORT+}SKIP(bal)" + else + _E2E_INST_SAVE="$INST" + INST="${E2E_PAIVANA_INSTANCE}" + set +e + e2e_one_pay_public_template \ + "${E2E_PAIVANA_TEMPLATE}" \ + "Paivana paywall" \ + "${E2E_PAIVANA_AMOUNT}" + pc=$? + set -e + INST="$_E2E_INST_SAVE" + unset _E2E_INST_SAVE + if [ "$pc" = "0" ]; then + PAIVANA_OK=1 + PAY_OK=1 + PAY_OK_N=$((PAY_OK_N + 1)) + PAIVANA_REPORT="${PAIVANA_REPORT:+$PAIVANA_REPORT+}PAY_OK" + ok "paivana" "template ${E2E_PAIVANA_TEMPLATE} paid ${E2E_PAIVANA_AMOUNT}" + else + PAIVANA_REPORT="${PAIVANA_REPORT:+$PAIVANA_REPORT+}PAY_FAIL" + warn "paivana" "template pay failed (${E2E_PAIVANA_TEMPLATE} · ${E2E_PAIVANA_AMOUNT})" + fi + fi + info "paivana summary" "${PAIVANA_REPORT:-?} · instance ${E2E_PAIVANA_INSTANCE}" +fi + +set_group report +section "e2e · report" +info "user" "$USER" +info "ATM withdraws" "$WITHDRAW_REPORT" +info "payments" "$PAY_REPORT" +info "shop" "${SHOP_REPORT:-(n/a)}" +info "paivana" "${PAIVANA_REPORT:-(n/a)}" +info "final balance" "$(fmt_bal "$SCRATCH/bal-live.out" 2>/dev/null || echo "(n/a)")" +info "scratch" "$SCRATCH" +# Success if we had coins and at least one pay, OR coins + only pay skips (nothing affordable) +if [ "${#BLOCKERS[@]}" -eq 0 ]; then + if [ "$WITHDRAW_OK" = "1" ] && [ "$PAY_OK" = "1" ]; then + exit 0 + fi + if [ "$WITHDRAW_OK" = "1" ] && [ "$PAY_OK_N" = "0" ] && [ "$PAY_FAIL_N" = "0" ]; then + warn "pay" "no payment tried/completed — withdraw OK" + exit 0 + fi + if [ "$WITHDRAW_OK" = "1" ] && [ "$PAY_OK" != "1" ]; then + # money there, pays failed → soft exit 1 without inventing blockers if already warned + exit 1 + fi +fi +exit 1 diff --git a/check_goa_ladder.sh b/check_goa_ladder.sh new file mode 100755 index 0000000..d7c60ae --- /dev/null +++ b/check_goa_ladder.sh @@ -0,0 +1,1364 @@ +#!/usr/bin/env bash +# check_goa_ladder.sh — withdraw/pay amount ladder (GOA + stage TESTPAYSAN) +# +# Flow (bank landings): +# 1) GET /intro/auto-account.json → personal *account-* (balance 0) +# 2) Mint pool withdrawals as explorer + confirm when selected +# 3) wallet-cli accept-uri only (no run-until-done) +# 4) bank confirm when selected; settle = balance + transfer_done +# +# Amounts: random strictly increasing; pins 0, max-1, max. +# Phase A: withdraw ladder (cumulative wallet). Phase B: pay ladder. +# +# Stacks: +# GOA — LADDER_MAX_AMOUNT = libeufin ceiling; explorer from koopa-admin-secrets +# TESTPAYSAN stage — auto max_wire from bank /config, min denom from /keys; +# explorer from stagepaysan secrets / SSH +# +# Usage: +# ./taler-monitoring.sh ladder +# ./taler-monitoring.sh -d stage.lefrancpaysan.ch ladder +# LADDER_MAX_AMOUNT=100 LADDER_STEPS=7 ./taler-monitoring.sh -d stage.lefrancpaysan.ch ladder +# +# Env (see body): LADDER_STEPS, LADDER_MAX_AMOUNT, LADDER_MIN_AMOUNT, LADDER_STACK_AUTO, +# LADDER_PAY, EXP_PW / EXP_PW_FILE, CLI_JS, MERCHANT_INSTANCE, … +set -euo pipefail +ROOT=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=lib.sh +source "$ROOT/lib.sh" +# When invoked standalone (not via taler-monitoring.sh), still load secrets.env +load_monitoring_secrets_env 2>/dev/null || true + +# Area ladder.* — withdraw/pay amount ladder (GOA + TESTPAYSAN) +# Groups: ladder.plan / ladder.load / ladder.withdraw / ladder.pay / ladder.report +set_area ladder +set_group plan +SECTION_T0=$(date +%s) +now_ms() { python3 -c 'import time; print(int(time.time()*1000))'; } +elapsed_ms() { + # elapsed_ms START_MS + python3 -c 'import sys; print(int(sys.argv[1]) - int(sys.argv[2]))' "$(now_ms)" "$1" +} + +: "${LADDER_TIMEOUT_S:=3600}" +: "${LADDER_SETTLE_ROUNDS:=18}" +: "${LADDER_SETTLE_SLEEP:=2}" +: "${EXP_USER:=explorer}" +# EXP_PW_FILE / EXP_PW optional overrides; otherwise read_secret + SECRETS_ROOT / stage SSH +: "${EXP_PW_FILE:=}" +: "${EXP_PW:=}" +: "${CLI_JS:=}" +# GOA libeufin amount ceiling (hacktivism). Stage auto-replaces via bank /config. +: "${LADDER_MAX_AMOUNT:=4503599627370496}" +: "${LADDER_STEPS:=23}" +: "${LADDER_LOAD:=1}" +: "${LADDER_PAY:=1}" +: "${LADDER_STACK_AUTO:=1}" +# Withdraw mids = pay mids × scale (so wallet can afford the pay ladder) +: "${LADDER_WITHDRAW_SCALE:=1.5}" +# Floor for random mids (must be ≥ smallest exchange coin; GOA min denom ≈ 0.000001) +: "${LADDER_MIN_AMOUNT:=0.000001}" +: "${LADDER_CONFIRM_POLLS:=40}" +: "${LADDER_PAY_SETTLE_ROUNDS:=6}" +: "${MERCHANT_INSTANCE:=goa-demo-cp4zqk}" +# Public variable template for stage pays (optional; fixed templates used when amount maps) +: "${LADDER_PAY_TEMPLATE:=}" +: "${LADDER_PAY_TEMPLATE_INSTANCE:=fermes-des-collines}" + +GOA_LADDER_CEILING="4503599627370496" + +# Resolve wallet-cli .mjs (no hardcoded laptop path) +if [ -z "${CLI_JS}" ] || [ ! -f "${CLI_JS}" ]; then + CLI_JS=$(find_wallet_cli 2>/dev/null || true) +fi +if [ -z "${CLI_JS}" ] || [ ! -f "${CLI_JS}" ]; then + # allow PATH wrapper as last resort (wcli falls back to taler-wallet-cli) + CLI_JS="" +fi + +CUR="${EXPECT_CURRENCY:-GOA}" +BANK="${BANK_PUBLIC%/}" +EX="${EXCHANGE_PUBLIC%/}/" +MER="${MERCHANT_PUBLIC%/}" +INST="${MERCHANT_INSTANCE}" + +# --- Stack-specific ladder maxima (TESTPAYSAN stage) --- +# Uses bank max_wire_transfer_amount and exchange min denom when still on GOA defaults. +apply_ladder_stack_defaults() { + [ "${LADDER_STACK_AUTO}" = "1" ] || return 0 + if [ "${CUR}" != "TESTPAYSAN" ]; then + case "${TALER_DOMAIN:-}" in + stage.lefrancpaysan.ch|stage.bank.lefrancpaysan.ch|stage.exchange.lefrancpaysan.ch|stage.monnaie.lefrancpaysan.ch) + CUR="TESTPAYSAN" + ;; + *) return 0 ;; + esac + fi + + local bank_cfg max_wire min_denom + bank_cfg=$(curl -sS -m 12 "${BANK}/config" 2>/dev/null || true) + max_wire=$(printf '%s' "$bank_cfg" | python3 -c ' +import json,sys +try: + d=json.load(sys.stdin) +except Exception: + print("") + raise SystemExit(0) +a=d.get("max_wire_transfer_amount") or "" +print(a.split(":",1)[-1] if a else "") +' 2>/dev/null || true) + + min_denom=$(curl -sS -m 25 -H 'Accept: application/json' "${EX%/}/keys" 2>/dev/null | python3 -c ' +import json,sys +try: + d=json.load(sys.stdin) +except Exception: + print("") + raise SystemExit(0) +den=d.get("denoms") or d.get("denominations") or [] +vals=[] +for x in den if isinstance(den,list) else []: + v=x.get("value") or x.get("amount") or "" + if not v: continue + try: vals.append(float(str(v).split(":")[-1])) + except Exception: pass +print(min(vals) if vals else "") +' 2>/dev/null || true) + + # Fallback maxima if public config unreachable + [ -n "$max_wire" ] || max_wire="2000" + [ -n "$min_denom" ] || min_denom="0.01" + + # Only rewrite when caller left GOA defaults (explicit LADDER_MAX_AMOUNT=… kept) + if [ -n "$max_wire" ] && { [ "${LADDER_MAX_AMOUNT}" = "${GOA_LADDER_CEILING}" ] || [ "${LADDER_MAX_AMOUNT}" = "4503599627370496" ]; }; then + LADDER_MAX_AMOUNT="$max_wire" + fi + if [ -n "$min_denom" ] && { [ "${LADDER_MIN_AMOUNT}" = "0.000001" ] || [ "${LADDER_MIN_AMOUNT}" = "0.00000001" ]; }; then + LADDER_MIN_AMOUNT="$min_denom" + fi + # Slightly fewer rungs on stage (full GOA 23 still ok if user set LADDER_STEPS) + if [ "${LADDER_STEPS}" = "23" ]; then + LADDER_STEPS=15 + fi + # Stage farmer shops: private goa-demo instance does not exist + if [ "${MERCHANT_INSTANCE}" = "goa-demo-cp4zqk" ]; then + MERCHANT_INSTANCE="${LADDER_PAY_TEMPLATE_INSTANCE:-fermes-des-collines}" + INST="$MERCHANT_INSTANCE" + fi + # Prefer public templates for pays when no merchant token (set later) + : "${E2E_USE_TEMPLATES:=1}" + export E2E_USE_TEMPLATES + export LADDER_MAX_AMOUNT LADDER_MIN_AMOUNT LADDER_STEPS MERCHANT_INSTANCE + info "ladder stack" "TESTPAYSAN · max=${CUR}:${LADDER_MAX_AMOUNT} min=${CUR}:${LADDER_MIN_AMOUNT} steps=${LADDER_STEPS} (from bank max_wire / keys denoms)" +} +apply_ladder_stack_defaults +SCRATCH=$(mktemp -d) +WDB="$SCRATCH/wallet.sqlite3" +REPORT_DIR="${LADDER_REPORT_DIR:-$SCRATCH}" +mkdir -p "$REPORT_DIR" +TSV="$REPORT_DIR/ladder-results.tsv" +PAY_TSV="$REPORT_DIR/ladder-pay-results.tsv" +JSON="$REPORT_DIR/ladder-report.json" +LOAD_BEFORE="$REPORT_DIR/load-before.json" +LOAD_AFTER="$REPORT_DIR/load-after.json" +echo -e "rung\trange\tamount\tstatus\tms_mint\tms_accept\tms_confirm\tms_settle\tms_total\twid\tnote" >"$TSV" +echo -e "rung\trange\tamount\tstatus\tms_order\tms_handle\tms_settle\tms_total\toid\tnote" >"$PAY_TSV" + +ladder_over() { + local now + now=$(date +%s) + [ $((now - SECTION_T0)) -ge "$LADDER_TIMEOUT_S" ] +} + +wcli() { + if [ -n "${CLI_JS:-}" ] && [ -f "$CLI_JS" ]; then + node "$CLI_JS" --wallet-db="$WDB" --no-throttle "$@" + else + taler-wallet-cli --wallet-db="$WDB" --no-throttle "$@" + fi +} + +# Explorer pool password: env → file override → stack secrets → SSH +resolve_explorer_pw() { + local pw="" f="" host="" + if [ -n "${EXP_PW:-}" ]; then + printf '%s' "$EXP_PW" + return 0 + fi + if [ -n "${EXP_PW_FILE:-}" ] && [ -f "$EXP_PW_FILE" ]; then + tr -d '\n\r' <"$EXP_PW_FILE" + return 0 + fi + + # Stage TESTPAYSAN — never use GOA koopa-admin-secrets explorer pw first + if [ "${CUR}" = "TESTPAYSAN" ] \ + || [[ "${TALER_DOMAIN:-}" == stage.*lefrancpaysan* ]] \ + || [[ "${TALER_DOMAIN:-}" == *stage.lefrancpaysan* ]]; then + for f in \ + "${FRANCPAYSAN_SECRETS:-}/stage/bank-explorer-password.txt" \ + "${HOME}/francpaysan-secrets/stage/bank-explorer-password.txt" \ + "${HOME}/src/francpaysan-secrets/stage/bank-explorer-password.txt" \ + "${HOME}/taler/src/francpaysan-secrets/stage/bank-explorer-password.txt" \ + "${HOME}/.config/taler-landing/stage-bank-explorer-password.txt" + do + [ -n "$f" ] && [ -f "$f" ] || continue + tr -d '\n\r' <"$f" + return 0 + done + for host in \ + "${INSIDE_SSH:-}" \ + francpaysan-stage-user \ + francpaysan-host + do + [ -n "$host" ] || continue + pw=$(ssh -o BatchMode=yes -o ConnectTimeout=12 "$host" \ + 'tr -d "\n\r" /dev/null \ + || sudo tr -d "\n\r" /dev/null' \ + 2>/dev/null || true) + if [ -n "$pw" ]; then + printf '%s' "$pw" + return 0 + fi + done + return 1 + fi + + # GOA / default: SECRETS_ROOT / ~/.config / koopa SSH + if pw=$(read_secret "taler-bank/bank-explorer-password.txt" 2>/dev/null) && [ -n "$pw" ]; then + printf '%s' "$pw" + return 0 + fi + for f in \ + "${SECRETS_ROOT:+${SECRETS_ROOT}/taler-bank/bank-explorer-password.txt}" \ + "${HOME}/src/koopa/koopa-admin-secrets/koopa/host-root/taler-bank/bank-explorer-password.txt" \ + "${HOME}/.config/taler-landing/bank-explorer-password.txt" + do + [ -n "$f" ] && [ -f "$f" ] || continue + tr -d '\n\r' <"$f" + return 0 + done + return 1 +} + +wallet_avail() { + wcli balance 2>/dev/null | python3 -c ' +import json,sys +t=sys.stdin.read() +i=t.find("{") +if i<0: + print("0"); raise SystemExit +d=json.loads(t[i:t.rfind("}")+1]) +cur=sys.argv[1] +for b in d.get("balances") or []: + a=b.get("available") or "" + if a.startswith(cur+":"): + print(a.split(":",1)[1]); raise SystemExit +print("0") +' "$CUR" 2>/dev/null || echo "0" +} + +# Build paired pay + withdraw ladders (same step count, shared random shape). +# Pay: [0] + log-uniform mids + [max-1] + [max] +# Wd: [0] + max(mid, mid×scale) mids + [max-1] + [max] (mids higher → enough to spend) +# Writes: $1 = withdraw list file, $2 = pay list file (space-separated CUR:amt lines as one line each) +build_ladder_pair() { + local wd_out="$1" pay_out="$2" + python3 - <<'PY' "$CUR" "${LADDER_MAX_AMOUNT}" "${LADDER_STEPS}" "${LADDER_MIN_AMOUNT}" "${LADDER_WITHDRAW_SCALE}" "$wd_out" "$pay_out" +import math, random, sys +from decimal import Decimal, ROUND_HALF_UP, ROUND_UP +from pathlib import Path + +cur = sys.argv[1] +max_amt = Decimal(sys.argv[2]) +steps = max(1, int(sys.argv[3])) +min_amt = Decimal(sys.argv[4]) +scale = Decimal(sys.argv[5]) +wd_path, pay_path = Path(sys.argv[6]), Path(sys.argv[7]) +if min_amt <= 0: + min_amt = Decimal("0.000001") +if min_amt >= max_amt: + min_amt = max_amt / Decimal(1000) +if scale < 1: + scale = Decimal(1) +max_m1 = max_amt - Decimal(1) if max_amt > 1 else max_amt + +def fmt(v: Decimal) -> str: + q = v.quantize(Decimal("0.00000001"), rounding=ROUND_HALF_UP) + if q == q.to_integral(): + return format(int(q), "d") + return format(q, "f").rstrip("0").rstrip(".") + +def quant(v: Decimal) -> Decimal: + if v >= 1: + return v.quantize(Decimal(1), rounding=ROUND_HALF_UP) + if v < min_amt: + return min_amt + return v.quantize(Decimal("0.00000001"), rounding=ROUND_UP) + +def amt(v: Decimal) -> str: + return "%s:%s" % (cur, fmt(v)) + +def make_mids(n_mid: int, hi_cap: Decimal): + if n_mid <= 0 or hi_cap <= min_amt: + return [] + lo, hi = float(min_amt), float(hi_cap) * 0.999999 + if hi <= lo: + hi = lo * 10 + cuts = sorted(math.exp(random.uniform(math.log(lo), math.log(hi))) for _ in range(n_mid)) + out, prev = [], Decimal(0) + for c in cuts: + v = quant(Decimal(str(c))) + if v <= prev: + step = min_amt if prev < 1 else max(prev * Decimal("1e-6"), Decimal(1)) + v = quant(prev + step) + if v >= hi_cap: + v = quant(hi_cap - (Decimal(1) if hi_cap > 1 else min_amt)) + if v <= prev or v >= hi_cap: + continue + out.append(v) + prev = v + return out + +# Build withdraw ladder first (full range), then pay mids = withdraw/scale +# so each pay mid is always cheaper than the matching withdraw mid. +if steps == 1: + wd_vals = [max_amt] +elif steps == 2: + wd_vals = [Decimal(0), max_amt] +else: + n_mid = max(0, steps - 3) + wd_vals = [Decimal(0)] + make_mids(n_mid, max_m1) + [max_m1, max_amt] + while len(wd_vals) > steps and len(wd_vals) > 3: + wd_vals.pop(len(wd_vals) // 2) + while len(wd_vals) < steps and len(wd_vals) >= 2: + i = max(1, len(wd_vals) - 2) + a, b = wd_vals[i - 1], wd_vals[i] + if a <= 0: + m = max(min_amt, b / 2 if b > 0 else min_amt) + else: + m = (a * b).sqrt() if a * b > 0 else (a + b) / 2 + m = quant(m) + if m <= a or m >= b: + break + wd_vals.insert(i, m) + wd_vals = wd_vals[:steps] + +pay_vals = [] +prev_p = Decimal(-1) +for i, w in enumerate(wd_vals): + is_first = i == 0 + is_last = i == len(wd_vals) - 1 + is_m1 = (not is_last) and w == max_m1 and i == len(wd_vals) - 2 + if is_first and w == 0: + pay_vals.append(Decimal(0)) + prev_p = Decimal(0) + continue + if is_last: + pay_vals.append(max_amt) + continue + if is_m1 or w == max_m1: + pay_vals.append(max_m1) + prev_p = max_m1 + continue + # pay mid = withdraw / scale (strictly less funding needed per step) + p = quant(w / scale) if scale > 0 else w + if p < min_amt and w >= min_amt: + p = min_amt + if p <= prev_p: + p = quant(prev_p + (min_amt if prev_p < 1 else Decimal(1))) + if p >= w: + # keep pay strictly below this withdraw rung when possible + p = quant(w - (Decimal(1) if w > 1 else min_amt)) if w > prev_p else prev_p + if p <= prev_p: + p = prev_p # flat ok only if stuck; still ≤ w + pay_vals.append(p) + prev_p = p + +assert len(pay_vals) == len(wd_vals) +# sanity: every non-pin pay mid ≤ matching withdraw mid +for i, (p, w) in enumerate(zip(pay_vals, wd_vals)): + if i == 0 or i >= len(wd_vals) - 2: + continue + if p > w: + pay_vals[i] = w + +wd_path.write_text(" ".join(amt(v) for v in wd_vals) + "\n") +pay_path.write_text(" ".join(amt(v) for v in pay_vals) + "\n") +print(" ".join(amt(v) for v in wd_vals)) +PY +} + +# shellcheck source=metrics.sh +source "$ROOT/metrics.sh" +METRICS_DIR="$REPORT_DIR" +ALT_UNITS_FILE="${REPORT_DIR}/alt_unit_names.json" +export METRICS_DIR CUR WDB CLI_JS ALT_UNITS_FILE +# Ladder can disable host load without killing coin metrics +if [ "${LADDER_LOAD:-1}" = "0" ]; then + METRICS_LOAD=0 + export METRICS_LOAD +fi + +section "ladder · ${CUR} withdraw (0 → random → max · ${LADDER_STEPS} steps)" +info "bank" "$BANK" +info "exchange" "$EX" +info "currency" "$CUR" +# alt_unit_names for human amounts (Kilo-GOA / Mega-GOA / … + GOA in parentheses) +if metrics_load_alt_units "${EX%/}/config"; then + info "alt_unit_names" "from ${EX%/}/config → $ALT_UNITS_FILE" +else + warn "alt_unit_names" "using built-in SI fallback ($ALT_UNITS_FILE)" +fi +info "budget" "${LADDER_TIMEOUT_S}s" +_max_m1=$(python3 -c 'import sys; from decimal import Decimal; m=Decimal(sys.argv[1]); print(m-1 if m>1 else m)' "${LADDER_MAX_AMOUNT}" 2>/dev/null || echo "${LADDER_MAX_AMOUNT}-1") +info "steps" "${LADDER_STEPS} (0 + random≥${LADDER_MIN_AMOUNT} + max-1=${CUR}:${_max_m1} + max=${CUR}:${LADDER_MAX_AMOUNT})" +info "withdraw_scale" "${LADDER_WITHDRAW_SCALE}× pay mids (fund pay ladder)" +info "pay_phase" "$([ "${LADDER_PAY}" = "1" ] && echo enabled || echo disabled)" +info "currency/domain" "${CUR} · ${TALER_DOMAIN:-?} · bank=${BANK}" + +set_group load +section "ladder · load snapshot (before withdraws)" +metrics_report_load "$LOAD_BEFORE" "ladder-start" || true +# Fresh wallet per rung — baseline empty (or last-rung DB if re-used later) +metrics_report_coins "ladder-start" || true + +if ! EXP_PW=$(resolve_explorer_pw); then + err bank "explorer password missing" \ + "set EXP_PW / EXP_PW_FILE or SECRETS_ROOT=…/koopa/host-root (taler-bank/bank-explorer-password.txt)${SECRETS_ROOT:+ · SECRETS_ROOT=${SECRETS_ROOT}}" + secrets_hint 2>/dev/null || true + exit 1 +fi +if [ -n "${SECRETS_ROOT:-}" ]; then + info "explorer secret" "resolved (SECRETS_ROOT=${SECRETS_ROOT} · stage uses stagepaysan secret first when CUR=TESTPAYSAN)" +else + info "explorer secret" "resolved via EXP_PW / EXP_PW_FILE / stage SSH / koopa" +fi +if [ -n "${CLI_JS:-}" ] && [ -f "${CLI_JS}" ]; then + info "wallet-cli" "$CLI_JS" +else + info "wallet-cli" "PATH taler-wallet-cli ($(command -v taler-wallet-cli 2>/dev/null || echo missing))" +fi + +# --- auto-account --- +t0=$(now_ms) +if ! curl -sS -m 30 -o "$SCRATCH/auto-account.json" "${BANK}/intro/auto-account.json"; then + err bank "auto-account.json unreachable" + exit 1 +fi +ms_auto=$(elapsed_ms "$t0") +if ! python3 -c 'import json; d=json.load(open("'"$SCRATCH"'/auto-account.json")); assert d.get("ok") or d.get("username")' 2>/dev/null; then + err bank "auto-account create failed" "$(head -c 120 "$SCRATCH/auto-account.json" | tr '\n' ' ')" + exit 1 +fi +ACCT_USER=$(python3 -c 'import json; print(json.load(open("'"$SCRATCH"'/auto-account.json"))["username"])') +ok "auto-account ${ACCT_USER} (${ms_auto}ms) — personal ${CUR}:0; pool=explorer" +info "auto-account password" "(see $SCRATCH/auto-account.json — not logged)" + +# --- explorer token --- +t0=$(now_ms) +TOK=$(curl -sS -m 20 -u "${EXP_USER}:${EXP_PW}" \ + -H 'Content-Type: application/json' \ + -d '{"scope":"readwrite","duration":{"d_us":3600000000}}' \ + "${BANK}/accounts/${EXP_USER}/token" \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["access_token"])') +ms_tok=$(elapsed_ms "$t0") +[ -n "$TOK" ] || { err bank "explorer token failed"; exit 1; } +ok "explorer token (${ms_tok}ms)" + +# ONE cumulative wallet for all withdraws + pays (need balance to spend). +# force-select uses *last* reserve_pub from the current accept output. +wallet_prepare() { + local label="${1:-wallet}" + WDB="$SCRATCH/wallet-${label}.sqlite3" + export WDB + if [ ! -f "$WDB" ]; then + wcli exchanges add "$EX" >"$SCRATCH/ex-add-$label.out" 2>&1 || true + wcli exchanges update "$EX" >"$SCRATCH/ex-upd-$label.out" 2>&1 || true + wcli exchanges accept-tos "$EX" >"$SCRATCH/ex-tos-$label.out" 2>&1 || true + fi +} + +t0=$(now_ms) +rm -f "$SCRATCH/wallet-main.sqlite3" +wallet_prepare "main" +ms_tos=$(elapsed_ms "$t0") +ok "wallet exchange + ToS (${ms_tos}ms) — cumulative DB for withdraw+pay (no run-until-done)" + +build_ladder_pair "$SCRATCH/ladder-wd-plan.txt" "$SCRATCH/ladder-pay-plan.txt" +LADDER_LIST=$(tr -d '\n' <"$SCRATCH/ladder-wd-plan.txt") +PAY_LIST=$(tr -d '\n' <"$SCRATCH/ladder-pay-plan.txt") +: "${LADDER_MAX_RUNGS:=99}" +# shellcheck disable=SC2086 +set -- $LADDER_LIST +if [ "$#" -gt "$LADDER_MAX_RUNGS" ]; then + # shellcheck disable=SC2046 + set -- $(printf '%s\n' "$@" | head -n "$LADDER_MAX_RUNGS") +fi +LADDER_N=$# +info "withdraw plan" "$*" +info "withdraw plan (alt)" "$(format_amount_list_alt "$@")" +info "pay plan" "$PAY_LIST" +# shellcheck disable=SC2086 +info "pay plan (alt)" "$(format_amount_list_alt $PAY_LIST)" +printf '%s\n' "$@" >"$SCRATCH/ladder-plan.txt" +printf '%s\n' $PAY_LIST >"$SCRATCH/ladder-pay-plan-lines.txt" 2>/dev/null || true + +OK_N=0 +FAIL_N_L=0 +PAY_OK_N=0 +PAY_FAIL_N=0 +STOP_REASON="" +STOP_AMOUNT="" +declare -a RUNG_JSON=() + +set_group withdraw +section "ladder · phase A · withdraw (${LADDER_N} rungs)" +rung=0 +for AMT in "$@"; do + rung=$((rung + 1)) + if ladder_over; then + STOP_REASON="timeout budget ${LADDER_TIMEOUT_S}s" + warn ladder "time budget exhausted" \ + "problem: LADDER_TIMEOUT_S=${LADDER_TIMEOUT_S}s reached after ${OK_N} ok rungs; remaining amounts not tried" + break + fi + + section "ladder · rung $rung $AMT · $(format_amount_alt "$AMT")" + tag=$(printf '%s' "$AMT" | tr '.:' '__') + # TSV "range" column: fixed pins at ends, random mids (refined after AMT_NUM) + if [ "$rung" -eq 1 ]; then + range_note="pin:0" + elif [ "$rung" -eq "$LADDER_N" ]; then + range_note="pin:max" + elif [ "$rung" -eq $((LADDER_N - 1)) ] && [ "$LADDER_N" -ge 3 ]; then + range_note="pin:max-1" + else + range_note="random" + fi + t_rung=$(now_ms) + ms_mint=0 ms_accept=0 ms_confirm=0 ms_settle=0 + note="" + status="FAIL" + WID="-" + FORCE_SEL_409_LOGGED=0 + + # keep cumulative main wallet + wallet_prepare "main" + + # mint from explorer pool + t0=$(now_ms) + code=$(curl -sS -m 30 -o "$SCRATCH/wd-$tag.json" -w '%{http_code}' \ + -H "Authorization: Bearer ${TOK}" \ + -H 'Content-Type: application/json' \ + -d "{\"amount\":\"${AMT}\"}" \ + "${BANK}/accounts/${EXP_USER}/withdrawals") + ms_mint=$(elapsed_ms "$t0") + WID=$(python3 -c 'import json;d=json.load(open("'"$SCRATCH"'/wd-'"$tag"'.json"));print(d.get("withdrawal_id") or "")' 2>/dev/null || true) + URI=$(python3 -c 'import json;u=json.load(open("'"$SCRATCH"'/wd-'"$tag"'.json")).get("taler_withdraw_uri") or "";print(u.replace(":443/","/"))' 2>/dev/null || true) + # numeric amount (for zero / settle / ceiling special-cases) + AMT_NUM=$(python3 -c 'import sys; print(sys.argv[1].split(":",1)[-1])' "$AMT") + IS_ZERO=0 + python3 -c 'import sys; from decimal import Decimal; sys.exit(0 if Decimal(sys.argv[1])==0 else 1)' "$AMT_NUM" 2>/dev/null && IS_ZERO=1 + IS_MAX_PIN=0 + IS_MAX_M1_PIN=0 + if [ "$AMT_NUM" = "${LADDER_MAX_AMOUNT}" ]; then + IS_MAX_PIN=1 + range_note="pin:max" + elif [ "$AMT_NUM" = "$((LADDER_MAX_AMOUNT - 1))" ] 2>/dev/null || \ + [ "$AMT_NUM" = "$(python3 -c 'print(int("'"$LADDER_MAX_AMOUNT"'")-1)')" ]; then + IS_MAX_M1_PIN=1 + range_note="pin:max-1" + fi + + if [ "$code" != "200" ] && [ "$code" != "201" ] || [ -z "$WID" ] || [ -z "$URI" ]; then + # strip quotes so STOP_REASON never breaks later shell/python argv + note="mint HTTP $code $(head -c 100 "$SCRATCH/wd-$tag.json" 2>/dev/null | tr '\n\"' ' ')" + ms_total=$(elapsed_ms "$t_rung") + if [ "$IS_ZERO" = "1" ]; then + # Probe only: bank may reject GOA:0 — record and continue ladder + status="ZERO_REJECT" + note="zero-withdraw rejected (expected possible): $note" + warn bank "mint $AMT rejected" \ + "problem: bank will not create a GOA:0 withdrawal (zero amount probe). Ladder continues. detail: $note" + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + OK_N=$((OK_N + 1)) + continue + fi + # Absolute max is a ceiling probe — bank often returns SQL P0001/5110; do not abort. + if [ "$IS_MAX_PIN" = "1" ]; then + status="CEILING_REJECT" + note="absolute max rejected (ceiling probe; max-1 is the hard pin): $note" + warn bank "mint $AMT rejected (ceiling)" \ + "problem: bank rejects absolute LADDER_MAX_AMOUNT (often SQL P0001/5110). max-1 rung is the last expected success. Ladder continues to report. detail: $note" + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + continue + fi + err bank "mint $AMT failed" "$note" + status="FAIL_MINT" + STOP_REASON="$note" + STOP_AMOUNT="$AMT" + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + FAIL_N_L=$((FAIL_N_L + 1)) + break + fi + ok "mint $AMT ($WID) ${ms_mint}ms" + + before=$(wallet_avail) + + # accept + t0=$(now_ms) + if wcli withdraw accept-uri --exchange "$EX" "$URI" >"$SCRATCH/accept-$tag.out" 2>&1; then + ms_accept=$(elapsed_ms "$t0") + ok "accept-uri $AMT ${ms_accept}ms" + else + ms_accept=$(elapsed_ms "$t0") + note="accept-uri failed: $(tail -c 200 "$SCRATCH/accept-$tag.out" | tr '\n' ' ')" + ms_total=$(elapsed_ms "$t_rung") + # Wallet 7006: no denominations for this amount (0, sub-denom dust, etc.) — warn & continue + if [ "$IS_ZERO" = "1" ] || grep -qE 'code: 7006|"code"[[:space:]]*:[[:space:]]*7006|No denominations could be selected' \ + "$SCRATCH/accept-$tag.out" 2>/dev/null; then + if [ "$IS_ZERO" = "1" ]; then + status="ZERO_SKIP" + note="zero-withdraw skip (7006 / no denoms): $note" + warn wallet "accept $AMT skipped" \ + "problem: wallet code 7006 — no coin denominations for GOA:0 (zero amount cannot be withdrawn as coins). Ladder continues. detail: $note" + else + status="SKIP_DENOM" + note="skip amount (wallet 7006 no denoms): $note" + warn wallet "accept $AMT skipped" \ + "problem: wallet code 7006 — no denominations match this amount (below smallest coin or not combinable). Ladder continues with next rung. detail: $note" + fi + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + continue + fi + err wallet "accept $AMT" "$note" + status="FAIL_ACCEPT" + STOP_REASON="$note" + STOP_AMOUNT="$AMT" + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + FAIL_N_L=$((FAIL_N_L + 1)) + break + fi + + # Confirm ASAP when bank status is selected. No run-until-done (hangs on macOS/wallet). + # Server-side auto-confirm only watches landing withdraw-watch.ids — ladder must confirm itself. + bank_st() { + curl -sS -m 8 "${BANK}/taler-integration/withdrawal-operation/${WID}" 2>/dev/null \ + | python3 -c 'import json,sys; print((json.load(sys.stdin).get("status") or "").strip())' 2>/dev/null || true + } + do_confirm() { + curl -sS -m 15 -o "$SCRATCH/conf-$tag.json" -w '%{http_code}' -X POST \ + -H "Authorization: Bearer ${TOK}" -H 'Content-Type: application/json' -d '{}' \ + "${BANK}/accounts/${EXP_USER}/withdrawals/${WID}/confirm" + } + # Collect reserve_pub candidates for *this* withdrawal (WID + amount). + # Cumulative wallets re-print old reserves in accept/tx dumps — never trust a single "last" blindly. + # Prints unique pubs one per line, preferred order first. + extract_rpubs_for_wid() { + python3 - "$WID" "$AMT" "$SCRATCH/accept-$tag.out" "$SCRATCH/tx-$tag.json" "$SCRATCH/used-rpubs.txt" <<'PY' +import json, re, sys +from pathlib import Path + +wid = sys.argv[1] +amt = sys.argv[2] +paths = sys.argv[3:5] +used_path = sys.argv[5] +used = set() +if Path(used_path).is_file(): + used = {ln.strip() for ln in open(used_path) if ln.strip()} + +def walk_collect(o, bag, ctx=None): + ctx = dict(ctx or {}) + if isinstance(o, dict): + for k, v in o.items(): + kl = str(k).lower() + if kl in ("withdrawal_id", "withdraw_id", "wopid", "id") and isinstance(v, str): + ctx["id"] = v + if kl in ("amount", "rawamount", "instructedamount") and isinstance(v, str): + ctx["amount"] = v + if kl in ("taler_withdraw_uri", "talerwithdrawuri", "uri") and isinstance(v, str): + ctx["uri"] = v + if kl in ("reserve_pub", "reservepub") and isinstance(v, str) and len(v) >= 40: + bag.append((v, dict(ctx))) + walk_collect(v, bag, ctx) + elif isinstance(o, list): + for i in o: + walk_collect(i, bag, ctx) + +raw_pubs = [] # ordered as found +scored = [] # (score, pub) higher better +blob_all = "" +for p in paths: + try: + blob_all += open(p, errors="replace").read() + "\n" + except Exception: + pass + +# 1) full JSON objects in files +for p in paths: + try: + t = open(p, errors="replace").read() + except Exception: + continue + # whole-file JSON + for m in re.finditer(r"\{", t): + try: + o = json.loads(t[m.start():]) + except Exception: + continue + bag = [] + walk_collect(o, bag) + for pub, ctx in bag: + raw_pubs.append(pub) + score = 0 + cid = str(ctx.get("id") or "") + camt = str(ctx.get("amount") or "") + curi = str(ctx.get("uri") or "") + if wid and wid in cid: + score += 100 + if wid and wid in curi: + score += 80 + if amt and (camt == amt or camt.endswith(amt.split(":", 1)[-1])): + score += 40 + if pub in used: + score -= 200 + scored.append((score, pub)) + +# 2) regex fallback on accept file only (more current) +try: + acc = open(paths[0], errors="replace").read() +except Exception: + acc = "" +for pat in ( + r"\"reserve_pub\"\s*:\s*\"([A-Z0-9]{40,})\"", + r"\"reservePub\"\s*:\s*\"([A-Z0-9]{40,})\"", + r"reserve_pub[\"\s:=]+([A-Z0-9]{40,})", +): + for m in re.finditer(pat, acc, re.I): + pub = m.group(1) + raw_pubs.append(pub) + score = 10 + # proximity to WID in accept output + window = acc[max(0, m.start() - 400) : m.end() + 400] + if wid and wid in window: + score += 100 + if amt and amt in window: + score += 30 + if pub in used: + score -= 200 + scored.append((score, pub)) + +# prefer high score, then later occurrence +order = [] +seen = set() +for score, pub in sorted(scored, key=lambda x: (-x[0],), reverse=False): + # sort by score desc: use reverse sorted + pass +for score, pub in sorted(scored, key=lambda x: x[0], reverse=True): + if pub in seen or pub in used: + continue + seen.add(pub) + order.append(pub) +# append unused raw in reverse (newest-ish) +for pub in reversed(raw_pubs): + if pub in seen or pub in used: + continue + seen.add(pub) + order.append(pub) +for pub in order: + print(pub) +PY + } + + mark_rpub_used() { + local p="$1" + [ -n "$p" ] || return 0 + mkdir -p "$SCRATCH" 2>/dev/null || true + grep -qxF "$p" "$SCRATCH/used-rpubs.txt" 2>/dev/null || echo "$p" >>"$SCRATCH/used-rpubs.txt" + } + + force_select_if_needed() { + local st_now="$1" + [ "$st_now" = "pending" ] || [ -z "$st_now" ] || return 0 + local rpub epayto code_fs any=0 + # refresh tx dump each try (wallet may attach reserve late) + wcli transactions >"$SCRATCH/tx-$tag.json" 2>&1 || true + epayto=$(curl -sS -m 10 "${EX%/}/keys" 2>/dev/null | python3 -c ' +import json,sys +d=json.load(sys.stdin) +acc=d.get("accounts") or [] +for a in acc: + p=a.get("payto_uri") or a.get("payto_address") or "" + if "x-taler-bank" in p or "exchange" in p: + print(p); break +else: + if acc: print(acc[0].get("payto_uri") or "") +' 2>/dev/null || true) + if [ -z "$epayto" ]; then + warn bank "force-select skipped" "problem: exchange payto empty from /keys" + return 0 + fi + # Try candidates until bank leaves pending (200/204) or we exhaust + while IFS= read -r rpub; do + [ -n "$rpub" ] || continue + any=1 + code_fs=$(curl -sS -m 12 -o "$SCRATCH/force-sel-$tag.json" -w '%{http_code}' -X POST \ + -H 'Content-Type: application/json' \ + -d "{\"reserve_pub\":\"${rpub}\",\"selected_exchange\":\"${epayto}\"}" \ + "${BANK}/taler-integration/withdrawal-operation/${WID}" 2>/dev/null || echo "000") + if [ "$code_fs" = "200" ] || [ "$code_fs" = "204" ]; then + info "force-select" "HTTP ${code_fs} rpub=${rpub:0:12}… (ok for WID ${WID:0:8})" + mark_rpub_used "$rpub" + return 0 + fi + if [ "$code_fs" = "409" ]; then + # 5114 = this reserve already bound to another op — not "out of money" + info "force-select" "HTTP 409 rpub=${rpub:0:12}… (stale/used reserve — not balance; trying next)" + mark_rpub_used "$rpub" + continue + fi + info "force-select" "HTTP ${code_fs} rpub=${rpub:0:12}… body=$(tr '\n' ' ' <"$SCRATCH/force-sel-$tag.json" 2>/dev/null | head -c 120)" + # other errors: still try next candidate + done < <(extract_rpubs_for_wid) + if [ "$any" != "1" ]; then + warn bank "force-select skipped" \ + "problem: no reserve_pub for this withdraw (WID=${WID:0:8}…); wallet may not have selected yet" + fi + } + + # No run-until-done (hangs / banned). Read transactions only for reserve_pub candidates. + wcli transactions >"$SCRATCH/tx-$tag.json" 2>&1 || true + + t0=$(now_ms) + conf_ok=0 + st="" + # Poll bank status; force-select while pending; confirm as soon as selected + for i in $(seq 1 "${LADDER_CONFIRM_POLLS}"); do + ladder_over && break + st=$(bank_st) + case "$st" in + selected) + ccode=$(do_confirm) + if [ "$ccode" = "204" ] || [ "$ccode" = "200" ]; then + conf_ok=1 + info "confirm" "HTTP $ccode after selected (poll $i)" + else + note="confirm HTTP $ccode" + fi + break + ;; + confirmed) + conf_ok=1 + break + ;; + aborted) + note="withdrawal aborted by bank" + break + ;; + esac + # force-select while pending — try alternate rpubs on 5114 (not out-of-money) + if [ "$st" = "pending" ] || [ -z "$st" ]; then + if [ "$i" -eq 1 ] || [ "$i" -eq 2 ] || [ $((i % 3)) -eq 0 ]; then + force_select_if_needed "$st" + fi + fi + sleep 0.35 + done + ms_confirm=$(elapsed_ms "$t0") + st=$(printf '%s' "${st:-}" | tr -d '\r\n' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + if [ "$conf_ok" != "1" ]; then + note="${note:-confirm timeout last=${st:-empty}}" + # Soft: not confirmed — usually stale reserve_pub (5114), NOT empty pool balance + status="SKIP_CONFIRM" + warn bank "confirm $AMT skipped" \ + "problem: bank status='${st:-empty}' after ${LADDER_CONFIRM_POLLS} polls (want selected/confirmed). Usually reserve_pub mismatch (5114), not out-of-money — mint/accept already OK. detail: $note" + ms_total=$(elapsed_ms "$t_rung") + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + continue + fi + ok "confirm $AMT ${ms_confirm}ms (client, on selected)" + + # settle: poll wallet balance + bank transfer_done only — never run-until-done + t0=$(now_ms) + settled=0 + xfer="?" + if [ "$IS_ZERO" = "1" ]; then + settled=1 + note="zero-amount: no coin delta expected" + else + for r in $(seq 1 "$LADDER_SETTLE_ROUNDS"); do + ladder_over && break + after=$(wallet_avail) + if python3 -c " +from decimal import Decimal +import sys +sys.exit(0 if Decimal(sys.argv[1]) > Decimal(sys.argv[2]) else 1) +" "$after" "$before" 2>/dev/null; then + settled=1 + break + fi + xfer=$(curl -sS -m 10 "${BANK}/taler-integration/withdrawal-operation/${WID}" \ + | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("transfer_done"), d.get("status"))' 2>/dev/null || echo "?") + # bank done is enough to leave settle without hanging on wallet + if echo "$xfer" | grep -qi True; then + note="bank transfer_done (no run-until-done) avail=${after} $xfer" + break + fi + sleep "$LADDER_SETTLE_SLEEP" + done + fi + ms_settle=$(elapsed_ms "$t0") + after=$(wallet_avail) + ms_total=$(elapsed_ms "$t_rung") + if [ -z "${xfer:-}" ] || [ "$xfer" = "?" ]; then + xfer=$(curl -sS -m 10 "${BANK}/taler-integration/withdrawal-operation/${WID}" \ + | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("transfer_done"), d.get("status"))' 2>/dev/null || echo "?") + fi + + if [ "$settled" = "1" ]; then + status="OK" + note="${note:-avail=${CUR}:${after}}" + ok "settle $AMT → ${CUR}:${after} (settle ${ms_settle}ms, rung ${ms_total}ms)" + OK_N=$((OK_N + 1)) + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + metrics_report_coins "ladder-r${rung}-after-${tag}" || true + metrics_record_flow withdrawn "$AMT" || true + elif echo "$xfer" | grep -qi True; then + status="OK_BANK" + note="bank transfer_done avail=${after} $xfer (no run-until-done)" + ok "settle $AMT bank transfer_done (wallet avail=${CUR}:${after}, ${ms_settle}ms)" + OK_N=$((OK_N + 1)) + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + metrics_report_coins "ladder-r${rung}-after-${tag}" || true + metrics_record_flow withdrawn "$AMT" || true + else + note="no coins / no transfer_done avail=${after} $xfer" + err wallet "settle $AMT" "$note" + status="FAIL_SETTLE" + STOP_REASON="$note" + STOP_AMOUNT="$AMT" + FAIL_N_L=$((FAIL_N_L + 1)) + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + metrics_report_coins "ladder-r${rung}-fail-${tag}" || true + break + fi +done + +# --------------------------------------------------------------------------- +# Phase B — payment ladder (same shape: 0 → low → … → max-1 → max) +# --------------------------------------------------------------------------- +PAY_OK_N=0 +PAY_FAIL_N=0 +if [ "${LADDER_PAY}" = "1" ] && [ -n "${PAY_LIST:-}" ] && [ "$FAIL_N_L" -eq 0 ]; then + set_group pay + section "ladder · phase B · pay" + metrics_report_coins "before-pay-ladder" || true + # Merchant secret (same as e2e) — stage often has only public shop templates + MPW="${E2E_MERCHANT_TOKEN:-${MERCHANT_TOKEN:-}}" + if [ -z "$MPW" ]; then + MPW=$(read_secret "taler-merchant/merchant-${INST}-password.txt" 2>/dev/null || true) + fi + if [ -z "$MPW" ]; then + MPW=$(read_secret "taler-merchant/merchant-goa-demo-cp4zqk-password.txt" 2>/dev/null || true) + fi + if [ -z "$MPW" ] && [ "${CUR}" = "TESTPAYSAN" ]; then + for f in \ + "${FRANCPAYSAN_SECRETS:-}/stage/default-instance-token.txt" \ + "${HOME}/francpaysan-secrets/stage/default-instance-token.txt" \ + "${HOME}/src/francpaysan-secrets/stage/default-instance-token.txt" + do + [ -f "$f" ] || continue + MPW=$(tr -d '\n\r' <"$f") + break + done + if [ -z "$MPW" ]; then + for host in "${INSIDE_SSH:-}" francpaysan-stage-user francpaysan-host; do + [ -n "$host" ] || continue + MPW=$(ssh -o BatchMode=yes -o ConnectTimeout=12 "$host" \ + 'tr -d "\n\r" /dev/null || true' \ + 2>/dev/null || true) + [ -n "$MPW" ] && break + done + fi + fi + if [ -z "$MPW" ]; then + if [ "${CUR}" = "TESTPAYSAN" ]; then + # Stage: withdraw ladder is the main probe; pays need instance token or + # public templates for *exact* face values (not continuous ladder amounts). + info pay "no merchant token — skip pay ladder on TESTPAYSAN (withdraw maxima still covered)" + info pay "hint: set E2E_MERCHANT_TOKEN or use ./taler-monitoring.sh -d stage.lefrancpaysan.ch e2e for shop templates" + else + warn pay "no merchant token — skip pay ladder (set E2E_MERCHANT_TOKEN or secrets)" + fi + else + ok "merchant token" "instance ${INST}" + case "$MPW" in + secret-token:*) AUTH="Authorization: Bearer ${MPW}" ;; + *) AUTH="Authorization: Bearer secret-token:${MPW}" ;; + esac + wallet_prepare "main" + # shellcheck disable=SC2086 + set -- $PAY_LIST + PAY_N=$# + info "pay rungs" "$PAY_N · $*" + prung=0 + for PAMT in "$@"; do + prung=$((prung + 1)) + if ladder_over; then + warn pay "time budget exhausted" "after ${PAY_OK_N} ok pays" + break + fi + section "ladder · pay rung $prung $PAMT · $(format_amount_alt "$PAMT")" + ptag=$(printf '%s' "$PAMT" | tr '.:' '__') + if [ "$prung" -eq 1 ]; then + prange="pin:0" + elif [ "$prung" -eq "$PAY_N" ]; then + prange="pin:max" + elif [ "$prung" -eq $((PAY_N - 1)) ] && [ "$PAY_N" -ge 3 ]; then + prange="pin:max-1" + else + prange="random" + fi + PNUM=$(python3 -c 'import sys; print(sys.argv[1].split(":",1)[-1])' "$PAMT") + IS_PZERO=0 + python3 -c 'import sys; from decimal import Decimal; sys.exit(0 if Decimal(sys.argv[1])==0 else 1)' "$PNUM" 2>/dev/null && IS_PZERO=1 + IS_PMAX=0 + [ "$PNUM" = "${LADDER_MAX_AMOUNT}" ] && IS_PMAX=1 + t_pay=$(now_ms) + ms_order=0 ms_handle=0 ms_psettle=0 + pnote="" + pstatus="FAIL" + OID="-" + + metrics_report_coins "before-pay-${ptag}" || true + bal=$(wallet_avail) + + if [ "$IS_PZERO" = "1" ]; then + pstatus="ZERO_SKIP" + pnote="zero pay probe skipped" + warn pay "pay $PAMT skipped" "problem: zero amount order not useful. Ladder continues." + ms_total=$(elapsed_ms "$t_pay") + echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t${ms_handle}\t${ms_psettle}\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" + PAY_OK_N=$((PAY_OK_N + 1)) + continue + fi + + # Soft: absolute max pay is a ceiling probe (unlikely affordable / merchant may reject) + if [ "$IS_PMAX" = "1" ]; then + # try once; on fail CEILING_REJECT + : + fi + + # Insufficient balance → soft skip for max pin only; hard fail for mid/max-1 + if ! python3 -c 'import sys; from decimal import Decimal; sys.exit(0 if Decimal(sys.argv[1])>=Decimal(sys.argv[2]) else 1)' "$bal" "$PNUM" 2>/dev/null; then + pnote="insufficient balance avail=${CUR}:${bal} need=${PAMT}" + ms_total=$(elapsed_ms "$t_pay") + if [ "$IS_PMAX" = "1" ]; then + pstatus="CEILING_SKIP" + warn pay "pay $PAMT skipped (ceiling)" "$pnote" + echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t0\t0\t0\t${ms_total}\t-\t${pnote}" >>"$PAY_TSV" + continue + fi + err pay "pay $PAMT" "$pnote" + pstatus="FAIL_BALANCE" + PAY_FAIL_N=$((PAY_FAIL_N + 1)) + FAIL_N_L=$((FAIL_N_L + 1)) + STOP_REASON="$pnote" + STOP_AMOUNT="$PAMT" + echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t0\t0\t0\t${ms_total}\t-\t${pnote}" >>"$PAY_TSV" + break + fi + + t0=$(now_ms) + SUM_JSON=$(python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "ladder pay ${PAMT}" 2>/dev/null || echo '"ladder pay"') + curl -skS -m 20 -o "$SCRATCH/ord-$ptag.json" -X POST \ + -H "$AUTH" -H 'Content-Type: application/json' \ + -d "{\"order\":{\"summary\":${SUM_JSON},\"amount\":\"${PAMT}\",\"fulfillment_message\":\"ok\"},\"create_token\":true}" \ + "${MER}/instances/${INST}/private/orders" 2>"$SCRATCH/ord-$ptag.err" || true + ms_order=$(elapsed_ms "$t0") + OID=$(python3 -c 'import json,re,sys;t=open(sys.argv[1]).read() +try: print(json.loads(t).get("order_id") or "") +except Exception: + m=re.search(r"\"order_id\"\s*:\s*\"([^\"]+)\"",t); print(m.group(1) if m else "") +' "$SCRATCH/ord-$ptag.json" 2>/dev/null || true) + OTOK=$(python3 -c 'import json,re,sys;t=open(sys.argv[1]).read() +try: print(json.loads(t).get("token") or "") +except Exception: + m=re.search(r"\"token\"\s*:\s*\"([^\"]+)\"",t); print(m.group(1) if m else "") +' "$SCRATCH/ord-$ptag.json" 2>/dev/null || true) + if [ -z "$OID" ]; then + pnote="order create failed $(head -c 80 "$SCRATCH/ord-$ptag.json" 2>/dev/null | tr '\n\"' ' ')" + ms_total=$(elapsed_ms "$t_pay") + if [ "$IS_PMAX" = "1" ]; then + pstatus="CEILING_REJECT" + warn pay "pay $PAMT rejected (ceiling)" "$pnote" + echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t0\t0\t${ms_total}\t-\t${pnote}" >>"$PAY_TSV" + continue + fi + err pay "order $PAMT" "$pnote" + pstatus="FAIL_ORDER" + PAY_FAIL_N=$((PAY_FAIL_N + 1)) + FAIL_N_L=$((FAIL_N_L + 1)) + STOP_REASON="$pnote" + STOP_AMOUNT="$PAMT" + echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t0\t0\t${ms_total}\t-\t${pnote}" >>"$PAY_TSV" + break + fi + ok "order $OID ($PAMT) ${ms_order}ms" + + curl -skS -m 12 -o "$SCRATCH/ord-det-$ptag.json" -H "$AUTH" \ + "${MER}/instances/${INST}/private/orders/${OID}" 2>/dev/null || true + PAYURI=$(python3 -c 'import json,sys +try: print(json.load(open(sys.argv[1])).get("taler_pay_uri") or "") +except Exception: print("") +' "$SCRATCH/ord-det-$ptag.json" 2>/dev/null || true) + if [ -z "$PAYURI" ] && [ -n "$OTOK" ]; then + MH=$(python3 -c 'from urllib.parse import urlparse; print(urlparse("'"$MER"'").hostname or "taler.hacktivism.ch")' 2>/dev/null || echo "taler.hacktivism.ch") + PAYURI="taler://pay/${MH}/instances/${INST}/${OID}/?c=${OTOK}" + fi + PAYURI=$(printf '%s' "$PAYURI" | sed 's/:443\//\//g; s/:443?/?/g') + if [ -z "$PAYURI" ]; then + pnote="no pay URI for $OID" + ms_total=$(elapsed_ms "$t_pay") + err pay "uri $PAMT" "$pnote" + pstatus="FAIL_URI" + PAY_FAIL_N=$((PAY_FAIL_N + 1)) + FAIL_N_L=$((FAIL_N_L + 1)) + STOP_REASON="$pnote" + STOP_AMOUNT="$PAMT" + echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t0\t0\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" + break + fi + + t0=$(now_ms) + if ! wcli handle-uri --yes "$PAYURI" >"$SCRATCH/pay-$ptag.out" 2>&1; then + ms_handle=$(elapsed_ms "$t0") + pnote="handle-uri failed $(tail -c 100 "$SCRATCH/pay-$ptag.out" | tr '\n\"' ' ')" + ms_total=$(elapsed_ms "$t_pay") + if [ "$IS_PMAX" = "1" ]; then + pstatus="CEILING_REJECT" + warn pay "pay $PAMT handle failed (ceiling)" "$pnote" + echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t${ms_handle}\t0\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" + continue + fi + err pay "handle $PAMT" "$pnote" + pstatus="FAIL_HANDLE" + PAY_FAIL_N=$((PAY_FAIL_N + 1)) + FAIL_N_L=$((FAIL_N_L + 1)) + STOP_REASON="$pnote" + STOP_AMOUNT="$PAMT" + echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t${ms_handle}\t0\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" + break + fi + ms_handle=$(elapsed_ms "$t0") + ok "handle-uri $PAMT ${ms_handle}ms" + + # Short settle polls: merchant order + wallet tx only — never run-until-done + t0=$(now_ms) + settled=0 + r=0 + while [ "$r" -lt "${LADDER_PAY_SETTLE_ROUNDS}" ]; do + r=$((r + 1)) + wcli transactions >"$SCRATCH/tx-$ptag.out" 2>&1 || true + curl -skS -m 8 -o "$SCRATCH/ord-paid-$ptag.json" -H "$AUTH" \ + "${MER}/instances/${INST}/private/orders/${OID}" 2>/dev/null || true + if grep -qiE 'payment|paid|Payment' "$SCRATCH/tx-$ptag.out" 2>/dev/null \ + || python3 -c 'import json,sys +d=json.load(open(sys.argv[1])) +sys.exit(0 if d.get("paid") is True or str(d.get("order_status","")).lower()=="paid" else 1) +' "$SCRATCH/ord-paid-$ptag.json" 2>/dev/null; then + settled=1 + break + fi + sleep 0.5 + done + ms_psettle=$(elapsed_ms "$t0") + ms_total=$(elapsed_ms "$t_pay") + after=$(wallet_avail) + if [ "$settled" = "1" ]; then + pstatus="OK" + pnote="avail=${CUR}:${after}" + ok "pay settled $PAMT → bal ${CUR}:${after} (total ${ms_total}ms)" + PAY_OK_N=$((PAY_OK_N + 1)) + echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t${ms_handle}\t${ms_psettle}\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" + metrics_report_coins "after-pay-${ptag}" || true + metrics_record_flow spent "$PAMT" || true + else + pnote="not settled order=$OID avail=${CUR}:${after}" + if [ "$IS_PMAX" = "1" ]; then + pstatus="CEILING_REJECT" + warn pay "pay $PAMT not settled (ceiling)" "$pnote" + echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t${ms_handle}\t${ms_psettle}\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" + continue + fi + err pay "settle $PAMT" "$pnote" + pstatus="FAIL_SETTLE" + PAY_FAIL_N=$((PAY_FAIL_N + 1)) + FAIL_N_L=$((FAIL_N_L + 1)) + STOP_REASON="$pnote" + STOP_AMOUNT="$PAMT" + echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t${ms_handle}\t${ms_psettle}\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" + metrics_report_coins "after-pay-fail-${ptag}" || true + break + fi + done + metrics_report_coins "after-pay-ladder" || true + info "pay summary" "ok=$PAY_OK_N fail=$PAY_FAIL_N tsv=$PAY_TSV" + fi +elif [ "${LADDER_PAY}" = "1" ] && [ "$FAIL_N_L" -gt 0 ]; then + warn pay "skipped pay ladder" "withdraw phase already failed" +fi + +ms_phase=$(python3 -c 'import sys,time; print(int((time.time()-float(sys.argv[1]))*1000))' "$SECTION_T0") + +# --- report --- +set_group report +section "ladder · report" +info "auto-account" "$ACCT_USER" +info "ok_rungs" "$OK_N" +info "fail_rungs" "$FAIL_N_L" +info "pay_ok" "$PAY_OK_N" +info "pay_fail" "$PAY_FAIL_N" +info "phase_ms" "$ms_phase" +info "tsv" "$TSV" +info "pay_tsv" "$PAY_TSV" + +# speed summary via python — free-text stop reason via env (JSON " in bank errors +# used to break shell argv quoting → "syntax error near unexpected token '('") +export LADDER_REPORT_STOP_AMOUNT="${STOP_AMOUNT:-}" +export LADDER_REPORT_STOP_REASON="${STOP_REASON:-}" +python3 - "$TSV" "$PAY_TSV" "$JSON" "$OK_N" "$FAIL_N_L" "$PAY_OK_N" "$PAY_FAIL_N" "$ms_phase" "$ACCT_USER" "$CUR" <<'PY' +import csv, json, os, sys, statistics +tsv, pay_tsv, jpath = sys.argv[1:4] +ok_n, fail_n, pay_ok, pay_fail, phase_ms, acct, cur = sys.argv[4:11] +stop_amt = os.environ.get("LADDER_REPORT_STOP_AMOUNT") or None +stop_reason = os.environ.get("LADDER_REPORT_STOP_REASON") or None + +def load_rows(path): + rows = [] + try: + with open(path, newline="") as f: + for row in csv.DictReader(f, delimiter="\t"): + rows.append(row) + except Exception: + pass + return rows + +rows = load_rows(tsv) +prows = load_rows(pay_tsv) + +def nums(rs, key): + out = [] + for row in rs: + try: + out.append(int(row[key])) + except Exception: + pass + return out + +def stats(xs): + if not xs: + return {"n": 0} + return { + "n": len(xs), + "min_ms": min(xs), + "max_ms": max(xs), + "avg_ms": int(sum(xs) / len(xs)), + "p50_ms": int(statistics.median(xs)), + } + +report = { + "currency": cur, + "auto_account": acct, + "ok_rungs": int(ok_n), + "fail_rungs": int(fail_n), + "pay_ok": int(pay_ok), + "pay_fail": int(pay_fail), + "stopped_at_amount": stop_amt or None, + "stop_reason": stop_reason or None, + "phase_ms": int(phase_ms), + "timing": { + "mint": stats(nums(rows, "ms_mint")), + "accept": stats(nums(rows, "ms_accept")), + "confirm": stats(nums(rows, "ms_confirm")), + "settle": stats(nums(rows, "ms_settle")), + "rung_total": stats(nums(rows, "ms_total")), + "pay_order": stats(nums(prows, "ms_order")), + "pay_handle": stats(nums(prows, "ms_handle")), + "pay_settle": stats(nums(prows, "ms_settle")), + "pay_total": stats(nums(prows, "ms_total")), + }, + "rungs": rows, + "pays": prows, +} +json.dump(report, open(jpath, "w"), indent=2) +print("JSON", jpath) +print("--- withdraw speed (ms) ---") +for k in ("mint", "accept", "confirm", "settle", "rung_total"): + v = report["timing"][k] + if v.get("n"): + print(" %s n=%s min=%s p50=%s avg=%s max=%s" % (k.ljust(12), v["n"], v["min_ms"], v["p50_ms"], v["avg_ms"], v["max_ms"])) +print("--- pay speed (ms) ---") +for k in ("pay_order", "pay_handle", "pay_settle", "pay_total"): + v = report["timing"][k] + if v.get("n"): + print(" %s n=%s min=%s p50=%s avg=%s max=%s" % (k.ljust(12), v["n"], v["min_ms"], v["p50_ms"], v["avg_ms"], v["max_ms"])) +print("--- withdraw rungs ---") +for row in rows: + print(" %2s %-16s %-12s total=%sms" % (row.get("rung"), row.get("amount"), row.get("status"), row.get("ms_total"))) +print("--- pay rungs ---") +for row in prows: + print(" %2s %-16s %-12s total=%sms oid=%s" % (row.get("rung"), row.get("amount"), row.get("status"), row.get("ms_total"), row.get("oid"))) +if stop_amt: + print("STOPPED at %s: %s" % (stop_amt, stop_reason)) +else: + print("Completed without hard failure (or budget stop without fail).") +PY + +wcli balance 2>&1 | tee "$REPORT_DIR/balance-final.out" | tail -20 || true + +set_group load +section "ladder · load snapshot (after withdraws)" +metrics_report_load "$LOAD_AFTER" "ladder-end" || true +if [ -f "$LOAD_BEFORE" ] && [ -f "$LOAD_AFTER" ]; then + section "metrics · ladder load delta" + metrics_print_load_delta "$LOAD_BEFORE" "$LOAD_AFTER" || true +fi +# Speed timings → metrics overall (min/p50/avg/max per phase) +if [ -f "$JSON" ]; then + python3 - "$JSON" "${METRICS_DIR}/perf-summary.json" <<'PY' 2>/dev/null || true +import json, sys +rep = json.load(open(sys.argv[1])) +out = {} +for k, v in (rep.get("timing") or {}).items(): + if isinstance(v, dict) and v.get("n"): + out[k] = v +json.dump(out, open(sys.argv[2], "w"), indent=2) +PY +fi +export METRICS_WITHDRAW_TSV="$TSV" +export METRICS_PAY_TSV="$PAY_TSV" +metrics_report_coins "ladder-end" || true +metrics_print_overall "ladder final" || true + +# Keep scratch if LADDER_REPORT_DIR set; else copy key files to /tmp +if [ -z "${LADDER_REPORT_DIR:-}" ]; then + KEEP="/tmp/goa-ladder-report-$(date +%Y%m%d-%H%M%S)" + mkdir -p "$KEEP" + cp -a "$TSV" "$PAY_TSV" "$JSON" "$SCRATCH/auto-account.json" "$SCRATCH/ladder-plan.txt" \ + "$SCRATCH/ladder-wd-plan.txt" "$SCRATCH/ladder-pay-plan.txt" \ + "$REPORT_DIR/balance-final.out" "$LOAD_BEFORE" "$LOAD_AFTER" "$KEEP/" 2>/dev/null || true + info "report_dir" "$KEEP" + echo "$KEEP" >"$SCRATCH/KEEP_PATH" +fi + +if [ "$FAIL_N_L" -gt 0 ]; then + blocker "ladder" "stopped at ${STOP_AMOUNT:-?} — ${STOP_REASON:-error}" + exit 1 +fi +if [ "$OK_N" -eq 0 ]; then + blocker "ladder" "no successful withdraw rungs" + exit 1 +fi +ok "ladder finished withdraw_ok=$OK_N pay_ok=$PAY_OK_N phase=${ms_phase}ms" +exit 0 diff --git a/check_inside.sh b/check_inside.sh new file mode 100755 index 0000000..7f62880 --- /dev/null +++ b/check_inside.sh @@ -0,0 +1,546 @@ +#!/usr/bin/env bash +# Inside status for bank / exchange / merchant containers. +# +# Access modes (shown in the log as flags): +# host-podman — this host runs podman exec into containers (no SSH). +# INSIDE_PODMAN=1 or INSIDE_MODE=local-podman, or auto on GOA host. +# ssh — from a laptop/remote: SSH to KOOPA_SSH / INSIDE_SSH, then podman there. +# +# Profiles: +# koopa (default LOCAL_STACK=1) — taler-hacktivism* containers +# stage-lfp (TESTPAYSAN) — SSH INSIDE_SSH (stagepaysan), low-priv podman +set -euo pipefail +ROOT=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=lib.sh +source "$ROOT/lib.sh" +# shellcheck source=metrics.sh +source "$ROOT/metrics.sh" + +set_area inside + +PROFILE="${INSIDE_PROFILE:-}" +if [ -z "$PROFILE" ]; then + if [ "${LOCAL_STACK:-0}" = "1" ]; then + PROFILE=koopa + elif [ "${EXPECT_CURRENCY:-}" = "TESTPAYSAN" ] || [ -n "${INSIDE_SSH:-}" ]; then + PROFILE=stage-lfp + else + PROFILE=koopa + fi +fi + +# Resolve host-podman vs ssh before first check ID +_use_local_podman=0 +if [ "$PROFILE" != "stage-lfp" ]; then + if [ "${INSIDE_PODMAN:-0}" = "1" ] || [ "${INSIDE_MODE:-}" = "local-podman" ]; then + _use_local_podman=1 + elif command -v podman >/dev/null 2>&1 \ + && podman ps --format '{{.Names}}' 2>/dev/null | grep -qE 'taler-hacktivism'; then + _use_local_podman=1 + fi +fi +if [ "$_use_local_podman" = "1" ]; then + INSIDE_ACCESS=host-podman + set_group host +else + INSIDE_ACCESS=ssh + set_group ssh +fi +export INSIDE_ACCESS + +section "inside · collect (${PROFILE} · access=${INSIDE_ACCESS})" +info "flags" "INSIDE_ACCESS=${INSIDE_ACCESS} INSIDE_PODMAN=${INSIDE_PODMAN:-0} INSIDE_MODE=${INSIDE_MODE:-} LOCAL_STACK=${LOCAL_STACK:-0} SKIP_SSH=${SKIP_SSH:-0} KOOPA_SSH=${KOOPA_SSH:-} INSIDE_SSH=${INSIDE_SSH:-}" + +# --------------------------------------------------------------------------- +# stage-lfp: low-priv stagepaysan on francpaysan-host +# --------------------------------------------------------------------------- +if [ "$PROFILE" = "stage-lfp" ]; then + SSH_HOST="${INSIDE_SSH:-francpaysan-stage-user}" + BANK_CTR="${INSIDE_BANK_CTR:-stage-lfp-bank}" + EX_CTR="${INSIDE_EXCHANGE_CTR:-stage-lfp-exchange-ansible}" + MER_CTR="${INSIDE_MERCHANT_CTR:-stage-lfp-merchant}" + BANK_PORT="${INSIDE_BANK_PORT:-9032}" + EX_PORT="${INSIDE_EXCHANGE_PORT:-9031}" + MER_PORT="${INSIDE_MERCHANT_PORT:-9030}" + DNS_BANK="${INSIDE_DNS_BANK:-stage.bank.lefrancpaysan.ch}" + DNS_EX="${INSIDE_DNS_EXCHANGE:-stage.exchange.lefrancpaysan.ch}" + DNS_MER="${INSIDE_DNS_MERCHANT:-stage.monnaie.lefrancpaysan.ch}" + # Stage remote has more podman execs than koopa; allow a bit more wall time + STAGE_SSH_T="${INSIDE_SSH_TIMEOUT:-${SSH_CMD_TIMEOUT:-24}}" + if [ "${STAGE_SSH_T}" -lt 24 ] 2>/dev/null; then STAGE_SSH_T=24; fi + + if ! mon_ssh_ok "$SSH_HOST"; then + err "ssh" "cannot reach ${SSH_HOST} (stagepaysan low-priv) — set INSIDE_SSH= or SKIP" + summary + exit 1 + fi + ok "ssh ${SSH_HOST}" "stagepaysan (podman, no sudo)" + + # Inject names/ports into remote (ssh bash -s does not inherit local env). + RAW=$( + { + printf 'BANK_CTR=%q; EX_CTR=%q; MER_CTR=%q\n' "$BANK_CTR" "$EX_CTR" "$MER_CTR" + printf 'BANK_PORT=%q; EX_PORT=%q; MER_PORT=%q\n' "$BANK_PORT" "$EX_PORT" "$MER_PORT" + printf 'DNS_BANK=%q; DNS_EX=%q; DNS_MER=%q\n' "$DNS_BANK" "$DNS_EX" "$DNS_MER" + cat <<'REMOTE' +set +e +emit() { printf 'E|%s|%s|%s|%s\n' "$1" "$2" "$3" "$(printf '%s' "${4:-}" | tr '\n\r' ' ' | head -c 200)"; } +hc() { curl -skS -m 3 -o /tmp/mb -w '%{http_code}' "$1" 2>/dev/null || echo 000; } +hasp() { podman exec "$1" pgrep -f "$2" >/dev/null 2>&1; } + +BANK="${BANK_CTR}" +EX="${EX_CTR}" +MER="${MER_CTR}" +podman ps --format '{{.Names}}' 2>/dev/null | grep -qx "$BANK" || BANK=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -iE 'stage.*bank|lfp-bank' | head -1) +podman ps --format '{{.Names}}' 2>/dev/null | grep -qx "$EX" || EX=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -iE 'stage.*exchange|lfp-exchange' | head -1) +podman ps --format '{{.Names}}' 2>/dev/null | grep -qx "$MER" || MER=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -iE 'stage.*merchant|lfp-merchant' | head -1) + +check_dns() { + local comp="$1" ctr="$2" host="$3" + local line ip code + [ -n "$host" ] || { emit "$comp" WARN "dns" "empty host"; return 1; } + line=$(podman exec "$ctr" getent ahostsv4 "$host" 2>/dev/null | head -1) + [ -z "$line" ] && line=$(podman exec "$ctr" getent hosts "$host" 2>/dev/null | head -1) + ip=$(echo "$line" | awk '{print $1}') + if [ -z "$ip" ]; then + emit "$comp" WARN "dns $host" "no resolve inside container" + return 1 + fi + if [ "$ip" = "127.0.0.1" ] || [ "$ip" = "::1" ]; then + emit "$comp" WARN "dns $host" "$ip (loopback)" + return 1 + fi + # 169.254.x = pasta host-gateway pin is OK for stage + code=$(podman exec "$ctr" curl -skS -m 3 -o /dev/null -w '%{http_code}' "https://${host}/config" 2>/dev/null || echo 000) + if [ "$code" = "200" ]; then + emit "$comp" OK "dns $host" "→ $ip /config=$code" + else + emit "$comp" WARN "dns $host" "→ $ip /config=$code" + fi +} + +if [ -z "$BANK" ]; then emit bank ERROR container "not running" +else + emit bank INFO container "$(podman ps --filter name=^${BANK}$ --format '{{.Names}} {{.Status}}' | head -1)" + emit bank INFO ports "$(podman ps --filter name=^${BANK}$ --format '{{.Ports}}' | head -1)" + hasp "$BANK" 'MainKt serve|libeufin-bank serve' && emit bank OK libeufin "running" || emit bank ERROR libeufin "not running" + podman exec "$BANK" pg_isready -q 2>/dev/null && emit bank OK postgres "ready" || emit bank WARN postgres "pg_isready failed" + c=$(hc "http://127.0.0.1:${BANK_PORT}/config") + [ "$c" = "200" ] && emit bank OK "host :${BANK_PORT}/config" "HTTP $c" || emit bank ERROR "host :${BANK_PORT}/config" "HTTP $c" + c=$(hc "http://127.0.0.1:${BANK_PORT}/taler-integration/config") + [ "$c" = "200" ] && emit bank OK "host integration" "HTTP $c" || emit bank ERROR "host integration" "HTTP $c" + check_dns bank "$BANK" "$DNS_BANK" || true + check_dns bank "$BANK" "$DNS_EX" || true +fi + +if [ -z "$EX" ]; then emit exchange ERROR container "not running" +else + emit exchange INFO container "$(podman ps --filter name=^${EX}$ --format '{{.Names}} {{.Status}}' | head -1)" + c=$(hc "http://127.0.0.1:${EX_PORT}/config") + [ "$c" = "200" ] && emit exchange OK "host :${EX_PORT}/config" "HTTP $c" || emit exchange ERROR "host :${EX_PORT}/config" "HTTP $c" + c=$(curl -sS -m 5 -o /dev/null -w '%{http_code}' "http://127.0.0.1:${EX_PORT}/keys" 2>/dev/null || echo 000) + [ "$c" = "200" ] && emit exchange OK "host :${EX_PORT}/keys" "HTTP $c" || emit exchange ERROR "host :${EX_PORT}/keys" "HTTP $c" + hasp "$EX" 'taler-exchange-httpd' && emit exchange OK httpd "running" || emit exchange ERROR httpd "not running" + hasp "$EX" 'taler-exchange-wirewatch' && emit exchange OK wirewatch "running" || emit exchange ERROR wirewatch "not running" + hasp "$EX" 'taler-exchange-aggregator' && emit exchange OK aggregator "running" || emit exchange WARN aggregator "not running" + hasp "$EX" 'taler-exchange-transfer' && emit exchange OK transfer "running" || emit exchange WARN transfer "not running" + check_dns exchange "$EX" "$DNS_BANK" || true + check_dns exchange "$EX" "$DNS_EX" || true + check_dns exchange "$EX" "$DNS_MER" || true +fi + +if [ -z "$MER" ]; then emit merchant ERROR container "not running" +else + emit merchant INFO container "$(podman ps --filter name=^${MER}$ --format '{{.Names}} {{.Status}}' | head -1)" + c=$(hc "http://127.0.0.1:${MER_PORT}/config") + [ "$c" = "200" ] && emit merchant OK "host :${MER_PORT}/config" "HTTP $c" || emit merchant ERROR "host :${MER_PORT}/config" "HTTP $c" + hasp "$MER" 'taler-merchant-httpd' && emit merchant OK httpd "running" || emit merchant ERROR httpd "not running" + # Stage merchant often has no wirewatch unit — INFO not WARN + hasp "$MER" 'taler-merchant-wirewatch' && emit merchant OK wirewatch "running" || emit merchant INFO wirewatch "not running (optional on stage)" + hasp "$MER" 'taler-merchant-depositcheck' && emit merchant OK depositcheck "running" || emit merchant INFO depositcheck "not running (optional on stage)" + hasp "$MER" 'taler-merchant-exchangekeyupdate' && emit merchant OK exchangekeyupdate "running" || emit merchant INFO exchangekeyupdate "not running" + check_dns merchant "$MER" "$DNS_BANK" || true + check_dns merchant "$MER" "$DNS_EX" || true + check_dns merchant "$MER" "$DNS_MER" || true +fi + +# Caddy is host-wide (often root); stagepaysan can only see the process +if pgrep -x caddy >/dev/null 2>&1; then + emit caddy OK process "running (host)" +else + emit caddy WARN process "not seen as stagepaysan (may still run as root)" +fi + +echo DONE +REMOTE + } | mon_ssh_bash "$SSH_HOST" "${STAGE_SSH_T}" || true + ) + + if [ -z "$RAW" ] || ! echo "$RAW" | grep -q '^E|'; then + err "ssh" "stage remote timed out or empty (cap ${STAGE_SSH_T}s · host=${SSH_HOST})" + summary + exit 1 + fi + + _last_inside_grp="" + while IFS= read -r line; do + case "$line" in + E\|*) + IFS='|' read -r _ comp level key detail <<<"$line" + case "$comp" in + bank|exchange|merchant|caddy) _g="$comp" ;; + *) _g="ssh" ;; + esac + if [ "$_g" != "$_last_inside_grp" ]; then + set_group "$_g" + _last_inside_grp="$_g" + fi + case "$level" in + OK) ok "[$comp] $key${detail:+ ($detail)}" ;; + ERROR) err "$comp" "$key" "$detail" ;; + WARN) warn "[$comp] $key" "$detail" ;; + INFO) info "[$comp] $key" "$detail" ;; + esac + ;; + esac + done <<<"$RAW" + + # Stats from outside (laptop) — public HTTPS, no stagepaysan write needed + set_group stats + section "inside · public stats (outside-in)" + _probe_public_stats() { + local comp="$1" url="$2" + local body code age unix now + body=$(mktemp) + code=$(curl -skS -m 8 -o "$body" -w '%{http_code}' "$url" 2>/dev/null || echo 000) + if [ "$code" != "200" ]; then + warn "[$comp] public stats.json" "HTTP $code · $url" + rm -f "$body" + return + fi + unix=$(python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); print(d.get("generated_at_unix") or 0)' "$body" 2>/dev/null || echo 0) + now=$(date +%s) + if [ "${unix:-0}" -gt 0 ] 2>/dev/null; then + age=$((now - unix)) + if [ "$age" -lt 0 ]; then age=0; fi + if [ "$age" -le "${STATS_STALE_SECS:-900}" ]; then + ok "[$comp] public stats.json" "HTTP 200 age=${age}s · $url" + elif [ "$age" -le "${STATS_FAIL_SECS:-3600}" ]; then + warn "[$comp] public stats.json" "stale age=${age}s · $url" + else + err "$comp" "public stats.json" "too old age=${age}s · $url" + fi + else + ok "[$comp] public stats.json" "HTTP 200 · $url" + fi + rm -f "$body" + } + _probe_public_stats bank "https://${DNS_BANK}/intro/stats.json" + _probe_public_stats exchange "https://${DNS_EX}/intro/stats.json" + _probe_public_stats merchant "https://${DNS_MER}/intro/stats.json" + + # Host load as stagepaysan (no container RSS from koopa metrics) + set_group load + section "inside · load (stagepaysan host)" + LOAD_LINE=$(mon_ssh_bash "$SSH_HOST" 8 <<'EOF' || true +python3 - <<'PY' +import os +la=os.getloadavg() +print("loadavg=%.2f,%.2f,%.2f" % la) +try: + with open("/proc/meminfo") as f: + d={} + for line in f: + k,v=line.split(":")[0], line.split(":")[1].strip().split()[0] + d[k]=int(v) + total=d.get("MemTotal",0)/1024/1024 + avail=d.get("MemAvailable",0)/1024/1024 + used=total-avail + print("mem_used=%.2fGiB avail=%.2fGiB total=%.2fGiB" % (used, avail, total)) +except Exception: + print("mem=?") +PY +EOF +) + if [ -n "$LOAD_LINE" ]; then + info "stage host" "$(echo "$LOAD_LINE" | tr '\n' ' ')" + else + info "stage host" "load probe empty" + fi + + set_group disk + section "inside · disk free space (stage host + containers)" + _disk_raw=$(mon_ssh_bash "$SSH_HOST" "${STAGE_SSH_T:-24}" <<'DISK' || true +set +e +echo "###HOST###" +df -Pk / /var /home /tmp /mnt/data 2>/dev/null || df -Pk +echo "###CTRS###" +for c in $(podman ps --format '{{.Names}}' 2>/dev/null); do + echo "###CTR $c###" + podman exec "$c" df -Pk / /var /tmp 2>/dev/null || podman exec "$c" df -Pk 2>/dev/null +done +DISK +) + _host_df=$(printf '%s\n' "$_disk_raw" | sed -n '/^###HOST###$/,/^###CTRS###$/p' | sed '1d;$d') + mon_disk_check_remote_text "ssh:${SSH_HOST}" "$_host_df" || true + _ctr=""; _buf="" + while IFS= read -r _line || [ -n "$_line" ]; do + case "$_line" in + '###CTR '*) + if [ -n "$_ctr" ]; then mon_disk_check_remote_text "ctr:${_ctr}" "$_buf" || true; fi + _ctr=${_line####CTR }; _ctr=${_ctr%###}; _buf="" + ;; + '###CTRS###'|'###HOST###') ;; + *) [ -n "$_ctr" ] && _buf="${_buf}${_line}"$'\n' ;; + esac + done <<<"$_disk_raw" + [ -n "$_ctr" ] && mon_disk_check_remote_text "ctr:${_ctr}" "$_buf" || true + + summary + exit 0 +fi + +# --------------------------------------------------------------------------- +# koopa (default) — host-podman exec (on GOA host) or SSH to KOOPA_SSH (laptop) +# --------------------------------------------------------------------------- +section "inside · collect from koopa (access=${INSIDE_ACCESS})" + +if [ "$_use_local_podman" = "1" ]; then + set_group host + if ! command -v podman >/dev/null 2>&1; then + err "host" "INSIDE_PODMAN/host-podman but podman missing" + summary + exit 1 + fi + ok "host→container" "podman exec on this host (INSIDE_ACCESS=host-podman · no SSH)" +elif [ "${SKIP_SSH}" = "1" ] && [ "${LOCAL_STACK:-0}" != "1" ]; then + set_group ssh + warn "ssh" "SKIP_SSH=1 and not host-podman — skipped" + summary + exit 0 +elif ! koopa_ssh_ok; then + set_group ssh + err "ssh" "cannot reach ${KOOPA_SSH} — from laptop use KOOPA_SSH=; on host set INSIDE_PODMAN=1" + summary + exit 1 +else + set_group ssh + ok "ssh ${KOOPA_SSH}" "remote host then podman (INSIDE_ACCESS=ssh · laptop/remote)" +fi + +# Collect script (file → bash local or ssh bash -s) +_INSIDE_SCRIPT=$(mktemp) +trap 'rm -f "${_INSIDE_SCRIPT:-}"' RETURN +cat >"$_INSIDE_SCRIPT" <<'REMOTE' +set +e +emit() { printf 'E|%s|%s|%s|%s\n' "$1" "$2" "$3" "$(printf '%s' "${4:-}" | tr '\n\r' ' ' | head -c 200)"; } +# quick curl +hc() { curl -skS -m 3 -o /tmp/mb -w '%{http_code}' "$1" 2>/dev/null || echo 000; } +# quick process check inside container (pgrep only) +hasp() { podman exec "$1" pgrep -f "$2" >/dev/null 2>&1; } + +BANK=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -iE 'hacktivism-bank|taler-bank' | head -1) +[ -z "$BANK" ] && BANK=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -i bank | head -1) +EX=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -i exchange | head -1) +# Exact merchant container name first — never fall through to *-bank +MER=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -E '^taler-hacktivism$' | head -1) +[ -z "$MER" ] && MER=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -i merchant | grep -viE 'bank|exchange' | head -1) +[ -z "$MER" ] && MER=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -E 'hacktivism' | grep -viE 'bank|exchange' | head -1) + +# Domain resolve inside container (wirewatch needs bank.hacktivism.ch → real IP) +# emit: comp LEVEL dns "host → ip" or fail +check_dns() { + local comp="$1" ctr="$2" host="$3" + local line ip code + # Prefer IPv4 (wirewatch/libcurl often happier; avoid dead AAAA) + line=$(podman exec "$ctr" getent ahostsv4 "$host" 2>/dev/null | head -1) + [ -z "$line" ] && line=$(podman exec "$ctr" getent hosts "$host" 2>/dev/null | head -1) + ip=$(echo "$line" | awk '{print $1}') + if [ -z "$ip" ]; then + emit "$comp" ERROR "dns $host" "no resolve — run pin-container-hosts.sh" + return 1 + fi + # 127.0.0.1 is almost always wrong for public bank/exchange from inside pasta + if [ "$ip" = "127.0.0.1" ] || [ "$ip" = "::1" ]; then + emit "$comp" ERROR "dns $host" "$ip (loopback — wirewatch will fail)" + return 1 + fi + code=$(podman exec "$ctr" curl -skS -m 3 -o /dev/null -w '%{http_code}' "https://${host}/config" 2>/dev/null || echo 000) + if [ "$code" = "200" ]; then + emit "$comp" OK "dns $host" "→ $ip /config=$code" + else + emit "$comp" WARN "dns $host" "→ $ip /config=$code" + fi +} + +if [ -z "$BANK" ]; then emit bank ERROR container "not running" +else + emit bank INFO container "$(podman ps --filter name=$BANK --format '{{.Names}} {{.Status}}' | head -1)" + emit bank INFO ports "$(podman ps --filter name=$BANK --format '{{.Ports}}' | head -1)" + hasp "$BANK" 'MainKt serve|libeufin-bank serve' && emit bank OK libeufin "running" || emit bank ERROR libeufin "not running — API/withdraw dead" + podman exec "$BANK" pg_isready -q 2>/dev/null && emit bank OK postgres "ready" || emit bank ERROR postgres "not ready" + c=$(hc http://127.0.0.1:9012/config) + [ "$c" = "200" ] && emit bank OK "local /config" "HTTP $c" || emit bank ERROR "local /config" "HTTP $c" + c=$(hc http://127.0.0.1:9012/taler-integration/config) + [ "$c" = "200" ] && emit bank OK "local integration" "HTTP $c" || emit bank ERROR "local integration" "HTTP $c" + hasp "$BANK" 'nginx' && emit bank OK nginx ":9013" || emit bank WARN nginx "not running" + check_dns bank "$BANK" bank.hacktivism.ch || true + check_dns bank "$BANK" exchange.hacktivism.ch || true +fi + +if [ -z "$EX" ]; then emit exchange ERROR container "not running" +else + emit exchange INFO container "$(podman ps --filter name=$EX --format '{{.Names}} {{.Status}}' | head -1)" + c=$(hc http://127.0.0.1:9011/config) + [ "$c" = "200" ] && emit exchange OK "local /config" "HTTP $c" || emit exchange ERROR "local /config" "HTTP $c" + c=$(curl -sS -m 5 -o /dev/null -w '%{http_code}' http://127.0.0.1:9011/keys 2>/dev/null || echo 000) + [ "$c" = "200" ] && emit exchange OK "local /keys" "HTTP $c" || emit exchange ERROR "local /keys" "HTTP $c" + hasp "$EX" 'taler-exchange-httpd' && emit exchange OK httpd "running" || emit exchange ERROR httpd "not running" + hasp "$EX" 'taler-exchange-wirewatch' && emit exchange OK wirewatch "running" || emit exchange ERROR wirewatch "not running — withdraw stuck after bank confirm" + hasp "$EX" 'taler-exchange-aggregator' && emit exchange OK aggregator "running" || emit exchange WARN aggregator "not running" + hasp "$EX" 'taler-exchange-transfer' && emit exchange OK transfer "running" || emit exchange WARN transfer "not running" + # critical for wire gateway + check_dns exchange "$EX" bank.hacktivism.ch || true + check_dns exchange "$EX" exchange.hacktivism.ch || true + check_dns exchange "$EX" taler.hacktivism.ch || true +fi + +if [ -z "$MER" ]; then emit merchant ERROR container "not running" +else + emit merchant INFO container "$(podman ps --filter name=$MER --format '{{.Names}} {{.Status}}' | head -1)" + c=$(hc https://127.0.0.1:9010/config) + [ "$c" = "200" ] && emit merchant OK "local /config" "HTTP $c" || emit merchant ERROR "local /config" "HTTP $c" + hasp "$MER" 'taler-merchant-httpd' && emit merchant OK httpd "running" || emit merchant ERROR httpd "not running" + hasp "$MER" 'taler-merchant-wirewatch' && emit merchant OK wirewatch "running" || emit merchant WARN wirewatch "not running" + # depositcheck: may abort on exchange track 404 (merchant assertion) — try ensure once + if hasp "$MER" 'taler-merchant-depositcheck'; then + emit merchant OK depositcheck "running" + else + podman exec "$MER" /usr/local/bin/ensure_merchant_helpers.sh >/dev/null 2>&1 || true + sleep 1 + if hasp "$MER" 'taler-merchant-depositcheck'; then + emit merchant OK depositcheck "running (after ensure_merchant_helpers)" + elif hasp "$MER" 'taler-merchant-wirewatch'; then + # Known: depositcheck can SIGABRT on exchange 404 for stale coins — not startable cleanly + emit merchant INFO depositcheck "not running (often aborts on track 404; wirewatch OK · settlement partial)" + else + emit merchant WARN depositcheck "not running" + fi + fi + check_dns merchant "$MER" bank.hacktivism.ch || true + check_dns merchant "$MER" exchange.hacktivism.ch || true + check_dns merchant "$MER" taler.hacktivism.ch || true +fi + +if systemctl is-active caddy >/dev/null 2>&1 || pgrep -x caddy >/dev/null 2>&1; then + emit caddy OK process "active" +else + emit caddy ERROR process "not active" +fi +echo DONE +REMOTE + +if [ "$_use_local_podman" = "1" ]; then + RAW=$(bash "$_INSIDE_SCRIPT" || true) +else + RAW=$(koopa_ssh_bash "${SSH_CMD_TIMEOUT}" <"$_INSIDE_SCRIPT" || true) +fi +rm -f "$_INSIDE_SCRIPT" + +if [ -z "$RAW" ] || ! echo "$RAW" | grep -q '^E|'; then + if [ "$_use_local_podman" = "1" ]; then + err "host" "collect empty (INSIDE_ACCESS=host-podman · podman exec failed)" + else + err "ssh" "collect empty (INSIDE_ACCESS=ssh · cap ${SSH_CMD_TIMEOUT}s · host=${KOOPA_SSH})" + fi + summary + exit 1 +fi + +# Fallback group for non-component rows: host (podman) vs ssh (remote) +_access_grp=ssh +[ "$_use_local_podman" = "1" ] && _access_grp=host + +_last_inside_grp="" +while IFS= read -r line; do + case "$line" in + E\|*) + IFS='|' read -r _ comp level key detail <<<"$line" + # Group IDs by component: inside.bank-02, inside.exchange-04; access = host|ssh + case "$comp" in + bank|exchange|merchant|caddy) _g="$comp" ;; + *) _g="$_access_grp" ;; + esac + if [ "$_g" != "$_last_inside_grp" ]; then + set_group "$_g" + _last_inside_grp="$_g" + fi + case "$level" in + OK) ok "[$comp] $key${detail:+ ($detail)}" ;; + ERROR) err "$comp" "$key" "$detail" ;; + WARN) warn "[$comp] $key" "$detail" ;; + INFO) info "[$comp] $key" "$detail" ;; + esac + ;; + esac +done <<<"$RAW" + +# Host loadavg + RAM + per-container RSS/CPU (same probe as e2e/ladder) +set_group load +section "inside · load / memory" +METRICS_DIR="${METRICS_DIR:-$(mktemp -d)}" +export METRICS_DIR +metrics_report_load "${METRICS_DIR}/load-inside.json" "inside" || true + +# Disk free space: host + taler containers (WARN tight, ERROR full) +set_group disk +section "inside · disk free space" +_disk_ec=0 +if [ "$_use_local_podman" = "1" ]; then + mon_disk_check_host "host" || _disk_ec=1 + while read -r _cname; do + [ -n "$_cname" ] || continue + mon_disk_check_podman "$_cname" || _disk_ec=1 + done < <(podman ps --format '{{.Names}}' 2>/dev/null | grep -iE 'hacktivism|taler-|stage-lfp|lfp-' || true) +else + # remote host via SSH: df on host + each container + _disk_raw=$(koopa_ssh_bash "${SSH_CMD_TIMEOUT:-20}" <<'DISK' || true +set +e +echo "###HOST###" +df -Pk / /var /home /tmp 2>/dev/null || df -Pk +echo "###CTRS###" +for c in $(podman ps --format '{{.Names}}' 2>/dev/null); do + echo "###CTR $c###" + podman exec "$c" df -Pk / /var /tmp 2>/dev/null || podman exec "$c" df -Pk 2>/dev/null +done +DISK +) + _host_df=$(printf '%s\n' "$_disk_raw" | sed -n '/^###HOST###$/,/^###CTRS###$/p' | sed '1d;$d') + mon_disk_check_remote_text "ssh:${KOOPA_SSH:-remote}" "$_host_df" || _disk_ec=1 + _ctr="" + _buf="" + while IFS= read -r _line || [ -n "$_line" ]; do + case "$_line" in + '###CTR '*) + if [ -n "$_ctr" ]; then + mon_disk_check_remote_text "ctr:${_ctr}" "$_buf" || _disk_ec=1 + fi + _ctr=${_line####CTR } + _ctr=${_ctr%###} + _buf="" + ;; + '###CTRS###'|'###HOST###') ;; + *) + if [ -n "$_ctr" ]; then + _buf="${_buf}${_line}"$'\n' + fi + ;; + esac + done <<<"$_disk_raw" + if [ -n "$_ctr" ]; then + mon_disk_check_remote_text "ctr:${_ctr}" "$_buf" || _disk_ec=1 + fi +fi +unset _disk_raw _host_df _ctr _buf _line _cname _disk_ec + +summary diff --git a/check_qr_payloads.py b/check_qr_payloads.py new file mode 100755 index 0000000..74efde4 --- /dev/null +++ b/check_qr_payloads.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python3 +"""Collect + validate QR-related payloads for taler-monitoring (www.qr-*). + +Sources (CLI paths / URLs handled by caller writing local files): + - landing HTML (href, data-qr-url, data-qr-taler, raw text) + - demo-withdraw.json / auto-account.json (taler_withdraw_uri / qr_payload) + - optional: already-decoded lines from zbarimg (kind=decoded) + +Stdout TSV: source\\tkind\\turi\\tstatus\\tdetail + kind: withdraw | pay | pay-template | refund | payto | https | other | empty + status: ok | bad + +Exit 0 always (caller tallies ok/bad lines). stderr for hard parse errors only. +""" +from __future__ import annotations + +import json +import re +import sys +from html.parser import HTMLParser +from pathlib import Path +from urllib.parse import unquote, urlparse + +# Schemes we care about for wallet / settlement QRs +TALER_RE = re.compile(r"taler://[^\s\"'<>]+", re.I) +PAYTO_RE = re.compile(r"payto://[^\s\"'<>]+", re.I) +HTTPS_RE = re.compile(r"https://[^\s\"'<>]+", re.I) + +WITHDRAW_RE = re.compile( + r"^taler://withdraw/([^/]+)/taler-integration/([0-9a-fA-F-]+)/?$" +) +# order pay / template — host may include path segments after +PAY_RE = re.compile(r"^taler://pay/([^/?#]+)/", re.I) +PAY_TEMPLATE_RE = re.compile(r"^taler://pay-template/([^/?#]+)/", re.I) +REFUND_RE = re.compile(r"^taler://refund/", re.I) +PAYTO_OK_RE = re.compile(r"^payto://[a-z0-9][a-z0-9+.-]*/", re.I) + + +def looks_like_real_uri(uri: str) -> bool: + """Drop JS regex fragments, docs ellipses, incomplete placeholders.""" + if not uri or len(uri) < 12: + return False + if any(c in uri for c in "[]{}<>\n\r\t"): + return False + if "…" in uri or "..." in uri: + return False + # incomplete withdraw stubs in docs + if re.match(r"^taler://withdraw/?$", uri, re.I): + return False + if re.match(r"^taler://pay/?$", uri, re.I): + return False + # trailing junk / JS regex fragments (allow ? & = in query strings) + if uri.endswith((")", "(", "\\", "^", "*")): + return False + if re.search(r"[\\^$*]", uri): + return False + # lone incomplete `taler://…?` without path + if re.match(r"^taler://[^/]+$", uri, re.I): + return False + return True + + +def strip_default_port_host(host: str) -> str | None: + if not host or host.startswith(":"): + return None + if ":" in host: + h, _, p = host.rpartition(":") + if not h or not p.isdigit(): + return None + if p in ("443", "80"): + return None # must be stripped for wallets + return host + + +def classify_and_validate(uri: str) -> tuple[str, str, str]: + """Return (kind, status, detail).""" + uri = (uri or "").strip() + if not uri: + return "empty", "bad", "empty" + # strip accidental trailing punctuation from HTML scrape + uri = uri.rstrip(").,;]") + low = uri.lower() + + if low.startswith("taler://withdraw/"): + m = WITHDRAW_RE.match(uri) + if not m: + return "withdraw", "bad", "shape need taler://withdraw/HOST/taler-integration/UUID" + host = m.group(1) + if strip_default_port_host(host) is None: + return "withdraw", "bad", "host/port invalid or default :443/:80 not stripped" + return "withdraw", "ok", f"host={host} id={m.group(2)[:12]}…" + + if low.startswith("taler://pay-template/"): + m = PAY_TEMPLATE_RE.match(uri) + if not m: + return "pay-template", "bad", "need taler://pay-template/HOST/…" + host = m.group(1) + if strip_default_port_host(host) is None: + return "pay-template", "bad", "host/port invalid or default port not stripped" + return "pay-template", "ok", f"host={host}" + + if low.startswith("taler://pay/"): + m = PAY_RE.match(uri) + if not m: + return "pay", "bad", "need taler://pay/HOST/…" + host = m.group(1) + if strip_default_port_host(host) is None: + return "pay", "bad", "host/port invalid or default port not stripped" + if "instances" not in uri and "/orders/" not in uri: + # still often valid private order URIs + return "pay", "ok", f"host={host}" + return "pay", "ok", f"host={host}" + + if low.startswith("taler://refund/"): + if not REFUND_RE.match(uri): + return "refund", "bad", "bad refund URI" + return "refund", "ok", "refund" + + if low.startswith("taler://"): + return "other", "ok", "other taler:// (accepted)" + + if low.startswith("payto://"): + if not PAYTO_OK_RE.match(uri): + return "payto", "bad", "need payto://method/…" + # wallet pay QR must not be payto for GOA demo-withdraw (checked elsewhere) + return "payto", "ok", uri.split("?", 1)[0][:72] + + if low.startswith("https://") or low.startswith("http://"): + p = urlparse(uri) + if p.scheme not in ("http", "https") or not p.netloc: + return "https", "bad", "bad URL" + return "https", "ok", p.netloc + (p.path or "/")[:40] + + return "other", "bad", "unsupported scheme" + + +class LinkHarvester(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.uris: list[tuple[str, str]] = [] # (source_hint, uri) + + def handle_starttag(self, tag, attrs): + ad = {k.lower(): (v or "") for k, v in attrs} + for key in ("href", "src", "data-qr-url", "data-qr-taler", "data-uri", "data-payto"): + if key in ad and ad[key]: + self.uris.append((f"attr:{key}", ad[key].strip())) + for k, v in ad.items(): + if k.startswith("data-") and v and ( + "taler://" in v.lower() or "payto://" in v.lower() or k.endswith("qr-url") + ): + self.uris.append((f"attr:{k}", v.strip())) + + def handle_data(self, data): + if not data or "://" not in data: + return + for m in TALER_RE.finditer(data): + self.uris.append(("text", m.group(0))) + for m in PAYTO_RE.finditer(data): + self.uris.append(("text", m.group(0))) + + +def harvest_html(path: Path, source: str) -> list[tuple[str, str, str]]: + raw = path.read_text(encoding="utf-8", errors="replace") + h = LinkHarvester() + try: + h.feed(raw) + except Exception: + pass + out: list[tuple[str, str, str]] = [] + for hint, u in h.uris: + lu = u.lower() + if lu.startswith(("taler://", "payto://", "https://", "http://")): + if looks_like_real_uri(u): + out.append((f"{source}:{hint}", u, "html")) + # raster image paths that look like QR assets (not .js) + elif re.search(r"\.(png|jpe?g|gif)(\?|$)", lu) and "qr" in lu: + out.append((f"{source}:imgpath", u, "imgpath")) + # Prefer attribute/text harvest; light regex only for complete-looking URIs in pre/code + for m in TALER_RE.finditer(raw): + u = m.group(0).rstrip(").,;]'\"") + if looks_like_real_uri(u) and ( + WITHDRAW_RE.match(u) + or PAY_RE.match(u) + or PAY_TEMPLATE_RE.match(u) + or u.lower().startswith("taler://refund/") + ): + out.append((f"{source}:re", u, "html")) + for m in PAYTO_RE.finditer(raw): + u = m.group(0).rstrip(").,;]'\"") + if looks_like_real_uri(u) and PAYTO_OK_RE.match(u): + out.append((f"{source}:re", u, "html")) + return out + + +def harvest_json(path: Path, source: str) -> list[tuple[str, str, str]]: + try: + d = json.loads(path.read_text(encoding="utf-8", errors="replace")) + except Exception as e: + return [(source, "", f"json-error:{e}")] + out: list[tuple[str, str, str]] = [] + if not isinstance(d, dict): + return out + for key in ( + "taler_withdraw_uri", + "qr_payload", + "taler_pay_uri", + "payto_uri", + "payto", + "uri", + ): + v = d.get(key) + if isinstance(v, str) and v.strip(): + out.append((f"{source}:{key}", v.strip(), "json")) + # nested + for k, v in d.items(): + if isinstance(v, str) and ( + v.startswith("taler://") or v.startswith("payto://") + ): + out.append((f"{source}:{k}", v.strip(), "json")) + return out + + +def main(argv: list[str]) -> int: + # args: pairs of source_label path + if len(argv) < 3 or len(argv) % 2 == 0: + print( + "usage: check_qr_payloads.py LABEL path [LABEL path ...]", + file=sys.stderr, + ) + return 2 + seen: set[str] = set() + rows: list[tuple[str, str, str, str, str]] = [] + i = 1 + while i < len(argv): + label, path_s = argv[i], argv[i + 1] + i += 2 + path = Path(path_s) + if not path.is_file(): + rows.append((label, "empty", "", "bad", f"missing file {path_s}")) + continue + if path.suffix.lower() in (".html", ".htm") or "html" in path.name: + items = harvest_html(path, label) + elif path.suffix.lower() == ".json" or path.name.endswith(".json"): + items = harvest_json(path, label) + else: + # plain text: one URI per line or raw + text = path.read_text(encoding="utf-8", errors="replace").strip() + items = [] + for line in text.splitlines(): + line = line.strip() + if line: + items.append((label, line, "text")) + if not items and text: + items.append((label, text, "text")) + + for src, uri, _origin in items: + if not uri: + continue + # skip pure fragment / relative without scheme for validate + if uri.startswith("#") or uri == "/": + continue + lu = uri.lower() + if lu.startswith("/") or "imgpath" in src: + continue + if not looks_like_real_uri(uri): + continue + # https: app-store / wallet / explicit data-qr-* only + if lu.startswith("http"): + if "data-qr" not in src and not any( + x in lu + for x in ( + "play.google", + "apps.apple", + "f-droid", + "wallet.taler", + "taler.net/wallet", + ) + ): + continue + if uri in seen: + continue + seen.add(uri) + kind, status, detail = classify_and_validate(uri) + rows.append((src, kind, uri, status, detail)) + + for src, kind, uri, status, detail in rows: + # TSV — uri may contain tabs rarely; replace + safe = uri.replace("\t", " ").replace("\n", " ") + print(f"{src}\t{kind}\t{safe}\t{status}\t{detail}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/check_sanity.sh b/check_sanity.sh new file mode 100755 index 0000000..1e3b922 --- /dev/null +++ b/check_sanity.sh @@ -0,0 +1,285 @@ +#!/usr/bin/env bash +# Sanity checks for bank · exchange · merchant (public + server-side). +# Sections are independent; continues after failures. +set -euo pipefail +ROOT=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=lib.sh +source "$ROOT/lib.sh" + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +expect_code() { + local label="$1" want="$2" url="$3" + local code + code=$(http_code "$url") + case ",$want," in + *",$code,"*) ok "$label" ;; + *) fail "$label" "HTTP $code (want $want) $url" ;; + esac +} + +json_currency() { + python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); print(d.get("currency") or "")' "$1" 2>/dev/null || true +} + +# Area sanity.* — public + optional server-side per component +# Groups: sanity.bank / sanity.exchange / sanity.merchant +set_area sanity + +# --------------------------------------------------------------------------- +set_group bank +section "sanity · bank" +# --------------------------------------------------------------------------- +expect_code "bank public /config" 200 "$BANK_PUBLIC/config" +expect_code "bank public /taler-integration/config" 200 "$BANK_PUBLIC/taler-integration/config" +expect_code "bank public /webui/" 200 "$BANK_PUBLIC/webui/" + +code=$(http_body "$BANK_PUBLIC/config" "$tmp/bank-config.json") +if [ "$code" = "200" ]; then + cur=$(json_currency "$tmp/bank-config.json") + [ "$cur" = "GOA" ] && ok "bank currency GOA" || fail "bank currency" "got ${cur:-?}" + # wire type / name if present + python3 - "$tmp/bank-config.json" <<'PY' 2>/dev/null && ok "bank config JSON object" || fail "bank config JSON" +import json,sys +d=json.load(open(sys.argv[1])) +sys.exit(0 if isinstance(d, dict) and d.get("currency") else 1) +PY + if json_has_alt_unit_names "$tmp/bank-config.json" >/tmp/alt-bank-s.$$ 2>&1; then + ok "bank /config alt_unit_names" "$(tr '\n' '; ' /dev/null || echo 000) +echo "LOCAL_CONFIG=$code" +code2=$(curl -sS -m 5 -o /dev/null -w '%{http_code}' http://127.0.0.1:9012/taler-integration/config 2>/dev/null || echo 000) +echo "LOCAL_INT=$code2" +if podman exec "$BANK" bash -c 'pgrep -f "MainKt serve|libeufin-bank serve" >/dev/null' 2>/dev/null; then + echo "LIBEUFIN=1" +else + echo "LIBEUFIN=0" +fi +if podman exec "$BANK" bash -c 'pg_isready -q' 2>/dev/null; then + echo "PG=1" +else + echo "PG=0" +fi +# in-container health if present +if podman exec "$BANK" test -x /usr/local/bin/check_bank-health.sh 2>/dev/null; then + podman exec "$BANK" /usr/local/bin/check_bank-health.sh 2>&1 | sed 's/^/HEALTH /' | tail -20 + echo "HEALTH_EC=${PIPESTATUS[0]}" +fi +REMOTE +) + echo "$BOUT" | grep -q '^CTR=.\+' && ok "bank container $(echo "$BOUT" | sed -n 's/^CTR=//p' | head -1)" || fail "bank container" "not found" + echo "$BOUT" | grep -q 'LOCAL_CONFIG=200' && ok "bank local :9012/config" || fail "bank local :9012/config" + echo "$BOUT" | grep -q 'LOCAL_INT=200' && ok "bank local :9012/taler-integration/config" || fail "bank local integration" + echo "$BOUT" | grep -q 'LIBEUFIN=1' && ok "bank libeufin-bank process" || fail "bank libeufin-bank process" + echo "$BOUT" | grep -q 'PG=1' && ok "bank postgres ready" || warn "bank postgres" "pg_isready failed" + if echo "$BOUT" | grep -q 'HEALTH '; then + if echo "$BOUT" | grep -qE 'HEALTH_EC=0|ALL CRITICAL CHECKS PASSED'; then + ok "bank check_bank-health.sh" + else + # health script may false-fail process grep; warn not fail if local config ok + warn "bank check_bank-health.sh" "non-zero or incomplete" + fi + fi +else + warn "bank server-side" "ssh ${KOOPA_SSH} unavailable" +fi + +# --------------------------------------------------------------------------- +set_group exchange +section "sanity · exchange" +# --------------------------------------------------------------------------- +expect_code "exchange public /config" 200 "$EXCHANGE_PUBLIC/config" +expect_code "exchange public /keys" 200 "$EXCHANGE_PUBLIC/keys" +expect_code "exchange public /terms" 200 "$EXCHANGE_PUBLIC/terms" + +code=$(http_body "$EXCHANGE_PUBLIC/config" "$tmp/ex-config.json") +if [ "$code" = "200" ]; then + cur=$(json_currency "$tmp/ex-config.json") + [ "$cur" = "GOA" ] && ok "exchange currency GOA" || fail "exchange currency" "got ${cur:-?}" + if json_has_alt_unit_names "$tmp/ex-config.json" "GOA" >/tmp/alt-ex-s.$$ 2>&1; then + ok "exchange /config alt_unit_names" "$(tr '\n' '; ' /dev/null || echo 000) +echo "LOCAL_CONFIG=$code" +codek=$(curl -sS -m 8 -o /dev/null -w '%{http_code}' http://127.0.0.1:9011/keys 2>/dev/null || echo 000) +echo "LOCAL_KEYS=$codek" +if [ -n "$EX" ]; then + if podman exec "$EX" bash -c 'pgrep -f taler-exchange-httpd >/dev/null' 2>/dev/null; then + echo "HTTPD=1" + else + echo "HTTPD=0" + fi + for p in taler-exchange-secmod-rsa taler-exchange-secmod-eddsa taler-exchange-wirewatch taler-exchange-aggregator; do + if podman exec "$EX" bash -c "pgrep -f $p >/dev/null" 2>/dev/null; then + echo "PROC_$p=1" + else + echo "PROC_$p=0" + fi + done + if podman exec "$EX" test -x /usr/local/bin/check_exchange-health.sh 2>/dev/null; then + SKIP_ENSURE=1 podman exec -e SKIP_ENSURE=1 "$EX" /usr/local/bin/check_exchange-health.sh 2>&1 | sed 's/^/HEALTH /' | tail -25 + fi +fi +REMOTE +) + echo "$EOUT" | grep -q '^CTR=.\+' && ok "exchange container $(echo "$EOUT" | sed -n 's/^CTR=//p' | head -1)" || fail "exchange container" + echo "$EOUT" | grep -q 'LOCAL_CONFIG=200' && ok "exchange local :9011/config" || fail "exchange local :9011/config" + echo "$EOUT" | grep -q 'LOCAL_KEYS=200' && ok "exchange local :9011/keys" || fail "exchange local :9011/keys" + echo "$EOUT" | grep -q 'HTTPD=1' && ok "exchange-httpd process" || warn "exchange-httpd process" "not detected" + echo "$EOUT" | grep -q 'PROC_taler-exchange-wirewatch=1' && ok "exchange wirewatch" || warn "exchange wirewatch" "not running" + echo "$EOUT" | grep -q 'PROC_taler-exchange-aggregator=1' && ok "exchange aggregator" || warn "exchange aggregator" "not running" +else + warn "exchange server-side" "ssh unavailable" +fi + +# --------------------------------------------------------------------------- +set_group merchant +section "sanity · merchant" +# --------------------------------------------------------------------------- +expect_code "merchant public /config" 200 "$MERCHANT_PUBLIC/config" +expect_code "merchant public /webui/" 200 "$MERCHANT_PUBLIC/webui/" + +code=$(http_body "$MERCHANT_PUBLIC/config" "$tmp/mer-config.json") +if [ "$code" = "200" ]; then + if python3 - "$tmp/mer-config.json" "$EXCHANGE_PUBLIC" <<'PY' +import json,sys +d=json.load(open(sys.argv[1])) +want=sys.argv[2].rstrip("/") +curs=list((d.get("currencies") or {}).keys()) +ex=d.get("exchanges") or [] +urls=[] +for e in ex: + if isinstance(e, dict): + u=e.get("base_url") or e.get("url") or e.get("exchange_base_url") or "" + if u: urls.append(u.rstrip("/")) + elif isinstance(e, str): + urls.append(e.rstrip("/")) +ok_goa = "GOA" in curs or any((e.get("currency") if isinstance(e, dict) else None)=="GOA" for e in ex) +ok_ex = any(want in u or "exchange.hacktivism.ch" in u for u in urls) +print("currencies", curs) +print("exchanges", urls[:5]) +sys.exit(0 if ok_goa and ok_ex else 1) +PY + then + ok "merchant config GOA + exchange.hacktivism.ch" + else + fail "merchant config GOA + exchange" "see currencies/exchanges" + fi + if json_has_alt_unit_names "$tmp/mer-config.json" >/tmp/alt-mer-s.$$ 2>&1; then + ok "merchant currencies alt_unit_names" "$(tr '\n' '; ' /dev/null || echo 000) +echo "LOCAL_CONFIG=$code" +if [ -n "$MER" ]; then + if podman exec "$MER" bash -c 'pgrep -f taler-merchant-httpd >/dev/null' 2>/dev/null; then + echo "HTTPD=1" + else + echo "HTTPD=0" + fi + if podman exec "$MER" test -x /usr/local/bin/check_merchant-health.sh 2>/dev/null; then + SKIP_ENSURE=1 podman exec -e SKIP_ENSURE=1 "$MER" /usr/local/bin/check_merchant-health.sh 2>&1 | sed 's/^/HEALTH /' | tail -30 + fi +fi +REMOTE +) + echo "$MOUT" | grep -q '^CTR=.\+' && ok "merchant container $(echo "$MOUT" | sed -n 's/^CTR=//p' | head -1)" || fail "merchant container" + echo "$MOUT" | grep -q 'LOCAL_CONFIG=200' && ok "merchant local :9010/config" || fail "merchant local :9010/config" + echo "$MOUT" | grep -q 'HTTPD=1' && ok "merchant-httpd process" || warn "merchant-httpd process" "not detected" + if echo "$MOUT" | grep -q 'HEALTH '; then + if echo "$MOUT" | grep -qiE 'ALL CRITICAL|HEALTH_EC=0|\[OK\]'; then + ok "merchant check_merchant-health.sh (sample OK)" + else + warn "merchant check_merchant-health.sh" "see remote output" + fi + fi +else + warn "merchant server-side" "ssh unavailable" +fi + +summary diff --git a/check_server.sh b/check_server.sh new file mode 100755 index 0000000..3a9618a --- /dev/null +++ b/check_server.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# Server-side component checks on koopa (via SSH). +set -euo pipefail +ROOT=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=lib.sh +source "$ROOT/lib.sh" + +# Area server.* — host/container ports via SSH +set_area server +set_group ssh +section "server · ssh ${KOOPA_SSH}" + +if ! koopa_ssh_ok; then + fail "ssh ${KOOPA_SSH}" "unreachable within ${SSH_CONNECT_TIMEOUT}s (SKIP_SSH=1 to skip)" + summary + exit 1 +fi +ok "ssh ${KOOPA_SSH}" + +# Run a remote script; collect structured lines +REMOTE=$(koopa_ssh_bash "${SSH_CMD_TIMEOUT}" <<'REMOTE' +set +e +report() { printf 'R|%s|%s|%s\n' "$1" "$2" "$3"; } + +# containers +for name in taler-hacktivism-bank taler-hacktivism-exchange-ansible taler-hacktivism; do + if podman ps --format '{{.Names}}' 2>/dev/null | grep -qx "$name"; then + st=$(podman ps --filter "name=^${name}$" --format '{{.Status}}' | head -1) + report OK "container $name" "$st" + else + # soft match + hit=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -E "$name|bank|exchange|hacktivism" | head -3 | tr '\n' ' ') + if podman ps --format '{{.Names}}' 2>/dev/null | grep -q bank && [ "$name" = taler-hacktivism-bank ]; then + bn=$(podman ps --format '{{.Names}}' | grep -i bank | head -1) + report OK "container bank ($bn)" "$(podman ps --filter name="$bn" --format '{{.Status}}' | head -1)" + elif podman ps --format '{{.Names}}' 2>/dev/null | grep -qi exchange && echo "$name" | grep -qi exchange; then + en=$(podman ps --format '{{.Names}}' | grep -i exchange | head -1) + report OK "container exchange ($en)" "$(podman ps --filter name="$en" --format '{{.Status}}' | head -1)" + elif podman ps --format '{{.Names}}' 2>/dev/null | grep -qx taler-hacktivism && [ "$name" = taler-hacktivism ]; then + report OK "container taler-hacktivism" "$(podman inspect -f '{{.State.Status}}' taler-hacktivism 2>/dev/null)" + else + report FAIL "container $name" "not running (seen: $hit)" + fi + fi +done + +# local HTTP on pasta ports +for spec in \ + "bank-api|http://127.0.0.1:9012/config|200" \ + "bank-integration|http://127.0.0.1:9012/taler-integration/config|200" \ + "landing|http://127.0.0.1:9013/intro/|200" \ + "exchange|http://127.0.0.1:9011/config|200" \ + "merchant|https://127.0.0.1:9010/config|200" +do + IFS='|' read -r id url want <<<"$spec" + code=$(curl -skS -m 5 -o /tmp/mon-s.out -w '%{http_code}' "$url" 2>/dev/null || echo 000) + if [ "$code" = "$want" ]; then + report OK "local $id : ${url#*//}" "HTTP $code" + else + report FAIL "local $id" "HTTP $code want $want" + fi +done + +# processes inside bank +BANK=$(podman ps --format '{{.Names}}' | grep -iE 'bank|hacktivism-bank' | head -1) +if [ -n "$BANK" ]; then + if podman exec "$BANK" bash -c 'pgrep -f "MainKt serve|libeufin-bank serve" >/dev/null' 2>/dev/null; then + report OK "libeufin-bank process" "in $BANK" + else + report FAIL "libeufin-bank process" "not running in $BANK" + fi + if podman exec "$BANK" bash -c 'pg_isready -q' 2>/dev/null; then + report OK "postgres (bank)" "ready" + else + report WARN "postgres (bank)" "pg_isready failed" + fi + if podman exec "$BANK" bash -c 'pgrep -x nginx >/dev/null' 2>/dev/null; then + report OK "nginx landing" "in $BANK" + else + report WARN "nginx landing" "not seen in $BANK" + fi +else + report FAIL "bank container" "none" +fi + +# exchange process / systemd if any +EX=$(podman ps --format '{{.Names}}' | grep -i exchange | head -1) +if [ -n "$EX" ]; then + if podman exec "$EX" bash -c 'pgrep -f taler-exchange-httpd >/dev/null || systemctl is-active taler-exchange-httpd 2>/dev/null | grep -q active' 2>/dev/null; then + report OK "exchange-httpd" "in $EX" + else + # config answering is enough + report WARN "exchange-httpd process" "not detected; port check above" + fi +fi + +MER=$(podman ps --format '{{.Names}}' | grep -E '^taler-hacktivism$' | head -1) +[ -z "$MER" ] && MER=$(podman ps --format '{{.Names}}' | grep -i merchant | head -1) +if [ -n "$MER" ]; then + if podman exec "$MER" bash -c 'pgrep -f taler-merchant-httpd >/dev/null || true; curl -sk -m 3 -o /dev/null -w %{http_code} https://127.0.0.1:9010/config' 2>/dev/null | grep -q 200; then + report OK "merchant-httpd" "responds in $MER" + else + report WARN "merchant-httpd" "check manually in $MER" + fi +fi + +# caddy on host +if systemctl is-active caddy >/dev/null 2>&1 || pgrep -x caddy >/dev/null 2>&1; then + report OK "caddy" "active" +else + report WARN "caddy" "not detected as active" +fi + +# disk free (host + containers) — lines for mon_disk_check_remote_text on laptop side +echo "###DISK_HOST###" +df -Pk / /var /home /tmp 2>/dev/null || df -Pk +echo "###DISK_CTRS###" +for c in $(podman ps --format '{{.Names}}' 2>/dev/null); do + echo "###CTR $c###" + podman exec "$c" df -Pk / /var /tmp 2>/dev/null || podman exec "$c" df -Pk 2>/dev/null +done +REMOTE +) + +while IFS= read -r line; do + case "$line" in + R\|*) + IFS='|' read -r _ st label detail <<<"$line" + case "$st" in + OK) ok "$label${detail:+ ($detail)}" ;; + FAIL) fail "$label" "$detail" ;; + WARN) warn "$label" "$detail" ;; + esac + ;; + esac +done <<<"$REMOTE" + +set_group disk +section "server · disk free space" +_host_df=$(printf '%s\n' "$REMOTE" | sed -n '/^###DISK_HOST###$/,/^###DISK_CTRS###$/p' | sed '1d;$d') +mon_disk_check_remote_text "ssh:${KOOPA_SSH}" "$_host_df" || true +_ctr=""; _buf="" +while IFS= read -r _line || [ -n "$_line" ]; do + case "$_line" in + '###CTR '*) + if [ -n "$_ctr" ]; then mon_disk_check_remote_text "ctr:${_ctr}" "$_buf" || true; fi + _ctr=${_line####CTR }; _ctr=${_ctr%###}; _buf="" + ;; + '###DISK_CTRS###'|'###DISK_HOST###') ;; + *) + # only buffer after we entered CTR section + if [ -n "$_ctr" ]; then _buf="${_buf}${_line}"$'\n'; fi + ;; + esac +done <<<"$REMOTE" +[ -n "$_ctr" ] && mon_disk_check_remote_text "ctr:${_ctr}" "$_buf" || true + +summary diff --git a/check_surface.sh b/check_surface.sh new file mode 100755 index 0000000..1452d3b --- /dev/null +++ b/check_surface.sh @@ -0,0 +1,692 @@ +#!/usr/bin/env bash +# check_surface.sh — REMOTE-ONLY public surface / ecosystem scan. +# +# NOT in default phases. Explicit: +# ./taler-monitoring.sh surface +# ./taler-monitoring.sh -d hacktivism.ch surface +# +# Without -d / with generic ecosystem: catalog of taler.net, gnunet.org, +# taler-systems.com, taler-ops, mattermost, … +# With -d DOMAIN: expand domains.conf stack hosts + catalog entries matching +# that domain + common subdomain guesses; port/protocol probes only remote. +# +# Rules: +# - no SSH into targets, no local podman on targets +# - ICMP/port alone never proves "down" — confirm via expected protocol +# - catalogued host unreachable via protocol → ERROR +# - version unknown after probes → WARN +# - CVE hit (OSV) for identified software+version → ERROR +# +set -euo pipefail +ROOT=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=lib.sh +source "$ROOT/lib.sh" + +set_area surface +section "surface · remote ecosystem / domain inventory (outside-in only)" + +CATALOG="${SURFACE_CATALOG:-$ROOT/surface-catalog.conf}" +# Common ports when scanning a host discovered without explicit list +DEFAULT_SCAN_PORTS="${SURFACE_SCAN_PORTS:-22,80,443,993,8443,9418}" +PORT_TIMEOUT="${SURFACE_PORT_TIMEOUT:-2}" +HTTP_TIMEOUT="${SURFACE_HTTP_TIMEOUT:-12}" +# CVE via OSV (public, no key). Disable: SURFACE_CVE=0 +: "${SURFACE_CVE:=1}" +# Extra ports when -d domain mode +DOMAIN_EXTRA_PORTS="${SURFACE_DOMAIN_PORTS:-22,80,443,8443}" + +declare -A HOST_PORTS # host -> comma ports +declare -A HOST_PROTO # host -> expect proto +declare -A HOST_LABEL # host -> short label +declare -A HOST_EXPECT # host -> 1 if catalogued (must respond) +ORDERED_HOSTS=() + +add_host() { + local h="$1" ports="${2:-}" proto="${3:-https}" label="${4:-}" expect="${5:-1}" + h=$(printf '%s' "$h" | tr 'A-Z' 'a-z' | sed 's#^https\?://##;s#/.*##;s/\.$//') + [ -n "$h" ] || return 0 + if [ -z "${HOST_PORTS[$h]+x}" ]; then + ORDERED_HOSTS+=("$h") + HOST_PORTS[$h]="${ports:-443}" + HOST_PROTO[$h]="${proto:-https}" + HOST_LABEL[$h]="${label:-$h}" + HOST_EXPECT[$h]="$expect" + else + # merge ports + local p + for p in ${ports//,/ }; do + case ",${HOST_PORTS[$h]}," in + *",$p,"*) ;; + *) HOST_PORTS[$h]="${HOST_PORTS[$h]},$p" ;; + esac + done + [ "${HOST_EXPECT[$h]}" = "1" ] || HOST_EXPECT[$h]="$expect" + fi +} + +load_catalog() { + local line h ports proto label + [ -f "$CATALOG" ] || { + warn "catalog" "missing $CATALOG — using empty base list" + return 0 + } + while IFS= read -r line || [ -n "$line" ]; do + line=${line%%#*} + line=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + [ -z "$line" ] && continue + # host ports proto label + h=$(echo "$line" | awk '{print $1}') + ports=$(echo "$line" | awk '{print $2}') + proto=$(echo "$line" | awk '{print $3}') + label=$(echo "$line" | awk '{print $4}') + [ "$ports" = "-" ] && ports="" + add_host "$h" "$ports" "${proto:-https}" "${label:-}" 1 + done <"$CATALOG" + info "catalog" "loaded $CATALOG · ${#ORDERED_HOSTS[@]} hosts" +} + +# Clear inventory (used when switching to domain-only scope) +clear_hosts() { + ORDERED_HOSTS=() + unset HOST_PORTS HOST_PROTO HOST_LABEL HOST_EXPECT + declare -gA HOST_PORTS HOST_PROTO HOST_LABEL HOST_EXPECT +} + +# Expand for -d domain: ONLY that domain (remote). No whole-ecosystem catalog. +expand_domain_scope() { + local d="$1" h + d=${d#https://}; d=${d%/} + d=${d#http://} + info "scope" "domain mode −d $d (remote only · not full ecosystem catalog)" + + clear_hosts + + # stack endpoints from applied profile (always expected) + for h in \ + "${BANK_PUBLIC:-}" \ + "${EXCHANGE_PUBLIC:-}" \ + "${MERCHANT_PUBLIC:-}" \ + "${PAIVANA_PUBLIC:-}" + do + [ -n "$h" ] || continue + add_host "$h" "80,443,$DOMAIN_EXTRA_PORTS" https "stack" 1 + done + + # apex + common subdomains (expected if DNS exists — set expect after DNS in scan, + # but catalogued stack already expected; guesses start optional) + for sub in "" www bank exchange taler merchant backend shop shops stage \ + git docs www2 api static media landing mon401 + do + if [ -z "$sub" ]; then + h="$d" + else + h="${sub}.${d}" + fi + # stack hosts already added as expect=1; guesses optional until we promote + if [ -z "${HOST_PORTS[$h]+x}" ]; then + add_host "$h" "$DOMAIN_EXTRA_PORTS" https "guess" 0 + fi + done + + # also pick catalog lines that belong to this domain only + if [ -f "$CATALOG" ]; then + local line ports proto label + while IFS= read -r line || [ -n "$line" ]; do + line=${line%%#*} + line=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + [ -z "$line" ] && continue + h=$(echo "$line" | awk '{print $1}') + h=$(printf '%s' "$h" | tr 'A-Z' 'a-z' | sed 's#^https\?://##;s#/.*##') + if [ "$h" = "$d" ] || [[ "$h" == *".$d" ]]; then + ports=$(echo "$line" | awk '{print $2}') + proto=$(echo "$line" | awk '{print $3}') + label=$(echo "$line" | awk '{print $4}') + [ "$ports" = "-" ] && ports="" + add_host "$h" "$ports" "${proto:-https}" "${label:-dom}" 1 + fi + done <"$CATALOG" + fi +} + +# Ecosystem mode: full catalog; all entries expected +expand_ecosystem_scope() { + info "scope" "ecosystem mode (taler.net / gnunet.org / taler-systems.com / taler-ops / mattermost, … · remote only)" + local h + for h in "${ORDERED_HOSTS[@]}"; do + HOST_EXPECT[$h]=1 + done +} + +# --- remote probes (no SSH) --- +dns_ok() { + local h="$1" + getent ahosts "$h" >/dev/null 2>&1 || getent hosts "$h" >/dev/null 2>&1 +} + +# TCP connect only (not proof of service) +tcp_open() { + local h="$1" port="$2" + if command -v timeout >/dev/null 2>&1; then + timeout "$PORT_TIMEOUT" bash -c "echo >/dev/tcp/${h}/${port}" 2>/dev/null + else + bash -c "echo >/dev/tcp/${h}/${port}" 2>/dev/null + fi +} + +# ICMP optional — never sole grounds for ERROR +ping_host() { + local h="$1" + ping -c 1 -W 2 "$h" >/dev/null 2>&1 || ping -c 1 -w 2 "$h" >/dev/null 2>&1 +} + +# HTTPS probe: code, server header, version hints, cert subject/dates +probe_https() { + local h="$1" port="${2:-443}" + local url hdr body code server via cert_end subj san + url="https://${h}/" + [ "$port" != "443" ] && url="https://${h}:${port}/" + body=$(mktemp) + hdr=$(mktemp) + code=$(curl -skS -m "$HTTP_TIMEOUT" -D "$hdr" -o "$body" -w '%{http_code}' \ + --connect-timeout "$PORT_TIMEOUT" "$url" 2>/dev/null || echo 000) + server=$(awk 'BEGIN{IGNORECASE=1} /^server:/{sub(/\r$/,""); sub(/^server:[[:space:]]*/,""); print; exit}' "$hdr" 2>/dev/null || true) + via=$(awk 'BEGIN{IGNORECASE=1} /^x-powered-by:/{sub(/\r$/,""); sub(/^[^:]+:[[:space:]]*/,""); print; exit}' "$hdr" 2>/dev/null || true) + # cert + cert_end=$(echo | openssl s_client -servername "$h" -connect "${h}:${port}" 2>/dev/null \ + | openssl x509 -noout -enddate 2>/dev/null | sed 's/notAfter=//') + subj=$(echo | openssl s_client -servername "$h" -connect "${h}:${port}" 2>/dev/null \ + | openssl x509 -noout -subject 2>/dev/null | head -1) + # taler version hints in body + local taler_hint + taler_hint=$(grep -oE 'taler[^"[:space:]]{0,40}|GNU Taler|libeufin|gnunet' "$body" 2>/dev/null | head -3 | tr '\n' ' ' || true) + # config JSON version if /config works + local cfg_ver="" + local ccode + ccode=$(curl -skS -m "$HTTP_TIMEOUT" -o "$body" -w '%{http_code}' \ + "https://${h}/config" 2>/dev/null || echo 000) + if [ "$ccode" = "200" ]; then + cfg_ver=$(python3 -c 'import json,sys +try: + d=json.load(open(sys.argv[1])) + print(d.get("version") or d.get("name") or d.get("currency") or "config-json") +except Exception: + print("")' "$body" 2>/dev/null || true) + fi + rm -f "$hdr" "$body" + printf 'code=%s server=%s powered=%s cert_end=%s cfg=%s hint=%s' \ + "$code" "${server:-}" "${via:-}" "${cert_end:-}" "${cfg_ver:-}" "${taler_hint:-}" + # return 0 if HTTP answered (any 2xx/3xx/4xx — service is up) + case "$code" in + 2??|3??|4??) return 0 ;; + *) return 1 ;; + esac +} + +probe_http() { + local h="$1" port="${2:-80}" + local url code + url="http://${h}/" + [ "$port" != "80" ] && url="http://${h}:${port}/" + code=$(curl -sS -m "$HTTP_TIMEOUT" -o /dev/null -w '%{http_code}' \ + --connect-timeout "$PORT_TIMEOUT" "$url" 2>/dev/null || echo 000) + printf 'code=%s' "$code" + case "$code" in 2??|3??|4??) return 0 ;; *) return 1 ;; esac +} + +probe_ssh_banner() { + local h="$1" port="${2:-22}" + local ban + ban=$(timeout "$PORT_TIMEOUT" bash -c "exec 3<>/dev/tcp/${h}/${port}; dd bs=256 count=1 <&3 2>/dev/null" 2>/dev/null \ + | tr -d '\r' | head -1 || true) + printf 'banner=%s' "${ban:-}" + [ -n "$ban" ] +} + +# Map Server header → package name guess for OSV +guess_package() { + local server="$1" + local s + s=$(printf '%s' "$server" | tr 'A-Z' 'a-z') + case "$s" in + *nginx*) echo "nginx" ;; + *apache*|*httpd*) echo "apache" ;; + *caddy*) echo "caddy" ;; + *openbsd\ httpd*) echo "openbsd-httpd" ;; + *) echo "" ;; + esac +} + +extract_version() { + local server="$1" + # nginx/1.22.1 → 1.22.1 + printf '%s' "$server" | sed -n 's/.*\/\([0-9][0-9.]*\).*/\1/p' | head -1 +} + +# OSV / CVE check for software version taken from Server headers. +# +# Important: headers only expose upstream versions (nginx/1.26.3), never the +# Debian package revision (1.26.3-3+deb13u7). Querying OSV ecosystem=Debian +# with the bare version yields massive false positives (ancient DEBIAN-CVE-* +# with introduced:0 and no fixed event, plus every package revision that +# merely *starts with* 1.26.3). +# +# Policy: +# - bare upstream version → upstream SEMVER signal only (WARN by default) +# - full Debian package version (contains '-') → Debian OSV, filtered +# - SURFACE_CVE=0 disables; SURFACE_CVE_LEVEL=error|warn (default warn for bare) +check_cves() { + local pkg="$1" ver="$2" host="$3" + [ "$SURFACE_CVE" = "1" ] || return 0 + [ -n "$pkg" ] && [ -n "$ver" ] || return 0 + local level out rc + # bare header versions are approximate → WARN unless overridden + if [[ "$ver" == *-* ]] || [[ "$ver" == *+* ]]; then + level="${SURFACE_CVE_LEVEL:-error}" + else + level="${SURFACE_CVE_LEVEL:-warn}" + fi + out=$( + SURFACE_CVE_PKG="$pkg" SURFACE_CVE_VER="$ver" SURFACE_CVE_HOST="$host" python3 - <<'PY' 2>/dev/null || true +import json, os, re, urllib.request + +pkg = os.environ.get("SURFACE_CVE_PKG", "") +ver = os.environ.get("SURFACE_CVE_VER", "") +host = os.environ.get("SURFACE_CVE_HOST", "") + +def ver_tuple(s: str): + nums = [int(x) for x in re.findall(r"\d+", s or "")[:5]] + return tuple(nums) if nums else () + +def vt_cmp(a, b): + n = max(len(a), len(b)) + a = a + (0,) * (n - len(a)) + b = b + (0,) * (n - len(b)) + return (a > b) - (a < b) + +def in_semver_events(ver, events): + """True if ver is in [introduced, fixed) for SEMVER-like event list.""" + vt = ver_tuple(ver) + if not vt: + return False + # process sequential introduced/fixed pairs + intro = None + for e in events or []: + if "introduced" in e: + intro = e.get("introduced") + elif "fixed" in e or "last_affected" in e: + fixed = e.get("fixed") + last = e.get("last_affected") + lo = ver_tuple("0" if intro in (None, "0") else str(intro)) + if vt_cmp(vt, lo) < 0: + intro = None + continue + if fixed is not None: + if vt_cmp(vt, ver_tuple(str(fixed))) < 0: + return True + elif last is not None: + if vt_cmp(vt, ver_tuple(str(last))) <= 0: + return True + intro = None + # open-ended introduced without fixed → ignore (never clears; Debian noise) + return False + +def osv_post(body): + req = urllib.request.Request( + "https://api.osv.dev/v1/query", + data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=20) as r: + return json.load(r) + except Exception: + return {} + +def osv_get(vid): + try: + with urllib.request.urlopen(f"https://api.osv.dev/v1/vulns/{vid}", timeout=15) as r: + return json.load(r) + except Exception: + return {} + +bare = not (("-" in ver) or ("+" in ver)) +hits = [] + +if bare: + # Upstream signal via CVE records that carry extracted_events for this package + # (Server header has no Debian revision — do NOT use ecosystem=Debian). + seed = { + "nginx": [ + "CVE-2025-23419", + "CVE-2025-53859", + "CVE-2024-7347", + "CVE-2024-34161", + "CVE-2024-32760", + "CVE-2024-31079", + "CVE-2024-24989", + "CVE-2024-24990", + ], + "apache": [ + "CVE-2024-38474", + "CVE-2024-38476", + "CVE-2024-38477", + "CVE-2023-31122", + "CVE-2023-43622", + ], + "caddy": [ + "CVE-2022-29718", + "CVE-2023-50463", + ], + }.get(pkg, []) + for vid in seed: + doc = osv_get(vid) + if not doc: + continue + ok_hit = False + for a in doc.get("affected") or []: + # prefer extracted_events (human SEMVER) on GIT/nginx ranges + for rg in a.get("ranges") or []: + db = rg.get("database_specific") or {} + extracted = db.get("extracted_events") or [] + if extracted and in_semver_events(ver, extracted): + # package name filter when present + cpes = db.get("cpe") or [] + if isinstance(cpes, str): + cpes = [cpes] + blob = json.dumps(a).lower() + json.dumps(cpes).lower() + if pkg == "nginx" and "nginx" not in blob and "f5" not in blob: + continue + ok_hit = True + break + if rg.get("type") == "SEMVER" and in_semver_events(ver, rg.get("events") or []): + ok_hit = True + break + if ok_hit: + break + if ok_hit: + hits.append(vid) +else: + # Full package version — Debian ecosystem is meaningful + data = osv_post({"package": {"name": pkg, "ecosystem": "Debian"}, "version": ver}) + for v in data.get("vulns") or []: + vid = v.get("id") or "?" + actionable = False + for a in v.get("affected") or []: + for rg in a.get("ranges") or []: + events = rg.get("events") or [] + has_end = any(("fixed" in e) or ("last_affected" in e) for e in events) + if not has_end: + continue # open-ended Debian noise + if rg.get("type") in ("ECOSYSTEM", "SEMVER"): + # version string is full deb version; trust OSV query match + # but only if a fixed event exists (actionable) + actionable = True + break + if actionable: + break + # exact version listed + versions = a.get("versions") or [] + if ver in versions: + actionable = True + break + if actionable: + hits.append(vid) + +# de-dup preserve order +seen = set() +uniq = [] +for h in hits: + if h not in seen: + seen.add(h) + uniq.append(h) + +if uniq: + print(f"HIT {len(uniq)} " + ",".join(uniq[:12])) +else: + print("CLEAN") +PY + ) + rc=0 + case "$out" in + HIT\ *) + local n ids + n=$(printf '%s' "$out" | awk '{print $2}') + ids=$(printf '%s' "$out" | cut -d' ' -f3-) + if [ "$level" = "error" ]; then + err "cve" "$host $pkg $ver — $n actionable vuln(s)" "$ids" + rc=1 + else + warn "cve" "$host $pkg $ver — $n actionable vuln(s) (header version · not Debian pkg)" "$ids" + rc=0 + fi + ;; + CLEAN) + if [[ "$ver" == *-* ]] || [[ "$ver" == *+* ]]; then + info "cve" "$host $pkg $ver — OSV clean (Debian package version)" + else + info "cve" "$host $pkg $ver — no actionable upstream CVE for bare Server-header version" + fi + ;; + *) + info "cve" "$host $pkg $ver — CVE probe skipped/unavailable" + ;; + esac + return "$rc" +} + +cert_expiry_check() { + local h="$1" end="$2" + [ -n "$end" ] || return 0 + local end_epoch now left days + end_epoch=$(date -d "$end" +%s 2>/dev/null || date -j -f "%b %e %T %Y %Z" "$end" +%s 2>/dev/null || echo 0) + now=$(date +%s) + [ "$end_epoch" -gt 0 ] || return 0 + left=$((end_epoch - now)) + days=$((left / 86400)) + if [ "$left" -le 0 ]; then + err "tls" "$h certificate EXPIRED" "notAfter=$end" + return 1 + fi + if [ "$days" -le 14 ]; then + warn "tls" "$h certificate expires in ${days}d" "notAfter=$end" + else + info "tls" "$h cert ok · ${days}d left · notAfter=$end" + fi + return 0 +} + +scan_host() { + local h="$1" + local ports proto label expect + local p open_ports=() any_proto_ok=0 ping_ok=0 dns=0 + local probe_detail server_hdr pkg ver + + ports=${HOST_PORTS[$h]:-443} + proto=${HOST_PROTO[$h]:-https} + label=${HOST_LABEL[$h]:-$h} + expect=${HOST_EXPECT[$h]:-0} + + set_group "$label" + + if dns_ok "$h"; then + dns=1 + ok "dns" "$h resolves" + else + if [ "$expect" = "1" ]; then + err "dns" "$h does not resolve (catalogued)" + else + info "dns" "$h no resolve (optional guess) — skip" + fi + return 0 + fi + + if ping_host "$h"; then + ping_ok=1 + info "ping" "$h ICMP ok" + else + info "ping" "$h ICMP no reply (not decisive)" + fi + + # port scan (unique ports) + local _seen_ports=" " + for p in ${ports//,/ }; do + [ -n "$p" ] || continue + case "$_seen_ports" in *" $p "*) continue ;; esac + _seen_ports="$_seen_ports$p " + if tcp_open "$h" "$p"; then + open_ports+=("$p") + info "port" "$h:$p open (TCP)" + else + info "port" "$h:$p closed/filtered (TCP)" + fi + done + + # Protocol verification on open ports (and always try 443/80 for https/http expect) + local try_ports=("${open_ports[@]}") + if [ ${#try_ports[@]} -eq 0 ]; then + # still try expected protocol ports even if scan said closed (scan false negatives) + case "$proto" in + https) try_ports=(443) ;; + http) try_ports=(80) ;; + ssh) try_ports=(22) ;; + *) try_ports=(443 80) ;; + esac + fi + + server_hdr="" + ver="" + for p in "${try_ports[@]}"; do + case "$p" in + 443|8443) + if probe_detail=$(probe_https "$h" "$p"); then + any_proto_ok=1 + ok "https" "$h:$p up · $probe_detail" + server_hdr=$(printf '%s' "$probe_detail" | sed -n 's/.*server=\([^ ]*\).*/\1/p') + # cert + local cend + cend=$(printf '%s' "$probe_detail" | sed -n 's/.*cert_end=\([^ ]*\).*/\1/p') + cert_expiry_check "$h" "$cend" || true + local cfg + cfg=$(printf '%s' "$probe_detail" | sed -n 's/.*cfg=\([^ ]*\).*/\1/p') + if [ -n "$cfg" ]; then + info "version" "$h config/api hint: $cfg" + ver="$cfg" + fi + else + info "https" "$h:$p no HTTP response · ${probe_detail:-}" + fi + ;; + 80|8080) + if probe_detail=$(probe_http "$h" "$p"); then + any_proto_ok=1 + ok "http" "$h:$p up · $probe_detail" + else + info "http" "$h:$p no HTTP response" + fi + ;; + 22) + if probe_detail=$(probe_ssh_banner "$h" "$p"); then + any_proto_ok=1 + ok "ssh" "$h:$p banner · $probe_detail" + ver=$(printf '%s' "$probe_detail" | sed 's/banner=//') + else + info "ssh" "$h:$p no SSH banner" + fi + ;; + *) + if tcp_open "$h" "$p"; then + info "tcp" "$h:$p open · protocol unknown" + fi + ;; + esac + done + + # Version summary + if [ -n "$server_hdr" ]; then + info "server-header" "$h · $server_hdr" + pkg=$(guess_package "$server_hdr") + ver_soft=$(extract_version "$server_hdr") + if [ -n "$pkg" ] && [ -n "$ver_soft" ]; then + info "version" "$h software $pkg $ver_soft (from Server header)" + check_cves "$pkg" "$ver_soft" "$h" || true + elif [ -n "$server_hdr" ]; then + warn "version" "$h Server header present but version unknown · $server_hdr" + fi + elif [ -n "$ver" ]; then + info "version" "$h · $ver" + else + if [ "$any_proto_ok" = "1" ]; then + warn "version" "$h reachable but software version unknown" + fi + fi + + # Expected service must answer protocol (not just ping/port) + # Note: ${array[*]:-x} is NOT valid default syntax (bash treats : as slice). + local ports_txt="${open_ports[*]}" + ports_txt=${ports_txt:-none} + if [ "$expect" = "1" ]; then + if [ "$any_proto_ok" = "1" ]; then + ok "reachability" "$h catalogued service OK (protocol confirmed)" + else + err "reachability" "$h catalogued but not reachable via ${proto}/protocol" \ + "dns=$dns ping=$ping_ok open_ports=${ports_txt} (ICMP/TCP alone not enough; protocol failed)" + fi + else + if [ "$any_proto_ok" = "1" ]; then + info "reachability" "$h optional host responds" + else + info "reachability" "$h optional · no protocol response (ok)" + fi + fi +} + +# --- main --- +# Scope (remote only): +# ./taler-monitoring.sh surface → full ecosystem catalog +# ./taler-monitoring.sh -d hacktivism.ch surface → only that domain +if [ "${SURFACE_SCOPE:-}" = "ecosystem" ]; then + load_catalog + expand_ecosystem_scope +elif [ "${SURFACE_SCOPE:-}" = "domain" ] || [ "${TALER_DOMAIN_FROM_CLI:-0}" = "1" ]; then + # domain mode: do not load whole ecosystem first + CATALOG="${SURFACE_CATALOG:-$ROOT/surface-catalog.conf}" + expand_domain_scope "${TALER_DOMAIN:-hacktivism.ch}" +elif [ "${SURFACE_SCOPE:-auto}" = "auto" ]; then + # no -d → ecosystem; with -d → domain (TALER_DOMAIN_FROM_CLI) + if [ "${TALER_DOMAIN_FROM_CLI:-0}" = "1" ]; then + CATALOG="${SURFACE_CATALOG:-$ROOT/surface-catalog.conf}" + expand_domain_scope "$TALER_DOMAIN" + else + load_catalog + expand_ecosystem_scope + fi +else + load_catalog + expand_ecosystem_scope +fi + +info "inventory" "${#ORDERED_HOSTS[@]} hosts to probe (remote only · no SSH · no local podman on targets)" +info "flags" "SURFACE_CVE=${SURFACE_CVE:-1} PORT_TIMEOUT=${PORT_TIMEOUT:-2} HTTP_TIMEOUT=${HTTP_TIMEOUT:-12}" + +for h in "${ORDERED_HOSTS[@]}"; do + [ -n "$h" ] || continue + scan_host "$h" || true +done + +# mon_disk on the *runner* only (optional) — not remote servers' disks +if [ "${SURFACE_CHECK_RUNNER_DISK:-0}" = "1" ]; then + set_group disk + if declare -F mon_disk_check_host >/dev/null 2>&1; then + mon_disk_check_host "runner" || true + else + warn "disk" "mon_disk_check_host not available" + fi +fi + +# summary returns non-zero when FAIL_N>0; do not trip set -e mid-script +summary || true +if [ "${FAIL_N:-0}" -eq 0 ]; then + exit 0 +fi +exit 1 diff --git a/check_urls.sh b/check_urls.sh new file mode 100755 index 0000000..36520d4 --- /dev/null +++ b/check_urls.sh @@ -0,0 +1,1595 @@ +#!/usr/bin/env bash +# Outside-in public HTTPS checks (no SSH). +set -euo pipefail +ROOT=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=lib.sh +source "$ROOT/lib.sh" + +# Area www.* — public HTTPS (outside-in) +# Groups: www.exchange / www.perf / www.stats / www.bank / www.merchant / +# www.paivana / www.landing +set_area www +section "www · public URLs · ${TALER_DOMAIN:-?} (outside-in, no SSH)" + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +check_url() { + local label="$1" expect="$2" url="$3" + local code + code=$(http_code "$url") + case ",$expect," in + *",$code,"*) ok "$label $url" ;; + *) fail "$label $url" "got $code want $expect" ;; + esac +} + +# Pre-launch remote stack: local=0 and no expected currency (e.g. prod LFP not live yet). +# Missing endpoints → INFO, not WARN (infra not supposed to serve yet). +is_prelaunch_stack() { + [ "${LOCAL_STACK:-1}" = "0" ] && [ -z "${EXPECT_CURRENCY:-}" ] +} + +# Soft check: OK on expect, WARN on foreign stack if down, ERROR on local stack. +# Always treats 301/302/303/307/308→200 (follow) as success when 200 is expected. +# Pre-launch (no currency): INFO instead of WARN for missing endpoints. +check_url_soft() { + local label="$1" expect="$2" url="$3" + local code + code=$(http_code "$url") + case ",$expect," in + *",$code,"*) ok "$label $url" ;; + *) + # Follow redirects (308 Permanent Redirect is common for trailing slash) + case "$code" in + 301|302|303|307|308) + code=$(curl -skS --max-redirs 5 -L -m "${TIMEOUT}" -o /dev/null -w '%{http_code}' "$url" 2>/dev/null || echo 000) + case ",$expect," in + *",$code,"*) ok "$label $url" "via redirect → HTTP $code"; return ;; + esac + ;; + esac + if [ "${LOCAL_STACK:-1}" = "0" ]; then + if is_prelaunch_stack; then + info "$label $url" "HTTP $code — pre-launch / not serving yet" + else + warn "$label $url" "got $code (optional on remote domain)" + fi + else + fail "$label $url" "got $code want $expect" + fi + ;; + esac +} + +expect_currency() { + local label="$1" file="$2" want="${EXPECT_CURRENCY:-}" + local cur + cur=$(python3 -c 'import json,sys;print(json.load(open(sys.argv[1])).get("currency",""))' "$file" 2>/dev/null || true) + if [ -z "$want" ]; then + info "$label currency" "${cur:-?}" + return + fi + if [ "$cur" = "$want" ]; then + ok "$label currency=$want" + else + fail "$label currency" "got ${cur:-?} want $want" + fi +} + +# Legal docs: /terms and /privacy must be HTTP 200 with a real document body +# (not empty, not "not configured", not JSON API error). +# $1=label $2=url $3=optional needle regex (case-insensitive) for local stack +check_legal_doc() { + local label="$1" url="$2" needle="${3:-}" + local f code soft + soft=0 + [ "${LOCAL_STACK:-1}" = "0" ] && soft=1 + f=$(mktemp) + code=$(curl -skS --max-redirs 5 -L -m "${TIMEOUT}" \ + -H "Accept: text/html,text/markdown,text/plain,*/*" \ + -o "$f" -w '%{http_code}' "$url" 2>/dev/null || echo 000) + if [ "$code" != "200" ]; then + rm -f "$f" + if [ "$soft" = "1" ]; then + if is_prelaunch_stack; then + info "$label" "HTTP $code — pre-launch / not serving yet · $url" + else + warn "$label" "HTTP $code — $url" + fi + else + fail "$label" "HTTP $code — $url" + fi + return + fi + if [ ! -s "$f" ]; then + rm -f "$f" + fail "$label" "empty body — $url" + return + fi + # merchant returns plain "not configured" when PRIVACY_ETAG missing + if grep -qiE '^(not configured)\s*$' "$f" 2>/dev/null \ + || grep -qiE '"code"\s*:\s*21' "$f" 2>/dev/null; then + rm -f "$f" + fail "$label" "not configured / API error — $url" + return + fi + if [ -n "$needle" ] && [ "${LOCAL_STACK:-1}" = "1" ]; then + if ! grep -qiE "$needle" "$f" 2>/dev/null; then + warn "$label content" "missing /$needle/ — $url" + rm -f "$f" + return + fi + fi + ok "$label" "HTTP 200 · $(wc -c <"$f" | tr -d ' ') bytes" + rm -f "$f" +} + +# Shared shape for bank mint JSON (demo-withdraw + auto-account). +# Args: $1=json file $2=auto|demo +# stdout (ok): one detail line; for demo, optional "\twithdrawal_id" suffix. +# stderr (fail): short reason. Exit 0/1. +validate_bank_withdraw_json() { + python3 - "$1" "$2" <<'PY' +import json, re, sys +from urllib.parse import urlparse + +def check_withdraw_uri(wuri: str): + """HOST or HOST:non-default-port; default :443/:80 must be stripped (wallet fix).""" + m = re.match( + r"^taler://withdraw/([^/]+)/taler-integration/([0-9a-fA-F-]+)$", + wuri or "", + ) + if not m: + print( + "need taler://withdraw/HOST/taler-integration/ID:", + (wuri or "")[:120], + file=sys.stderr, + ) + return None + host = m.group(1) + if not host or host.startswith(":"): + print("bad withdraw host:", host, file=sys.stderr) + return None + if ":" in host: + h, _, p = host.rpartition(":") + if not h or not p.isdigit(): + print("bad withdraw host:port:", host, file=sys.stderr) + return None + if p in ("443", "80"): + print("withdraw must strip default port :%s:" % p, host, file=sys.stderr) + return None + return host, m.group(2), wuri + +path, mode = sys.argv[1], sys.argv[2] +d = json.load(open(path)) +wuri = d.get("taler_withdraw_uri") or ( + d.get("qr_payload") if mode == "auto" else "" +) or "" + +if mode == "auto": + if not d.get("ok"): + print("ok!=true", file=sys.stderr) + sys.exit(1) + if d.get("payto_uri"): + print("payto_uri must not be present", file=sys.stderr) + sys.exit(1) + if not check_withdraw_uri(wuri): + sys.exit(1) + webui = d.get("login_url") or d.get("webui") or d.get("account_url") or "" + u = urlparse(webui) + if u.scheme not in ("http", "https") or "webui" not in (u.path or ""): + print("login webui missing:", webui[:80], file=sys.stderr) + sys.exit(1) + print("%s · %s" % (d.get("username", ""), wuri[:72])) +elif mode == "demo": + if not d.get("ok", True) and "taler_withdraw_uri" not in d: + print("not ok", file=sys.stderr) + sys.exit(1) + if not check_withdraw_uri(wuri): + sys.exit(1) + print("%s\t%s" % (wuri[:80], d.get("withdrawal_id") or "")) +else: + print("bad mode:", mode, file=sys.stderr) + sys.exit(2) +sys.exit(0) +PY +} + +# --- exchange (core; hard on local stack, soft on remote) --- www.exchange-NN +set_group exchange +if [ "${LOCAL_STACK:-1}" = "1" ]; then + check_url "exchange /config" 200 "$EXCHANGE_PUBLIC/config" +else + check_url_soft "exchange /config" 200 "$EXCHANGE_PUBLIC/config" +fi +code=$(http_body "$EXCHANGE_PUBLIC/config" "$tmp/ec.json") +if [ "$code" = "200" ]; then + expect_currency "exchange" "$tmp/ec.json" + # currency_specification.alt_unit_names (wallet codec) + if json_has_alt_unit_names "$tmp/ec.json" "${EXPECT_CURRENCY:-}" >/tmp/alt-ex.$$ 2>&1; then + ok "exchange /config alt_unit_names" "$(tr '\n' '; ' "$PERF_TSV" + +# Measure one URL: require HTTP expect (default 200), report time_total in ms. +# $1=label $2=url $3=optional expected codes (default 200) +check_perf() { + local label="$1" url="$2" expect="${3:-200}" + local out code t_s ms + out=$(curl -skS --max-redirs 3 -L -m "${PERF_CURL_TIMEOUT:-25}" \ + -o /dev/null -w '%{http_code} %{time_total}' "$url" 2>/dev/null || echo "000 0") + code=$(printf '%s' "$out" | awk '{print $1}') + t_s=$(printf '%s' "$out" | awk '{print $2}') + ms=$(awk -v t="${t_s:-0}" 'BEGIN{ + ms=(t+0)*1000 + if (ms>0 && ms<1) ms=1 + printf "%d", int(ms+0.5) + }') + # Always record sample for rollup (label, ms, http, url) + printf '%s\t%s\t%s\t%s\n' "$label" "$ms" "$code" "$url" >>"$PERF_TSV" + case ",$expect," in + *",$code,"*) + if [ "$ms" -ge "${PERF_FAIL_MS}" ] 2>/dev/null; then + fail "$label" "HTTP $code · ${ms} ms ≥ fail ${PERF_FAIL_MS} ms · $url" + elif [ "$ms" -ge "${PERF_WARN_MS}" ] 2>/dev/null; then + warn "$label" "HTTP $code · ${ms} ms ≥ warn ${PERF_WARN_MS} ms · $url" + else + ok "$label" "HTTP $code · ${ms} ms · $url" + fi + ;; + *) + if [ "${LOCAL_STACK:-1}" = "1" ]; then + fail "$label" "HTTP $code want $expect · ${ms} ms · $url" + elif is_prelaunch_stack; then + info "$label" "HTTP $code · ${ms} ms · pre-launch · $url" + else + warn "$label" "HTTP $code want $expect · ${ms} ms · $url" + fi + ;; + esac +} + +# Bank first (wallet-critical paths before UI chrome) +check_perf "perf bank /taler-integration/config" "$BANK_PUBLIC/taler-integration/config" +check_perf "perf bank /config" "$BANK_PUBLIC/config" +if [ "${CHECK_LANDING:-1}" = "1" ]; then + check_perf "perf bank /intro/" "$BANK_PUBLIC/intro/" + check_perf "perf bank /intro/stats.json" "$BANK_PUBLIC/intro/stats.json" 200 +fi +check_perf "perf bank /webui/" "$BANK_PUBLIC/webui/" 200,301,302 + +# Exchange +check_perf "perf exchange /config" "$EXCHANGE_PUBLIC/config" +check_perf "perf exchange /keys" "$EXCHANGE_PUBLIC/keys" +if [ "${CHECK_LANDING:-1}" = "1" ]; then + check_perf "perf exchange /intro/" "$EXCHANGE_PUBLIC/intro/" +fi + +# Merchant +check_perf "perf merchant /config" "$MERCHANT_PUBLIC/config" +check_perf "perf merchant /webui/" "$MERCHANT_PUBLIC/webui/" 200,301,302 +if [ "${CHECK_LANDING:-1}" = "1" ]; then + check_perf "perf merchant /intro/" "$MERCHANT_PUBLIC/intro/" +fi + +info "perf note" "RTT measured from this host (outside-in)" + +# --------------------------------------------------------------------------- +# Landing stats.json (public outside-in) — freshness + display fields. +# Timestamps (required for external measurement): +# generated_at_unix — epoch seconds (preferred) +# generated_at — ISO-8601 +# generated_at_human — footer / “updated” UI string +# Bank collector always sets all three; exchange/merchant scripts set unix too. +# Env: +# STATS_STALE_SECS warn if age > this (default 900 = 15m; timer often 5–15m) +# STATS_FAIL_SECS hard fail if age > this (default 3600); 0 = never hard-fail on age +# --------------------------------------------------------------------------- +: "${STATS_STALE_SECS:=900}" +: "${STATS_FAIL_SECS:=3600}" + +_fetch_landing_stats_json() { + # $1=name bank|exchange|merchant $2=base URL → writes $tmp/stats-$1.json, exit 0 if ok + local name="$1" base="$2" + local f="$tmp/stats-${name}.json" + local code ctr path + rm -f "$f" + code=$(http_body "${base}/intro/stats.json" "$f" 2>/dev/null || echo 000) + if [ "$code" = "200" ] && [ -s "$f" ]; then + return 0 + fi + # Optional inside-container fallback (no public path or empty) + if [ "${LOCAL_STACK:-0}" = "1" ] && [ "${SKIP_SSH:-0}" != "1" ] && koopa_ssh_ok 2>/dev/null; then + case "$name" in + bank) ctr=taler-hacktivism-bank; path=/var/www/bank-landing/stats.json ;; + exchange) ctr=taler-hacktivism-exchange-ansible; path=/var/www/exchange-landing/stats.json ;; + merchant) ctr=taler-hacktivism; path=/var/www/merchant-landing/stats.json ;; + *) return 1 ;; + esac + if koopa_ssh_run 12 "podman exec ${ctr} cat ${path} 2>/dev/null" >"$f" 2>/dev/null \ + && [ -s "$f" ]; then + return 0 + fi + fi + return 1 +} + +# Validate one stats.json for monitoring display + outside age measurement. +# Prints lines: OK|WARN|ERR|STALE|FAIL +# Sets $tmp/stats-meta-$name.txt with gen_unix|source for shared-feed compare. +check_landing_stats_json() { + local name="$1" base="$2" + local f="$tmp/stats-${name}.json" + local line hard_missing=0 + + if ! _fetch_landing_stats_json "$name" "$base"; then + if [ "${CHECK_LANDING:-1}" = "1" ] || [ "${LOCAL_STACK:-0}" = "1" ]; then + fail "stats ${name}" "HTTP/body missing · ${base}/intro/stats.json" + else + warn "stats ${name}" "stats.json not available" + fi + return 1 + fi + ok "stats ${name} reachable" "${base}/intro/stats.json" + + # Multi-line verdict from python (STATUS detail) + while IFS= read -r line; do + case "$line" in + META\ *) + printf '%s\n' "${line#META }" >"$tmp/stats-meta-${name}.txt" + ;; + OK\ *) + ok "stats ${name}" "${line#OK }" + ;; + WARN\ *) + warn "stats ${name}" "${line#WARN }" + ;; + STALE\ *) + warn "stats ${name} freshness" "stale · ${line#STALE }" + ;; + FAIL\ *) + fail "stats ${name}" "${line#FAIL }" + hard_missing=1 + ;; + ERR\ *) + if [ "${LOCAL_STACK:-0}" = "1" ] || [ "${CHECK_LANDING:-1}" = "1" ]; then + fail "stats ${name}" "${line#ERR }" + else + warn "stats ${name}" "${line#ERR }" + fi + hard_missing=1 + ;; + *) + [ -n "$line" ] && warn "stats ${name}" "unparsed · $line" + ;; + esac + done < <(python3 - "$f" "$name" "${STATS_STALE_SECS}" "${STATS_FAIL_SECS}" <<'PY' 2>/dev/null || echo "ERR python check failed" +import json, sys, time, re +from datetime import datetime, timezone + +path, role, stale_s, fail_s = sys.argv[1], sys.argv[2], int(sys.argv[3]), int(sys.argv[4]) +now = int(time.time()) + +def out(kind, msg): + print("%s %s" % (kind, msg)) + +try: + d = json.load(open(path)) +except Exception as e: + out("ERR", "JSON parse: %s" % str(e)[:100]) + sys.exit(0) + +if not isinstance(d, dict): + out("ERR", "body is not a JSON object") + sys.exit(0) + +if d.get("ok") is False: + out("ERR", "ok=false · %s" % str(d.get("error") or d.get("hint") or "")[:80]) + sys.exit(0) + +# --- timestamps (outside-in age measurement) --- +gu = d.get("generated_at_unix") +giso = d.get("generated_at") or d.get("generated_at_iso") +ghum = d.get("generated_at_human") +age = None +src_u = None + +if gu is not None and str(gu).strip() != "": + try: + src_u = int(float(gu)) + # tolerate ms accidental timestamps + if src_u > 10_000_000_000: + src_u //= 1000 + age = now - src_u + except Exception: + out("ERR", "generated_at_unix not numeric: %r" % gu) + src_u = None +elif giso: + # parse ISO when unix missing (legacy exchange/merchant before unix field) + try: + s = str(giso).strip() + if s.endswith("Z"): + s = s[:-1] + "+00:00" + # allow +0200 without colon + s = re.sub(r"([+-]\d{2})(\d{2})$", r"\1:\2", s) + dt = datetime.fromisoformat(s) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + src_u = int(dt.timestamp()) + age = now - src_u + out("WARN", "no generated_at_unix — derived age from generated_at ISO (deploy collector with unix)") + except Exception as e: + out("ERR", "cannot parse generated_at=%r (%s)" % (giso, e)) +else: + out("ERR", "missing generated_at_unix and generated_at — cannot measure freshness outside") + +if not ghum and not giso: + out("ERR", "missing generated_at_human/generated_at — UI cannot show “updated” time") +elif ghum: + out("OK", "timestamp human=%s" % ghum) +else: + out("OK", "timestamp iso=%s" % giso) + +if src_u is not None: + out("META", "%s|%s|%s" % (src_u, d.get("source") or "", d.get("currency") or "")) + if age is not None and age < -120: + out("WARN", "clock skew · generated_at_unix in the future by %ss" % (-age)) + if age is not None: + out("OK", "generated_at_unix=%s age=%ss (now=%s)" % (src_u, age, now)) + if fail_s > 0 and age > fail_s: + out("FAIL", "age=%ss > STATS_FAIL_SECS=%s · collector not updating" % (age, fail_s)) + elif age > stale_s: + out("STALE", "age=%ss > STATS_STALE_SECS=%s · timer lag or stuck publish" % (age, stale_s)) + +# --- display fields for landings / monitoring --- +src = str(d.get("source") or "") +# Bank-shaped (incl. stage shared feed on all three landings) +bankish = ( + role == "bank" + or "collect_bank_stats" in src + or "shared" in src + or isinstance(d.get("withdraws"), dict) + or isinstance(d.get("bank_accounts"), dict) +) +if bankish: + missing = [] + if not isinstance(d.get("withdraws"), dict): + missing.append("withdraws") + ba = d.get("bank_accounts") + wl = d.get("wallets") + if not isinstance(ba, dict) and not isinstance(wl, dict): + missing.append("bank_accounts|wallets") + if d.get("currency") in (None, ""): + missing.append("currency") + if missing: + out("ERR", "bank-shaped display missing: %s" % ",".join(missing)) + else: + w = d.get("withdraws") or {} + ba = ba if isinstance(ba, dict) else {} + n_acc = ba.get("total") if ba.get("total") is not None else ba.get("users") + out( + "OK", + "display bank-shape currency=%s accounts=%s withdraws.count=%s source=%s" + % (d.get("currency"), n_acc, w.get("count"), (src or "?")[:48]), + ) +elif src == "exchange-db" or (role == "exchange" and "reserves" in d): + need = [] + for k in ("reserves", "wire_in_count", "known_coins"): + if k not in d and k.replace("known_coins", "coins_live") not in d: + if k == "known_coins" and ("coins_live" in d or "coins_remaining_amount" in d): + continue + need.append(k) + if need: + out("WARN", "exchange display keys missing: %s" % ",".join(need)) + else: + out( + "OK", + "display exchange-db reserves=%s wire_in=%s coins=%s" + % (d.get("reserves"), d.get("wire_in_count"), d.get("known_coins") or d.get("coins_live")), + ) +elif src == "merchant-db" or (role == "merchant" and "instances" in d): + if "instances" not in d and "orders" not in d: + out("ERR", "merchant display missing instances/orders") + else: + out( + "OK", + "display merchant-db instances=%s orders=%s paid=%s" + % (d.get("instances"), d.get("orders"), d.get("paid")), + ) +else: + out("WARN", "unknown stats schema role=%s source=%s keys=%s" % (role, src[:40], ",".join(list(d.keys())[:8]))) + +# performance block optional (stage shared bank feed may omit heavy probes) +p = d.get("performance") +if isinstance(p, dict) and p: + mem = p.get("memory") if isinstance(p.get("memory"), dict) else {} + rss = mem.get("container_rss_human") or mem.get("proc_sum_rss_human") or "—" + load = p.get("loadavg") or "—" + bits = [] + for k in ("config_ms", "integration_ms", "webui_ms", "keys_ms", "terms_ms"): + if k in p and p[k] is not None: + bits.append("%s=%s" % (k.replace("_ms", ""), p[k])) + lat = " ".join(bits) if bits else "latency=—" + out("OK", "performance loadavg=%s RSS=%s %s" % (load, rss, lat)) +else: + out("OK", "performance block optional/absent") +PY +) + + return 0 +} + +if [ "${CHECK_LANDING:-1}" = "1" ] || [ "${LOCAL_STACK:-0}" = "1" ]; then + set_group stats + section "www · landing stats.json (freshness + display fields · outside-in)" + info "stats policy" "STALE≥${STATS_STALE_SECS}s WARN · FAIL≥${STATS_FAIL_SECS}s ERROR (0=disable fail) · need generated_at_unix" + check_landing_stats_json "bank" "$BANK_PUBLIC" || true + check_landing_stats_json "exchange" "$EXCHANGE_PUBLIC" || true + check_landing_stats_json "merchant" "$MERCHANT_PUBLIC" || true + + # Shared feed: stage-style publish copies same bank stats to all three URLs + if [ -f "$tmp/stats-meta-bank.txt" ] && [ -f "$tmp/stats-meta-exchange.txt" ] && [ -f "$tmp/stats-meta-merchant.txt" ]; then + shared_line=$(python3 - "$tmp/stats-meta-bank.txt" "$tmp/stats-meta-exchange.txt" "$tmp/stats-meta-merchant.txt" <<'PY' 2>/dev/null || true +import sys +def parse(p): + try: + t = open(p).read().strip().split("|", 2) + return t[0], (t[1] if len(t) > 1 else ""), (t[2] if len(t) > 2 else "") + except Exception: + return "", "", "" +b, bs, bc = parse(sys.argv[1]) +e, es, ec = parse(sys.argv[2]) +m, ms, mc = parse(sys.argv[3]) +if not b or not e or not m: + print("SKIP incomplete meta") +elif b == e == m: + print("SHARED gen_unix=%s bank_src=%s" % (b, (bs or "?")[:50])) +else: + # Shared feed only when all three claim the same bank-collector source + # (e.g. stage TESTPAYSAN "… shared"). GOA uses independent exchange-db / + # merchant-db collectors — different gen_unix is expected. + def bankish(s): + s = (s or "").lower() + return "shared" in s or "collect_bank_stats" in s + if bankish(bs) and bankish(es) and bankish(ms): + print("DRIFT bank=%s exchange=%s merchant=%s (expected equal for shared feed)" % (b, e, m)) + else: + print("INDEPENDENT bank=%s exchange=%s merchant=%s" % (b, e, m)) +PY +) + case "$shared_line" in + SHARED\ *) + ok "stats shared feed" "${shared_line#SHARED }" + ;; + DRIFT\ *) + warn "stats shared feed" "timestamps differ · ${shared_line#DRIFT }" + ;; + INDEPENDENT\ *) + info "stats feeds" "independent collectors · ${shared_line#INDEPENDENT }" + ;; + *) + info "stats feeds" "${shared_line:-n/a}" + ;; + esac + fi + info "stats note" "public GET /intro/stats.json; measure age via generated_at_unix vs wall clock" +fi +# Rollup + optional JSON for metrics_print_overall +if [ -s "$PERF_TSV" ]; then + PERF_JSON="$tmp/perf-summary.json" + perf_line=$(python3 - "$PERF_TSV" "$PERF_JSON" "$PERF_WARN_MS" "$PERF_FAIL_MS" <<'PY' +import json, sys +rows = [] +for line in open(sys.argv[1]): + parts = line.rstrip("\n").split("\t") + if len(parts) < 3: + continue + label, ms_s, code = parts[0], parts[1], parts[2] + try: + ms = int(float(ms_s)) + except Exception: + continue + rows.append({"label": label, "ms": ms, "http": code}) +vals = sorted(r["ms"] for r in rows) +n = len(vals) +if n == 0: + print("n=0") + json.dump({"n": 0}, open(sys.argv[2], "w")) + raise SystemExit(0) +def pct(p): + if n == 1: + return vals[0] + i = min(n - 1, max(0, int(round((p / 100.0) * (n - 1))))) + return vals[i] +avg = int(round(sum(vals) / n)) +warn_ms = int(sys.argv[3]) +fail_ms = int(sys.argv[4]) +slow = [r for r in rows if r["ms"] >= warn_ms] +rep = { + "n": n, + "min_ms": vals[0], + "p50_ms": pct(50), + "avg_ms": avg, + "max_ms": vals[-1], + "warn_ms": warn_ms, + "fail_ms": fail_ms, + "slow": [{"label": r["label"], "ms": r["ms"]} for r in slow], + "samples": rows, +} +# metrics_print_overall expects named buckets with n/min/p50/avg/max +out = { + "www_public_https": { + "n": n, + "min_ms": vals[0], + "p50_ms": pct(50), + "avg_ms": avg, + "max_ms": vals[-1], + }, + **{r["label"].replace(" ", "_"): {"n": 1, "min_ms": r["ms"], "p50_ms": r["ms"], "avg_ms": r["ms"], "max_ms": r["ms"]} for r in rows}, +} +json.dump(out, open(sys.argv[2], "w"), indent=2) +extra = "" +if slow: + extra = " slow: " + ", ".join("%s=%dms" % (r["label"].replace("perf ", ""), r["ms"]) for r in slow) +print( + "n=%d min=%dms p50=%dms avg=%dms max=%dms (warn≥%dms fail≥%dms)%s" + % (n, vals[0], pct(50), avg, vals[-1], warn_ms, fail_ms, extra) +) +PY + ) + info "perf summary" "$perf_line" + # Keep a copy if METRICS_DIR is set (e2e/ladder overall stats) + if [ -n "${METRICS_DIR:-}" ] && [ -d "${METRICS_DIR}" ]; then + cp -f "$PERF_JSON" "${METRICS_DIR}/perf-summary.json" 2>/dev/null || true + fi +fi +info "perf note" "measured from this host (outside-in); thresholds PERF_WARN_MS=${PERF_WARN_MS} PERF_FAIL_MS=${PERF_FAIL_MS}" + + +# Exchange terms + privacy (same www.exchange group — issues map to exchange) +set_group exchange +check_legal_doc "exchange /terms" "$EXCHANGE_PUBLIC/terms" "terms|GOA|exploration|FADP|revDSG|privacy" +check_legal_doc "exchange /privacy" "$EXCHANGE_PUBLIC/privacy" "privacy|FADP|revDSG|data|GOA|exploration" +# trailing slash: 200 or redirect to bare path +code=$(http_code "$EXCHANGE_PUBLIC/terms/") +case "$code" in + 200|301|302|303|307|308) ok "exchange /terms/" "HTTP $code" ;; + *) + if is_prelaunch_stack; then + info "exchange /terms/" "HTTP $code — pre-launch" + else + warn "exchange /terms/" "HTTP $code" + fi + ;; +esac + +# --- bank --- +set_group bank +if [ "${LOCAL_STACK:-1}" = "0" ]; then + check_url_soft "bank /config" 200 "$BANK_PUBLIC/config" +else + check_url "bank /config" 200 "$BANK_PUBLIC/config" +fi +code=$(http_body "$BANK_PUBLIC/config" "$tmp/bc.json") +if [ "$code" = "200" ]; then + expect_currency "bank" "$tmp/bc.json" + if json_has_alt_unit_names "$tmp/bc.json" >/tmp/alt-bank.$$ 2>&1; then + ok "bank /config alt_unit_names" "$(tr '\n' '; ' "$tmp/aa-val"); then + ok "bank /intro/auto-account.json" "$aa_detail" + else + fail "bank /intro/auto-account.json" "invalid withdraw/login ($(tr '\n' ' ' <"$tmp/aa-val" | sed 's/[[:space:]]*$//'))" + fi + ;; + 404|405|501) + if [ "${EXPECT_CURRENCY:-}" = "GOA" ] || [ "${LOCAL_STACK:-0}" = "1" ]; then + fail "bank /intro/auto-account.json" "HTTP $aa_code (want 200; 405/501 = broken)" + else + info "bank /intro/auto-account.json" "HTTP $aa_code — skip (not a GOA auto-account stack)" + fi + ;; + 502|503|000) + fail "bank /intro/auto-account.json" "HTTP $aa_code want 200" + ;; + *) + if [ "${EXPECT_CURRENCY:-}" = "GOA" ] || [ "${LOCAL_STACK:-0}" = "1" ]; then + fail "bank /intro/auto-account.json" "HTTP $aa_code want 200" + else + warn "bank /intro/auto-account.json" "HTTP $aa_code (optional off-GOA)" + fi + ;; + esac +fi + +# Bank legal docs (landing nginx via Caddy /terms* /privacy* or /intro/*) +check_legal_doc "bank /terms" "$BANK_PUBLIC/terms" "terms|GOA|exploration|bank|FADP|revDSG" +# Prefer /privacy; fall back to /intro/privacy.html for older deploys +code=$(http_code "$BANK_PUBLIC/privacy") +if [ "$code" = "200" ]; then + check_legal_doc "bank /privacy" "$BANK_PUBLIC/privacy" "privacy|FADP|revDSG|data|GOA|bank" +else + if [ "${LOCAL_STACK:-1}" = "1" ]; then + check_legal_doc "bank /privacy (or /intro/privacy.html)" \ + "$BANK_PUBLIC/intro/privacy.html" "privacy|FADP|revDSG|data|GOA|bank" + # still report bare /privacy failure for local + warn "bank /privacy" "HTTP $code — prefer Caddy handle /privacy* → landing" + else + check_url_soft "bank /privacy" 200 "$BANK_PUBLIC/privacy" + fi +fi + +# --- merchant --- +set_group merchant +if [ "${LOCAL_STACK:-1}" = "0" ]; then + check_url_soft "merchant /config" 200 "$MERCHANT_PUBLIC/config" +else + check_url "merchant /config" 200 "$MERCHANT_PUBLIC/config" +fi +code=$(http_body "$MERCHANT_PUBLIC/config" "$tmp/mc.json") +if [ "$code" = "200" ]; then + want="${EXPECT_CURRENCY:-}" + if python3 - "$tmp/mc.json" "$want" <<'PY' +import json,sys +try: + d=json.load(open(sys.argv[1])) +except Exception: + sys.exit(2) +want=sys.argv[2] +if not want: + sys.exit(0) +curs=list((d.get("currencies") or {}).keys()) +ex=d.get("exchanges") or [] +ok = want in curs or any((e.get("currency") if isinstance(e,dict) else None)==want for e in ex) +if not ok and isinstance(d.get("currency"), str): + ok = d["currency"]==want +sys.exit(0 if ok else 1) +PY + then + ok "merchant /config (${want:-currency} ok)" + else + ec=$? + if [ "$ec" = "2" ]; then + warn "merchant /config" "non-JSON body" + elif [ -n "$want" ]; then + fail "merchant /config currency" "want $want" + else + info "merchant /config" "ok" + fi + fi + # merchant-local currency maps (GOA + CHF, …) + if json_has_alt_unit_names "$tmp/mc.json" >/tmp/alt-mer.$$ 2>&1; then + ok "merchant /config currencies alt_unit_names" "$(tr '\n' '; ' /dev/null || true) + # take first 3 digits only (avoid 200000 glue when curl exits non-zero after writing code) + printf '%s' "$raw" | tr -cd '0-9' | head -c 3 + [ -n "$(printf '%s' "$raw" | tr -cd '0-9')" ] || printf '000' + } + for asset in index.html index.js; do + tmo="${TIMEOUT}" + [ "$asset" = "index.js" ] && tmo="${WEBUI_INDEX_JS_TIMEOUT:-60}" + acode=$(_webui_asset_code "$MERCHANT_PUBLIC/webui/${asset}" "$tmo") + case "$acode" in + 200) ok "merchant /webui/${asset}" "HTTP 200" ;; + *) + if [ "${LOCAL_STACK:-1}" = "1" ]; then + fail "merchant /webui/${asset}" "HTTP ${acode:-000}" + else + warn "merchant /webui/${asset}" "HTTP ${acode:-000}" + fi + ;; + esac + done + # return group for any later merchant checks in this block + set_group merchant + fi +fi + +# Merchant legal docs +check_legal_doc "merchant /terms" "$MERCHANT_PUBLIC/terms" "terms|dual|GOA|CHF|explorational|merchant" +check_legal_doc "merchant /privacy" "$MERCHANT_PUBLIC/privacy" "privacy|FADP|revDSG|data|GOA|CHF|merchant" +code=$(http_code "$MERCHANT_PUBLIC/terms/") +case "$code" in + 200|301|302|303|307|308) ok "merchant /terms/" "HTTP $code" ;; + *) + if is_prelaunch_stack; then + info "merchant /terms/" "HTTP $code — pre-launch" + else + warn "merchant /terms/" "HTTP $code (expect 302 → /terms)" + fi + ;; +esac + +# --------------------------------------------------------------------------- +# Paivana paywall (local GOA stack) — public front only; pay path is e2e +# --------------------------------------------------------------------------- +if [ "${LOCAL_STACK:-1}" = "1" ] && [ "${E2E_PAIVANA:-1}" != "0" ]; then + set_group paivana + section "www · paivana paywall" + : "${PAIVANA_PUBLIC:=https://paivana.hacktivism.ch}" + PAIVANA_PUBLIC="${PAIVANA_PUBLIC%/}" + hdr=$(curl -skS -m "${TIMEOUT}" -D - -o /dev/null "${PAIVANA_PUBLIC}/" 2>/dev/null || true) + pcode=$(printf '%s' "$hdr" | awk 'BEGIN{c="000"} /^HTTP/{c=$2} END{print c}') + loc=$(printf '%s' "$hdr" | awk 'BEGIN{IGNORECASE=1} /^location:/{sub(/\r$/,""); sub(/^location:[[:space:]]*/,""); print; exit}') + case "$pcode" in + 301|302|303|307|308) + if printf '%s' "$loc" | grep -qiE 'paivana|templates|well-known'; then + ok "paivana /" "HTTP $pcode → template flow" + else + ok "paivana /" "HTTP $pcode redirect" + fi + info "paivana Location" "${loc:0:140}" + ;; + 402) + # Paywall may answer Payment Required with body (healthy, not 502) + ok "paivana /" "HTTP 402 Payment Required (paywall up)" + ;; + 200) + warn "paivana /" "HTTP 200 (expected paywall redirect/402 to template)" + ;; + *) + warn "paivana /" "HTTP ${pcode:-000} — ${PAIVANA_PUBLIC}/ (e2e pay may still work via template)" + ;; + esac +fi + +# --------------------------------------------------------------------------- +# Landing pages: every HTTPS link exposed on bank / merchant / exchange intros +# + required static assets + bank withdraw mint (taler://withdraw only) +# Skipped when CHECK_LANDING=0 (e.g. taler-ops.ch — no GOA-style landings). +# --------------------------------------------------------------------------- +if [ "${CHECK_LANDING:-1}" != "1" ]; then + section "www · landing pages" + info "landing checks" "skipped (CHECK_LANDING=0 · stack has no public /intro landings)" + summary + exit 0 +fi + +set_group landing +section "www · landing exposed links · bank / merchant / exchange" + +# Probe one URL: print code to stdout (200 after following redirects counts as 200). +# Sets _landing_code. Exit 0 if OK (200 or redirect→200), 1 otherwise. +_landing_probe() { + local url="$1" + local code + code=$(http_code "$url") + case "$code" in + 200) _landing_code=200; return 0 ;; + 301|302|303|307|308) + code=$(curl -skS --max-redirs 5 -L -m "${TIMEOUT}" -o /dev/null -w '%{http_code}' "$url" 2>/dev/null || echo 000) + _landing_code="$code" + [ "$code" = "200" ] && return 0 + return 1 + ;; + *) + _landing_code="$code" + return 1 + ;; + esac +} + +# Known-good landing static paths — one report line via caller aggregate, or +# soft=1 single warn. Returns 0 if ok. +check_landing_asset() { + local label="$1" url="$2" soft="${3:-0}" + if _landing_probe "$url"; then + return 0 + fi + if [ "$soft" = "1" ]; then + warn "$label" "HTTP ${_landing_code:-?} — $url" + else + fail "$label" "HTTP ${_landing_code:-?} — $url" + fi + return 1 +} + +# Soft external: never ERROR; used only for failures in aggregated external probe. +check_external_soft() { + local label="$1" url="$2" + local code + code=$(curl -skS --max-redirs 5 -L -m "${TIMEOUT}" -o /dev/null -w '%{http_code}' "$url" 2>/dev/null || echo 000) + case "$code" in + 200|204|301|302|303|307|308) return 0 ;; + *) warn "$label" "HTTP $code · $url"; return 1 ;; + esac +} + +# Parse one landing HTML: collect absolute https + root-relative href/src; +# resolve against base; classify own-stack vs external. +# Writes lists: $1.own $1.ext (one URL per line) +extract_landing_urls() { + local base="$1" html="$2" out_prefix="$3" + python3 - "$base" "$html" "$out_prefix" <<'PY' +import re, sys +from urllib.parse import urljoin, urlparse + +base, html_path, out = sys.argv[1], sys.argv[2], sys.argv[3] +html = open(html_path, encoding="utf-8", errors="replace").read() +base = base.rstrip("/") + "/" +parsed_base = urlparse(base) +# Hard-check only the three public Taler hosts for this stack (not git.* etc.) +own_hosts = { + (parsed_base.hostname or "").lower(), + "bank.hacktivism.ch", + "exchange.hacktivism.ch", + "taler.hacktivism.ch", +} +# include configured public hosts when domain differs (demo / ops) +for envu in ( + __import__("os").environ.get("BANK_PUBLIC", ""), + __import__("os").environ.get("EXCHANGE_PUBLIC", ""), + __import__("os").environ.get("MERCHANT_PUBLIC", ""), +): + h = urlparse(envu).hostname if envu else None + if h: + own_hosts.add(h.lower()) + +raw = set() +for m in re.finditer( + r'''(?:href|src|content)=["']([^"'#]+)["']''', html, re.I +): + raw.add(m.group(1).strip()) +# bare absolute URLs in scripts (fetch, template strings) +for m in re.finditer(r'''https://[^\s"'<>\\]+''', html): + u = m.group(0).rstrip("\\).,;'\"") + # strip trailing punctuation leftovers + while u and u[-1] in ".,);]}\"'": + u = u[:-1] + if u.startswith("https://"): + raw.add(u) + +own, ext = set(), set() +skip_prefix = ("data:", "javascript:", "mailto:", "taler://", "blob:") +skip_exact = {"website", "summary_large_image", "image/png", "en_US"} +for r in raw: + if not r or r in skip_exact: + continue + if r.startswith(skip_prefix): + continue + # meta content noise + if re.fullmatch(r"\d+", r) or r.startswith("width="): + continue + if " " in r and not r.startswith("http"): + continue + if r.startswith("//"): + absu = "https:" + r + elif r.startswith("http://") or r.startswith("https://"): + absu = r + elif r.startswith("/"): + absu = urljoin(base, r) + else: + # relative asset + if "/" in r or r.endswith((".js", ".css", ".png", ".svg", ".html", ".json", ".uri")): + absu = urljoin(base + "intro/", r) + else: + continue + # drop query-only noise / anchors already stripped + p = urlparse(absu) + if p.scheme not in ("http", "https"): + continue + # normalize: drop fragment + absu = absu.split("#", 1)[0] + host = (p.hostname or "").lower() + # og image query ok + if host in own_hosts: + own.add(absu) + else: + ext.add(absu) + +open(out + ".own", "w").write("\n".join(sorted(own)) + ("\n" if own else "")) +open(out + ".ext", "w").write("\n".join(sorted(ext)) + ("\n" if ext else "")) +print(f"own={len(own)} ext={len(ext)}") +PY +} + +check_one_landing() { + local name="$1" base="$2" + local html="$tmp/landing-${name}.html" + local pref="$tmp/urls-${name}" + local code n own_n ext_n + local a_ok=0 a_fail=0 a_soft=0 + local own_ok=0 own_fail=0 + local ext_ok=0 ext_fail=0 + local fail_sample="" soft_sample="" + + code=$(http_body "${base}/intro/" "$html") + if [ "$code" != "200" ]; then + if [ "${LOCAL_STACK:-1}" = "1" ]; then + fail "landing ${name}" "/intro/ HTTP $code — skip assets/links" + else + warn "landing ${name}" "/intro/ HTTP $code — skip assets/links" + fi + return + fi + + # Required static: qrcode.min.js. og-goa-shop.png only on GOA/local. + if _landing_probe "${base}/intro/qrcode.min.js"; then + a_ok=$((a_ok + 1)) + else + a_fail=$((a_fail + 1)) + fail_sample="${fail_sample}${fail_sample:+; }HTTP ${_landing_code} ${base}/intro/qrcode.min.js" + fi + if [ "${EXPECT_CURRENCY:-}" = "GOA" ] || [ "${LOCAL_STACK:-0}" = "1" ]; then + if _landing_probe "${base}/intro/og-goa-shop.png"; then + a_ok=$((a_ok + 1)) + else + a_fail=$((a_fail + 1)) + fail_sample="${fail_sample}${fail_sample:+; }HTTP ${_landing_code} ${base}/intro/og-goa-shop.png" + fi + fi + # qr-logo: hard on GOA (wallet QR frame); soft elsewhere if missing + if _landing_probe "${base}/intro/qr-logo.png"; then + a_ok=$((a_ok + 1)) + else + if [ "${EXPECT_CURRENCY:-}" = "GOA" ] || [ "${LOCAL_STACK:-0}" = "1" ]; then + a_soft=$((a_soft + 1)) + soft_sample="${soft_sample}${soft_sample:+; }HTTP ${_landing_code} qr-logo.png" + fi + fi + + n=$(extract_landing_urls "$base" "$html" "$pref" 2>/dev/null || echo "own=0 ext=0") + own_n=0 + ext_n=0 + [ -f "${pref}.own" ] && own_n=$(grep -c . "${pref}.own" 2>/dev/null || echo 0) + [ -f "${pref}.ext" ] && ext_n=$(grep -c . "${pref}.ext" 2>/dev/null || echo 0) + # strip newlines from grep -c edge cases + own_n=${own_n//[^0-9]/} + ext_n=${ext_n//[^0-9]/} + own_n=${own_n:-0} + ext_n=${ext_n:-0} + + # Probe own-stack URLs — count only; list failures + if [ -f "${pref}.own" ]; then + while IFS= read -r u; do + [ -n "$u" ] || continue + if _landing_probe "$u"; then + own_ok=$((own_ok + 1)) + else + own_fail=$((own_fail + 1)) + # keep a few samples (max ~3) + if [ "$own_fail" -le 3 ]; then + fail_sample="${fail_sample}${fail_sample:+; }own HTTP ${_landing_code} $u" + fi + fi + done < "${pref}.own" + fi + + # External: soft counts (stores can be slow — longer timeout + browser UA) + if [ -f "${pref}.ext" ]; then + local _ext_to="${EXT_TIMEOUT:-25}" + while IFS= read -r u; do + [ -n "$u" ] || continue + code=$(curl -skS --max-redirs 5 -L -m "${_ext_to}" \ + -A "Mozilla/5.0 (compatible; taler-monitoring/1.0)" \ + -o /dev/null -w '%{http_code}' "$u" 2>/dev/null || echo 000) + case "$code" in + 200|204|301|302|303|307|308) ext_ok=$((ext_ok + 1)) ;; + *) + ext_fail=$((ext_fail + 1)) + if [ "$ext_fail" -le 3 ]; then + soft_sample="${soft_sample}${soft_sample:+; }ext HTTP $code $u" + fi + ;; + esac + done < "${pref}.ext" + fi + + # One primary line per landing + local detail + detail="/intro $(wc -c <"$html" | tr -d ' ')B · assets ${a_ok}/$((a_ok + a_fail + a_soft)) · own-links ${own_ok}/${own_n} · external ${ext_ok}/${ext_n}" + if [ "$a_fail" -gt 0 ] || [ "$own_fail" -gt 0 ] || [ "${own_n:-0}" -lt 1 ]; then + if [ "${LOCAL_STACK:-1}" = "1" ]; then + fail "landing ${name}" "$detail${fail_sample:+ · $fail_sample}" + else + warn "landing ${name}" "$detail${fail_sample:+ · $fail_sample}" + fi + else + ok "landing ${name}" "$detail" + fi + if [ "$a_soft" -gt 0 ] || [ "$ext_fail" -gt 0 ]; then + warn "landing ${name} soft" "${soft_sample:-soft issues}" + fi +} + +check_one_landing "bank" "$BANK_PUBLIC" +check_one_landing "merchant" "$MERCHANT_PUBLIC" +check_one_landing "exchange" "$EXCHANGE_PUBLIC" + +# Cross-links: one line +if [ "${LOCAL_STACK:-1}" = "1" ]; then + _cx_ok=0 + _cx_fail=0 + _cx_detail="" + for pair in \ + "bank→merchant|$MERCHANT_PUBLIC/intro/" \ + "bank→exchange|$EXCHANGE_PUBLIC/intro/" \ + "merchant→bank|$BANK_PUBLIC/intro/" \ + "exchange→bank|$BANK_PUBLIC/intro/" + do + _cx_name="${pair%%|*}" + _cx_url="${pair#*|}" + if _landing_probe "$_cx_url"; then + _cx_ok=$((_cx_ok + 1)) + else + _cx_fail=$((_cx_fail + 1)) + _cx_detail="${_cx_detail}${_cx_detail:+; }${_cx_name} HTTP ${_landing_code}" + fi + done + if [ "$_cx_fail" -eq 0 ]; then + ok "landing cross-links" "${_cx_ok}/4 intros reachable" + else + fail "landing cross-links" "${_cx_ok}/4 ok · ${_cx_detail}" + fi +fi + +# Bank withdraw mint + shop assets — compact +if [ "${LOCAL_STACK:-1}" = "1" ] || [ -n "${BANK_PUBLIC:-}" ]; then + _ba_ok=0 + _ba_soft=0 + _ba_msg="" + # GOA landings ship shop-pay.*; stage TESTPAYSAN uses monnaies /assets/shop-ui.js + if [ "${EXPECT_CURRENCY:-}" = "GOA" ] || [ "${LOCAL_STACK:-0}" = "1" ]; then + if _landing_probe "$BANK_PUBLIC/intro/shop-pay.js"; then _ba_ok=$((_ba_ok + 1)); else _ba_soft=$((_ba_soft + 1)); _ba_msg="${_ba_msg}shop-pay.js; "; fi + if _landing_probe "$BANK_PUBLIC/intro/shop-pay.css"; then _ba_ok=$((_ba_ok + 1)); else _ba_soft=$((_ba_soft + 1)); _ba_msg="${_ba_msg}shop-pay.css; "; fi + fi + dw_code=$(http_body "$BANK_PUBLIC/intro/demo-withdraw.json" "$tmp/dw.json") + case "$dw_code" in + 200) + if dw_out=$(validate_bank_withdraw_json "$tmp/dw.json" demo 2>"$tmp/dw-val"); then + dw_detail=${dw_out%%$'\t'*} + wid=${dw_out#*$'\t'} + [ "$wid" = "$dw_out" ] && wid= + ok "bank /intro/demo-withdraw.json" "$dw_detail" + if [ -n "$wid" ]; then + if _landing_probe "$BANK_PUBLIC/taler-integration/withdrawal-operation/${wid}"; then + _ba_ok=$((_ba_ok + 1)) + else + fail "landing bank withdraw-op" "HTTP ${_landing_code} · id=$wid" + fi + fi + ok "landing bank withdraw/shop" "demo-withdraw + shop assets ok (${_ba_ok} checks)" + else + fail "landing bank demo-withdraw" "invalid taler://withdraw shape" + fail "bank /intro/demo-withdraw.json" "invalid taler://withdraw ($(tr '\n' ' ' <"$tmp/dw-val" | sed 's/[[:space:]]*$//'))" + fi + ;; + 404|405|501) + # GOA bank landing mints demo-withdraw.json; stage TESTPAYSAN uses ATM/wallet guide only + if [ "${EXPECT_CURRENCY:-}" = "GOA" ] || [ "${LOCAL_STACK:-0}" = "1" ]; then + fail "landing bank demo-withdraw" "HTTP $dw_code (want 200)" + else + info "landing bank demo-withdraw" "HTTP $dw_code — skip (not a GOA demo-withdraw stack)" + fi + ;; + 502|503|000) + fail "landing bank demo-withdraw" "HTTP $dw_code (want 200)" + ;; + *) + if [ "${LOCAL_STACK:-1}" = "1" ]; then + fail "landing bank demo-withdraw" "HTTP $dw_code want 200" + else + warn "landing bank demo-withdraw" "HTTP $dw_code" + fi + ;; + esac + if [ "$_ba_soft" -gt 0 ]; then + warn "landing bank shop assets" "${_ba_msg}soft-missing" + fi +fi + +# Merchant shop assets — GOA: shop-pay.*; stage TESTPAYSAN: /assets/shop-ui.js + +# shops.css + QR encoder (shop-ui modal needs /intro/qrcode.min.js or /qrcode.min.js). +_ma=0 +_ma_need=2 +_ma_label="shop-pay.js + .css" +if [ "${EXPECT_CURRENCY:-}" = "GOA" ] || [ "${LOCAL_STACK:-0}" = "1" ]; then + _landing_probe "$MERCHANT_PUBLIC/intro/shop-pay.js" && _ma=$((_ma + 1)) + _landing_probe "$MERCHANT_PUBLIC/intro/shop-pay.css" && _ma=$((_ma + 1)) +else + # stage farmer shops: UI + stylesheet + qrcode (pay modal QR_Taler) + _ma_need=3 + _ma_label="shop-ui.js + shops.css + qrcode.min.js" + _landing_probe "$MERCHANT_PUBLIC/assets/shop-ui.js" && _ma=$((_ma + 1)) + _landing_probe "$MERCHANT_PUBLIC/assets/shops.css" && _ma=$((_ma + 1)) + # Prefer /intro/ (always on landings tree); root alias is Caddy-only convenience + if _landing_probe "$MERCHANT_PUBLIC/intro/qrcode.min.js" \ + || _landing_probe "$MERCHANT_PUBLIC/qrcode.min.js"; then + _ma=$((_ma + 1)) + fi +fi +if [ "$_ma" -eq "$_ma_need" ]; then + ok "landing merchant shop assets" "${_ma_label}" +else + if [ "${EXPECT_CURRENCY:-}" = "GOA" ] || [ "${LOCAL_STACK:-0}" = "1" ]; then + warn "landing merchant shop assets" "${_ma}/${_ma_need} present (soft) · want ${_ma_label}" + else + # stage: hard-ish — missing qrcode broke pay QR («bibliothèque manquante») + if [ "$_ma" -lt 2 ]; then + fail "landing merchant shop assets" "${_ma}/${_ma_need} · want ${_ma_label}" + elif [ "$_ma" -lt "$_ma_need" ]; then + warn "landing merchant shop assets" "${_ma}/${_ma_need} · want ${_ma_label}" + else + info "landing merchant shop assets" "${_ma}/${_ma_need} · ${_ma_label}" + fi + fi +fi + +# --------------------------------------------------------------------------- +# QR payloads — form + encode/decode roundtrip (qrencode + zbarimg) +# Collect taler:// + payto:// (+ app-store https from data-qr-url) from landings +# and bank mint JSON; re-encode PNG; decode; require exact payload match. +# Env: QR_CHECK=0 to skip; QR_ECC=M (default) for qrencode -l +# --------------------------------------------------------------------------- +if [ "${QR_CHECK:-1}" = "1" ] && [ "${CHECK_LANDING:-1}" = "1" ]; then + set_group qr + section "www · QR payloads (form + encode/decode · taler:// payto://)" + + QR_ECC="${QR_ECC:-M}" + qr_dir="$tmp/qr-check" + mkdir -p "$qr_dir" + _qr_tools=1 + if ! command -v qrencode >/dev/null 2>&1; then + warn "qr tools" "qrencode missing — form checks only (apt install qrencode)" + _qr_tools=0 + fi + if ! command -v zbarimg >/dev/null 2>&1; then + warn "qr tools" "zbarimg missing — form checks only (apt install zbar-tools)" + _qr_tools=0 + fi + if [ "$_qr_tools" = "1" ]; then + ok "qr tools" "qrencode + zbarimg" + fi + + # Fetch sources (HTML landings + mint JSON when present) + _qr_args=() + for pair in \ + "bank-intro|$BANK_PUBLIC/intro/" \ + "merchant-intro|$MERCHANT_PUBLIC/intro/" \ + "exchange-intro|$EXCHANGE_PUBLIC/intro/" + do + _qn="${pair%%|*}" + _qu="${pair#*|}" + _qf="$qr_dir/${_qn}.html" + _qc=$(http_body "$_qu" "$_qf" 2>/dev/null || echo 000) + if [ "$_qc" = "200" ] && [ -s "$_qf" ]; then + _qr_args+=("$_qn" "$_qf") + fi + done + for pair in \ + "demo-withdraw|$BANK_PUBLIC/intro/demo-withdraw.json" \ + "auto-account|$BANK_PUBLIC/intro/auto-account.json" + do + _qn="${pair%%|*}" + _qu="${pair#*|}" + _qf="$qr_dir/${_qn}.json" + _qc=$(http_body "$_qu" "$_qf" 2>/dev/null || echo 000) + if [ "$_qc" = "200" ] && [ -s "$_qf" ]; then + _qr_args+=("$_qn" "$_qf") + fi + done + + if [ "${#_qr_args[@]}" -eq 0 ]; then + warn "qr sources" "no landing HTML / mint JSON fetched — skip payload checks" + else + _qr_tsv="$qr_dir/payloads.tsv" + if ! python3 "$ROOT/check_qr_payloads.py" "${_qr_args[@]}" >"$_qr_tsv" 2>"$qr_dir/harvest.err"; then + warn "qr harvest" "collector failed · $(tr '\n' ' ' <"$qr_dir/harvest.err" | head -c 160)" + fi + + _qr_n=0 + _qr_form_ok=0 + _qr_form_bad=0 + _qr_rt_ok=0 + _qr_rt_bad=0 + _qr_taler=0 + _qr_payto=0 + _qr_bad_sample="" + _qr_i=0 + + while IFS=$'\t' read -r src kind uri status detail || [ -n "${src:-}" ]; do + [ -n "${src:-}" ] || continue + _qr_n=$((_qr_n + 1)) + case "$kind" in + withdraw|pay|pay-template|refund|other) _qr_taler=$((_qr_taler + 1)) ;; + payto) _qr_payto=$((_qr_payto + 1)) ;; + esac + + if [ "$status" = "ok" ]; then + _qr_form_ok=$((_qr_form_ok + 1)) + else + _qr_form_bad=$((_qr_form_bad + 1)) + _qr_bad_sample="${_qr_bad_sample}${_qr_bad_sample:+; }form/${kind} ${uri:0:56}" + # taler:// and payto:// form errors are hard on local/GOA + case "$kind" in + withdraw|pay|pay-template|payto) + if [ "${LOCAL_STACK:-1}" = "1" ] || [ "${EXPECT_CURRENCY:-}" = "GOA" ]; then + fail "qr form ${kind}" "${src} · ${detail} · ${uri:0:80}" + else + warn "qr form ${kind}" "${src} · ${detail} · ${uri:0:80}" + fi + ;; + *) + warn "qr form ${kind}" "${src} · ${detail} · ${uri:0:80}" + ;; + esac + continue + fi + + # Encode → PNG → zbarimg (exact match) + if [ "$_qr_tools" = "1" ] && [ -n "$uri" ]; then + _qr_i=$((_qr_i + 1)) + _png="$qr_dir/p-${_qr_i}.png" + # ECC: H for wallet-style long URIs, L for long https store links, else QR_ECC + _ecc="$QR_ECC" + case "$kind" in + https) _ecc=L ;; + withdraw|pay|pay-template) _ecc=M ;; + payto) _ecc=M ;; + esac + if ! qrencode -l "$_ecc" -s 4 -m 2 -o "$_png" "$uri" 2>/dev/null; then + _qr_rt_bad=$((_qr_rt_bad + 1)) + fail "qr encode ${kind}" "qrencode failed · ${uri:0:72}" + continue + fi + _got=$(zbarimg --raw -q "$_png" 2>/dev/null | head -n1 | tr -d '\r' || true) + if [ "$_got" = "$uri" ]; then + _qr_rt_ok=$((_qr_rt_ok + 1)) + else + _qr_rt_bad=$((_qr_rt_bad + 1)) + _qr_bad_sample="${_qr_bad_sample}${_qr_bad_sample:+; }decode/${kind}" + fail "qr decode ${kind}" "zbar mismatch · want ${uri:0:48}… · got ${_got:0:48}" + fi + fi + done <"$_qr_tsv" + + # Download any obvious static QR images from bank intro and decode (soft) + if [ "$_qr_tools" = "1" ] && [ -f "$qr_dir/bank-intro.html" ]; then + python3 - "$qr_dir/bank-intro.html" "$BANK_PUBLIC" "$qr_dir" <<'PY' >"$qr_dir/img-urls.txt" 2>/dev/null || true +import re, sys +from urllib.parse import urljoin +html = open(sys.argv[1], encoding="utf-8", errors="replace").read() +base = sys.argv[2].rstrip("/") + "/" +out = sys.argv[3] +seen = set() +for m in re.finditer(r'''(?:src|href)=["']([^"']+\.(?:png|jpe?g|gif|svg))["']''', html, re.I): + u = m.group(1) + if "qr" not in u.lower() and "withdraw" not in u.lower(): + continue + if u.startswith("data:"): + continue + absu = urljoin(base + "intro/", u) + if absu in seen: + continue + seen.add(absu) + print(absu) +PY + _img_n=0 + _img_ok=0 + while read -r _img_url; do + [ -n "$_img_url" ] || continue + _img_n=$((_img_n + 1)) + _imgf="$qr_dir/img-${_img_n}.bin" + _ic=$(http_body "$_img_url" "$_imgf" 2>/dev/null || echo 000) + if [ "$_ic" != "200" ] || [ ! -s "$_imgf" ]; then + warn "qr image fetch" "HTTP ${_ic} · ${_img_url}" + continue + fi + # zbarimg needs image format; skip non-raster + if ! file "$_imgf" 2>/dev/null | grep -qiE 'PNG|JPEG|PBM|PGM|PPM|GIF'; then + info "qr image skip" "not raster · ${_img_url}" + continue + fi + _dec=$(zbarimg --raw -q "$_imgf" 2>/dev/null | head -n1 | tr -d '\r' || true) + if [ -z "$_dec" ]; then + warn "qr image decode" "no barcode · ${_img_url}" + continue + fi + _img_ok=$((_img_ok + 1)) + # validate decoded payload form + printf '%s\n' "$_dec" >"$qr_dir/img-decoded.txt" + _line=$(python3 "$ROOT/check_qr_payloads.py" "img" "$qr_dir/img-decoded.txt" 2>/dev/null | head -1 || true) + if [ -n "$_line" ]; then + _st=$(printf '%s' "$_line" | cut -f4) + _kd=$(printf '%s' "$_line" | cut -f2) + if [ "$_st" = "ok" ]; then + ok "qr image ${_kd}" "${_dec:0:64}" + else + case "$_kd" in + withdraw|pay|pay-template|payto) + fail "qr image ${_kd}" "bad form · ${_dec:0:72}" + ;; + *) + warn "qr image ${_kd}" "bad form · ${_dec:0:72}" + ;; + esac + fi + else + ok "qr image decode" "${_dec:0:64}" + fi + done <"$qr_dir/img-urls.txt" + if [ "$_img_n" -gt 0 ]; then + info "qr images" "fetched ${_img_n} · decoded ${_img_ok}" + fi + fi + + # Summaries + if [ "$_qr_n" -eq 0 ]; then + warn "qr payloads" "none found in landings/mint JSON" + else + if [ "$_qr_form_bad" -eq 0 ]; then + ok "qr form" "${_qr_form_ok}/${_qr_n} ok · taler=${_qr_taler} payto=${_qr_payto}" + else + if [ "${LOCAL_STACK:-1}" = "1" ]; then + fail "qr form" "${_qr_form_ok}/${_qr_n} ok · ${_qr_form_bad} bad${_qr_bad_sample:+ · $_qr_bad_sample}" + else + warn "qr form" "${_qr_form_ok}/${_qr_n} ok · ${_qr_form_bad} bad" + fi + fi + if [ "$_qr_tools" = "1" ]; then + if [ "$_qr_rt_bad" -eq 0 ] && [ "$_qr_rt_ok" -gt 0 ]; then + ok "qr encode/decode" "${_qr_rt_ok} roundtrips (ecc default ${QR_ECC})" + elif [ "$_qr_rt_ok" -eq 0 ] && [ "$_qr_rt_bad" -eq 0 ]; then + info "qr encode/decode" "no payloads to encode" + else + fail "qr encode/decode" "${_qr_rt_ok} ok · ${_qr_rt_bad} fail" + fi + fi + fi + fi +else + if [ "${QR_CHECK:-1}" != "1" ]; then + info "qr checks" "skipped (QR_CHECK=0)" + fi +fi + +summary diff --git a/check_versions.sh b/check_versions.sh new file mode 100755 index 0000000..8fc7c66 --- /dev/null +++ b/check_versions.sh @@ -0,0 +1,563 @@ +#!/usr/bin/env bash +# Area versions-### — Taler packages vs deb.taler.net (trixie) + repo availability. +# +# Checks: +# 1) deb.taler.net apt endpoints reachable (InRelease / Packages / sample .deb) +# 2) containers can reach deb.taler.net (install path from inside) +# 3) installed taler*/libeufin*/libtaler*/libdonau* versions vs suite index +# +# Env: +# TALER_APT_SUITE=trixie +# TALER_APT_BASE=https://deb.taler.net/apt/debian +# TALER_APT_INDEX=…/dists/trixie/main/binary-amd64/Packages +# TALER_APT_TESTING_INDEX=…/dists/trixie-testing/… +# TALER_PKG_BEHIND=warn|error (default: core packages ERROR if behind, else warn) +# SKIP_SSH=1 skip container install checks +set -euo pipefail +ROOT=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=lib.sh +source "$ROOT/lib.sh" + +set_area versions +set_group outside + +SUITE="${TALER_APT_SUITE:-trixie}" +APT_BASE="${TALER_APT_BASE:-https://deb.taler.net/apt/debian}" +APT_BASE="${APT_BASE%/}" +INDEX="${TALER_APT_INDEX:-${APT_BASE}/dists/${SUITE}/main/binary-amd64/Packages}" +INRELEASE="${TALER_APT_INRELEASE:-${APT_BASE}/dists/${SUITE}/InRelease}" +TESTING_INDEX="${TALER_APT_TESTING_INDEX:-${APT_BASE}/dists/${SUITE}-testing/main/binary-amd64/Packages}" +BEHIND_MODE="${TALER_PKG_BEHIND:-warn}" + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +http_get() { + # usage: http_get URL [outfile] → prints http_code; body to outfile or /dev/null + local url="$1" out="${2:-/dev/null}" + curl -sS --max-time 30 -L -o "$out" -w '%{http_code}' "$url" 2>/dev/null || echo 000 +} + +# --------------------------------------------------------------------------- +# 1) OUTSIDE — deb.taler.net install source (runner / laptop, no SSH) +# --------------------------------------------------------------------------- +section "versions · outside · deb.taler.net (${SUITE})" + +# DNS +HOST_APT="${APT_BASE#https://}"; HOST_APT="${HOST_APT#http://}"; HOST_APT="${HOST_APT%%/*}" +if getent hosts "$HOST_APT" >/dev/null 2>&1 \ + || python3 -c "import socket; socket.getaddrinfo('${HOST_APT}', 443)" >/dev/null 2>&1; then + ip=$(python3 -c "import socket; print(socket.getaddrinfo('${HOST_APT}',443)[0][4][0])" 2>/dev/null || true) + ok "outside DNS ${HOST_APT}" "${ip:-resolved}" +else + fail "outside DNS ${HOST_APT}" "unresolvable — cannot use as apt source" +fi + +# HTTPS portal / apt root +code=$(http_get "https://${HOST_APT}/") +[ "$code" = "200" ] && ok "outside https://${HOST_APT}/" "HTTP $code" \ + || warn "outside https://${HOST_APT}/" "HTTP $code" + +code=$(http_get "${APT_BASE}/") +case "$code" in + 200|301|302) ok "outside apt base" "${APT_BASE}/ HTTP $code" ;; + *) fail "outside apt base" "${APT_BASE}/ HTTP $code" ;; +esac + +# Suite metadata (what apt update needs) +code=$(http_get "$INRELEASE" "$tmp/InRelease") +if [ "$code" = "200" ] && [ -s "$tmp/InRelease" ]; then + ok "outside InRelease" "${SUITE} HTTP 200 · $(wc -c <"$tmp/InRelease" | tr -d ' ') bytes" +else + fail "outside InRelease" "HTTP $code — $INRELEASE (apt update will fail)" +fi + +RELEASE_URL="${APT_BASE}/dists/${SUITE}/Release" +code=$(http_get "$RELEASE_URL" "$tmp/Release") +[ "$code" = "200" ] && ok "outside Release" "${SUITE} HTTP 200" \ + || warn "outside Release" "HTTP $code — $RELEASE_URL" + +# Packages (plain) — required for version compare +code=$(http_get "$INDEX" "$tmp/Packages") +if [ "$code" = "200" ] && [ -s "$tmp/Packages" ] && grep -q '^Package: ' "$tmp/Packages"; then + n_pkg=$(grep -c '^Package: ' "$tmp/Packages" || true) + ok "outside Packages" "${SUITE} HTTP 200 · ${n_pkg} packages" +else + fail "outside Packages" "HTTP $code — $INDEX" + # still try more probes, but cannot compare versions without index +fi + +# Packages.gz (apt often prefers this) +PKGZ_URL="${APT_BASE}/dists/${SUITE}/main/binary-amd64/Packages.gz" +code=$(http_get "$PKGZ_URL" "$tmp/Packages.gz") +if [ "$code" = "200" ] && [ -s "$tmp/Packages.gz" ]; then + ok "outside Packages.gz" "${SUITE} HTTP 200 · $(wc -c <"$tmp/Packages.gz" | tr -d ' ') bytes" +else + warn "outside Packages.gz" "HTTP $code — apt may still use plain Packages" +fi + +# Sample pool .deb downloadable (proves packages are installable, not just listed) +if [ -s "$tmp/Packages" ]; then + DEB_PATH=$(awk ' + /^Package: taler-exchange$/ { p=1 } + p && /^Filename: / { sub(/^Filename: /,""); print; exit } + p && /^$/ { p=0 } + ' "$tmp/Packages") + [ -z "$DEB_PATH" ] && DEB_PATH=$(awk ' + /^Package: taler-merchant$/ { p=1 } + p && /^Filename: / { sub(/^Filename: /,""); print; exit } + p && /^$/ { p=0 } + ' "$tmp/Packages") + if [ -n "$DEB_PATH" ]; then + DEB_URL="${APT_BASE}/${DEB_PATH}" + code=$(curl -sS --max-time 30 -o /dev/null -w '%{http_code}' -r 0-128 "$DEB_URL" 2>/dev/null || echo 000) + case "$code" in + 200|206) + ok "outside pool .deb" "$(basename "$DEB_PATH") HTTP $code" + ;; + *) + fail "outside pool .deb" "HTTP $code — $DEB_URL (index ok but debs not fetchable)" + ;; + esac + else + warn "outside pool .deb" "no Filename for taler-exchange/merchant in index" + fi + + for pkg in taler-exchange taler-merchant libeufin-bank; do + ver=$(awk -v p="$pkg" ' + $0=="Package: "p { hit=1; next } + hit && /^Version: / { sub(/^Version: /,""); print; exit } + hit && /^$/ { hit=0 } + ' "$tmp/Packages") + if [ -n "$ver" ]; then + ok "outside suite offers ${pkg}" "$ver" + else + fail "outside suite offers ${pkg}" "missing from ${SUITE} Packages" + fi + done +fi + +HAVE_TESTING=0 +code=$(http_get "$TESTING_INDEX" "$tmp/Packages-testing") +if [ "$code" = "200" ] && [ -s "$tmp/Packages-testing" ] && grep -q '^Package: ' "$tmp/Packages-testing"; then + HAVE_TESTING=1 + ok "outside ${SUITE}-testing Packages" "HTTP 200 (optional compare)" +else + info "outside ${SUITE}-testing Packages" "HTTP ${code:-000} (optional)" +fi + +# TLS: certificate verify (curl default) already used above; explicit openssl probe if available +if command -v openssl >/dev/null 2>&1; then + if echo | openssl s_client -servername "$HOST_APT" -connect "${HOST_APT}:443" 2>/dev/null \ + | grep -q 'Verify return code: 0'; then + ok "outside TLS ${HOST_APT}" "verify ok" + else + # curl succeeded with default CA — soft warn only + warn "outside TLS ${HOST_APT}" "openssl verify not clean (curl may still work)" + fi +fi + +# Need Packages index for later compare +if [ ! -s "$tmp/Packages" ] || ! grep -q '^Package: ' "$tmp/Packages"; then + fail "outside Packages usable" "cannot continue version compare without suite index" + summary + exit 1 +fi + +# --------------------------------------------------------------------------- +# 2) INSIDE — containers can reach deb.taler.net (pasta / install path) +# host-podman: INSIDE_PODMAN=1 on GOA host · ssh: KOOPA_SSH / INSIDE_SSH from laptop +# --------------------------------------------------------------------------- +_VERS_LOCAL=0 +_VERS_SSH_HOST="" +_VERS_BANK="taler-hacktivism-bank" +_VERS_EX="taler-hacktivism-exchange-ansible" +_VERS_MER="taler-hacktivism" +if [ "${INSIDE_PODMAN:-0}" = "1" ] || [ "${INSIDE_MODE:-}" = "local-podman" ]; then + _VERS_LOCAL=1 +elif [ "${INSIDE_PROFILE:-}" = "stage-lfp" ] \ + || [ "${EXPECT_CURRENCY:-}" = "TESTPAYSAN" ] \ + || [ -n "${INSIDE_SSH:-}" ]; then + _VERS_SSH_HOST="${INSIDE_SSH:-francpaysan-stage-user}" + _VERS_BANK="${INSIDE_BANK_CTR:-stage-lfp-bank}" + _VERS_EX="${INSIDE_EXCHANGE_CTR:-stage-lfp-exchange-ansible}" + _VERS_MER="${INSIDE_MERCHANT_CTR:-stage-lfp-merchant}" +elif [ "${LOCAL_STACK:-0}" = "1" ] && [ "${SKIP_SSH:-0}" != "1" ]; then + # laptop → koopa via SSH (not host-podman) + _VERS_SSH_HOST="${KOOPA_SSH}" +fi + +if [ "$_VERS_LOCAL" = "1" ]; then + VERS_ACCESS=host-podman + set_group host +else + VERS_ACCESS=ssh + set_group ssh +fi + +section "versions · inside · containers → deb.taler.net (access=${VERS_ACCESS})" +info "flags" "VERS_ACCESS=${VERS_ACCESS} INSIDE_PODMAN=${INSIDE_PODMAN:-0} INSIDE_MODE=${INSIDE_MODE:-} LOCAL_STACK=${LOCAL_STACK:-0} SKIP_SSH=${SKIP_SSH:-0} KOOPA_SSH=${KOOPA_SSH:-} INSIDE_SSH=${INSIDE_SSH:-}" + +if [ "$_VERS_LOCAL" != "1" ] && [ -z "$_VERS_SSH_HOST" ]; then + info "access" "no host-podman and no SSH target — outside-only package index" + info "outside-only" "deb.taler.net public checks completed above" + summary + exit 0 +fi + +if [ "$_VERS_LOCAL" = "1" ]; then + if ! command -v podman >/dev/null 2>&1; then + err "host" "INSIDE_PODMAN/host-podman but podman missing" + info "outside-only" "deb.taler.net public checks completed above" + summary + exit 1 + fi + ok "host→container" "podman exec on this host (VERS_ACCESS=host-podman · no SSH)" +elif [ "$_VERS_SSH_HOST" = "${KOOPA_SSH:-}" ] || [ "${INSIDE_PROFILE:-}" = "koopa" ]; then + if ! koopa_ssh_ok; then + err "ssh" "cannot reach ${KOOPA_SSH} — from laptop set KOOPA_SSH=; on host set INSIDE_PODMAN=1" + info "outside-only" "deb.taler.net public checks completed above" + summary + exit 1 + fi + ok "ssh ${KOOPA_SSH}" "remote then podman (VERS_ACCESS=ssh)" +else + if ! mon_ssh_ok "$_VERS_SSH_HOST"; then + err "ssh" "cannot reach ${_VERS_SSH_HOST} (stagepaysan) — skip container package compare" + info "outside-only" "deb.taler.net public checks completed above" + summary + exit 1 + fi + ok "ssh ${_VERS_SSH_HOST}" "stagepaysan (VERS_ACCESS=ssh)" +fi + +INRELEASE_URL="${APT_BASE}/dists/${SUITE}/InRelease" +# Write remote script to a file (avoids bash 3.2 parse bugs with case/;; inside $(…)</dev/null | grep -Fx "$want" | head -1) + if [ -n "$c" ]; then echo "$c"; return; fi + case "$want" in + *exchange*) podman ps --format '{{.Names}}' 2>/dev/null | grep -i exchange | head -1 ;; + *bank*) podman ps --format '{{.Names}}' 2>/dev/null | grep -iE 'stage.*bank|hacktivism-bank|taler-bank|lfp-bank' | head -1 ;; + *) + c=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -E '^taler-hacktivism$|^stage-lfp-merchant$' | head -1) + if [ -n "$c" ]; then echo "$c"; return; fi + podman ps --format '{{.Names}}' 2>/dev/null | grep -iE 'merchant|hacktivism' | grep -viE 'bank|exchange' | head -1 + ;; + esac +} +for want in "$WANT_EX" "$WANT_BANK" "$WANT_MER"; do + c=$(resolve_ctr "$want") + [ -n "$c" ] || continue + role=other + case "$c" in + *exchange*) role=exchange ;; + *bank*) role=bank ;; + *) role=merchant ;; + esac + code=$(podman exec "$c" curl -sS -m 12 -o /dev/null -w '%{http_code}' "$INRELEASE_URL" 2>/dev/null || echo 000) + echo "R|${role}|${c}|${code}" + if podman exec "$c" bash -lc 'grep -Rqs deb.taler.net /etc/apt/sources.list /etc/apt/sources.list.d 2>/dev/null'; then + echo "S|${role}|${c}|yes" + else + echo "S|${role}|${c}|no" + fi + podman exec "$c" dpkg-query -W -f '${Package}\t${Version}\n' 2>/dev/null \ + | awk -v role="$role" -v ctr="$c" ' + $1 ~ /^(taler-|libeufin-|libtaler|libdonau)/ { + printf "P|%s|%s|%s|%s\n", role, ctr, $1, $2 + }' +done +REMOTE +} >"$tmp/remote-versions.sh" + +if [ "$_VERS_LOCAL" = "1" ]; then + REMOTE_OUT=$(bash "$tmp/remote-versions.sh" 2>/dev/null || true) +elif [ "$_VERS_SSH_HOST" = "${KOOPA_SSH:-}" ] || [ "${INSIDE_PROFILE:-}" = "koopa" ]; then + REMOTE_OUT=$(koopa_ssh_bash 60 <"$tmp/remote-versions.sh" || true) +else + REMOTE_OUT=$(mon_ssh_bash "$_VERS_SSH_HOST" 60 <"$tmp/remote-versions.sh" || true) +fi + +printf '%s\n' "$REMOTE_OUT" | grep -E '^R\|' >"$tmp/reach.tsv" || true +printf '%s\n' "$REMOTE_OUT" | grep -E '^S\|' >"$tmp/sources.tsv" || true +printf '%s\n' "$REMOTE_OUT" | grep -E '^P\|' >"$tmp/installed.tsv" || true + +if [ -s "$tmp/reach.tsv" ]; then + while IFS='|' read -r _ role ctr code; do + if [ "$code" = "200" ]; then + ok "container ${role} → deb.taler.net" "${ctr} InRelease HTTP $code" + else + fail "container ${role} → deb.taler.net" "${ctr} InRelease HTTP $code — apt install/update will fail" + fi + done <"$tmp/reach.tsv" +else + warn "container → deb.taler.net" "no reachability rows" +fi + +if [ -s "$tmp/sources.tsv" ]; then + while IFS='|' read -r _ role ctr has; do + if [ "$has" = "yes" ]; then + ok "container ${role} apt source" "${ctr} lists deb.taler.net" + else + warn "container ${role} apt source" "${ctr} no deb.taler.net in sources.list*" + fi + done <"$tmp/sources.tsv" +fi + +if [ ! -s "$tmp/installed.tsv" ]; then + fail "installed packages" "none found in taler containers" + summary + exit 1 +fi +n_inst=$(wc -l <"$tmp/installed.tsv" | tr -d ' ') +ok "collected installed packages" "${n_inst} rows" + +# --------------------------------------------------------------------------- +# 3) compare installed vs trixie (and note testing) +# --------------------------------------------------------------------------- +set_group compare +section "versions · compare installed vs ${SUITE}" + +export BEHIND_MODE HAVE_TESTING +export TALER_APT_SUITE="$SUITE" +CORE_PKGS="taler-exchange taler-exchange-database taler-merchant taler-merchant-webui libeufin-bank libeufin-common libtalerexchange libtalermerchant taler-terms-generator" +export CORE_PKGS + +python3 - "$tmp/Packages" "$tmp/Packages-testing" "$tmp/installed.tsv" <<'PY' >"$tmp/py.out" +import sys, os +from collections import defaultdict + +main_idx, test_idx, inst_path = sys.argv[1:4] +have_testing = os.environ.get("HAVE_TESTING", "0") == "1" +core = set(os.environ.get("CORE_PKGS", "").split()) +suite = os.environ.get("TALER_APT_SUITE", "trixie") + +def _ord_ch(c): + if c == "~": + return -1 + if c.isdigit(): + return 0 + if c.isalpha(): + return ord(c) + return ord(c) + 256 + +def _split_ver(v): + epoch = 0 + if ":" in v: + e, v = v.split(":", 1) + try: + epoch = int(e) + except ValueError: + epoch = 0 + if "-" in v: + upstream, deb = v.rsplit("-", 1) + else: + upstream, deb = v, "" + return epoch, upstream, deb + +def _cmp_part(a, b): + ia = ib = 0 + while ia < len(a) or ib < len(b): + while True: + ca = a[ia] if ia < len(a) and not a[ia].isdigit() else "" + cb = b[ib] if ib < len(b) and not b[ib].isdigit() else "" + if ca == "" and cb == "": + break + if ca == "" and cb: + return -1 if _ord_ch(cb) > 0 else (1 if cb == "~" else -1) + if cb == "" and ca: + return 1 if _ord_ch(ca) > 0 else (-1 if ca == "~" else 1) + oa, ob = _ord_ch(ca), _ord_ch(cb) + ia += 1 + ib += 1 + if oa != ob: + return (oa > ob) - (oa < ob) + sa = sb = "" + while ia < len(a) and a[ia].isdigit(): + sa += a[ia]; ia += 1 + while ib < len(b) and b[ib].isdigit(): + sb += b[ib]; ib += 1 + na = int(sa) if sa else 0 + nb = int(sb) if sb else 0 + if na != nb: + return (na > nb) - (na < nb) + return 0 + +def deb_cmp(a, b): + if a == b: + return 0 + ea, ua, da = _split_ver(a) + eb, ub, db = _split_ver(b) + if ea != eb: + return (ea > eb) - (ea < eb) + c = _cmp_part(ua, ub) + if c: + return c + return _cmp_part(da, db) + +def parse_packages(path): + pkgs = {} + if not path or not os.path.isfile(path) or os.path.getsize(path) == 0: + return pkgs + cur = ver = None + with open(path, encoding="utf-8", errors="replace") as f: + for line in f: + line = line.rstrip("\n") + if line.startswith("Package: "): + if cur and ver and (cur not in pkgs or deb_cmp(ver, pkgs[cur]) > 0): + pkgs[cur] = ver + cur = line[9:].strip() + ver = None + elif line.startswith("Version: ") and cur: + ver = line[9:].strip() + elif line == "" and cur: + if ver and (cur not in pkgs or deb_cmp(ver, pkgs[cur]) > 0): + pkgs[cur] = ver + cur = ver = None + if cur and ver and (cur not in pkgs or deb_cmp(ver, pkgs[cur]) > 0): + pkgs[cur] = ver + return pkgs + +main = parse_packages(main_idx) +testing = parse_packages(test_idx) if have_testing else {} + +by_pkg = defaultdict(list) +with open(inst_path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line.startswith("P|"): + continue + parts = line.split("|") + if len(parts) < 5: + continue + _, role, ctr, pkg, ver = parts[:5] + by_pkg[pkg].append((role, ctr, ver)) + +for pkg in sorted(by_pkg.keys()): + versions = sorted({v for _, _, v in by_pkg[pkg]}, key=lambda v: v) + # pick "highest" via deb_cmp + inst = versions[0] + for v in versions[1:]: + if deb_cmp(v, inst) > 0: + inst = v + roles = ",".join(sorted({r for r, _, _ in by_pkg[pkg]})) + multi = len(set(versions)) > 1 + trixie = main.get(pkg) + testv = testing.get(pkg) + status = "ok" + detail = "" + if multi: + status = "warn" + detail = "multiple installed: " + ", ".join(sorted(set(versions))) + if trixie is None: + if status == "ok": + status = "info" + detail = (detail + "; " if detail else "") + f"not in {suite} main index" + else: + c = deb_cmp(inst, trixie) + if c == 0: + detail = (detail + "; " if detail else "") + f"= {suite} {trixie}" + elif c < 0: + status = "behind" + detail = f"installed {inst} < {suite} {trixie}" + else: + status = "ahead" + detail = f"installed {inst} > {suite} {trixie}" + if testv and deb_cmp(inst, testv) == 0: + detail += f" (={suite}-testing)" + elif testv: + detail += f" (testing has {testv})" + core_flag = "1" if pkg in core else "0" + print(f"{status}|{pkg}|{inst}|{trixie or '-'}|{roles}|{core_flag}|{detail}") +PY + +ok_n=0; ahead_n=0; behind_n=0 +while IFS='|' read -r status pkg inst trixie roles core detail; do + [ -n "${status:-}" ] || continue + label="pkg ${pkg} (${roles})" + case "$status" in + ok) + ok "$label" "${inst}" + ok_n=$((ok_n + 1)) + ;; + ahead) + info "$label" "$detail" + ahead_n=$((ahead_n + 1)) + ;; + behind) + behind_n=$((behind_n + 1)) + if [ "$core" = "1" ] || [ "$BEHIND_MODE" = "error" ]; then + fail "$label" "$detail" + else + warn "$label" "$detail" + fi + ;; + warn) + warn "$label" "$detail" + ;; + info) + info "$label" "installed ${inst}${detail:+ — $detail}" + ;; + *) + info "$label" "$status $detail" + ;; + esac +done <"$tmp/py.out" + +# Core packages must exist somewhere +for need in taler-exchange libeufin-bank taler-merchant; do + if grep -qE "^P\|[^|]+\|[^|]+\|${need}\|" "$tmp/installed.tsv"; then + ok "core installed ${need}" + else + fail "core installed ${need}" "not present in any taler container" + fi +done + +info "suite" "deb.taler.net ${SUITE} (testing_index=${HAVE_TESTING})" +info "tally" "match=${ok_n} ahead=${ahead_n} behind=${behind_n}" + +# Disk free on the machine(s) we just probed (host-podman or SSH target) +set_group disk +section "versions · disk free space" +if [ "${_VERS_LOCAL:-0}" = "1" ]; then + mon_disk_check_host "host" || true + while read -r _vc; do + [ -n "$_vc" ] || continue + mon_disk_check_podman "$_vc" || true + done < <(podman ps --format '{{.Names}}' 2>/dev/null | grep -iE 'hacktivism|taler-|stage-lfp|lfp-' || true) +elif [ -n "${_VERS_SSH_HOST:-}" ]; then + _vdisk=$(mon_ssh_bash "$_VERS_SSH_HOST" 20 <<'DISK' || true +set +e +echo "###HOST###" +df -Pk / /var /home /tmp 2>/dev/null || df -Pk +echo "###CTRS###" +for c in $(podman ps --format '{{.Names}}' 2>/dev/null); do + echo "###CTR $c###" + podman exec "$c" df -Pk / /var /tmp 2>/dev/null || true +done +DISK +) + mon_disk_check_remote_text "ssh:${_VERS_SSH_HOST}" \ + "$(printf '%s\n' "$_vdisk" | sed -n '/^###HOST###$/,/^###CTRS###$/p' | sed '1d;$d')" || true + _ctr=""; _buf="" + while IFS= read -r _line || [ -n "$_line" ]; do + case "$_line" in + '###CTR '*) + [ -n "$_ctr" ] && mon_disk_check_remote_text "ctr:${_ctr}" "$_buf" || true + _ctr=${_line####CTR }; _ctr=${_ctr%###}; _buf="" + ;; + '###CTRS###'|'###HOST###') ;; + *) [ -n "$_ctr" ] && _buf="${_buf}${_line}"$'\n' ;; + esac + done <<<"$_vdisk" + [ -n "$_ctr" ] && mon_disk_check_remote_text "ctr:${_ctr}" "$_buf" || true +fi + +summary diff --git a/domains.conf b/domains.conf new file mode 100644 index 0000000..1cc12af --- /dev/null +++ b/domains.conf @@ -0,0 +1,46 @@ +# taler-monitoring domain profiles +# +# When you add a stack, declare the three public endpoints here: +# name bank exchange merchant-backend currency local landing [canonical] +# +# - name: what -d / TALER_DOMAIN matches (aliases allowed as extra lines) +# - bank / exchange / merchant: hostname or full https:// URL +# - currency: GOA | KUDOS | CHF | … (empty = report only) +# - local: 1 = koopa SSH/inside/e2e stack; 0 = public-only +# - landing: 1 = check /intro landings + assets; 0 = skip (no GOA-style landings) +# - canonical: optional TALER_DOMAIN label after alias match +# +# CLI overrides after load: --bank --exchange --merchant --currency +# Alternate file: TALER_DOMAINS_CONF=/path/to/domains.conf + +# Local GOA (koopa) — landings matter +koopa bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch GOA 1 1 hacktivism.ch +hacktivism bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch GOA 1 1 hacktivism.ch +hacktivism.ch bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch GOA 1 1 + +# Public demo (KUDOS) — has intro-style pages; keep soft landing checks +taler.net bank.demo.taler.net exchange.demo.taler.net backend.demo.taler.net KUDOS 0 1 demo.taler.net +demo.taler.net bank.demo.taler.net exchange.demo.taler.net backend.demo.taler.net KUDOS 0 1 + +# Public test stack (TESTKUDOS) — no GOA-style landings +test.taler.net bank.test.taler.net exchange.test.taler.net backend.test.taler.net TESTKUDOS 0 0 + +# TOPS CHF — multi-tenant merchant my.taler-ops.ch; no public GOA landings +taler-ops.ch bank.taler-ops.ch exchange.taler-ops.ch my.taler-ops.ch CHF 0 0 +my.taler-ops.ch bank.taler-ops.ch exchange.taler-ops.ch my.taler-ops.ch CHF 0 0 taler-ops.ch +stage.taler-ops.ch bank.stage.taler-ops.ch exchange.stage.taler-ops.ch my.stage.taler-ops.ch CHF 0 0 +my.stage.taler-ops.ch bank.stage.taler-ops.ch exchange.stage.taler-ops.ch my.stage.taler-ops.ch CHF 0 0 stage.taler-ops.ch + +# Franc Paysan (Infomaniak) — merchant host is monnaie.* (not taler./backend.) +# Prod: no monetary launch yet; stage uses TESTPAYSAN (public HTTPS landings + shops) +lefrancpaysan.ch bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch - 0 0 +monnaie.lefrancpaysan.ch bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch - 0 0 lefrancpaysan.ch +bank.lefrancpaysan.ch bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch - 0 0 lefrancpaysan.ch +exchange.lefrancpaysan.ch bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch - 0 0 lefrancpaysan.ch + +# Stage TESTPAYSAN — landings on / + farmer shops under stage.monnaie…/shops/ +# name bank exchange merchant currency local landing [canonical] +stage.lefrancpaysan.ch stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch TESTPAYSAN 0 1 +stage.bank.lefrancpaysan.ch stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch TESTPAYSAN 0 1 stage.lefrancpaysan.ch +stage.exchange.lefrancpaysan.ch stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch TESTPAYSAN 0 1 stage.lefrancpaysan.ch +stage.monnaie.lefrancpaysan.ch stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch TESTPAYSAN 0 1 stage.lefrancpaysan.ch diff --git a/harness-live.txt b/harness-live.txt new file mode 100644 index 0000000..b7d5500 --- /dev/null +++ b/harness-live.txt @@ -0,0 +1,7 @@ +# Optional: taler-harness against *our* exchange (needs a harness version +# that accepts current /config JSON). Older installs often FAIL codec checks +# even when the exchange is fine — trust `./taler-monitoring.sh urls` first. +# +# idcommand + +lint-exchange-goa taler-harness deployment lint-exchange-url https://exchange.hacktivism.ch/ diff --git a/harness-tests.txt b/harness-tests.txt new file mode 100644 index 0000000..1c63e3b --- /dev/null +++ b/harness-tests.txt @@ -0,0 +1,41 @@ +# Real-life–relevant taler-harness integration tests +# (from: taler-harness list-integrationtests) +# +# Criteria: everyday user/merchant/bank paths — withdraw, pay, deposit, refund, +# libeufin bank, templates/paywall. Excludes KYC edge cases, experimental, +# timetravel, perf, backup, mailbox, most fault-injection. +# +# These spin up a local TESTKUDOS stack (same client code paths as production). +# For tests against *public* demos, see harness-live.txt / `./taler-monitoring.sh live`. + +# --- Core path: get money + spend it (the GOA story) --- +simple-payment +withdrawal-bank-integrated +payment + +# --- Withdraw variants users actually hit --- +withdrawal-manual +withdrawal-external +withdrawal-idempotent + +# --- Pay variants shops use --- +payment-template +paywall-flow +payment-idempotency +payment-zero +payment-abort +otp + +# --- After pay: coins / merchant ops --- +deposit +refund +refund-auto +wallet-refresh + +# --- Bank stack we run (libeufin / regional) --- +libeufin-bank +bank-api + +# --- Client hygiene --- +wallet-config +term-of-service-format diff --git a/host-agent/README.md b/host-agent/README.md new file mode 100644 index 0000000..0664f28 --- /dev/null +++ b/host-agent/README.md @@ -0,0 +1,183 @@ +# host-agent — reporting generation (GOA + FP + any stack) + +**Global pipeline** lives in **`run-host-report.sh`** (all stacks share it): + +| Behaviour | Default | +|-----------|---------| +| Suite refresh | `update-suite.sh` → `origin/main` | +| Wall clock | `RUN_TIMEOUT=600` | +| Log | line-buffered `run-*.log` (continuous write) | +| HTML | always (first run / empty tree too) + commit URL | +| Timeout notice | end of log + top jump on `/monitoring_err/` | + +Thin wrappers only set stack defaults: + +| Wrapper | Stack | Typical user | +|---------|-------|----------------| +| `run-hacktivism-monitoring.sh` | GOA `hacktivism.ch` | `hernani` @ koopa | +| `run-fp-stage-monitoring.sh` | FP stage | `stagepaysan` | +| `run-fp-prod-monitoring.sh` | FP prod | `francpaysan` | +| `run-aptdeploy.sh` | CLI: ensure + **aptdeploy** (no HTML) | local if hostname=koopa, else **ssh koopa-external** | +| `run-aptdeploy-monitoring.sh` | **aptdeploy** → `/taler-monitoring-aptdeploy*` | `hernani` @ koopa (4h timer, taler only) | +| `run-surface-monitoring.sh` | **surface** → `/taler-monitoring-surface*` | `hernani` @ koopa (hourly, taler only) | + +### apt-src deploy tests (hacktivism `/monitoring`) + +| Container | Mode | +|-----------|------| +| `koopa-taler-deploy-test-apt-src-trixie` | **fresh**: recreate when repo package fingerprint changes | +| `koopa-taler-deploy-test-apt-src-trixie-testing` | **fresh** for `trixie-testing` | +| `koopa-taler-deploy-test-apt-src-trixie-upgrade` | **upgrade**: initial install, then mytops-style `apt install` + `upgrade` | +| `koopa-taler-deploy-test-apt-src-trixie-testing-upgrade` | **upgrade** for testing suite | + +```bash +# from laptop / clementine → triggers on koopa-external +./host-agent/run-aptdeploy.sh +./host-agent/run-aptdeploy.sh upgrade +./host-agent/run-aptdeploy.sh fresh +./host-agent/run-aptdeploy.sh check-only + +# on koopa +./taler-monitoring.sh -d hacktivism.ch aptdeploy +``` + +Checks: package table, `httpd --version`, libtalerutil/`ldd` on failure, `systemctl` for +`taler-merchant.target`, basic local `/config` probe. No nginx/HTTPS in containers. + +### surface + aptdeploy HTML (taler.hacktivism.ch only) + +| URL | Role | Timer | +|-----|------|-------| +| `…/taler-monitoring-surface/` | ecosystem surface scan | 1h | +| `…/taler-monitoring-surface_err/` | errors + jump list | ″ | +| `…/taler-monitoring-aptdeploy/` | apt-src container deploy tests | 4h | +| `…/taler-monitoring-aptdeploy_err/` | errors + jump list | ″ | + +Not on bank/exchange. Staging: +`~/monitoring-sites-staging/taler.hacktivism.ch/taler-monitoring-{surface,aptdeploy}*`. + +```bash +systemctl --user enable --now taler-monitoring-surface.timer +systemctl --user enable --now taler-monitoring-aptdeploy.timer +systemctl --user start taler-monitoring-surface.service +systemctl --user start taler-monitoring-aptdeploy.service +# Caddy: handles only in taler.hacktivism.ch block (root) +``` + +Env examples: **`env/*.env.example`**. FP docs: `francpaysan-admin-log/docs/taler-monitoring-host-agents.md`. + +## GOA (koopa) — host-podman + +Runs **as user `hernani`** (systemd user, linger). **Primary** mode: + +- **`INSIDE_PODMAN=1`** / **`INSIDE_MODE=local-podman`**: access mode **`host-podman`** — + `podman exec` on this host into + `taler-hacktivism-bank` / `taler-hacktivism-exchange-ansible` / `taler-hacktivism` + (IDs `inside.host-*`, not `inside.ssh-*`). + From a **laptop**, leave `INSIDE_PODMAN` off → access mode **`ssh`** via `KOOPA_SSH` + (`inside.ssh-*`). +- Public **`urls` + `versions`** for the stack (`-d hacktivism.ch`). +- Writes console HTML for all three sites under + `~/monitoring-sites-staging/{bank,exchange,taler}.hacktivism.ch/`. + +Caddy (host root) can serve that tree at `/monitoring` — you configure Caddy. + +## Latest software + config isolation + +**Source of truth is always Forgejo** (`git.hacktivism.ch`), not a laptop rsync copy. + +Every host-agent / aptdeploy run: + +1. **`update-suite.sh`**: `git fetch` + **`SUITE_UPDATE_MODE=reset`** (default) → `origin/main` + into `SUITE_DIR` (prefer `~/src/koopa-admin-log`). +2. **Re-exec** host-agent scripts from that tree so the running code matches `COMMIT_URL`. +3. Local overrides only in **`~/.config/taler-monitoring/env`** — never overwritten by git. +4. HTML footer = exact commit: + `https://git.hacktivism.ch/hernani/taler-monitoring/src/commit/` +5. **`RUN_TIMEOUT=600`**, line-buffered logs, HTML always written. + +If fetch fails → previous tree + WARN. +**To ship features (aptdeploy, surface, …): push to `origin/main`.** Local-only edits vanish on the next reset. + +If pull fails (no network / auth), the previous tree is used and a WARN is logged. + +## Install (on koopa as hernani) + +```bash +# once: local config (optional) +mkdir -p ~/.config/taler-monitoring +cp ~/src/taler-monitoring/host-agent/env.example \ + ~/.config/taler-monitoring/env + +# laptop: +./host-agent/install-host-agent.sh --remote koopa-external + +# or on koopa: +~/src/taler-monitoring/host-agent/install-host-agent.sh +``` + +Units: + +| Unit | Role | +|------|------| +| `taler-monitoring-hacktivism.path` | fires on software stamp + landing/caddy config changes | +| `taler-monitoring-hacktivism.timer` | every **4h** fallback | +| `taler-monitoring-hacktivism.service` | oneshot → `/monitoring*` (urls inside versions) | +| `taler-monitoring-surface.timer` | every **1h** | +| `taler-monitoring-surface.service` | → `/taler-monitoring-surface*` (taler only) | +| `taler-monitoring-aptdeploy.timer` | every **4h** | +| `taler-monitoring-aptdeploy.service` | → `/taler-monitoring-aptdeploy*` (taler only) | + +## After package / image / landing upgrades + +```bash +~/src/taler-monitoring/host-agent/touch-software-stamp.sh +# → path unit starts the service +``` + +## Manual + +```bash +systemctl --user start taler-monitoring-hacktivism.service +systemctl --user status taler-monitoring-hacktivism.service --no-pager +journalctl --user -u taler-monitoring-hacktivism.service -n 50 --no-pager +ls ~/monitoring-sites-staging/bank.hacktivism.ch/monitoring_err/ +``` + +## Relation to firecuda outside-only timer + +| | host-agent (koopa) | firecuda site-gen | +|--|--------------------|-------------------| +| Access | **podman on host** | public HTTPS only | +| Phases | urls **inside** versions | urls versions | +| Trigger | software path + 4h | *could* launchd 4h | +| Primary for GOA | **yes** | **off** (optional only) | + +**firecuda-external is not scheduled** right now (launchd disabled). Scripts under +`site-gen/` can still one-shot a public run if needed; see `site-gen/README.md`. + +No secrets required for default phases. `secrets.env` optional for e2e later. + +## Git commit on HTML pages + +Header of each generated page includes a link like: + +`https://git.hacktivism.ch/hernani/taler-monitoring/src/commit/` + +That SHA is the tree **after** `update-suite.sh` for that run. + +## Packages on koopa + +See **[../DEPENDENCIES.md](../DEPENDENCIES.md)**. + +**Default host-agent needs no new packages** if curl/python3/podman/git are present (they are). + +**Root once** only for full QR checks in `urls`: + +```bash +zypper in qrencode zbar +``` + +Without them, monitoring still runs; QR group WARNs and skips encode/decode. + +Caddy `/var/www/monitoring-sites` is separate (you configured hernani write access). diff --git a/host-agent/ROOT-APPLY-MONITORING.md b/host-agent/ROOT-APPLY-MONITORING.md new file mode 100644 index 0000000..17dd386 --- /dev/null +++ b/host-agent/ROOT-APPLY-MONITORING.md @@ -0,0 +1,57 @@ +# Root: make monitoring pages live on koopa (GOA) + +## Symptom if not applied + +Public URLs return **merchant/bank JSON** like: + +```json +{ "code": 21, "hint": "There is no endpoint defined for the URL provided..." } +``` + +or a **404 HTML** page from the app reverse_proxy. + +Cause: live `/etc/caddy/Caddyfile` has **no** `handle /monitoring*` (falls through to +API). Prepared config is only under **`/home/hernani/koopa-caddy/Caddyfile`**. + +## One-shot as root (recommended) + +```bash +sudo /home/hernani/koopa-caddy/apply-monitoring-live.sh +``` + +That script: + +1. `rsync` staging HTML → `/var/www/monitoring-sites/` (incl. surface + aptdeploy) +2. Makes tree writable for `hernani` (group `caddy` + ACL) so later runs self-sync +3. Installs prepared Caddyfile → `/etc/caddy/Caddyfile`, validates, reloads Caddy +4. Smoke-curls all monitoring URLs (must **not** be JSON `code:21`) + +## Paths + +| Path | Host | Source timer | +|------|------|----------------| +| `/monitoring/` + `/monitoring_err/` | bank + exchange + taler | `taler-monitoring-hacktivism.timer` (4h) | +| `/taler-monitoring-surface/` + `/_err/` | **taler.hacktivism.ch only** | `taler-monitoring-surface.timer` (1h) | +| `/taler-monitoring-aptdeploy/` + `/_err/` | **taler.hacktivism.ch only** | `taler-monitoring-aptdeploy.timer` (4h) | + +HTML is generated as **hernani** into +`/home/hernani/monitoring-sites-staging/`. + +## Manual equivalent + +```bash +rsync -a --delete /home/hernani/monitoring-sites-staging/ /var/www/monitoring-sites/ +chown -R hernani:caddy /var/www/monitoring-sites && chmod -R g+rwX /var/www/monitoring-sites +sudo /home/hernani/koopa-caddy/apply.sh # or apply-monitoring-live.sh +``` + +Expect **200** (HTML with sticky bar). Not merchant JSON `code:21`. + +## After each monitoring run (if www is not hernani-writable) + +```bash +rsync -a --delete /home/hernani/monitoring-sites-staging/ /var/www/monitoring-sites/ +chown -R caddy:caddy /var/www/monitoring-sites +``` + +Or grant `hernani` write on `/var/www/monitoring-sites` once so the host-agent can rsync itself. diff --git a/host-agent/ensure-apt-deploy-test-containers.sh b/host-agent/ensure-apt-deploy-test-containers.sh new file mode 100755 index 0000000..75fa3e4 --- /dev/null +++ b/host-agent/ensure-apt-deploy-test-containers.sh @@ -0,0 +1,373 @@ +#!/usr/bin/env bash +# ensure-apt-deploy-test-containers.sh +# +# Four rootless podman containers on koopa (no nginx / no HTTPS): +# +# Fresh rebuild when repo fingerprint changes: +# koopa-taler-deploy-test-apt-src-trixie +# koopa-taler-deploy-test-apt-src-trixie-testing +# +# Persistent upgrade path (mytops upgrade.sh style): +# koopa-taler-deploy-test-apt-src-trixie-upgrade +# koopa-taler-deploy-test-apt-src-trixie-testing-upgrade +# +# Run as hernani on koopa. From elsewhere use run-aptdeploy.sh (ssh koopa-external). +# +set -euo pipefail + +PODMAN_BIN="${APT_DEPLOY_PODMAN:-podman}" +IMG="${APT_DEPLOY_BASE_IMAGE:-docker.io/library/debian:trixie}" +STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/taler-apt-deploy" +URI_BASE="${APT_DEPLOY_URI:-https://deb.taler.net/apt/debian}" +mkdir -p "$STATE_DIR" + +# Packages we track for "new version → rebuild fresh containers" +TRACK_PKGS="${APT_DEPLOY_TRACK_PKGS:-taler-merchant libtalermerchant libtalerexchange libdonau}" + +# Canonical four containers (keep). Everything else matching legacy names is removed. +CANONICAL_CONTAINERS=( + koopa-taler-deploy-test-apt-src-trixie + koopa-taler-deploy-test-apt-src-trixie-testing + koopa-taler-deploy-test-apt-src-trixie-upgrade + koopa-taler-deploy-test-apt-src-trixie-testing-upgrade +) + +# Legacy / typo names from earlier iterations — always delete if present +LEGACY_CONTAINERS=( + koopa-taler-build-test-apt-trixie + koopa-taler-build-test-apt-trixie-testing + koopa-taler-deploy-test-apt-trixie + koopa-taler-deploy-test-apt-trixie-testing + koopa-taler-buidl-test-apt-trixie + koopa-taler-buidl-test-apt-trixie-testing +) + +cleanup_legacy() { + local n img + echo "======== cleanup legacy containers / images ========" + for n in "${LEGACY_CONTAINERS[@]}"; do + if "$PODMAN_BIN" container exists "$n" 2>/dev/null; then + echo "remove legacy container $n" + "$PODMAN_BIN" rm -f "$n" 2>/dev/null || true + fi + # old setup images for that name + if "$PODMAN_BIN" image exists "localhost/${n}:setup" 2>/dev/null; then + echo "remove legacy image localhost/${n}:setup" + "$PODMAN_BIN" rmi -f "localhost/${n}:setup" 2>/dev/null || true + fi + done + # dangling setup images not in canonical set + while read -r img; do + [ -n "$img" ] || continue + local base keep=0 + base=${img#localhost/} + base=${base%:setup} + for n in "${CANONICAL_CONTAINERS[@]}"; do + [ "$base" = "$n" ] && keep=1 && break + done + if [ "$keep" -eq 0 ] && [[ "$img" == localhost/koopa-taler-*-test-apt* ]]; then + echo "remove non-canonical image $img" + "$PODMAN_BIN" rmi -f "$img" 2>/dev/null || true + fi + done < <("$PODMAN_BIN" images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null | grep -E 'localhost/koopa-taler-.*-test-apt' || true) +} + +# Prefer single suite tree: if both ~/src/koopa-admin-log and ~/koopa-admin-log are +# full git clones, keep the newer (by HEAD commit time) and replace the older with a symlink. +dedupe_suite_repos() { + local a="$HOME/src/koopa-admin-log" b="$HOME/koopa-admin-log" + local ta tb + [ "${APT_DEPLOY_DEDUPE_REPOS:-1}" = "1" ] || return 0 + if [ -d "$a/.git" ] && [ -d "$b/.git" ] && [ ! -L "$a" ] && [ ! -L "$b" ]; then + ta=$(git -C "$a" log -1 --format=%ct 2>/dev/null || echo 0) + tb=$(git -C "$b" log -1 --format=%ct 2>/dev/null || echo 0) + echo "======== dedupe suite repos (both are git clones) ========" + echo " $a HEAD_ts=$ta" + echo " $b HEAD_ts=$tb" + if [ "$ta" -ge "$tb" ]; then + echo "keep $a · remove $b · symlink $b → $a" + rm -rf "$b" + ln -sfn "$a" "$b" + else + echo "keep $b · remove $a · symlink $a → $b" + rm -rf "$a" + mkdir -p "$(dirname "$a")" + ln -sfn "$b" "$a" + fi + elif [ -d "$a/scripts/taler-monitoring" ] && [ -d "$b/scripts/taler-monitoring" ] \ + && [ ! -d "$a/.git" ] && [ -d "$b/.git" ] && [ ! -L "$a" ]; then + echo "remove non-git tree $a · symlink → $b" + rm -rf "$a" + mkdir -p "$(dirname "$a")" + ln -sfn "$b" "$a" + elif [ -d "$b/scripts/taler-monitoring" ] && [ -d "$a/scripts/taler-monitoring" ] \ + && [ ! -d "$b/.git" ] && [ -d "$a/.git" ] && [ ! -L "$b" ]; then + echo "remove non-git tree $b · symlink → $a" + rm -rf "$b" + ln -sfn "$a" "$b" + fi +} + +cleanup_legacy +dedupe_suite_repos +# --------------------------------------------------------------------------- +# Repo fingerprint: Version: lines from suite Packages index +# --------------------------------------------------------------------------- +repo_pkg_version() { + local suite=$1 pkg=$2 + curl -fsS --max-time 45 \ + "${URI_BASE}/dists/${suite}/main/binary-amd64/Packages" 2>/dev/null \ + | awk -v p="$pkg" ' + $0 == "Package: " p { want=1; next } + /^Package: / { want=0 } + want && /^Version: / { print $2; exit } + ' +} + +repo_fingerprint() { + local suite=$1 pkg v line="" + for pkg in $TRACK_PKGS; do + v=$(repo_pkg_version "$suite" "$pkg" || true) + [ -n "$v" ] || v="?" + line="${line}${pkg}=${v};" + done + printf '%s' "$line" +} + +container_fingerprint() { + local name=$1 + "$PODMAN_BIN" exec "$name" bash -lc ' + for p in '"$TRACK_PKGS"'; do + v=$(dpkg-query -W -f="\${Version}" "$p" 2>/dev/null || echo missing) + printf "%s=%s;" "$p" "$v" + done + ' 2>/dev/null || echo "missing" +} + +# --------------------------------------------------------------------------- +# Bootstrap: install base + taler source + packages (shared) +# --------------------------------------------------------------------------- +install_base_and_merchant() { + local name=$1 suite=$2 srcfile=$3 + "$PODMAN_BIN" exec -u root "$name" bash -lc ' + set -euo pipefail + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y -qq ca-certificates curl wget gnupg systemd systemd-sysv \ + dbus procps iproute2 + ' + "$PODMAN_BIN" exec -u root "$name" bash -lc " + set -euo pipefail + export DEBIAN_FRONTEND=noninteractive + mkdir -p /etc/apt/keyrings + wget -q -O /etc/apt/keyrings/taler-systems.gpg https://taler.net/taler-systems.gpg + cat >/etc/apt/sources.list.d/${srcfile} </dev/null + systemctl enable taler-merchant.target 2>/dev/null + systemctl start taler-merchant.target 2>/dev/null + sleep 2 + systemctl is-active taler-merchant-httpd.service 2>/dev/null || true + ' || true +} + +create_from_scratch() { + local name=$1 suite=$2 srcfile=$3 + echo "======== CREATE (fresh) $name Suites: $suite ========" + "$PODMAN_BIN" pull "$IMG" >/dev/null + "$PODMAN_BIN" rm -f "$name" 2>/dev/null || true + # drop previous setup image so we do not accumulate duplicates + "$PODMAN_BIN" rmi -f "localhost/${name}:setup" 2>/dev/null || true + "$PODMAN_BIN" run -d --name "$name" --hostname "$name" "$IMG" sleep infinity + install_base_and_merchant "$name" "$suite" "$srcfile" + commit_and_systemd "$name" + printf '%s\n' "$(repo_fingerprint "$suite")" >"$STATE_DIR/${name}.fp" + printf '%s\n' "$(container_fingerprint "$name")" >"$STATE_DIR/${name}.installed" + echo "created $name fp=$(cat "$STATE_DIR/${name}.fp")" +} + +# --------------------------------------------------------------------------- +# Upgrade path (mytops-admin-log upgrade.sh merchant packages, no restic/nginx) +# Prints a markdown-ish table of before → after versions. +# --------------------------------------------------------------------------- +upgrade_inside() { + local name=$1 + local before after pkg v0 v1 + echo "======== UPGRADE $name (mytops-style apt install + upgrade) ========" + + before=$("$PODMAN_BIN" exec "$name" bash -lc ' + for p in taler-merchant libtalermerchant libtalerexchange libdonau libgnunet \ + taler-merchant-webui taler-merchant-typst taler-terms-generator; do + v=$(dpkg-query -W -f="\${Version}" "$p" 2>/dev/null || echo "-") + printf "%s %s\n" "$p" "$v" + done + ' 2>/dev/null || true) + + # capture upgrade output + set +e + "$PODMAN_BIN" exec -u root "$name" bash -lc ' + set -euo pipefail + export DEBIAN_FRONTEND=noninteractive + systemctl stop taler-merchant.target 2>/dev/null || true + systemctl stop taler-merchant-donaukeyupdate.service 2>/dev/null || true + apt-get update -qq + apt-get install -y \ + taler-merchant \ + taler-merchant-typst \ + taler-merchant-webui \ + taler-terms-generator \ + || true + if dpkg -l challenger-httpd 2>/dev/null | grep -q "^ii"; then + apt-get remove -y challenger-httpd || true + fi + apt-get upgrade -y + systemctl daemon-reload 2>/dev/null || true + systemctl enable taler-merchant.target 2>/dev/null || true + # try dbconfig if present (may no-op without full config) + command -v taler-merchant-dbconfig >/dev/null && taler-merchant-dbconfig 2>/dev/null || true + systemctl reset-failed "taler-merchant*" 2>/dev/null || true + systemctl start taler-merchant.target 2>/dev/null || true + sleep 2 + ' 2>&1 | tee "$STATE_DIR/${name}.upgrade.log" | tail -30 + set -e + + after=$("$PODMAN_BIN" exec "$name" bash -lc ' + for p in taler-merchant libtalermerchant libtalerexchange libdonau libgnunet \ + taler-merchant-webui taler-merchant-typst taler-terms-generator; do + v=$(dpkg-query -W -f="\${Version}" "$p" 2>/dev/null || echo "-") + printf "%s %s\n" "$p" "$v" + done + ' 2>/dev/null || true) + + echo + echo "| package | before | after | change |" + echo "|---------|--------|-------|--------|" + while read -r pkg v0; do + [ -n "${pkg:-}" ] || continue + v1=$(printf '%s\n' "$after" | awk -v p="$pkg" '$1==p {print $2; exit}') + v1=${v1:--} + if [ "$v0" = "$v1" ]; then + ch="=" + else + ch="UPGRADED" + fi + echo "| $pkg | $v0 | $v1 | $ch |" + done <<<"$before" + + printf '%s\n' "$(container_fingerprint "$name")" >"$STATE_DIR/${name}.installed" +} + +# --------------------------------------------------------------------------- +# Fresh mode: recreate if missing / not running / repo fp changed +# --------------------------------------------------------------------------- +ensure_fresh() { + local name=$1 suite=$2 srcfile=$3 + local want have running + + want=$(repo_fingerprint "$suite") + echo "repo fingerprint [$suite]: $want" + + if ! "$PODMAN_BIN" container exists "$name" 2>/dev/null; then + create_from_scratch "$name" "$suite" "$srcfile" + return + fi + + running=$("$PODMAN_BIN" inspect -f '{{.State.Running}}' "$name" 2>/dev/null || echo false) + have=$(container_fingerprint "$name") + echo "installed in $name: $have" + + if [ "$running" != "true" ] \ + || ! "$PODMAN_BIN" exec "$name" bash -lc 'command -v taler-merchant-httpd' >/dev/null 2>&1 \ + || [ "$want" != "$have" ]; then + echo "rebuild $name (running=$running fp_match=$([ "$want" = "$have" ] && echo yes || echo NO))" + create_from_scratch "$name" "$suite" "$srcfile" + else + echo "OK $name up-to-date (fresh container matches repo)" + fi +} + +# --------------------------------------------------------------------------- +# Upgrade mode: create once, then always upgrade_inside +# --------------------------------------------------------------------------- +ensure_upgrade() { + local name=$1 suite=$2 srcfile=$3 + local running + + if ! "$PODMAN_BIN" container exists "$name" 2>/dev/null; then + create_from_scratch "$name" "$suite" "$srcfile" + echo "(initial install for upgrade-track container $name)" + return + fi + + running=$("$PODMAN_BIN" inspect -f '{{.State.Running}}' "$name" 2>/dev/null || echo false) + if [ "$running" != "true" ]; then + echo "start $name" + "$PODMAN_BIN" start "$name" || { + echo "recreate broken $name" + create_from_scratch "$name" "$suite" "$srcfile" + return + } + sleep 3 + fi + + if ! "$PODMAN_BIN" exec "$name" bash -lc 'command -v taler-merchant-httpd' >/dev/null 2>&1; then + echo "recreate $name (no httpd after start)" + create_from_scratch "$name" "$suite" "$srcfile" + return + fi + + upgrade_inside "$name" +} + +MODE="${1:-all}" # all | fresh | upgrade + +case "$MODE" in + all|fresh) + ensure_fresh koopa-taler-deploy-test-apt-src-trixie trixie taler.sources + ensure_fresh koopa-taler-deploy-test-apt-src-trixie-testing trixie-testing taler-testing.sources + ;;& + all|upgrade) + ensure_upgrade koopa-taler-deploy-test-apt-src-trixie-upgrade trixie taler.sources + ensure_upgrade koopa-taler-deploy-test-apt-src-trixie-testing-upgrade trixie-testing taler-testing.sources + ;; + *) + echo "usage: $0 [all|fresh|upgrade]" >&2 + exit 2 + ;; +esac + +echo "done ensure-apt-deploy-test-containers mode=$MODE" diff --git a/host-agent/env.example b/host-agent/env.example new file mode 100644 index 0000000..b7aec8b --- /dev/null +++ b/host-agent/env.example @@ -0,0 +1,26 @@ +# Copy to: ~/.config/taler-monitoring/env +# This file is NOT inside the git tree and is never overwritten by update-suite.sh. +# +# Full stack examples (same reporting generation for all): +# env/hacktivism.env.example +# env/stagepaysan.env.example +# env/francpaysan.env.example +# +# SUITE_GIT_URL=https://git.hacktivism.ch/hernani/taler-monitoring.git +# SUITE_GIT_REF=main +# SUITE_DIR=$HOME/src/taler-monitoring +# SUITE_UPDATE_MODE=reset +# +# TALER_DOMAIN=hacktivism.ch +# INSIDE_PODMAN=1 +# INSIDE_MODE=local-podman +# CONTINUE_ON_ERROR=1 +# PHASES for the *default* hacktivism timer only (urls inside versions). +# Dedicated wrappers always override PHASES + HTML paths: +# run-surface-monitoring.sh → /taler-monitoring-surface* +# run-aptdeploy-monitoring.sh → /taler-monitoring-aptdeploy* +# PHASES="urls inside versions" +# RUN_TIMEOUT=600 +# HTML_OUT=$HOME/monitoring-sites-staging +# MON_HOSTS="bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch" +# SOURCE_REPO_WEB=https://git.hacktivism.ch/hernani/taler-monitoring diff --git a/host-agent/env/francpaysan.env.example b/host-agent/env/francpaysan.env.example new file mode 100644 index 0000000..d3132d8 --- /dev/null +++ b/host-agent/env/francpaysan.env.example @@ -0,0 +1,24 @@ +# Copy to: ~/.config/taler-monitoring/env (user francpaysan) +# Never inside the git tree — update-suite never overwrites this file. +# +# Shared reporting generation (run-host-report.sh / run-fp-prod-monitoring.sh): +# RUN_TIMEOUT, line-buffered log, always HTML, commit link on git.hacktivism.ch + +TALER_DOMAIN=lefrancpaysan.ch +INSIDE_PODMAN=1 +INSIDE_MODE=local-podman +LOCAL_STACK=0 +CONTINUE_ON_ERROR=1 +PHASES="urls inside versions" +RUN_TIMEOUT=600 + +HTML_OUT=$HOME/monitoring-sites-staging +MON_HOSTS="bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch" + +SUITE_DIR=$HOME/src/taler-monitoring +SUITE_GIT_URL=https://git.hacktivism.ch/hernani/taler-monitoring.git +SUITE_GIT_REF=main +SUITE_UPDATE_MODE=reset +SOURCE_REPO_WEB=https://git.hacktivism.ch/hernani/taler-monitoring + +# DEPLOY_WWW_ROOT=/var/www/monitoring-sites diff --git a/host-agent/env/hacktivism.env.example b/host-agent/env/hacktivism.env.example new file mode 100644 index 0000000..32c8e00 --- /dev/null +++ b/host-agent/env/hacktivism.env.example @@ -0,0 +1,24 @@ +# Copy to: ~/.config/taler-monitoring/env (user hernani on koopa) +# Shared reporting generation: RUN_TIMEOUT, line-buffered log, always HTML. + +TALER_DOMAIN=hacktivism.ch +INSIDE_PODMAN=1 +INSIDE_MODE=local-podman +LOCAL_STACK=1 +CONTINUE_ON_ERROR=1 +PHASES="urls inside versions" +# aptdeploy HTML: run-aptdeploy-monitoring.sh → /taler-monitoring-aptdeploy* +RUN_TIMEOUT=600 +# APT_DEPLOY_ENSURE=1 +# APT_DEPLOY_SKIP=0 + +HTML_OUT=$HOME/monitoring-sites-staging +MON_HOSTS="bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch" + +SUITE_DIR=$HOME/src/taler-monitoring +SUITE_GIT_URL=https://git.hacktivism.ch/hernani/taler-monitoring.git +SUITE_GIT_REF=main +SUITE_UPDATE_MODE=reset +SOURCE_REPO_WEB=https://git.hacktivism.ch/hernani/taler-monitoring + +DEPLOY_WWW_ROOT=/var/www/monitoring-sites diff --git a/host-agent/env/stagepaysan.env.example b/host-agent/env/stagepaysan.env.example new file mode 100644 index 0000000..e21180a --- /dev/null +++ b/host-agent/env/stagepaysan.env.example @@ -0,0 +1,27 @@ +# Copy to: ~/.config/taler-monitoring/env (user stagepaysan) +# Never inside the git tree — update-suite never overwrites this file. +# +# Shared reporting generation (run-host-report.sh / run-fp-stage-monitoring.sh): +# RUN_TIMEOUT, line-buffered log, always HTML, commit link on git.hacktivism.ch + +TALER_DOMAIN=stage.lefrancpaysan.ch +INSIDE_PODMAN=1 +INSIDE_MODE=local-podman +INSIDE_PROFILE=stage-lfp +LOCAL_STACK=0 +CONTINUE_ON_ERROR=1 +PHASES="urls inside versions" +RUN_TIMEOUT=600 + +HTML_OUT=$HOME/monitoring-sites-staging +MON_HOSTS="stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch" + +# Git clone of koopa-admin-log (repo root, not …/scripts/taler-monitoring) +SUITE_DIR=$HOME/src/taler-monitoring +SUITE_GIT_URL=https://git.hacktivism.ch/hernani/taler-monitoring.git +SUITE_GIT_REF=main +SUITE_UPDATE_MODE=reset +SOURCE_REPO_WEB=https://git.hacktivism.ch/hernani/taler-monitoring + +# Optional public docroot if this user can write it +# DEPLOY_WWW_ROOT=/var/www/monitoring-sites diff --git a/host-agent/install-host-agent.sh b/host-agent/install-host-agent.sh new file mode 100755 index 0000000..5b64cf2 --- /dev/null +++ b/host-agent/install-host-agent.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# install-host-agent.sh — install hernani user systemd units on koopa +# +# # on koopa as hernani: +# ~/src/taler-monitoring/host-agent/install-host-agent.sh +# # or from laptop: +# ./install-host-agent.sh --remote koopa-external +# +set -euo pipefail +ROOT=$(cd "$(dirname "$0")" && pwd) +REMOTE="${1:-}" +[ "${1:-}" = "--remote" ] && REMOTE="${2:-koopa-external}" + +install_local() { + local mon home_unit + mon=$(cd "$ROOT/.." && pwd) + # Prefer ~/koopa-admin-log on live host + if [ -d "$HOME/src/taler-monitoring" ]; then + mon="$HOME/src/taler-monitoring" + fi + home_unit="$HOME/.config/systemd/user" + mkdir -p "$home_unit" \ + "$HOME/.local/state/taler-hacktivism-monitoring" \ + "$HOME/monitoring-sites-staging" \ + "$HOME/.config/taler-monitoring" \ + "$HOME/src" + + # Local env (never in git) — create from example if missing + if [ ! -f "$HOME/.config/taler-monitoring/env" ] && [ -f "$ROOT/env.example" ]; then + cp "$ROOT/env.example" "$HOME/.config/taler-monitoring/env" + echo "created ~/.config/taler-monitoring/env" + fi + + # Prefer a real git clone for update-suite.sh + if [ ! -d "$HOME/src/koopa-admin-log/.git" ] && [ ! -d "$HOME/koopa-admin-log/.git" ]; then + if [ -d "$HOME/src/taler-monitoring" ] && [ ! -d "$HOME/koopa-admin-log/.git" ]; then + echo "note: $HOME/koopa-admin-log has no .git — update-suite will clone to ~/src/koopa-admin-log" + fi + git clone https://git.hacktivism.ch/hernani/taler-monitoring.git "$HOME/src/koopa-admin-log" 2>/dev/null \ + || echo "WARN: clone failed (auth/network) — will use existing tree if any" + fi + + # Ensure suite is executable + chmod +x "$mon"/taler-monitoring.sh "$mon"/check_*.sh 2>/dev/null || true + chmod +x "$mon/host-agent"/*.sh 2>/dev/null || true + chmod +x "$mon/site-gen"/*.sh "$mon/site-gen/console_to_html.py" 2>/dev/null || true + + # Units reference %h/koopa-admin-log — prefer git clone or symlink + if [ -d "$HOME/src/koopa-admin-log/.git" ] && [ ! -e "$HOME/koopa-admin-log" ]; then + ln -sfn "$HOME/src/koopa-admin-log" "$HOME/koopa-admin-log" + echo "symlink $HOME/koopa-admin-log → $HOME/src/koopa-admin-log" + elif [ ! -e "$HOME/koopa-admin-log" ]; then + admin_root=$(cd "$mon/../.." && pwd) + ln -sfn "$admin_root" "$HOME/koopa-admin-log" + echo "symlink $HOME/koopa-admin-log → $admin_root" + fi + + install -m 644 "$ROOT/taler-monitoring-hacktivism.service" "$home_unit/" + install -m 644 "$ROOT/taler-monitoring-hacktivism.path" "$home_unit/" + install -m 644 "$ROOT/taler-monitoring-hacktivism.timer" "$home_unit/" + install -m 644 "$ROOT/taler-monitoring-surface.service" "$home_unit/" + install -m 644 "$ROOT/taler-monitoring-surface.timer" "$home_unit/" + install -m 644 "$ROOT/taler-monitoring-aptdeploy.service" "$home_unit/" + install -m 644 "$ROOT/taler-monitoring-aptdeploy.timer" "$home_unit/" + + # Fix ExecStart if install path differs (units use %h/koopa-admin-log) + systemctl --user daemon-reload + systemctl --user enable --now taler-monitoring-hacktivism.path + systemctl --user enable --now taler-monitoring-hacktivism.timer + systemctl --user enable --now taler-monitoring-surface.timer + systemctl --user enable --now taler-monitoring-aptdeploy.timer + # initial stamp so path exists + "$ROOT/touch-software-stamp.sh" + systemctl --user start taler-monitoring-hacktivism.service || true + systemctl --user start taler-monitoring-surface.service || true + systemctl --user start taler-monitoring-aptdeploy.service || true + + echo "--- status ---" + systemctl --user status taler-monitoring-hacktivism.path --no-pager -l | head -15 + systemctl --user status taler-monitoring-hacktivism.timer --no-pager -l | head -12 + systemctl --user status taler-monitoring-surface.timer --no-pager -l | head -12 + systemctl --user status taler-monitoring-aptdeploy.timer --no-pager -l | head -12 + systemctl --user list-timers --all | grep taler-monitoring || true + echo "OK host-agent (hacktivism 4h + surface 1h + aptdeploy 4h)" + echo "manual: systemctl --user start taler-monitoring-hacktivism.service" + echo "manual surface: systemctl --user start taler-monitoring-surface.service" + echo "manual aptdeploy: systemctl --user start taler-monitoring-aptdeploy.service" + echo "after upgrades: $ROOT/touch-software-stamp.sh" +} + +if [ -n "$REMOTE" ] && [ "$REMOTE" != "--remote" ]; then + echo "install on $REMOTE …" + rsync -az --delete \ + --exclude secrets.env \ + --exclude 'android-test/artifacts' \ + "$ROOT/../../" \ + "${REMOTE}:src/taler-monitoring/" + # ensure full tree for %h/koopa-admin-log if missing configs etc. + ssh -o BatchMode=yes -o ConnectTimeout=20 "$REMOTE" \ + 'test -d $HOME/koopa-admin-log || git clone https://git.hacktivism.ch/hernani/taler-monitoring.git $HOME/koopa-admin-log 2>/dev/null || true' + rsync -az "$ROOT/" "${REMOTE}:src/taler-monitoring/host-agent/" + rsync -az "$ROOT/../site-gen/" "${REMOTE}:src/taler-monitoring/site-gen/" 2>/dev/null || true + ssh -o BatchMode=yes "$REMOTE" \ + 'bash $HOME/src/taler-monitoring/host-agent/install-host-agent.sh' +else + install_local +fi diff --git a/host-agent/run-aptdeploy-monitoring.sh b/host-agent/run-aptdeploy-monitoring.sh new file mode 100755 index 0000000..fe81257 --- /dev/null +++ b/host-agent/run-aptdeploy-monitoring.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# run-aptdeploy-monitoring.sh — apt-src deploy tests → public HTML +# Public only on taler.hacktivism.ch: +# https://taler.hacktivism.ch/taler-monitoring-aptdeploy/ +# https://taler.hacktivism.ch/taler-monitoring-aptdeploy_err/ +# +# CLI one-shot without HTML remains: run-aptdeploy.sh [all|fresh|upgrade|check-only] +# +set -uo pipefail +AGENT_DIR=$(cd "$(dirname "$0")" && pwd) + +export AGENT_LABEL="${AGENT_LABEL:-aptdeploy-host-agent}" +export STATE_NAME="${STATE_NAME:-taler-aptdeploy-monitoring}" +export TALER_DOMAIN="${TALER_DOMAIN:-hacktivism.ch}" +# containers are local on koopa +export INSIDE_PODMAN=1 +export INSIDE_MODE=local-podman +export LOCAL_STACK=1 +export SKIP_SSH=0 +export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" +# four containers + checks; allow long wall +export RUN_TIMEOUT="${RUN_TIMEOUT:-1800}" +# dedicated job: always aptdeploy only +export PHASES=aptdeploy +# Only merchant public front — not bank/exchange +export MON_HOSTS="${MON_HOSTS:-taler.hacktivism.ch}" +export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}" +export DEPLOY_WWW_ROOT="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" + +export HTML_OK_DIR="taler-monitoring-aptdeploy" +export HTML_ERR_DIR="taler-monitoring-aptdeploy_err" +export HTML_URL_OK="/taler-monitoring-aptdeploy/" +export HTML_URL_ERR="/taler-monitoring-aptdeploy_err/" +export PAGE_LABEL="taler-monitoring-aptdeploy" + +# Ensure all four apt-src containers (fresh rebuild + upgrade track) +if [ "${APT_DEPLOY_ENSURE:-1}" = "1" ] \ + && [ -x "$AGENT_DIR/ensure-apt-deploy-test-containers.sh" ]; then + bash "$AGENT_DIR/ensure-apt-deploy-test-containers.sh" all \ + || echo "WARN: ensure-apt-deploy-test-containers failed (aptdeploy may ERROR)" >&2 +fi + +exec bash "$AGENT_DIR/run-host-report.sh" "$@" diff --git a/host-agent/run-aptdeploy.sh b/host-agent/run-aptdeploy.sh new file mode 100755 index 0000000..3286379 --- /dev/null +++ b/host-agent/run-aptdeploy.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +# run-aptdeploy.sh — ensure containers + aptdeploy phase only. +# +# Keep monitoring current: +# 1) Prefer Forgejo origin/main (update-suite reset) +# 2) If Forgejo is behind (no aptdeploy yet), re-apply suite-overlay saved +# from this agent tree before reset (rsync from laptop fills that tree) +# 3) Run ensure + taler-monitoring.sh aptdeploy +# +# Host: on koopa → local; else ssh APT_DEPLOY_SSH + rsync mon tree first. +# +set -euo pipefail + +AGENT_DIR=$(cd "$(dirname "$0")" && pwd) +BOOT_MON=$(cd "$AGENT_DIR/.." && pwd) +MODE="${1:-all}" +APT_DEPLOY_SSH="${APT_DEPLOY_SSH:-koopa-external}" +OVERLAY_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring/suite-overlay" + +on_koopa() { + local h + h=$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo "") + case "$h" in koopa|koopa.*) return 0 ;; esac + [ "${KOOPA_FORCE_LOCAL:-0}" = "1" ] && return 0 + return 1 +} + +mon_has_aptdeploy() { + local mon=$1 + [ -f "$mon/taler-monitoring.sh" ] && grep -qE 'aptdeploy\|apt-deploy' "$mon/taler-monitoring.sh" +} + +save_overlay_from() { + local src=$1 + mon_has_aptdeploy "$src" || return 0 + mkdir -p "$OVERLAY_DIR" + rsync -a --delete \ + --exclude secrets.env \ + --exclude 'android-test/artifacts' \ + --exclude 'android-test/apks' \ + --exclude 'android-test/out*' \ + --exclude '__pycache__' \ + "$src/" "$OVERLAY_DIR/" + echo "saved suite-overlay from $src → $OVERLAY_DIR" +} + +apply_overlay_to() { + local dest=$1 + [ -f "$OVERLAY_DIR/taler-monitoring.sh" ] || return 1 + mon_has_aptdeploy "$OVERLAY_DIR" || return 1 + mkdir -p "$dest" + rsync -a \ + --exclude secrets.env \ + --exclude 'android-test/artifacts' \ + --exclude 'android-test/apks' \ + --exclude 'android-test/out*' \ + --exclude '__pycache__' \ + "$OVERLAY_DIR/" "$dest/" + echo "applied suite-overlay → $dest (Forgejo lag; push origin/main when possible)" + return 0 +} + +pick_mon_tree() { + # Preserve working mon before reset wipes tracked files + save_overlay_from "$BOOT_MON" + + # shellcheck source=update-suite.sh + source "$AGENT_DIR/update-suite.sh" + + local suite_mon="$SUITE_DIR" + mkdir -p "$suite_mon" + + if mon_has_aptdeploy "$suite_mon"; then + MON_DIR="$suite_mon" + echo "using Forgejo suite mon @ ${COMMIT_SHORT:-?} · $MON_DIR" + elif apply_overlay_to "$suite_mon" && mon_has_aptdeploy "$suite_mon"; then + MON_DIR="$suite_mon" + echo "using suite mon + overlay @ ${COMMIT_SHORT:-?} · $MON_DIR" + elif mon_has_aptdeploy "$BOOT_MON"; then + MON_DIR="$BOOT_MON" + echo "WARN: using boot mon $MON_DIR (overlay apply failed)" >&2 + else + echo "ERROR: no mon tree with aptdeploy (Forgejo + overlay empty)" >&2 + echo " rsync suite from laptop or push to git.hacktivism.ch" >&2 + exit 2 + fi + + AGENT_DIR="$MON_DIR/host-agent" + export MON_DIR AGENT_DIR SUITE_DIR COMMIT COMMIT_SHORT COMMIT_URL + chmod +x "$AGENT_DIR"/*.sh "$MON_DIR"/taler-monitoring.sh "$MON_DIR"/check_*.sh 2>/dev/null || true +} + +run_local() { + export PATH="${HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin:${PATH:-}" + export PYTHONUNBUFFERED=1 + export PROGRESS_OFF="${PROGRESS_OFF:-0}" + + echo "======== aptdeploy local on $(hostname) mode=$MODE ========" + pick_mon_tree + + if [ "$MODE" != "check-only" ]; then + bash "$AGENT_DIR/ensure-apt-deploy-test-containers.sh" "$MODE" + fi + + cd "$MON_DIR" + export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" + export INSIDE_PODMAN="${INSIDE_PODMAN:-1}" + export LOCAL_STACK="${LOCAL_STACK:-1}" + export RUN_TIMEOUT="${RUN_TIMEOUT:-600}" + echo "run: $MON_DIR/taler-monitoring.sh -d hacktivism.ch aptdeploy · suite=${COMMIT_SHORT:-?}" + ./taler-monitoring.sh -d hacktivism.ch aptdeploy +} + +run_remote() { + echo "======== aptdeploy remote via ssh ${APT_DEPLOY_SSH} mode=$MODE ========" + if [ "${APT_DEPLOY_RSYNC:-1}" = "1" ]; then + echo "rsync mon suite → ${APT_DEPLOY_SSH}:src/src/taler-monitoring/" + rsync -az \ + --exclude secrets.env \ + --exclude 'android-test/artifacts' \ + --exclude 'android-test/apks' \ + --exclude 'android-test/out*' \ + --exclude '__pycache__' \ + "$BOOT_MON/" \ + "${APT_DEPLOY_SSH}:src/src/taler-monitoring/" \ + || echo "WARN: rsync failed" >&2 + fi + ssh -o BatchMode=yes -o ConnectTimeout=25 "$APT_DEPLOY_SSH" bash -s -- "$MODE" <<'EOS' +set -euo pipefail +MODE=${1:-all} +export PATH="${HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin:${PATH:-}" +export KOOPA_FORCE_LOCAL=1 +AGENT="$HOME/src/taler-monitoring/host-agent" +if [ ! -x "$AGENT/run-aptdeploy.sh" ]; then + AGENT="$HOME/src/taler-monitoring/host-agent" +fi +if [ ! -x "$AGENT/run-aptdeploy.sh" ]; then + echo "ERROR: run-aptdeploy.sh missing on remote" >&2 + exit 1 +fi +echo "remote AGENT=$AGENT" +exec bash "$AGENT/run-aptdeploy.sh" "$MODE" +EOS +} + +if on_koopa; then + run_local +else + run_remote +fi diff --git a/host-agent/run-fp-prod-monitoring.sh b/host-agent/run-fp-prod-monitoring.sh new file mode 100755 index 0000000..be876a8 --- /dev/null +++ b/host-agent/run-fp-prod-monitoring.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# FrancPaysan PROD host-agent (user francpaysan on francpaysan-host). +# Shared reporting: run-host-report.sh (timeout, line-buffered log, first-run HTML). +set -uo pipefail +AGENT_DIR=$(cd "$(dirname "$0")" && pwd) + +export AGENT_LABEL="${AGENT_LABEL:-fp-prod-host-agent}" +export STATE_NAME="${STATE_NAME:-taler-monitoring-lfp-prod}" +export TALER_DOMAIN="${TALER_DOMAIN:-lefrancpaysan.ch}" +export INSIDE_PODMAN="${INSIDE_PODMAN:-1}" +export INSIDE_MODE="${INSIDE_MODE:-local-podman}" +export LOCAL_STACK="${LOCAL_STACK:-0}" +export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" +export RUN_TIMEOUT="${RUN_TIMEOUT:-600}" +export PHASES="${PHASES:-urls inside versions}" +export MON_HOSTS="${MON_HOSTS:-bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch}" +export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}" +export DEPLOY_WWW_ROOT="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" + +exec bash "$AGENT_DIR/run-host-report.sh" "$@" diff --git a/host-agent/run-fp-stage-monitoring.sh b/host-agent/run-fp-stage-monitoring.sh new file mode 100755 index 0000000..70518f0 --- /dev/null +++ b/host-agent/run-fp-stage-monitoring.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# FrancPaysan STAGE host-agent (user stagepaysan on francpaysan-host). +# Shared reporting: run-host-report.sh (timeout, line-buffered log, first-run HTML). +set -uo pipefail +AGENT_DIR=$(cd "$(dirname "$0")" && pwd) + +export AGENT_LABEL="${AGENT_LABEL:-fp-stage-host-agent}" +export STATE_NAME="${STATE_NAME:-taler-monitoring-stage-lfp}" +export TALER_DOMAIN="${TALER_DOMAIN:-stage.lefrancpaysan.ch}" +export INSIDE_PODMAN="${INSIDE_PODMAN:-1}" +export INSIDE_MODE="${INSIDE_MODE:-local-podman}" +export INSIDE_PROFILE="${INSIDE_PROFILE:-stage-lfp}" +export LOCAL_STACK="${LOCAL_STACK:-0}" +export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" +export RUN_TIMEOUT="${RUN_TIMEOUT:-600}" +export PHASES="${PHASES:-urls inside versions}" +export MON_HOSTS="${MON_HOSTS:-stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch}" +export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}" +export DEPLOY_WWW_ROOT="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" + +exec bash "$AGENT_DIR/run-host-report.sh" "$@" diff --git a/host-agent/run-hacktivism-monitoring.sh b/host-agent/run-hacktivism-monitoring.sh new file mode 100755 index 0000000..8fd98fd --- /dev/null +++ b/host-agent/run-hacktivism-monitoring.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# GOA / hacktivism host-agent on koopa (hernani). +# Stack defaults only — reporting pipeline is shared: run-host-report.sh +set -uo pipefail +AGENT_DIR=$(cd "$(dirname "$0")" && pwd) + +export AGENT_LABEL="${AGENT_LABEL:-hacktivism-host-agent}" +export STATE_NAME="${STATE_NAME:-taler-hacktivism-monitoring}" +export TALER_DOMAIN="${TALER_DOMAIN:-hacktivism.ch}" +export INSIDE_PODMAN="${INSIDE_PODMAN:-1}" +export INSIDE_MODE="${INSIDE_MODE:-local-podman}" +export LOCAL_STACK="${LOCAL_STACK:-1}" +export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" +export RUN_TIMEOUT="${RUN_TIMEOUT:-600}" +# stack checks only — aptdeploy has its own HTML path + timer: +# run-aptdeploy-monitoring.sh → /taler-monitoring-aptdeploy* (taler.hacktivism.ch) +export PHASES="${PHASES:-urls inside versions}" +export MON_HOSTS="${MON_HOSTS:-bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch}" +export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}" +export DEPLOY_WWW_ROOT="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" +export APT_DEPLOY_ENSURE=0 + +exec bash "$AGENT_DIR/run-host-report.sh" "$@" diff --git a/host-agent/run-host-report.sh b/host-agent/run-host-report.sh new file mode 100755 index 0000000..c38cdcf --- /dev/null +++ b/host-agent/run-host-report.sh @@ -0,0 +1,358 @@ +#!/usr/bin/env bash +# run-host-report.sh — global reporting generation for all stacks +# (GOA/hacktivism, FP stage, FP prod, …). +# +# Shared behaviour (always): +# 1) update-suite → latest git.hacktivism.ch (local ~/.config never overwritten) +# 2) line-buffered run log (written continuously) +# 3) RUN_TIMEOUT wall clock for taler-monitoring.sh (default 600s) +# 4) always write HTML (first run / empty staging too) + Forgejo commit link +# 5) optional rsync to DEPLOY_WWW_ROOT if writable +# +# Configure via env file (not in git): +# ${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring/env +# or TALER_MONITORING_ENV=/path/to/env +# +# Thin wrappers only set stack defaults then exec this script: +# run-hacktivism-monitoring.sh +# run-fp-stage-monitoring.sh +# run-fp-prod-monitoring.sh +# +set -uo pipefail + +export PATH="${HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin:${PATH:-}" +export PYTHONUNBUFFERED=1 + +AGENT_DIR=$(cd "$(dirname "$0")" && pwd) +AGENT_MON=$(cd "$AGENT_DIR/.." && pwd) +CFG_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring" +ENV_FILE="${TALER_MONITORING_ENV:-$CFG_DIR/env}" +mkdir -p "$CFG_DIR" + +# Thin wrappers (surface, aptdeploy-only, …) export stack knobs before exec. +# Snapshot them so ~/.config/taler-monitoring/env cannot clobber dedicated jobs +# (env often has PHASES="urls inside versions aptdeploy" for the main timer). +_wrap_keys=( + PHASES HTML_OUT HTML_OK_DIR HTML_ERR_DIR HTML_URL_OK HTML_URL_ERR PAGE_LABEL + RUN_TIMEOUT STATE_NAME AGENT_LABEL MON_HOSTS TALER_DOMAIN DEPLOY_WWW_ROOT + INSIDE_PODMAN INSIDE_MODE LOCAL_STACK SKIP_SSH CONTINUE_ON_ERROR AUTH401_CONTINUE + APT_DEPLOY_ENSURE SURFACE_SCOPE TALER_DOMAIN_FROM_CLI STRICT_EXIT SOURCE_SUITE_PATH +) +for _k in "${_wrap_keys[@]}"; do + if [ -n "${!_k+x}" ]; then + printf -v "_WRAP_SET_${_k}" '%s' 1 + printf -v "_WRAP_VAL_${_k}" '%s' "${!_k}" + fi +done +unset _k + +if [ -f "$ENV_FILE" ]; then + # shellcheck disable=SC1090 + set -a + # shellcheck source=/dev/null + source "$ENV_FILE" + set +a +fi +# Tell update-suite.sh not to re-source env (would clobber wrapper PHASES etc.) +export _TALER_MON_ENV_LOADED=1 + +# Restore wrapper overrides (dedicated jobs win over shared env file) +for _k in "${_wrap_keys[@]}"; do + _set_var="_WRAP_SET_${_k}" + _val_var="_WRAP_VAL_${_k}" + if [ "${!_set_var:-}" = "1" ]; then + printf -v "$_k" '%s' "${!_val_var}" + export "$_k" + fi +done +unset _k _set_var _val_var + +# --- global report defaults (env file / wrapper wins if already set) --- +export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" +export AUTH401_CONTINUE="${AUTH401_CONTINUE:-${CONTINUE_ON_ERROR}}" +export RUN_TIMEOUT="${RUN_TIMEOUT:-600}" +export TALER_DOMAIN="${TALER_DOMAIN:-hacktivism.ch}" +PHASES="${PHASES:-urls inside versions}" +HTML_BASE="${HTML_OUT:-$HOME/monitoring-sites-staging}" +MON_HOSTS="${MON_HOSTS:-}" +SUITE_PATH="${SOURCE_SUITE_PATH:-.}" +AGENT_LABEL="${AGENT_LABEL:-host-report}" +STATE_NAME="${STATE_NAME:-taler-monitoring-${TALER_DOMAIN}}" +STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/${STATE_NAME}" +LOG_DIR="$STATE_DIR/logs" +DEPLOY_WWW="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" +# HTML path layout under each host (monitoring / surface / …) +HTML_OK_DIR="${HTML_OK_DIR:-monitoring}" +HTML_ERR_DIR="${HTML_ERR_DIR:-monitoring_err}" +HTML_URL_OK="${HTML_URL_OK:-/monitoring/}" +HTML_URL_ERR="${HTML_URL_ERR:-/monitoring_err/}" +PAGE_LABEL="${PAGE_LABEL:-monitoring}" + +# Sensible MON_HOSTS from domain if not set +if [ -z "$MON_HOSTS" ]; then + case "$TALER_DOMAIN" in + hacktivism.ch|koopa) + MON_HOSTS="bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch" + ;; + stage.lefrancpaysan.ch|stage.*lefrancpaysan*) + MON_HOSTS="stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch" + ;; + lefrancpaysan.ch) + MON_HOSTS="bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch" + ;; + *) + MON_HOSTS="${TALER_DOMAIN}" + ;; + esac +fi + +mkdir -p "$LOG_DIR" "$HTML_BASE" + +STAMP=$(date +%Y%m%d-%H%M%S) +LOG="$LOG_DIR/run-${STAMP}.log" +: >"$LOG" +if command -v stdbuf >/dev/null 2>&1; then + exec > >(stdbuf -oL -eL tee -a "$LOG") 2>&1 +else + exec > >(tee -a "$LOG") 2>&1 +fi + +echo "======== $(date -Iseconds 2>/dev/null || date) ${AGENT_LABEL} ========" +echo "env_file=$ENV_FILE" +echo "domain=$TALER_DOMAIN · phases=$PHASES · RUN_TIMEOUT=${RUN_TIMEOUT}s" +echo "hosts=$MON_HOSTS" +echo "flags INSIDE_PODMAN=${INSIDE_PODMAN:-} INSIDE_MODE=${INSIDE_MODE:-} LOCAL_STACK=${LOCAL_STACK:-} SKIP_SSH=${SKIP_SSH:-} INSIDE_PROFILE=${INSIDE_PROFILE:-}" + +# --- refresh from Forgejo; re-apply suite-overlay if Forgejo lacks needed phases --- +OVERLAY_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring/suite-overlay" +# preserve capable mon before reset wipes tracked files +if [ -f "$AGENT_MON/taler-monitoring.sh" ]; then + mkdir -p "$OVERLAY_DIR" + rsync -a \ + --exclude secrets.env --exclude 'android-test/artifacts' --exclude 'android-test/apks' \ + --exclude 'android-test/out*' --exclude '__pycache__' \ + "$AGENT_MON/" "$OVERLAY_DIR/" 2>/dev/null || true +fi + +# shellcheck source=update-suite.sh +source "$AGENT_DIR/update-suite.sh" + +SUITE_MON="${SUITE_DIR}" +ROOT="$SUITE_MON" +_need_overlay=0 +# Detect missing features in Forgejo tree (untracked helper files alone are not enough — +# taler-monitoring.sh must accept the phase name). +case " $PHASES " in + *" aptdeploy "*|*" apt-deploy "*|*" apt_src "*) + if ! grep -qE 'aptdeploy\|apt-deploy' "$SUITE_MON/taler-monitoring.sh" 2>/dev/null; then + _need_overlay=1 + fi + ;; +esac +case " $PHASES " in + *" surface "*|*" ecosystem "*) + if ! grep -qE 'surface\|ecosystem' "$SUITE_MON/taler-monitoring.sh" 2>/dev/null \ + || [ ! -f "$SUITE_MON/check_surface.sh" ]; then + _need_overlay=1 + fi + ;; +esac +# Wrapper-preserve + HTML path knobs may live only in local overlay until pushed +if [ -f "$OVERLAY_DIR/host-agent/run-host-report.sh" ] \ + && ! grep -q '_wrap_keys\|_WRAP_SET_' "$SUITE_MON/host-agent/run-host-report.sh" 2>/dev/null \ + && grep -q '_wrap_keys\|_WRAP_SET_' "$OVERLAY_DIR/host-agent/run-host-report.sh" 2>/dev/null; then + _need_overlay=1 +fi +if [ "$_need_overlay" = "1" ] && [ -f "$OVERLAY_DIR/taler-monitoring.sh" ]; then + echo "WARN: Forgejo @ ${COMMIT_SHORT:-?} lagging local suite-overlay (phases=[$PHASES])" >&2 + echo " re-applying suite-overlay → $SUITE_MON" >&2 + mkdir -p "$SUITE_MON" + rsync -a \ + --exclude secrets.env --exclude 'android-test/artifacts' --exclude 'android-test/apks' \ + --exclude 'android-test/out*' --exclude '__pycache__' \ + "$OVERLAY_DIR/" "$SUITE_MON/" + ROOT="$SUITE_MON" +elif [ "$_need_overlay" = "1" ]; then + echo "WARN: Forgejo missing features and no suite-overlay — using agent mon $AGENT_MON" >&2 + ROOT="$AGENT_MON" +else + echo "using suite mon @ ${COMMIT_SHORT:-?} · $ROOT" +fi +# Always restore host-agent + site-gen from pre-reset snapshot when present +# (systemd ExecStart points at koopa-admin-log which may be the reset suite tree). +if [ -d "$OVERLAY_DIR/host-agent" ]; then + mkdir -p "$SUITE_MON/host-agent" 2>/dev/null || true + rsync -a "$OVERLAY_DIR/host-agent/" "$SUITE_MON/host-agent/" 2>/dev/null || true +fi +if [ -d "$OVERLAY_DIR/site-gen" ]; then + mkdir -p "$SUITE_MON/site-gen" + rsync -a "$OVERLAY_DIR/site-gen/" "$SUITE_MON/site-gen/" 2>/dev/null || true +fi +unset _need_overlay + +if [ -f "$ROOT/site-gen/console_to_html.py" ]; then + SITE_GEN="$ROOT/site-gen" +elif [ -f "$AGENT_MON/site-gen/console_to_html.py" ]; then + SITE_GEN="$AGENT_MON/site-gen" + echo "note: using agent site-gen at $SITE_GEN" +elif [ -f "$SUITE_MON/site-gen/console_to_html.py" ]; then + SITE_GEN="$SUITE_MON/site-gen" +else + SITE_GEN="" +fi + +if [ ! -x "$ROOT/taler-monitoring.sh" ]; then + if [ -x "$AGENT_MON/taler-monitoring.sh" ]; then + ROOT="$AGENT_MON" + echo "note: using agent mon tree $ROOT" + else + echo "ERROR: suite missing at $ROOT" >&2 + COMMIT="${COMMIT:-unknown}" + COMMIT_SHORT="${COMMIT_SHORT:-unknown}" + COMMIT_URL="${COMMIT_URL:-}" + fi +fi + +echo "commit=${COMMIT_SHORT:-?} · ${COMMIT_URL:-}" + +ec=1 +if [ -x "${ROOT:-}/taler-monitoring.sh" ]; then + cd "$ROOT" || exit 1 + chmod +x taler-monitoring.sh check_*.sh host-agent/*.sh site-gen/*.sh site-gen/*.py 2>/dev/null || true + + # Overlay newer inside/versions host-podman bits if suite clone is stale + for f in check_inside.sh check_versions.sh taler-monitoring.sh lib.sh; do + if [ -f "$AGENT_MON/$f" ] && [ -f "$ROOT/$f" ]; then + if ! grep -q 'host-podman\|INSIDE_ACCESS\|RUN_TIMEOUT' "$ROOT/$f" 2>/dev/null \ + && grep -q 'host-podman\|INSIDE_ACCESS\|RUN_TIMEOUT' "$AGENT_MON/$f" 2>/dev/null; then + echo "note: overlay $f from agent install" + cp -f "$AGENT_MON/$f" "$ROOT/$f" + chmod +x "$ROOT/$f" 2>/dev/null || true + fi + fi + done + + # shellcheck disable=SC2086 + ./taler-monitoring.sh -d "$TALER_DOMAIN" $PHASES + ec=$? +fi + +# --- always generate HTML (first run / missing pages / every run) --- +write_bootstrap_html() { + local out="$1" host="$2" mode="$3" + mkdir -p "$(dirname "$out")" + local cshort="${COMMIT_SHORT:-unknown}" + local curl="${COMMIT_URL:-}" + local clink="$cshort" + local gen iso + gen=$(date -u +"%Y-%m-%d %H:%M:%SZ" 2>/dev/null || date) + iso=$(date -u +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -Iseconds) + if [ -n "$curl" ]; then + clink="$cshort" + fi + cat >"$out" < + + +monitoring ${mode} · ${host} + + +
+

Bootstrap page (converter missing or first install). See host-agent log.

+
$(tail -n 80 "$LOG" 2>/dev/null | sed 's/&/\&/g;s/
+
+ +EOF + echo "bootstrap html → $out" +} + +htmlify_host() { + local host="$1" + local mon="$HTML_BASE/$host/${HTML_OK_DIR}/index.html" + local mon_err="$HTML_BASE/$host/${HTML_ERR_DIR}/index.html" + mkdir -p "$HTML_BASE/$host/${HTML_OK_DIR}" "$HTML_BASE/$host/${HTML_ERR_DIR}" + + if [ -n "$SITE_GEN" ] && [ -f "$SITE_GEN/console_to_html.py" ]; then + python3 "$SITE_GEN/console_to_html.py" \ + --log "$LOG" \ + --out "$mon_err" \ + --hostname "$host" \ + --mode err \ + --commit "${COMMIT:-}" \ + --commit-url "${COMMIT_URL:-}" \ + --suite-path "$SUITE_PATH" \ + --page-label "$PAGE_LABEL" \ + --path-err "$HTML_URL_ERR" \ + --link-other "$HTML_URL_OK" + if [ "$ec" -eq 0 ]; then + python3 "$SITE_GEN/console_to_html.py" \ + --log "$LOG" \ + --out "$mon" \ + --hostname "$host" \ + --mode ok \ + --commit "${COMMIT:-}" \ + --commit-url "${COMMIT_URL:-}" \ + --suite-path "$SUITE_PATH" \ + --page-label "$PAGE_LABEL" \ + --path-err "$HTML_URL_ERR" \ + --link-other "" + rm -rf "$HTML_BASE/$host/${HTML_ERR_DIR}" + echo "html $host → ${HTML_URL_OK} only (clean · ${COMMIT_SHORT:-?})" + else + python3 "$SITE_GEN/console_to_html.py" \ + --log "$LOG" \ + --out "$mon" \ + --hostname "$host" \ + --mode redirect \ + --commit "${COMMIT:-}" \ + --commit-url "${COMMIT_URL:-}" \ + --suite-path "$SUITE_PATH" \ + --page-label "$PAGE_LABEL" \ + --path-err "$HTML_URL_ERR" + echo "html $host → ${HTML_URL_OK} stub + ${HTML_URL_ERR} (ec=$ec · ${COMMIT_SHORT:-?})" + fi + else + echo "WARN: console_to_html.py missing — bootstrap pages" + write_bootstrap_html "$mon_err" "$host" "err" + if [ "$ec" -eq 0 ]; then + write_bootstrap_html "$mon" "$host" "ok" + rm -rf "$HTML_BASE/$host/${HTML_ERR_DIR}" + else + write_bootstrap_html "$mon" "$host" "redirect" + fi + fi + + if [ ! -f "$mon" ]; then + write_bootstrap_html "$mon" "$host" "err" + fi + + printf '%s\n' "${COMMIT:-unknown}" >"$HTML_BASE/$host/COMMIT" + printf '%s\n' "${COMMIT_URL:-}" >"$HTML_BASE/$host/COMMIT_URL" +} +for host in $MON_HOSTS; do + htmlify_host "$host" +done + +if [ -n "$DEPLOY_WWW" ] && [ -w "$DEPLOY_WWW" ] 2>/dev/null; then + rsync -a --delete "$HTML_BASE/" "$DEPLOY_WWW/" && \ + echo "synced → $DEPLOY_WWW/" +fi + +ls -1t "$LOG_DIR"/run-*.log 2>/dev/null | tail -n +30 | xargs rm -f 2>/dev/null || true + +echo "======== done ec=$ec · log=$LOG · ${COMMIT_URL:-} ========" +if [ "${STRICT_EXIT:-0}" = "1" ]; then + exit "$ec" +fi +exit 0 diff --git a/host-agent/run-surface-monitoring.sh b/host-agent/run-surface-monitoring.sh new file mode 100755 index 0000000..1b7a8b8 --- /dev/null +++ b/host-agent/run-surface-monitoring.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# run-surface-monitoring.sh — hourly remote-only ecosystem surface scan +# Public only on taler.hacktivism.ch: +# https://taler.hacktivism.ch/taler-monitoring-surface/ +# https://taler.hacktivism.ch/taler-monitoring-surface_err/ +# +set -uo pipefail +AGENT_DIR=$(cd "$(dirname "$0")" && pwd) + +export AGENT_LABEL="${AGENT_LABEL:-surface-host-agent}" +export STATE_NAME="${STATE_NAME:-taler-surface-monitoring}" +export TALER_DOMAIN="${TALER_DOMAIN:-hacktivism.ch}" +# surface is remote-only — force flags (ignore shared env) +export INSIDE_PODMAN=0 +export LOCAL_STACK=0 +export SKIP_SSH=1 +export INSIDE_MODE="${INSIDE_MODE:-none}" +export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" +# many hosts; allow up to 45 min wall (phase also has RUN_TIMEOUT) +export RUN_TIMEOUT="${RUN_TIMEOUT:-2400}" +# dedicated job: always surface only (never inherit PHASES from shared env) +export PHASES=surface +# Only merchant public front — not bank/exchange +export MON_HOSTS="${MON_HOSTS:-taler.hacktivism.ch}" +export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}" +export DEPLOY_WWW_ROOT="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" + +export HTML_OK_DIR="taler-monitoring-surface" +export HTML_ERR_DIR="taler-monitoring-surface_err" +export HTML_URL_OK="/taler-monitoring-surface/" +export HTML_URL_ERR="/taler-monitoring-surface_err/" +export PAGE_LABEL="taler-monitoring-surface" + +# ecosystem catalog by default (not domain-only) +export SURFACE_SCOPE="${SURFACE_SCOPE:-ecosystem}" +export TALER_DOMAIN_FROM_CLI=0 + +# Do not run apt-deploy ensure for surface-only jobs +export APT_DEPLOY_ENSURE=0 + +exec bash "$AGENT_DIR/run-host-report.sh" "$@" diff --git a/host-agent/taler-monitoring-aptdeploy.service b/host-agent/taler-monitoring-aptdeploy.service new file mode 100644 index 0000000..180802c --- /dev/null +++ b/host-agent/taler-monitoring-aptdeploy.service @@ -0,0 +1,18 @@ +[Unit] +Description=Taler monitoring aptdeploy (apt-src container deploy tests) +Documentation=file:%h/src/taler-monitoring/host-agent/README.md +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin +Environment=CONTINUE_ON_ERROR=1 +Environment=RUN_TIMEOUT=1800 +WorkingDirectory=%h/src/taler-monitoring +ExecStart=%h/src/taler-monitoring/host-agent/run-aptdeploy-monitoring.sh +Nice=10 +TimeoutStartSec=45min + +[Install] +WantedBy=default.target diff --git a/host-agent/taler-monitoring-aptdeploy.timer b/host-agent/taler-monitoring-aptdeploy.timer new file mode 100644 index 0000000..4ff4fa4 --- /dev/null +++ b/host-agent/taler-monitoring-aptdeploy.timer @@ -0,0 +1,14 @@ +[Unit] +Description=Periodic taler-monitoring aptdeploy (apt-src deploy tests) +Documentation=file:%h/src/taler-monitoring/host-agent/README.md + +[Timer] +OnBootSec=35min +OnUnitActiveSec=4h +AccuracySec=5min +Persistent=true +RandomizedDelaySec=10min +Unit=taler-monitoring-aptdeploy.service + +[Install] +WantedBy=timers.target diff --git a/host-agent/taler-monitoring-hacktivism.path b/host-agent/taler-monitoring-hacktivism.path new file mode 100644 index 0000000..d1318fd --- /dev/null +++ b/host-agent/taler-monitoring-hacktivism.path @@ -0,0 +1,19 @@ +[Unit] +Description=Watch hacktivism software/config changes → re-run monitoring +Documentation=file:%h/src/taler-monitoring/host-agent/README.md + +[Path] +# Explicit stamp after package/image upgrades (touch-software-stamp.sh) +PathChanged=%h/.local/state/taler-hacktivism-monitoring/software.stamp +PathModified=%h/.local/state/taler-hacktivism-monitoring/software.stamp +# Landing + caddy config in admin-log (when checked out under ~) +PathChanged=%h/koopa-admin-log/configs/bank-landing +PathChanged=%h/koopa-admin-log/configs/exchange-landing +PathChanged=%h/koopa-admin-log/configs/merchant-landing +PathChanged=%h/koopa-admin-log/configs/caddy +# Debounce: wait for quiet period before firing +TriggerLimitIntervalSec=120 +TriggerLimitBurst=3 + +[Install] +WantedBy=default.target diff --git a/host-agent/taler-monitoring-hacktivism.service b/host-agent/taler-monitoring-hacktivism.service new file mode 100644 index 0000000..3195284 --- /dev/null +++ b/host-agent/taler-monitoring-hacktivism.service @@ -0,0 +1,21 @@ +[Unit] +Description=Taler monitoring GOA/hacktivism (host-podman inside + public urls) +Documentation=file:%h/src/taler-monitoring/host-agent/README.md +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +# Primary: host can podman-exec into taler-hacktivism* containers +Environment=INSIDE_PODMAN=1 +Environment=CONTINUE_ON_ERROR=1 +Environment=RUN_TIMEOUT=600 +Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin +WorkingDirectory=%h/src/taler-monitoring +ExecStart=%h/src/taler-monitoring/host-agent/run-hacktivism-monitoring.sh +Nice=10 +# Suite budget is RUN_TIMEOUT (10m) + git/HTML overhead +TimeoutStartSec=15min + +[Install] +WantedBy=default.target diff --git a/host-agent/taler-monitoring-hacktivism.timer b/host-agent/taler-monitoring-hacktivism.timer new file mode 100644 index 0000000..fe3dc0c --- /dev/null +++ b/host-agent/taler-monitoring-hacktivism.timer @@ -0,0 +1,14 @@ +[Unit] +Description=Periodic hacktivism monitoring (fallback if no path events) +Documentation=file:%h/src/taler-monitoring/host-agent/README.md + +[Timer] +# Every 4h + boot +OnBootSec=10min +OnUnitActiveSec=4h +AccuracySec=5min +Persistent=true +Unit=taler-monitoring-hacktivism.service + +[Install] +WantedBy=timers.target diff --git a/host-agent/taler-monitoring-surface.service b/host-agent/taler-monitoring-surface.service new file mode 100644 index 0000000..db0b442 --- /dev/null +++ b/host-agent/taler-monitoring-surface.service @@ -0,0 +1,18 @@ +[Unit] +Description=Taler monitoring surface (remote ecosystem inventory) +Documentation=file:%h/src/taler-monitoring/host-agent/README.md +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin +Environment=CONTINUE_ON_ERROR=1 +Environment=RUN_TIMEOUT=2400 +WorkingDirectory=%h/src/taler-monitoring +ExecStart=%h/src/taler-monitoring/host-agent/run-surface-monitoring.sh +Nice=10 +TimeoutStartSec=50min + +[Install] +WantedBy=default.target diff --git a/host-agent/taler-monitoring-surface.timer b/host-agent/taler-monitoring-surface.timer new file mode 100644 index 0000000..17a046d --- /dev/null +++ b/host-agent/taler-monitoring-surface.timer @@ -0,0 +1,14 @@ +[Unit] +Description=Hourly taler-monitoring surface scan (ecosystem) +Documentation=file:%h/src/taler-monitoring/host-agent/README.md + +[Timer] +OnBootSec=20min +OnUnitActiveSec=1h +AccuracySec=2min +Persistent=true +RandomizedDelaySec=5min +Unit=taler-monitoring-surface.service + +[Install] +WantedBy=timers.target diff --git a/host-agent/touch-software-stamp.sh b/host-agent/touch-software-stamp.sh new file mode 100755 index 0000000..0aff400 --- /dev/null +++ b/host-agent/touch-software-stamp.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Call after upgrading hacktivism packages / redeploying landings / image rebuilds. +# systemd path unit watches this stamp → re-runs monitoring. +set -euo pipefail +STATE="${XDG_STATE_HOME:-$HOME/.local/state}/taler-hacktivism-monitoring" +mkdir -p "$STATE" +STAMP="$STATE/software.stamp" +date -Iseconds >"$STAMP" +# also record container image ids for debugging +if command -v podman >/dev/null 2>&1; then + { + echo "# images $(date -Iseconds)" + podman images --format '{{.Repository}}:{{.Tag}} {{.ID}} {{.Digest}}' 2>/dev/null | grep -i hacktivism || true + podman ps --format '{{.Names}} {{.ImageID}} {{.Status}}' 2>/dev/null | grep -i hacktivism || true + } >"$STATE/software-images.txt" || true +fi +echo "touched $STAMP" diff --git a/host-agent/update-suite.sh b/host-agent/update-suite.sh new file mode 100755 index 0000000..883079e --- /dev/null +++ b/host-agent/update-suite.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# update-suite.sh — latest taler-monitoring from Forgejo (standalone repo root). +set -uo pipefail +CFG_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring" +ENV_FILE="${TALER_MONITORING_ENV:-$CFG_DIR/env}" +if [ -f "$ENV_FILE" ] && [ "${_TALER_MON_ENV_LOADED:-0}" != "1" ]; then + set -a; source "$ENV_FILE"; set +a +fi +SUITE_GIT_URL="${SUITE_GIT_URL:-https://git.hacktivism.ch/hernani/taler-monitoring.git}" +SUITE_GIT_REF="${SUITE_GIT_REF:-main}" +SUITE_UPDATE_MODE="${SUITE_UPDATE_MODE:-reset}" +if [ -z "${SUITE_DIR:-}" ]; then + if [ -d "$HOME/src/taler-monitoring/.git" ]; then SUITE_DIR="$HOME/src/taler-monitoring" + elif [ -d "$HOME/taler-monitoring/.git" ]; then SUITE_DIR="$HOME/taler-monitoring" + else SUITE_DIR="$HOME/src/taler-monitoring"; fi +fi +export SUITE_DIR +mkdir -p "$(dirname "$SUITE_DIR")" "$CFG_DIR" +if [ ! -d "$SUITE_DIR/.git" ]; then + echo "clone $SUITE_GIT_URL → $SUITE_DIR" + git clone --branch "$SUITE_GIT_REF" "$SUITE_GIT_URL" "$SUITE_DIR" || git clone "$SUITE_GIT_URL" "$SUITE_DIR" +fi +cd "$SUITE_DIR" || exit 1 +git remote set-url origin "$SUITE_GIT_URL" 2>/dev/null || true +echo "fetch origin ($SUITE_GIT_REF) …" +if ! git fetch --tags origin 2>&1; then + echo "WARN: git fetch failed — using existing tree" >&2 + return 0 2>/dev/null || exit 0 +fi +case "$SUITE_UPDATE_MODE" in + reset) + git checkout -B "$SUITE_GIT_REF" "origin/$SUITE_GIT_REF" 2>/dev/null || git checkout -B "$SUITE_GIT_REF" FETCH_HEAD + git reset --hard "origin/$SUITE_GIT_REF" 2>/dev/null || git reset --hard FETCH_HEAD + ;; + *) git pull --ff-only origin "$SUITE_GIT_REF" 2>&1 || true ;; +esac +COMMIT=$(git rev-parse HEAD) +COMMIT_SHORT=$(git rev-parse --short=12 HEAD) +SOURCE_REPO_WEB="${SOURCE_REPO_WEB:-https://git.hacktivism.ch/hernani/taler-monitoring}" +COMMIT_URL="${SOURCE_REPO_WEB}/src/commit/${COMMIT}" +export COMMIT COMMIT_SHORT COMMIT_URL SOURCE_REPO_WEB +echo "suite @ $COMMIT_SHORT $COMMIT_URL" +echo " dir=$SUITE_DIR" +ln -sfn "$SUITE_DIR" "$HOME/taler-monitoring" 2>/dev/null || true +chmod +x taler-monitoring.sh check_*.sh host-agent/*.sh site-gen/*.sh site-gen/*.py 2>/dev/null || true diff --git a/lib.sh b/lib.sh new file mode 100755 index 0000000..8455c6b --- /dev/null +++ b/lib.sh @@ -0,0 +1,1257 @@ +# shellcheck shell=bash +# Shared helpers for taler-monitoring (laptop or koopa). + +# Default stack = GOA / hacktivism (overridden by TALER_DOMAIN / --domain) +: "${TALER_DOMAIN:=hacktivism.ch}" +: "${BANK_PUBLIC:=https://bank.hacktivism.ch}" +: "${EXCHANGE_PUBLIC:=https://exchange.hacktivism.ch}" +: "${MERCHANT_PUBLIC:=https://taler.hacktivism.ch}" +: "${BANK_LOCAL:=http://127.0.0.1:9012}" +: "${EXCHANGE_LOCAL:=http://127.0.0.1:9011}" +: "${MERCHANT_LOCAL:=https://127.0.0.1:9010}" +: "${LANDING_LOCAL:=http://127.0.0.1:9013}" +: "${KOOPA_SSH:=koopa}" +# When LAN Host "koopa" is unreachable, try WAN DNAT (see ~/.ssh/config Host koopa-external). +: "${KOOPA_SSH_FALLBACKS:=koopa-external}" +: "${MERCHANT_INSTANCE:=goa-demo-cp4zqk}" +# Merchant portal /private API: instance ids must be lowercase or you get odd 401s. +# (Display names may use capitals; path segment /instances/{id}/ must not.) +MERCHANT_INSTANCE=$(printf '%s' "${MERCHANT_INSTANCE}" | tr '[:upper:]' '[:lower:]') +: "${WITHDRAW_AMT:=GOA:20}" # single-shot fallback; e2e ladder uses ATM notes +: "${PAY_AMT:=GOA:0.01}" +: "${CREDIT_AMT:=GOA:4700}" # covers ATM ladder 20+50+100+200+4200 (paivana) +: "${TIMEOUT:=12}" +: "${E2E_TIMEOUT:=55}" # whole e2e budget; skip rest when exceeded (e2e raises as needed) +: "${E2E_PAY_SECS:=22}" # dedicated seconds for pay handle-uri (avoid Alarm clock) +# Whole-run wall clock for taler-monitoring.sh (seconds). 0 = unlimited. +# Host-agent default phases (urls/inside/versions) should finish under this. +: "${RUN_TIMEOUT:=600}" +# Local GOA: public paivana paywall (https://paivana.hacktivism.ch · template GOA:4200) +: "${PAIVANA_PUBLIC:=https://paivana.hacktivism.ch}" +: "${E2E_PAIVANA:=1}" # 0 = skip paivana section in e2e +# Devtest: inject reserve credit via wire-gateway admin/add-incoming (optional). +# Default off once wirewatch DNS works; set E2E_FAKE_INCOMING=1 to force. +: "${E2E_FAKE_INCOMING:=0}" +# SSH must never hang the monitoring run +: "${SSH_CONNECT_TIMEOUT:=3}" +: "${SSH_CMD_TIMEOUT:=12}" # hard cap for whole remote script (seconds) +: "${SKIP_SSH:=0}" +# Expected currency for public /config checks (empty = report only, don't fail) +# Only if unset (not if intentionally empty = pre-launch / report-only) +: "${EXPECT_CURRENCY=GOA}" +# 1 = this is the local koopa/hacktivism stack (inside/e2e/SSH make sense) +: "${LOCAL_STACK:=1}" +# Probe merchant host candidates when applying a generic domain (0=off) +: "${TALER_DOMAIN_PROBE:=1}" + +BANK_PUBLIC=${BANK_PUBLIC%/} +EXCHANGE_PUBLIC=${EXCHANGE_PUBLIC%/} +MERCHANT_PUBLIC=${MERCHANT_PUBLIC%/} + +# --------------------------------------------------------------------------- +# Domain profiles → bank / exchange / merchant base URLs +# +# Single place to declare a stack: scripts/taler-monitoring/domains.conf +# (or TALER_DOMAINS_CONF). Each profile names the three public endpoints. +# +# CLI still wins after profile load: +# --bank URL --exchange URL --merchant URL --currency CODE +# Env: BANK_PUBLIC EXCHANGE_PUBLIC MERCHANT_PUBLIC EXPECT_CURRENCY +# +# Unknown domains fall back to heuristics (see apply_taler_domain). +# --------------------------------------------------------------------------- + +_MONITOR_LIB_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +: "${TALER_DOMAINS_CONF:=${_MONITOR_LIB_DIR}/domains.conf}" + +# host_or_url → https://… (no trailing slash) +_to_https_base() { + local x="${1%/}" + case "$x" in + https://*|http://*) printf '%s' "$x" ;; + "") printf '' ;; + *) printf 'https://%s' "$x" ;; + esac +} + +# 1 = check GOA-style /intro landing pages (assets, link crawl, demo-withdraw). +# 0 = skip (e.g. taler-ops.ch / mytops — no public landings). +: "${CHECK_LANDING:=1}" + +# Declare bank + exchange + merchant (+ currency, local stack, landing, domain id). +# Usage: +# set_taler_stack BANK EXCHANGE MERCHANT [CURRENCY] [LOCAL 0|1] [LANDING 0|1] [TALER_DOMAIN] +# BANK/EXCHANGE/MERCHANT: hostname or full URL +set_taler_stack() { + local bank="${1:-}" exchange="${2:-}" merchant="${3:-}" + local currency="${4:-}" local_stack="${5:-0}" landing="${6:-}" domain_id="${7:-}" + + BANK_PUBLIC=$(_to_https_base "$bank") + EXCHANGE_PUBLIC=$(_to_https_base "$exchange") + MERCHANT_PUBLIC=$(_to_https_base "$merchant") + # currency "-" or empty → report-only (do not force GOA) + if [ -n "$currency" ] && [ "$currency" != "-" ]; then + EXPECT_CURRENCY="$currency" + elif [ "$currency" = "-" ]; then + EXPECT_CURRENCY="" + fi + LOCAL_STACK="$local_stack" + if [ "$LOCAL_STACK" = "1" ]; then + SKIP_SSH=0 + else + SKIP_SSH=1 + fi + # default: landings only matter on local GOA stack + if [ -n "$landing" ]; then + CHECK_LANDING="$landing" + elif [ "$LOCAL_STACK" = "1" ]; then + CHECK_LANDING=1 + else + CHECK_LANDING=0 + fi + [ -n "$domain_id" ] && TALER_DOMAIN="$domain_id" + + BANK_PUBLIC=${BANK_PUBLIC%/} + EXCHANGE_PUBLIC=${EXCHANGE_PUBLIC%/} + MERCHANT_PUBLIC=${MERCHANT_PUBLIC%/} +} + +# Load first matching profile from domains.conf. +# Fields (whitespace-separated; # comments; blank lines ignored): +# name bank exchange merchant currency local[0|1] landing[0|1] [canonical_domain] +# Returns 0 if found, 1 if not. +load_domain_profile() { + local want="$1" conf="${TALER_DOMAINS_CONF:-}" + local line name bank exchange merchant currency local_stack landing canon + [ -n "$want" ] || return 1 + [ -n "$conf" ] && [ -f "$conf" ] || return 1 + + while IFS= read -r line || [ -n "$line" ]; do + line="${line%%#*}" + # shellcheck disable=SC2086 + set -- $line + [ $# -ge 5 ] || continue + name="$1" + [ "$name" = "$want" ] || continue + bank="$2" + exchange="$3" + merchant="$4" + currency="$5" + local_stack="${6:-0}" + landing="${7:-}" + canon="${8:-$name}" + # Backward compat: old 7th field was canonical domain (contains a dot) + if [ -n "$landing" ] && [[ "$landing" == *.* && "$landing" != "0" && "$landing" != "1" ]]; then + canon="$landing" + landing="" + fi + set_taler_stack "$bank" "$exchange" "$merchant" "$currency" "$local_stack" "$landing" "$canon" + return 0 + done <"$conf" + return 1 +} + +_normalize_domain() { + local d="$1" + d="${d#https://}" + d="${d#http://}" + d="${d%%/*}" + d="${d%%:*}" + # Strip service host prefix only if a real domain remains (has a dot). + # e.g. bank.demo.taler.net → demo.taler.net, taler.hacktivism.ch → hacktivism.ch + # but NOT taler.net → net + case "$d" in + bank.*|exchange.*|taler.*|backend.*|merchant.*|shop.*|libeufin.*|my.*|map.*|monnaie.*) + rest="${d#*.}" + if [[ "$rest" == *.* ]]; then + d="$rest" + fi + ;; + esac + printf '%s' "$d" +} + +_probe_https_config() { + # 0 if https://$1/config returns 200 + local host="$1" code + code=$(curl -skS --max-redirs 0 -m 4 -o /dev/null -w '%{http_code}' "https://${host}/config" 2>/dev/null || echo 000) + [ "$code" = "200" ] +} + +apply_taler_domain() { + local raw="${1:-}" + local d + [ -n "$raw" ] || return 0 + d=$(_normalize_domain "$raw") + TALER_DOMAIN="$d" + + # 1) Explicit profile (domains.conf) — preferred way to add stacks + if load_domain_profile "$d" || load_domain_profile "$raw"; then + # Optional tiny defaults for e2e ladders by currency + case "${EXPECT_CURRENCY:-}" in + GOA) + : "${WITHDRAW_AMT:=GOA:20}" + : "${PAY_AMT:=GOA:0.01}" + : "${CREDIT_AMT:=GOA:400}" + ;; + KUDOS|TESTKUDOS) + MERCHANT_INSTANCE="${MERCHANT_INSTANCE:-sandbox}" + WITHDRAW_AMT="${WITHDRAW_AMT:-${EXPECT_CURRENCY}:20}" + PAY_AMT="${PAY_AMT:-${EXPECT_CURRENCY}:0.01}" + CREDIT_AMT="${CREDIT_AMT:-${EXPECT_CURRENCY}:100}" + ;; + TESTPAYSAN) + # Stage FrancPaysan: farmer shops (override GOA default goa-demo-cp4zqk) + case "${MERCHANT_INSTANCE:-}" in + ""|goa-demo*|goa-shop) + MERCHANT_INSTANCE=fermes-des-collines + ;; + esac + MERCHANT_INSTANCE=$(printf '%s' "${MERCHANT_INSTANCE}" | tr '[:upper:]' '[:lower:]') + WITHDRAW_AMT="${WITHDRAW_AMT:-TESTPAYSAN:50}" + PAY_AMT="${PAY_AMT:-TESTPAYSAN:5}" + # Cover oeufs(5)+fromage(8.5)+jus(6)+shop pick(~5–12) + fees + CREDIT_AMT="${CREDIT_AMT:-TESTPAYSAN:200}" + # Public template pays on by default (shops landings) + : "${E2E_USE_TEMPLATES:=1}" + # Map pay-ladder amounts → public shop templates (instance set in e2e) + : "${E2E_TEMPLATE_MAP:=5=oeufs-6 4.5=pain-seigle 6=jus-pomme 8.5=fromage-chevre 12=miel-printemps 25=panier-legumes}" + : "${E2E_PAY_VALUES:=5 8.5 6}" + : "${E2E_WITHDRAW_VALUES:=20 50}" + # Inside checks as low-priv stagepaysan (podman, no sudo) — not koopa + INSIDE_PROFILE="${INSIDE_PROFILE:-stage-lfp}" + INSIDE_SSH="${INSIDE_SSH:-francpaysan-stage-user}" + INSIDE_BANK_CTR="${INSIDE_BANK_CTR:-stage-lfp-bank}" + INSIDE_EXCHANGE_CTR="${INSIDE_EXCHANGE_CTR:-stage-lfp-exchange-ansible}" + INSIDE_MERCHANT_CTR="${INSIDE_MERCHANT_CTR:-stage-lfp-merchant}" + # Host pasta ports (127.0.0.1 on francpaysan-host) + INSIDE_BANK_PORT="${INSIDE_BANK_PORT:-9032}" + INSIDE_EXCHANGE_PORT="${INSIDE_EXCHANGE_PORT:-9031}" + INSIDE_MERCHANT_PORT="${INSIDE_MERCHANT_PORT:-9030}" + INSIDE_DNS_BANK="${INSIDE_DNS_BANK:-stage.bank.lefrancpaysan.ch}" + INSIDE_DNS_EXCHANGE="${INSIDE_DNS_EXCHANGE:-stage.exchange.lefrancpaysan.ch}" + INSIDE_DNS_MERCHANT="${INSIDE_DNS_MERCHANT:-stage.monnaie.lefrancpaysan.ch}" + ;; + CHF) + WITHDRAW_AMT="${WITHDRAW_AMT:-CHF:20}" + PAY_AMT="${PAY_AMT:-CHF:0.01}" + CREDIT_AMT="${CREDIT_AMT:-CHF:100}" + ;; + esac + else + # 2) Unknown domain — heuristics only (prefer profile in domains.conf) + # Override: --bank / --exchange / --merchant / --currency + # Do not keep default GOA: empty currency = report only, no hard fail. + BANK_PUBLIC="https://bank.${d}" + EXCHANGE_PUBLIC="https://exchange.${d}" + # TOPS-style multi-tenant merchant first, then legacy names + MERCHANT_PUBLIC="https://my.${d}" + EXPECT_CURRENCY="" + LOCAL_STACK=0 + SKIP_SSH=1 + CHECK_LANDING=0 + if [ "${TALER_DOMAIN_PROBE}" = "1" ]; then + local h + for h in "monnaie.${d}" "backend.${d}" "my.${d}" "taler.${d}" "merchant.${d}" "shop.${d}"; do + _probe_https_config "$h" && { MERCHANT_PUBLIC="https://$h"; break; } + done + for h in "bank.${d}" "libeufin.${d}"; do + _probe_https_config "$h" && { BANK_PUBLIC="https://$h"; break; } + done + _probe_https_config "exchange.${d}" || true + fi + fi + + BANK_PUBLIC=${BANK_PUBLIC%/} + EXCHANGE_PUBLIC=${EXCHANGE_PUBLIC%/} + MERCHANT_PUBLIC=${MERCHANT_PUBLIC%/} + + # Koopa SSH only for LOCAL_STACK=1. Stage LFP uses INSIDE_SSH (stagepaysan) separately. + if [ "${LOCAL_STACK}" != "1" ]; then + SKIP_SSH=1 + fi + # Default inside profile for local GOA + if [ "${LOCAL_STACK}" = "1" ]; then + INSIDE_PROFILE="${INSIDE_PROFILE:-koopa}" + fi +} + +# Apply TALER_DOMAIN from env once (CLI exports TALER_DOMAIN_APPLIED=1 after overrides). +if [ "${TALER_DOMAIN_APPLIED:-0}" != "1" ] \ + && [ -n "${TALER_DOMAIN:-}" ] && [ "${TALER_DOMAIN}" != "hacktivism.ch" ]; then + apply_taler_domain "$TALER_DOMAIN" + TALER_DOMAIN_APPLIED=1 +fi + +# Safe SSH: publickey only, short connect, overall alarm so we never block forever. +SSH_BASE_OPTS=( + -o BatchMode=yes + -o ConnectTimeout="${SSH_CONNECT_TIMEOUT}" + -o ConnectionAttempts=1 + -o ServerAliveInterval=3 + -o ServerAliveCountMax=2 + -o StrictHostKeyChecking=accept-new + -o PreferredAuthentications=publickey + -o PasswordAuthentication=no + -o KbdInteractiveAuthentication=no + -o GSSAPIAuthentication=no + -o NumberOfPasswordPrompts=0 +) + +# Hard wall-clock timeout so ssh/curl never block the monitoring run forever. +with_timeout() { + local secs="$1"; shift + if command -v gtimeout >/dev/null 2>&1; then + gtimeout --kill-after=2 "$secs" "$@" + return $? + fi + if command -v timeout >/dev/null 2>&1; then + timeout -k 2 "$secs" "$@" 2>/dev/null || timeout --kill-after=2 "$secs" "$@" + return $? + fi + # Portable: perl alarm + process group kill + perl -e ' + use strict; use warnings; + my $secs = shift @ARGV; + my $pid = fork(); + die "fork: $!" unless defined $pid; + if ($pid == 0) { + setpgrp(0, 0); + exec @ARGV; + exit 127; + } + $SIG{ALRM} = sub { + kill "TERM", -$pid; + select(undef, undef, undef, 1.0); + kill "KILL", -$pid; + exit 124; + }; + alarm $secs; + waitpid($pid, 0); + my $code = $? >> 8; + alarm 0; + exit $code; + ' "$secs" "$@" +} + +# Generic short SSH to a named host (low-priv stagepaysan or koopa). +# usage: mon_ssh_bash HOST [timeout] <<'EOF' ... EOF +mon_ssh_bash() { + local host="$1" t="${2:-$SSH_CMD_TIMEOUT}" + [ -n "$host" ] || return 1 + with_timeout "$t" ssh "${SSH_BASE_OPTS[@]}" "$host" bash -s +} +mon_ssh_ok() { + local host="$1" + [ -n "$host" ] || return 1 + with_timeout $((SSH_CONNECT_TIMEOUT + 5)) \ + ssh "${SSH_BASE_OPTS[@]}" "$host" true >/dev/null 2>&1 +} + +# Pick a working SSH host: KOOPA_SSH first, then KOOPA_SSH_FALLBACKS (koopa-external). +# Sets KOOPA_SSH to the first host that answers. 0 = ok, 1 = none. +KOOPA_SSH_RESOLVED=0 +resolve_koopa_ssh() { + [ "${SKIP_SSH:-0}" = "1" ] && return 1 + if [ "${KOOPA_SSH_RESOLVED}" = "1" ]; then + return 0 + fi + local cands=() c f seen=" " + cands+=("${KOOPA_SSH}") + # shellcheck disable=SC2086 + for f in ${KOOPA_SSH_FALLBACKS}; do + case "$seen" in *" $f "*) continue ;; esac + cands+=("$f") + seen="$seen$f " + done + for c in "${cands[@]}"; do + if with_timeout $((SSH_CONNECT_TIMEOUT + 5)) \ + ssh "${SSH_BASE_OPTS[@]}" "$c" 'echo ok' >/dev/null 2>&1; then + if [ "$c" != "${KOOPA_SSH}" ]; then + # surface once so operators know we used WAN jump + printf '[INFO] SSH host %s unreachable — using %s\n' "${KOOPA_SSH}" "$c" >&2 || true + fi + KOOPA_SSH="$c" + KOOPA_SSH_RESOLVED=1 + export KOOPA_SSH + return 0 + fi + done + return 1 +} + +# Probe: 0 if any koopa SSH host works quickly +koopa_ssh_ok() { + [ "${SKIP_SSH}" = "1" ] && return 1 + resolve_koopa_ssh +} + +# Run remote bash -s with optional stdin script; hard-capped +# usage: koopa_ssh_bash [timeout_secs] <<'EOF' ... EOF +# or: koopa_ssh_run timeout_secs 'remote command' +koopa_ssh_run() { + local t="${1:-$SSH_CMD_TIMEOUT}" + shift + resolve_koopa_ssh || return 1 + with_timeout "$t" ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" "$@" +} + +koopa_ssh_bash() { + local t="${1:-$SSH_CMD_TIMEOUT}" + resolve_koopa_ssh || return 1 + with_timeout "$t" ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" 'bash -s' +} + +# stdin → remote python3 - (for metrics load probe) +koopa_ssh_python() { + local t="${1:-60}" + resolve_koopa_ssh || return 1 + with_timeout "$t" ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" python3 - +} + +# ANSI colours — left badge (clear severity) + body label colour + dim detail. +# Off: NO_COLOR=1 or CLICOLOR=0. +# OK green badge · green label +# INFO blue badge · cyan label +# WARN yellow badge · yellow label (never red) +# ERROR red badge · red label +# BLOCKER magenta badge· magenta label +# progress cyan badge = " 42%" + bar fills left→right +if [ "${NO_COLOR:-0}" = "1" ] || [ "${CLICOLOR:-1}" = "0" ]; then + G= R= Y= C= M= D= W= B= N= + BG_OK= BG_INFO= BG_WARN= BG_ERR= BG_BLK= BG_SEC= BG_PROG= + BOX=0 +else + G=$'\e[1;32m' # green + R=$'\e[1;31m' # red + Y=$'\e[1;33m' # yellow + C=$'\e[1;36m' # cyan + M=$'\e[1;35m' # magenta + D=$'\e[2m' # dim tid / detail + W=$'\e[1;37m' # bold white + B=$'\e[1m' # bold + N=$'\e[0m' # reset + # High-contrast left badges (bright white / black on strong bg) + BG_OK=$'\e[1;97;42m' # white on green + BG_INFO=$'\e[1;97;44m' # white on blue + BG_WARN=$'\e[1;30;103m' # black on bright yellow — not red + BG_ERR=$'\e[1;97;41m' # white on red — errors only + BG_BLK=$'\e[1;97;45m' # white on magenta — blockers + BG_SEC=$'\e[1;97;44m' # white on blue (section) + BG_PROG=$'\e[1;30;106m' # black on bright cyan (progress) + BOX=1 +fi + +PASS_N=0 +FAIL_N=0 +WARN_N=0 +INFO_N=0 +BLOCKERS=() # human-readable payment/withdraw blockers +ERRORS=() # all ERROR lines (component scope) + +# Grouped test IDs: area.group-NN (e.g. www.exchange-01, e2e.pay-03) +# plus global run number: #042 (monotonic for the whole ./taler-monitoring.sh run) +# set_area www # phase: www | e2e | inside | versions | … +# set_group exchange # → www.exchange-01 +# set_group bank # → www.bank-01 +# Without set_group: area-01, area-02, … (flat within area) +# Line shape: ┌ OK ┐ #003 www.exchange-02 label · detail +TEST_AREA="" +TEST_GROUP="" +TEST_N=0 +LAST_TID="" +# Global / progress: may continue across check_*.sh processes via TALER_MON_STATE +GLOBAL_N="${GLOBAL_N:-0}" +LAST_GLOBAL="" +PROGRESS_DONE="${PROGRESS_DONE:-0}" +PROGRESS_TOTAL="${PROGRESS_TOTAL:-0}" +PROGRESS_SHOW_EVERY="${PROGRESS_SHOW_EVERY:-8}" +PROGRESS_LAST_SHOWN="${PROGRESS_LAST_SHOWN:-0}" +_mon_state_load() { + [ -n "${TALER_MON_STATE:-}" ] && [ -f "${TALER_MON_STATE}" ] || return 0 + # shellcheck disable=SC1090 + . "${TALER_MON_STATE}" +} +_mon_state_save() { + [ -n "${TALER_MON_STATE:-}" ] || return 0 + { + printf 'GLOBAL_N=%s\n' "$GLOBAL_N" + printf 'PROGRESS_DONE=%s\n' "$PROGRESS_DONE" + printf 'PROGRESS_TOTAL=%s\n' "$PROGRESS_TOTAL" + printf 'PROGRESS_LAST_SHOWN=%s\n' "$PROGRESS_LAST_SHOWN" + } >"${TALER_MON_STATE}" +} +_mon_state_load +# Per-group counters so re-entering www.exchange after perf continues NN +# (shell vars TEST_CNT__). +_tid_key() { + # $1=area $2=group → safe identifier + printf 'TEST_CNT_%s_%s' "$1" "$2" | tr -c 'A-Za-z0-9_' '_' +} +_tid_save() { + [ -z "${TEST_AREA:-}" ] || [ -z "${TEST_GROUP:-}" ] && return 0 + local k + k=$(_tid_key "$TEST_AREA" "$TEST_GROUP") + eval "$k=\$TEST_N" +} +_tid_load() { + local k + k=$(_tid_key "$TEST_AREA" "$1") + eval "TEST_N=\${$k:-0}" +} +set_area() { + # leave previous group counter saved + _tid_save + TEST_AREA="$1" + TEST_GROUP="" + TEST_N=0 + LAST_TID="" +} +# Start a logical sub-group. Short stable names: +# exchange bank merchant perf stats landing paivana +# prereq wallet atm settle pay shop dig load report +# ssh caddy outside inside compare withdraw +# Counter continues if the same group is re-entered later in the area. +# Prints a compact group chip so log + issue text map cleanly (www.bank-03). +set_group() { + local g="$1" + # no-op if same group already active (e.g. inside emit loop) + if [ "$g" = "${TEST_GROUP:-}" ] && [ -n "$g" ]; then + return 0 + fi + _tid_save + TEST_GROUP="$g" + LAST_TID="" + if [ -z "${TEST_AREA:-}" ] || [ -z "$g" ]; then + TEST_N=0 + return 0 + fi + _tid_load "$g" + local tag="${TEST_AREA}.${g}" + if [ "${BOX:-0}" = "1" ]; then + # cyan-outline group chip: ┌ www.exchange ┐ + printf '%s%s┌ %s ┐%s\n' "$D" "$C" "$tag" "$N" + else + printf -- '-- %s --\n' "$tag" + fi + # show progress when entering a new group (if totals known) + _progress_maybe_show 1 +} +set_progress_total() { + # Optional: expected number of numbered checks in this run. + # PROGRESS_TOTAL=0 → no percent, only "done=N". + # Does not reset GLOBAL_N / PROGRESS_DONE if already mid-run (state file). + local n="${1:-0}" + # Never shrink below already completed work (re-estimate mid-run safe) + if [ "${PROGRESS_DONE:-0}" -gt 0 ] && [ "$n" -gt 0 ] && [ "$n" -lt "$PROGRESS_DONE" ]; then + n=$PROGRESS_DONE + fi + PROGRESS_TOTAL="$n" + if [ "${PROGRESS_DONE:-0}" -eq 0 ]; then + PROGRESS_LAST_SHOWN=0 + fi + _mon_state_save +} +add_progress_total() { + local n="${1:-0}" + PROGRESS_TOTAL=$((PROGRESS_TOTAL + n)) + _mon_state_save +} +# When estimate was short, grow total so we never print done>total (e.g. 224/205). +# Keeps ~10% headroom so the bar is not stuck at 100% while checks continue. +_progress_rebalance() { + [ "${PROGRESS_TOTAL:-0}" -gt 0 ] || return 0 + [ "${PROGRESS_DONE:-0}" -gt 0 ] || return 0 + if [ "$PROGRESS_DONE" -gt "$PROGRESS_TOTAL" ]; then + local head=$((PROGRESS_DONE / 10)) + [ "$head" -lt 12 ] && head=12 + PROGRESS_TOTAL=$((PROGRESS_DONE + head)) + fi +} +_progress_bar_line() { + # Format (grows left → right; badge = percent, not "PROG"): + # ┌ 42% ┐ ████████░░░░░░░░░░░░░░░░ 31/74 + local done="$1" total="$2" width=28 pct=0 filled empty i bar pct_lab + # Defensive: never display done > total + if [ "$total" -gt 0 ] && [ "$done" -gt "$total" ]; then + total=$done + fi + if [ "$total" -gt 0 ]; then + # integer percent; while mid-run past a short estimate, cap display < 100% + pct=$((done * 100 / total)) + [ "$pct" -gt 100 ] && pct=100 + if [ "$done" -lt "$total" ] && [ "$pct" -ge 100 ]; then + pct=99 + fi + # fill cells from the left (progress moves rightward as filled grows) + filled=$((done * width / total)) + [ "$filled" -gt "$width" ] && filled=$width + # show at least 1 block once any work done (unless 0%) + if [ "$done" -gt 0 ] && [ "$filled" -eq 0 ]; then + filled=1 + fi + # at 100% fill complete bar + if [ "$done" -ge "$total" ]; then + filled=$width + pct=100 + fi + empty=$((width - filled)) + bar="" + i=0 + while [ "$i" -lt "$filled" ]; do bar="${bar}█"; i=$((i + 1)); done + i=0 + while [ "$i" -lt "$empty" ]; do bar="${bar}░"; i=$((i + 1)); done + pct_lab=$(printf '%3d%%' "$pct") + if [ "${BOX:-0}" = "1" ]; then + # badge = percent (clearer than "PROG"); bar grows ░→█ left to right + printf '%s┌%s┐%s %s%s%s %s%d/%d%s\n' \ + "$BG_PROG" "$pct_lab" "$N" \ + "$C" "$bar" "$N" \ + "$D" "$done" "$total" "$N" + else + printf -- '%s%4s%s %s %d/%d\n' \ + "$C" "$pct_lab" "$N" "$bar" "$done" "$total" + fi + else + if [ "${BOX:-0}" = "1" ]; then + printf '%s┌ … ┐%s %sdone=%d%s (set PROGRESS_TOTAL= for %%)\n' \ + "$BG_PROG" "$N" "$D" "$done" "$N" + else + printf -- '%sdone=%d%s (set PROGRESS_TOTAL= for %%)\n' "$D" "$done" "$N" + fi + fi +} +_progress_maybe_show() { + local force="${1:-0}" + [ "${PROGRESS_OFF:-0}" = "1" ] && return 0 + [ "$PROGRESS_DONE" -le 0 ] && [ "$force" != "1" ] && return 0 + _progress_rebalance + if [ "$force" = "1" ] || \ + [ $((PROGRESS_DONE - PROGRESS_LAST_SHOWN)) -ge "$PROGRESS_SHOW_EVERY" ] || \ + { [ "$PROGRESS_TOTAL" -gt 0 ] && [ "$PROGRESS_DONE" -ge "$PROGRESS_TOTAL" ]; }; then + _progress_bar_line "$PROGRESS_DONE" "$PROGRESS_TOTAL" + PROGRESS_LAST_SHOWN=$PROGRESS_DONE + fi +} +# Assign next id into LAST_TID (must not run in a subshell). +_take_tid() { + LAST_TID="" + LAST_GLOBAL="" + if [ -z "${TEST_AREA:-}" ]; then + return + fi + # Global monotonic number for the whole monitoring run (#001 …) + GLOBAL_N=$((GLOBAL_N + 1)) + LAST_GLOBAL=$(printf '#%03d' "$GLOBAL_N") + TEST_N=$((TEST_N + 1)) + if [ -n "${TEST_GROUP:-}" ]; then + LAST_TID=$(printf '%s.%s-%02d' "$TEST_AREA" "$TEST_GROUP" "$TEST_N") + _tid_save + else + LAST_TID=$(printf '%s-%02d' "$TEST_AREA" "$TEST_N") + fi + PROGRESS_DONE=$((PROGRESS_DONE + 1)) + _progress_rebalance + _mon_state_save + _progress_maybe_show 0 +} +# Dim ids: "#003 www.exchange-01 " or empty +_fmt_tid() { + if [ -n "${LAST_GLOBAL:-}" ]; then + printf '%s%s%s ' "$D" "$LAST_GLOBAL" "$N" + fi + if [ -n "${LAST_TID:-}" ]; then + printf '%s%s%s ' "$D" "$LAST_TID" "$N" + fi +} +# Boxed badge cell: ┌ OK ┐ with filled bg (tag left-padded, width 7 for BLOCKER) +# $1=badge style $2=tag text +_fmt_badge() { + local badge="$1" tag="$2" inner + inner=$(printf -- '%-7s' "$tag") + if [ "${BOX:-0}" = "1" ]; then + printf '%s┌ %s┐%s' "$badge" "$inner" "$N" + else + printf '%s[ %s]%s' "$badge" "$inner" "$N" + fi +} +# One concrete line: ┌ OK ┐ tid label · detail +# $1=badge style $2=tag text $3=label colour $4=label $5=detail +_msg_line() { + local badge="$1" tag="$2" lcol="$3" label="$4" detail="${5:-}" + if [ -n "$detail" ]; then + printf -- '%s %s%s%s%s %s·%s %s%s%s\n' \ + "$(_fmt_badge "$badge" "$tag")" "$(_fmt_tid)" "$lcol" "$label" "$N" "$D" "$N" "$D" "$detail" "$N" + else + printf -- '%s %s%s%s%s\n' \ + "$(_fmt_badge "$badge" "$tag")" "$(_fmt_tid)" "$lcol" "$label" "$N" + fi +} + +ok() { + # ok "what is good" ["concrete evidence: HTTP 200 · 12ms · …"] + local label="$1" detail="${2:-}" + _take_tid + _msg_line "$BG_OK" "OK" "$G" "$label" "$detail" + PASS_N=$((PASS_N + 1)) +} +# component-scoped error: err bank "what failed" "why / HTTP / path" +err() { + local comp="$1" msg="$2" detail="${3:-}" + _take_tid + _msg_line "$BG_ERR" "ERROR" "$R" "${comp}: ${msg}" "$detail" + FAIL_N=$((FAIL_N + 1)) + ERRORS+=("${LAST_TID:+$LAST_TID }[$comp] $msg${detail:+ · $detail}") +} +# fail "what failed" ["why / HTTP code / path"] +fail() { + local label="$1" detail="${2:-}" + _take_tid + _msg_line "$BG_ERR" "ERROR" "$R" "$label" "$detail" + FAIL_N=$((FAIL_N + 1)) + ERRORS+=("${LAST_TID:+$LAST_TID }$label${detail:+ · $detail}") +} +warn() { + # warn "what is soft-bad" ["why still ok to continue"] + # warn component "what" "why" + # Yellow only — never red (red = ERROR / BLOCKER path) + local a1="${1:-}" a2="${2:-}" a3="${3:-}" + local head detail + _take_tid + if [ -n "$a3" ]; then + head="${a1}: ${a2}" + detail="$a3" + elif [ -n "$a2" ]; then + head="$a1" + detail="$a2" + else + head="$a1" + detail="" + fi + _msg_line "$BG_WARN" "WARN" "$Y" "$head" "$detail" + WARN_N=$((WARN_N + 1)) +} +info() { + # info "topic" ["concrete fact / value / next step"] + local label="$1" detail="${2:-}" + _take_tid + _msg_line "$BG_INFO" "INFO" "$C" "$label" "$detail" + INFO_N=$((INFO_N + 1)) +} +blocker() { + # Hard stop on pay/withdraw path: blocker "step" "why it cannot continue" + local step="$1" msg="$2" + _take_tid + _msg_line "$BG_BLK" "BLOCKER" "$M" "${step}" "$msg" + BLOCKERS+=("${LAST_TID:+$LAST_TID }[$step] $msg") + FAIL_N=$((FAIL_N + 1)) + ERRORS+=("BLOCKER ${LAST_TID:+$LAST_TID }[$step] $msg") +} +section() { + # Boxed section header (3 lines when colour on) + local title="$*" + local w=${#title} + [ "$w" -lt 24 ] && w=24 + [ "$w" -gt 56 ] && w=56 + local pad line i + pad=$(printf -- "%-${w}s" "$title") + if [ "${BOX:-0}" = "1" ]; then + line="" + i=0 + while [ "$i" -lt $((w + 2)) ]; do + line="${line}═" + i=$((i + 1)) + done + printf -- '\n%s╔%s╗%s\n' "$BG_SEC" "$line" "$N" + printf -- '%s║%s %s%s%s %s║%s\n' "$BG_SEC" "$N" "$B" "$pad" "$N" "$BG_SEC" "$N" + printf -- '%s╚%s╝%s\n' "$BG_SEC" "$line" "$N" + else + printf -- '\n== %s ==\n' "$title" + fi +} + +summary() { + local blk_n=${#BLOCKERS[@]} + echo "" + # final progress line (no _take_tid — plain printf) + if [ "${PROGRESS_OFF:-0}" != "1" ] && [ "$PROGRESS_DONE" -gt 0 ]; then + # Snap total to actual so last line is exact N/N 100% + if [ "$PROGRESS_TOTAL" -gt 0 ] && [ "$PROGRESS_DONE" -ne "$PROGRESS_TOTAL" ]; then + PROGRESS_TOTAL=$PROGRESS_DONE + _mon_state_save + fi + _progress_bar_line "$PROGRESS_DONE" "$PROGRESS_TOTAL" + fi + if [ "$GLOBAL_N" -gt 0 ]; then + printf -- '%s numbered checks this run: #001…#%03d%s\n' "$D" "$GLOBAL_N" "$N" + fi + if [ "$blk_n" -gt 0 ]; then + if [ "${BOX:-0}" = "1" ]; then + printf -- '%s┌ BLOCKERS · pay/withdraw cannot finish ┐%s\n' "$BG_BLK" "$N" + else + printf -- '%s--- BLOCKERS (pay/withdraw cannot finish) ---%s\n' "$M" "$N" + fi + local b + for b in "${BLOCKERS[@]}"; do + printf -- ' %s•%s %s%s%s\n' "$M" "$N" "$W" "$b" "$N" + done + fi + if [ "${#ERRORS[@]}" -gt 0 ] && [ "$blk_n" -lt "${#ERRORS[@]}" ]; then + if [ "${BOX:-0}" = "1" ]; then + printf -- '%s┌ ERRORS · failed checks ┐%s\n' "$BG_ERR" "$N" + else + printf -- '%s--- ERRORS (failed checks) ---%s\n' "$R" "$N" + fi + local e + for e in "${ERRORS[@]}"; do + case "$e" in BLOCKER*) continue ;; esac + printf -- ' %s•%s %s%s%s\n' "$R" "$N" "$W" "$e" "$N" + done + fi + + # Coloured totals — same severity badges as check lines (always; NO_COLOR blanks ANSI) + if [ "${BOX:-0}" = "1" ]; then + printf -- '\n%s┌ SUMMARY ┐%s\n' "$BG_SEC" "$N" + _sum_badge() { # $1=bg $2=tag $3=fg $4=n + printf -- ' %s┌ %-5s┐%s %s%4d%s\n' "$1" "$2" "$N" "$3" "$4" "$N" + } + _sum_badge "$BG_OK" "OK" "$G" "$PASS_N" + [ "$FAIL_N" -gt 0 ] && _sum_badge "$BG_ERR" "ERROR" "$R" "$FAIL_N" + [ "$WARN_N" -gt 0 ] && _sum_badge "$BG_WARN" "WARN" "$Y" "$WARN_N" + [ "$INFO_N" -gt 0 ] && _sum_badge "$BG_INFO" "INFO" "$C" "$INFO_N" + [ "$blk_n" -gt 0 ] && _sum_badge "$BG_BLK" "BLOCK" "$M" "$blk_n" + else + printf -- '\n[ SUMMARY ]\n' + printf -- ' [ OK ] %s%4d%s\n' "$G" "$PASS_N" "$N" + [ "$FAIL_N" -gt 0 ] && printf -- ' [ ERROR ] %s%4d%s\n' "$R" "$FAIL_N" "$N" + [ "$WARN_N" -gt 0 ] && printf -- ' [ WARN ] %s%4d%s\n' "$Y" "$WARN_N" "$N" + [ "$INFO_N" -gt 0 ] && printf -- ' [ INFO ] %s%4d%s\n' "$C" "$INFO_N" "$N" + [ "$blk_n" -gt 0 ] && printf -- ' [ BLOCK ] %s%4d%s\n' "$M" "$blk_n" "$N" + fi + # one-line rollup (bold label, severity-coloured counts) + printf -- ' %stotals:%s %s%d OK%s' "$B" "$N" "$G" "$PASS_N" "$N" + [ "$FAIL_N" -gt 0 ] && printf -- ' · %s%d ERROR%s' "$R" "$FAIL_N" "$N" + [ "$WARN_N" -gt 0 ] && printf -- ' · %s%d WARN%s' "$Y" "$WARN_N" "$N" + [ "$INFO_N" -gt 0 ] && printf -- ' · %s%d INFO%s' "$C" "$INFO_N" "$N" + [ "$blk_n" -gt 0 ] && printf -- ' · %s%d BLOCKER%s' "$M" "$blk_n" "$N" + printf '\n' + # overall verdict + if [ "$FAIL_N" -eq 0 ] && [ "$blk_n" -eq 0 ] && [ "$WARN_N" -eq 0 ]; then + if [ "${BOX:-0}" = "1" ]; then + printf -- ' %s┌ OK ┐%s %sall clear%s\n' "$BG_OK" "$N" "$G" "$N" + else + printf -- ' %s[ OK ] all clear%s\n' "$G" "$N" + fi + elif [ "$FAIL_N" -eq 0 ] && [ "$blk_n" -eq 0 ]; then + if [ "${BOX:-0}" = "1" ]; then + printf -- ' %s┌ WARN ┐%s %swarnings only (no hard fail)%s\n' "$BG_WARN" "$N" "$Y" "$N" + else + printf -- ' %s[ WARN ] warnings only (no hard fail)%s\n' "$Y" "$N" + fi + else + if [ "${BOX:-0}" = "1" ]; then + printf -- ' %s┌ ERROR ┐%s %sfailed — see list above%s\n' "$BG_ERR" "$N" "$R" "$N" + else + printf -- ' %s[ ERROR ] failed — see list above%s\n' "$R" "$N" + fi + fi + [ "$FAIL_N" -eq 0 ] +} + +# --------------------------------------------------------------------------- +# Disk space on hosts / containers under test +# WARN when tight, ERROR when full / critically low +# DISK_WARN_USED_PCT=85 # free below 15% → WARN +# DISK_ERR_USED_PCT=95 # free below 5% → ERROR +# DISK_ERR_FREE_BYTES=0 # free == 0 always ERROR +# --------------------------------------------------------------------------- +: "${DISK_WARN_USED_PCT:=85}" +: "${DISK_ERR_USED_PCT:=95}" + +# Parse one POSIX `df -P` body line (no header): fs size used avail capacity mount +# size/used/avail in 1K-blocks when using df -Pk +_mon_disk_eval_line() { + local where="$1" fs="$2" size="$3" used="$4" avail="$5" cap="$6" mnt="$7" + local pct label detail + pct=${cap%%%} + # non-numeric capacity → skip + case "$pct" in + ''|*[!0-9]*) return 0 ;; + esac + label="disk ${where} ${mnt}" + detail="fs=$fs size_1k=$size used_1k=$used avail_1k=$avail use=${pct}%" + if [ "$avail" = "0" ] || [ "$pct" -ge 100 ]; then + err "disk" "${where} ${mnt} FULL / overflow" "$detail" + return 1 + fi + if [ "$pct" -ge "${DISK_ERR_USED_PCT}" ]; then + err "disk" "${where} ${mnt} critically low free space (≥${DISK_ERR_USED_PCT}% used)" "$detail" + return 1 + fi + if [ "$pct" -ge "${DISK_WARN_USED_PCT}" ]; then + warn "disk" "${where} ${mnt} free space tight (≥${DISK_WARN_USED_PCT}% used)" "$detail" + return 0 + fi + ok "$label" "$detail" + return 0 +} + +# Report all filesystems from `df -P` / `df -Pk` output (with header). +# $1 = where label (host, container:name, ssh:host) +# $2 = full df text +mon_disk_report_df() { + local where="$1" text="$2" line fs size used avail cap mnt rest ec=0 + [ -n "$text" ] || { + warn "disk" "${where}: no df output" + return 0 + } + while IFS= read -r line || [ -n "$line" ]; do + [ -n "$line" ] || continue + case "$line" in + Filesystem*|Filesystem*) continue ;; + esac + # df -P: Filesystem 1024-blocks Used Available Capacity Mounted on + # shellcheck disable=SC2086 + set -- $line + [ "$#" -ge 6 ] || continue + fs=$1 + size=$2 + used=$3 + avail=$4 + cap=$5 + shift 5 + mnt=$* + # skip special/pseudo if tiny and not root-ish + case "$fs" in + tmpfs|devtmpfs|overlay|shm|nsfs|proc|sysfs|cgroup*) + # still check root-like mounts that fill (overlay / in containers) + case "$mnt" in + /|/var|/var/*|/home|/home/*|/data|/mnt/*) ;; + *) continue ;; + esac + ;; + esac + if ! _mon_disk_eval_line "$where" "$fs" "$size" "$used" "$avail" "$cap" "$mnt"; then + ec=1 + fi + done <<<"$text" + return "$ec" +} + +# Local host: important mounts +mon_disk_check_host() { + local where text + where="${1:-host}" + text="" + text=$(df -Pk / /var /home /tmp 2>/dev/null | awk 'NR==1 || !seen[$1,$6]++' || true) + # fallback whole table + if [ -z "$text" ]; then + text=$(df -Pk 2>/dev/null || true) + fi + mon_disk_report_df "$where" "$text" +} + +# Run df inside podman container +mon_disk_check_podman() { + local cname="$1" text + local bin="${2:-podman}" + text=$("$bin" exec "$cname" df -Pk / /var /tmp 2>/dev/null || "$bin" exec "$cname" df -Pk 2>/dev/null || true) + mon_disk_report_df "ctr:${cname}" "$text" +} + +# df text already collected remotely +mon_disk_check_remote_text() { + local where="$1" text="$2" + mon_disk_report_df "$where" "$text" +} + +http_code() { + local url="$1"; shift + curl -skS --max-redirs 0 -m "${TIMEOUT}" -o /dev/null -w '%{http_code}' "$@" "$url" 2>/dev/null || echo 000 +} + +http_body() { + local url="$1" out="$2"; shift 2 + curl -skS --max-redirs 0 -m "${TIMEOUT}" -o "$out" -w '%{http_code}' "$@" "$url" 2>/dev/null || echo 000 +} + +# --------------------------------------------------------------------------- +# Currency unit map checks (wallet codec: alt_unit_names must include "0") +# --------------------------------------------------------------------------- +# Returns 0 if JSON body at $1 has usable alt_unit_names. +# Supports: +# - exchange/bank: currency_specification.alt_unit_names +# - merchant: currencies..alt_unit_names for each code +# Optional $2 = required currency code for currency_specification.currency +json_has_alt_unit_names() { + local file="$1" want_cur="${2:-}" + python3 - "$file" "$want_cur" <<'PY' +import json, sys +path, want = sys.argv[1], sys.argv[2] +try: + d = json.load(open(path)) +except Exception as e: + print(f"json-error: {e}") + sys.exit(2) + +def check_au(au, label): + if not isinstance(au, dict) or not au: + print(f"{label}: missing/empty alt_unit_names") + return False + if "0" not in au or not str(au.get("0") or "").strip(): + print(f"{label}: alt_unit_names missing non-empty key \"0\" (have {sorted(au.keys())})") + return False + print(f"{label}: alt_unit_names ok (0={au.get('0')!r}, n={len(au)})") + return True + +ok = True +cs = d.get("currency_specification") +if isinstance(cs, dict): + if want and cs.get("currency") and cs.get("currency") != want: + print(f"currency_specification.currency={cs.get('currency')!r} want {want!r}") + ok = False + if not check_au(cs.get("alt_unit_names"), "currency_specification"): + ok = False +elif "currency_specification" in d: + print("currency_specification: not an object") + ok = False + +curs = d.get("currencies") +if isinstance(curs, dict) and curs: + for code, spec in curs.items(): + if not isinstance(spec, dict): + print(f"currencies.{code}: not an object") + ok = False + continue + if not check_au(spec.get("alt_unit_names"), f"currencies.{code}"): + ok = False + +if not isinstance(cs, dict) and not (isinstance(curs, dict) and curs): + # neither shape — fail + print("no currency_specification or currencies map") + ok = False + +sys.exit(0 if ok else 1) +PY +} + +# Check one exchange base URL's /config for alt_unit_names. +# $1=label $2=base_url $3=expected currency (optional) $4=strict(1) or soft(0) +check_exchange_alt_units() { + local label="$1" base="$2" want_cur="${3:-}" strict="${4:-1}" + local f code + base="${base%/}" + f=$(mktemp) + code=$(http_body "${base}/config" "$f") + if [ "$code" != "200" ]; then + rm -f "$f" + if [ "$strict" = "1" ]; then + fail "$label /config" "HTTP $code ($base)" + else + warn "$label /config" "HTTP $code ($base)" + fi + return + fi + local out ec + set +e + out=$(json_has_alt_unit_names "$f" "$want_cur" 2>&1) + ec=$? + set -e + rm -f "$f" + if [ "$ec" -eq 0 ]; then + ok "$label alt_unit_names" "$(echo "$out" | tr '\n' '; ' | sed 's/; $//')" + else + if [ "$strict" = "1" ]; then + fail "$label alt_unit_names" "$(echo "$out" | tr '\n' '; ' | sed 's/; $//')" + else + warn "$label alt_unit_names" "$(echo "$out" | tr '\n' '; ' | sed 's/; $//')" + fi + fi +} + +# From merchant /config JSON file: walk exchanges[] and check each base_url/config. +# Local stack hosts (hacktivism.ch or $EXCHANGE_PUBLIC host) are strict; others soft. +check_merchant_listed_exchanges_alt_units() { + local mer_json="$1" + local list + list=$(python3 - "$mer_json" <<'PY' +import json, sys +d = json.load(open(sys.argv[1])) +for e in d.get("exchanges") or []: + if not isinstance(e, dict): + continue + u = (e.get("base_url") or e.get("url") or "").rstrip("/") + c = e.get("currency") or "" + if u: + print(f"{c}\t{u}") +PY +) + if [ -z "$list" ]; then + fail "merchant exchanges[]" "empty — no exchanges to check for alt_unit_names" + return + fi + local line cur url strict host + while IFS=$'\t' read -r cur url; do + [ -n "$url" ] || continue + host="${url#https://}"; host="${host#http://}"; host="${host%%/*}" + strict=1 + case "$host" in + *hacktivism.ch) strict=1 ;; + *) + # foreign exchange (e.g. taler-ops) — soft unless it is our configured EXCHANGE_PUBLIC + if [ "$url" = "${EXCHANGE_PUBLIC}" ] || [ "$url" = "${EXCHANGE_PUBLIC}/" ]; then + strict=1 + else + strict=0 + fi + ;; + esac + check_exchange_alt_units "exchange ${cur:-?} ${host}" "$url" "$cur" "$strict" + done <<<"$list" +} + +# Live bank/merchant passwords: sibling **koopa-admin-secrets** (never in admin-log). +# Override: SECRETS_ROOT=/path/to/koopa-admin-secrets/koopa/host-root +# or KOOPA_ADMIN_SECRETS=/path/to/koopa-admin-secrets +SECRETS_ROOT="${SECRETS_ROOT:-}" +_secrets_search_log() { :; } # placeholder if we later want debug +if [ -z "$SECRETS_ROOT" ]; then + _mon_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + _admin_log="$(cd "${_mon_dir}/../.." && pwd)" + # Prefer explicit env pointing at the secrets *repo* root + if [ -n "${KOOPA_ADMIN_SECRETS:-}" ] && [ -d "${KOOPA_ADMIN_SECRETS}/koopa/host-root/taler-bank" ]; then + SECRETS_ROOT="${KOOPA_ADMIN_SECRETS}/koopa/host-root" + fi +fi +if [ -z "$SECRETS_ROOT" ]; then + _mon_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + _admin_log="$(cd "${_mon_dir}/../.." && pwd)" + for d in \ + "${_admin_log}/../koopa-admin-secrets/koopa/host-root" \ + "${_admin_log}/../../koopa-admin-secrets/koopa/host-root" \ + "${HOME}/taler/src/koopa-admin-secrets/koopa/host-root" \ + "${HOME}/src/koopa/koopa-admin-secrets/koopa/host-root" \ + "${HOME}/koopa-admin-secrets/koopa/host-root" \ + "/Users/newkamek/src/koopa/koopa-admin-secrets/koopa/host-root" \ + "$(cd "${_admin_log}/../.." 2>/dev/null && pwd)/koopa-admin-secrets/koopa/host-root" + do + # resolve .. without requiring the dir to exist first for cd + if [ -d "$d/taler-bank" ]; then + SECRETS_ROOT=$(cd "$d" && pwd) + break + fi + done + unset _mon_dir _admin_log d +fi + +read_secret() { + local rel="$1" base val="" + base=$(basename "$rel") + if [ -n "${SECRETS_ROOT:-}" ]; then + if [ -f "${SECRETS_ROOT}/${rel}" ]; then + tr -d '\n\r' <"${SECRETS_ROOT}/${rel}" + return 0 + fi + # also accept flat layout under host-root/ + if [ -f "${SECRETS_ROOT}/${base}" ]; then + tr -d '\n\r' <"${SECRETS_ROOT}/${base}" + return 0 + fi + fi + # Optional: copy under ~/.config/taler-landing/ (landing-stats style) + if [ -f "${HOME}/.config/taler-landing/${base}" ]; then + tr -d '\n\r' <"${HOME}/.config/taler-landing/${base}" + return 0 + fi + # Live on koopa host /root (same basenames as deploy) + if koopa_ssh_ok; then + val=$(koopa_ssh_run 12 "tr -d '\\n\\r' /dev/null || true" 2>/dev/null || true) + if [ -n "$val" ]; then + printf '%s' "$val" + return 0 + fi + # container path used by some installs + val=$(koopa_ssh_run 12 \ + "podman exec taler-hacktivism-bank tr -d '\\n\\r' /dev/null || true" 2>/dev/null || true) + if [ -n "$val" ]; then + printf '%s' "$val" + return 0 + fi + fi + return 1 +} + +# Human hint when secrets missing (e2e prereq) +secrets_hint() { + cat </dev/null || true) + if [ -n "$cand" ] && [ -f "$cand" ]; then + if head -1 "$cand" 2>/dev/null | grep -q 'node\|mjs'; then + # shebang node script or .mjs + echo "$cand"; return 0 + fi + # follow symlink into package tree + if [ -L "$cand" ]; then + c=$(readlink -f "$cand" 2>/dev/null || true) + [ -n "$c" ] && [ -f "$c" ] && { echo "$c"; return 0; } + fi + fi + return 1 +} diff --git a/metrics.sh b/metrics.sh new file mode 100644 index 0000000..4c440ea --- /dev/null +++ b/metrics.sh @@ -0,0 +1,1431 @@ +# shellcheck shell=bash +# Shared Taler stack metrics for e2e + ladder (source after lib.sh). +# +# - Host/container load (RAM, processes, DB sizes) before/after a phase +# - Wallet coin inventory (total + by denomination) +# - Final overall statistics block +# +# Env: +# METRICS_DIR where JSON snapshots go (default $SCRATCH or /tmp) +# METRICS_LOAD=0 skip remote/host load probes +# KOOPA_SSH / SKIP_SSH as in lib.sh + +: "${METRICS_LOAD:=1}" +: "${METRICS_DIR:=${SCRATCH:-/tmp}}" +mkdir -p "$METRICS_DIR" 2>/dev/null || true + +# --------------------------------------------------------------------------- +# alt_unit_names — human amounts (Kilo-GOA / Mega-GOA / …) with base in parens +# From exchange/bank currency_specification.alt_unit_names: +# "0"→GOA, "3"→Kilo-GOA, "6"→Mega-GOA, … "-3"→Milli-GOA, … +# Example: GOA:5000 → 5 Kilo-GOA (GOA:5000) +# --------------------------------------------------------------------------- +: "${ALT_UNITS_FILE:=${METRICS_DIR}/alt_unit_names.json}" + +# Load map from public /config into ALT_UNITS_FILE. $1=optional config URL. +metrics_load_alt_units() { + local url="${1:-${EXCHANGE_PUBLIC:-https://exchange.hacktivism.ch}/config}" + local code + mkdir -p "$(dirname "$ALT_UNITS_FILE")" 2>/dev/null || true + code=$(curl -skS -m 12 -o "${ALT_UNITS_FILE}.raw" -w '%{http_code}' "$url" 2>/dev/null || echo 000) + if [ "$code" != "200" ]; then + # soft fallback: SI-style names if live config unreachable + printf '%s\n' '{"0":"GOA","3":"Kilo-GOA","6":"Mega-GOA","9":"Giga-GOA","12":"Tera-GOA","15":"Peta-GOA","18":"Exa-GOA","21":"Zetta-GOA","24":"Yotta-GOA","-1":"Deci-GOA","-2":"Centi-GOA","-3":"Milli-GOA","-6":"Micro-GOA","-8":"Atomic-GOA"}' \ + >"$ALT_UNITS_FILE" + return 1 + fi + python3 - "${ALT_UNITS_FILE}.raw" "$ALT_UNITS_FILE" <<'PY' +import json, sys +src, dst = sys.argv[1:3] +d = json.load(open(src)) +au = None +cs = d.get("currency_specification") +if isinstance(cs, dict): + au = cs.get("alt_unit_names") +if not au and isinstance(d.get("currencies"), dict): + # merchant-style: currencies.GOA.alt_unit_names + for _code, spec in d["currencies"].items(): + if isinstance(spec, dict) and spec.get("alt_unit_names"): + au = spec["alt_unit_names"] + break +if not isinstance(au, dict) or "0" not in au: + au = {"0": d.get("currency") or "GOA"} +json.dump(au, open(dst, "w"), indent=2, sort_keys=True) +PY + export ALT_UNITS_FILE + return 0 +} + +# Format one amount: "GOA:5000" → "5 Kilo-GOA (GOA:5000)" +# Uses ALT_UNITS_FILE if present. Pure stdout. +format_amount_alt() { + local amt="${1:-}" + [ -n "$amt" ] || return 0 + python3 - "$amt" "${ALT_UNITS_FILE:-}" <<'PY' +import json, sys +from decimal import Decimal, InvalidOperation, ROUND_HALF_UP + +amt = sys.argv[1].strip() +path = sys.argv[2] if len(sys.argv) > 2 else "" +alt = {} +if path: + try: + alt = json.load(open(path)) + except Exception: + alt = {} +if not alt: + alt = {"0": "GOA"} + +def parse(s): + if ":" in s: + c, v = s.split(":", 1) + return c, Decimal(v) + return "GOA", Decimal(s) + +def fmt_num(v: Decimal) -> str: + if v == v.to_integral(): + return format(int(v), "d") + s = format(v.quantize(Decimal("0.00000001"), rounding=ROUND_HALF_UP), "f") + return s.rstrip("0").rstrip(".") + +try: + cur, val = parse(amt) +except (InvalidOperation, ValueError): + print(amt) + raise SystemExit(0) + +base_name = alt.get("0") or cur +# always show canonical base form in parens +base_s = "%s:%s" % (cur, fmt_num(val)) +if val == 0: + print("0 %s (%s)" % (base_name, base_s)) + raise SystemExit(0) + +# scales: power of 10 relative to unit "0" +scales = [] +for k, name in alt.items(): + try: + scales.append((int(k), str(name))) + except Exception: + pass +scales.sort(key=lambda x: -x[0]) # largest unit first + +# pick largest scale where |value| >= 10^scale (for scale>=0), +# or for fractions the finest unit that makes the coefficient >= 1 +chosen = None # (scale, name, coeff) +absval = abs(val) +for sc, name in scales: + unit = Decimal(10) ** sc + if unit <= 0: + continue + coeff = absval / unit + if coeff >= 1: + chosen = (sc, name, coeff if val >= 0 else -coeff) + break +if chosen is None: + # smaller than smallest unit — use base + print("%s %s (%s)" % (fmt_num(val), base_name, base_s)) + raise SystemExit(0) + +sc, name, coeff = chosen +# if base unit, prefer "GOA:12" style still with parens only when alt differs +if sc == 0: + print("%s %s" % (fmt_num(val), name)) + raise SystemExit(0) +print("%s %s (%s)" % (fmt_num(coeff), name, base_s)) +PY +} + +# Format list of "CUR:n" amounts for plans / logs (space-separated → multiline or compact) +format_amount_list_alt() { + local a out="" + for a in "$@"; do + [ -n "$out" ] && out="$out " + out="${out}$(format_amount_alt "$a")" + done + printf '%s\n' "$out" +} + +# --- coin inventory from wallet-cli dump-coins --- +# Writes JSON to $1; prints one-line summary to stdout. +# Sets COINS_TOTAL, COINS_FRESH, COINS_SPENT, COINS_AMOUNT_CIRC, COINS_SUMMARY. +metrics_wallet_coins() { + local out="${1:-$METRICS_DIR/coins.json}" + local dump="${METRICS_DIR}/dump-coins.raw" + mkdir -p "$(dirname "$out")" 2>/dev/null || true + COINS_TOTAL=0 + COINS_FRESH=0 + COINS_SPENT=0 + COINS_AMOUNT_CIRC="0" + COINS_SUMMARY="(no coins)" + # Prefer wcli() from caller (e2e/ladder). Do not pass a leading timeout number — + # ladder wcli() has no optional secs arg (would become a wallet subcommand). + if type wcli >/dev/null 2>&1; then + wcli advanced dump-coins >"$dump" 2>/dev/null || true + elif [ -n "${CLI_JS:-}" ] && [ -f "${CLI_JS}" ]; then + node "$CLI_JS" --wallet-db="${WDB:-}" --no-throttle advanced dump-coins >"$dump" 2>/dev/null || true + elif [ -n "${WALLET_CLI:-}" ] && [ -n "${WDB:-}" ]; then + node "$WALLET_CLI" --wallet-db="${WDB}" --no-throttle --skip-defaults advanced dump-coins >"$dump" 2>/dev/null || true + else + echo "$COINS_SUMMARY" + printf '%s\n' '{"ok":false,"reason":"no-wcli"}' >"$out" + return 1 + fi + python3 - "$dump" "$out" "${CUR:-GOA}" "${ALT_UNITS_FILE:-}" <<'PY' +import json, re, sys +from collections import Counter, defaultdict +from decimal import Decimal, InvalidOperation, ROUND_HALF_UP + +raw_path, out_path, cur = sys.argv[1:4] +alt_path = sys.argv[4] if len(sys.argv) > 4 else "" +raw = open(raw_path).read() if raw_path else "" +d = None +for m in re.finditer(r"\{", raw): + try: + d = json.loads(raw[m.start():]) + if isinstance(d, dict) and ("coins" in d or "coin" in d): + break + except Exception: + d = None +coins = [] +if isinstance(d, dict): + coins = d.get("coins") or d.get("coin") or [] + +alt = {} +if alt_path: + try: + alt = json.load(open(alt_path)) + except Exception: + alt = {} +if not alt: + alt = {"0": cur} + +def parse_amt(s): + """'GOA:10' / '10' → (currency, Decimal) or (cur, 0).""" + s = str(s or "").strip() + if not s or s == "?": + return cur, Decimal(0) + if ":" in s: + c, v = s.split(":", 1) + try: + return c, Decimal(v) + except InvalidOperation: + return c, Decimal(0) + try: + return cur, Decimal(s) + except InvalidOperation: + return cur, Decimal(0) + +def fmt_num(v: Decimal) -> str: + if v == v.to_integral(): + return format(int(v), "d") + s = format(v.quantize(Decimal("0.00000001"), rounding=ROUND_HALF_UP), "f") + return s.rstrip("0").rstrip(".") + +def fmt_amt(c, v: Decimal) -> str: + return "%s:%s" % (c, fmt_num(v)) + +def fmt_amt_alt(c, v: Decimal) -> str: + """5 Kilo-GOA (GOA:5000) using alt_unit_names.""" + base_s = fmt_amt(c, v) + base_name = alt.get("0") or c + if v == 0: + return "0 %s (%s)" % (base_name, base_s) + scales = [] + for k, name in alt.items(): + try: + scales.append((int(k), str(name))) + except Exception: + pass + scales.sort(key=lambda x: -x[0]) + absval = abs(v) + chosen = None + for sc, name in scales: + unit = Decimal(10) ** sc + coeff = absval / unit + if coeff >= 1: + chosen = (sc, name, coeff if v >= 0 else -coeff) + break + if chosen is None: + return "%s %s (%s)" % (fmt_num(v), base_name, base_s) + sc, name, coeff = chosen + if sc == 0: + return "%s %s" % (fmt_num(v), name) + return "%s %s (%s)" % (fmt_num(coeff), name, base_s) + +by_denom_all = Counter() # all coins +by_denom_circ = Counter() # non-spent +by_denom_spent = Counter() +by_status = Counter() +amt_circ = defaultdict(lambda: Decimal(0)) # currency → amount +amt_spent = defaultdict(lambda: Decimal(0)) +amt_all = defaultdict(lambda: Decimal(0)) +circ_n = 0 +spent_n = 0 +for c in coins: + if not isinstance(c, dict): + continue + dv = c.get("denomValue") or c.get("value") or c.get("denom_value") or "?" + st = str(c.get("coinStatus") or c.get("status") or "?") + by_denom_all[dv] += 1 + by_status[st] += 1 + ac, av = parse_amt(dv) + amt_all[ac] += av + st_l = st.lower() + if "spent" in st_l or "delete" in st_l or "dirty" in st_l: + spent_n += 1 + by_denom_spent[dv] += 1 + amt_spent[ac] += av + else: + circ_n += 1 + by_denom_circ[dv] += 1 + amt_circ[ac] += av + +total = len(coins) + +def denom_key(k): + try: + return float(str(k).split(":", 1)[-1]) + except Exception: + return 0.0 + +def denom_list(counter, spent_counter=None): + out = [] + for k in sorted(counter.keys(), key=denom_key): + item = {"denom": k, "count": counter[k]} + ac, av = parse_amt(k) + item["unit_value"] = str(av) + item["amount"] = fmt_amt(ac, av * counter[k]) + if spent_counter is not None: + item["spent_count"] = spent_counter.get(k, 0) + out.append(item) + return out + +# human: "10 GOA×3 (=30 GOA) …" and alt: "3 Kilo-GOA (GOA:3000)×1" +parts = [] +parts_alt = [] +for item in denom_list(by_denom_circ): + ac, av = parse_amt(item["denom"]) + total_v = av * item["count"] + parts.append("%s×%d (=%s)" % (item["denom"], item["count"], item["amount"])) + parts_alt.append( + "%s×%d (=%s)" + % (fmt_amt_alt(ac, av), item["count"], fmt_amt_alt(ac, total_v)) + ) +circ_amt_s = " ".join(fmt_amt(c, amt_circ[c]) for c in sorted(amt_circ)) +circ_amt_alt = " ".join(fmt_amt_alt(c, amt_circ[c]) for c in sorted(amt_circ)) +spent_amt_s = " ".join(fmt_amt(c, amt_spent[c]) for c in sorted(amt_spent)) if amt_spent else "0" +spent_amt_alt = " ".join(fmt_amt_alt(c, amt_spent[c]) for c in sorted(amt_spent)) if amt_spent else "0" +if not circ_amt_s: + circ_amt_s = "%s:0" % cur + circ_amt_alt = "0 %s (%s:0)" % (alt.get("0") or cur, cur) +summary = ( + "coins=%d in_circ=%d spent=%d amount_circ=%s | %s" + % (total, circ_n, spent_n, circ_amt_alt, " ".join(parts_alt) if parts_alt else "(empty)") +) +# enrich denom lists with human labels +def enrich(lst): + out = [] + for item in lst: + ac, av = parse_amt(item["denom"]) + total_v = av * int(item["count"]) + item = dict(item) + item["denom_alt"] = fmt_amt_alt(ac, av) + item["amount_alt"] = fmt_amt_alt(ac, total_v) + out.append(item) + return out + +report = { + "ok": True, + "currency": cur, + "total_coins": total, + "in_circulation": circ_n, + "spent": spent_n, + "amount_in_circulation": {c: str(amt_circ[c]) for c in amt_circ}, + "amount_in_circulation_s": circ_amt_s, + "amount_in_circulation_alt": circ_amt_alt, + "amount_spent": {c: str(amt_spent[c]) for c in amt_spent}, + "amount_spent_s": spent_amt_s, + "amount_spent_alt": spent_amt_alt, + "by_status": dict(by_status), + "by_denom": enrich(denom_list(by_denom_all, by_denom_spent)), + "by_denom_circulation": enrich(denom_list(by_denom_circ)), + "by_denom_spent": enrich(denom_list(by_denom_spent)), + "summary": summary, + "alt_unit_names": alt, +} +json.dump(report, open(out_path, "w"), indent=2) +print(summary) +open(out_path + ".total", "w").write(str(total)) +open(out_path + ".circ", "w").write(str(circ_n)) +open(out_path + ".spent", "w").write(str(spent_n)) +open(out_path + ".amt", "w").write(circ_amt_s) +PY + COINS_TOTAL=$(cat "${out}.total" 2>/dev/null || echo 0) + COINS_FRESH=$(cat "${out}.circ" 2>/dev/null || echo 0) + COINS_SPENT=$(cat "${out}.spent" 2>/dev/null || echo 0) + COINS_AMOUNT_CIRC=$(cat "${out}.amt" 2>/dev/null || echo "0") + COINS_SUMMARY=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("summary","?"))' "$out" 2>/dev/null || echo "$COINS_SUMMARY") + # history TSV for end-of-run table + if [ -n "${METRICS_DIR:-}" ]; then + hist="${METRICS_DIR}/coins-history.tsv" + if [ ! -f "$hist" ]; then + printf 'ts\tlabel\ttotal\tin_circ\tspent\tamount_circ\tsummary\n' >"$hist" + fi + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date)" \ + "${METRICS_COINS_LABEL:-snap}" \ + "$COINS_TOTAL" "$COINS_FRESH" "$COINS_SPENT" "$COINS_AMOUNT_CIRC" \ + "$(printf '%s' "$COINS_SUMMARY" | tr '\t' ' ')" >>"$hist" + fi + echo "$COINS_SUMMARY" +} + +# Emit coin inventory as monitoring info lines (label = step marker). +# $1=label $2=optional json path (default METRICS_DIR/coins-.json) +metrics_report_coins() { + local label="$1" + local safe + safe=$(printf '%s' "$label" | tr -c 'A-Za-z0-9._-' '_' | head -c 80) + local out="${2:-${METRICS_DIR}/coins-${safe}.json}" + local line prev="${METRICS_DIR}/coins-prev.json" + if [ -z "${WDB:-}" ] && [ -z "${WALLET_CLI:-}" ] && ! type wcli >/dev/null 2>&1; then + info "coins ${label}" "skipped (no wallet yet)" + return 0 + fi + METRICS_COINS_LABEL="$label" + export METRICS_COINS_LABEL + if ! metrics_wallet_coins "$out" >/dev/null; then + warn "coins ${label}" "dump-coins failed / empty" + return 1 + fi + # multi-line detail from JSON + while IFS= read -r line; do + [ -n "$line" ] || continue + info "coins ${label}" "$line" + done < <(python3 - "$out" <<'PY' +import json, sys +d = json.load(open(sys.argv[1])) +if not d.get("ok"): + print("unavailable: %s" % d.get("reason", "?")) + raise SystemExit(0) +amt = d.get("amount_in_circulation_alt") or d.get("amount_in_circulation_s") or "0" +print( + "n=%s in_circulation=%s spent=%s amount_circ=%s" + % (d.get("total_coins"), d.get("in_circulation"), d.get("spent"), amt) +) +st = d.get("by_status") or {} +if st: + print("status " + " ".join("%s=%s" % (k, st[k]) for k in sorted(st))) +circ = d.get("by_denom_circulation") or [] +if circ: + bits = [] + for x in circ: + dlab = x.get("denom_alt") or x.get("denom") + alab = x.get("amount_alt") or x.get("amount") + bits.append("%s×%d (=%s)" % (dlab, x["count"], alab)) + print("denoms_in_circ " + " ".join(bits)) +else: + print("denoms_in_circ (none)") +spent = d.get("by_denom_spent") or [] +if spent: + bits = [] + for x in spent: + dlab = x.get("denom_alt") or x.get("denom") + alab = x.get("amount_alt") or x.get("amount") + bits.append("%s×%d (=%s)" % (dlab, x["count"], alab)) + print("denoms_spent " + " ".join(bits)) +sp = d.get("amount_spent_alt") or d.get("amount_spent_s") +if sp and sp not in ("0", "0 GOA (GOA:0)"): + print("amount_spent %s" % sp) +PY + ) + # delta vs previous snapshot at this run + if [ -f "$prev" ]; then + local dsum + dsum=$(metrics_coins_delta "$prev" "$out" "${METRICS_DIR}/coins-delta-${safe}.json" 2>/dev/null || true) + if [ -n "$dsum" ]; then + info "coins ${label} Δ" "$dsum" + fi + fi + cp -f "$out" "$prev" 2>/dev/null || true + cp -f "$out" "${METRICS_DIR}/coins-final.json" 2>/dev/null || true + return 0 +} + +# Diff two coin JSON snapshots → new coins this step +metrics_coins_delta() { + local before="${1:-}" after="${2:-}" out="${3:-$METRICS_DIR/coins-delta.json}" + python3 - "$before" "$after" "$out" <<'PY' +import json, sys +from collections import Counter +def load(p): + try: + d=json.load(open(p)) + return d + except Exception: + return {} +b, a = load(sys.argv[1]), load(sys.argv[2]) +outp = sys.argv[3] +def bag(d): + c=Counter() + for item in d.get("by_denom") or []: + c[item.get("denom") or "?"] += int(item.get("count") or 0) + return c +bb, aa = bag(b), bag(a) +delta = aa - bb +parts = ["%s×%+d" % (k, delta[k]) for k in sorted(delta.keys(), key=lambda x: float(str(x).split(":")[-1]) if ":" in str(x) or str(x).replace(".","").isdigit() else 0) if delta[k]] +new_total = int(a.get("total_coins") or 0) - int(b.get("total_coins") or 0) +rep = { + "new_coins": new_total, + "total_after": int(a.get("total_coins") or 0), + "circulation_after": int(a.get("in_circulation") or 0), + "delta_by_denom": {k: delta[k] for k in delta}, + "summary": ("new=%+d total_now=%s | %s" % ( + new_total, a.get("total_coins"), " ".join(parts) if parts else "(no denom change)")), +} +json.dump(rep, open(outp, "w"), indent=2) +print(rep["summary"]) +PY +} + +# --- Taler stack load (host + bank/exchange/merchant) --- +# Writes JSON to $1. RAM, process counts, DB sizes, disk I/O counters. +# koopa: KOOPA_SSH (+ fallbacks). stage-lfp: INSIDE_SSH (stagepaysan) + stage-lfp-* names. +metrics_taler_load() { + local out="${1:-$METRICS_DIR/load.json}" + local label="${2:-snap}" + mkdir -p "$(dirname "$out")" 2>/dev/null || true + if [ "${METRICS_LOAD}" = "0" ]; then + printf '%s\n' "{\"ok\":false,\"reason\":\"disabled\",\"label\":\"$label\"}" >"$out" + return 0 + fi + local raw="" + local remote_py + local _m_bank _m_mer _m_ex _m_ssh + _m_bank="taler-hacktivism-bank" + _m_mer="taler-hacktivism" + _m_ex="taler-hacktivism-exchange-ansible" + _m_ssh="" + if [ "${INSIDE_PROFILE:-}" = "stage-lfp" ] \ + || [ "${EXPECT_CURRENCY:-}" = "TESTPAYSAN" ] \ + || { [ -n "${INSIDE_SSH:-}" ] && [ "${LOCAL_STACK:-0}" != "1" ]; }; then + _m_bank="${INSIDE_BANK_CTR:-stage-lfp-bank}" + _m_mer="${INSIDE_MERCHANT_CTR:-stage-lfp-merchant}" + _m_ex="${INSIDE_EXCHANGE_CTR:-stage-lfp-exchange-ansible}" + _m_ssh="${INSIDE_SSH:-francpaysan-stage-user}" + fi + remote_py=$(cat </dev/null | head -c 240) + blob=$(printf '%s %s' "$nm" "$cmd" | tr 'A-Z' 'a-z') + case "$blob" in + *postgres*|*postmaster*) n_postgres=$((n_postgres+1)); rss_postgres=$((rss_postgres+r)) ;; + *libeufin*|*mainkt*) n_libeufin=$((n_libeufin+1)); rss_libeufin=$((rss_libeufin+r)) ;; + *taler-merchant*) n_taler_merchant=$((n_taler_merchant+1)); rss_taler_merchant=$((rss_taler_merchant+r)) ;; + *taler-exchange*) n_taler_exchange=$((n_taler_exchange+1)); rss_taler_exchange=$((rss_taler_exchange+r)) ;; + *nginx*) n_nginx=$((n_nginx+1)); rss_nginx=$((rss_nginx+r)) ;; + *java*) n_java=$((n_java+1)); rss_java=$((rss_java+r)) ;; + *) n_other=$((n_other+1)); rss_other=$((rss_other+r)) ;; + esac +done +printf 'TOTAL %s %s\n' "$n" "$rss" +[ "$n_postgres" -gt 0 ] && printf 'ROLE postgres %s %s\n' "$n_postgres" "$rss_postgres" +[ "$n_libeufin" -gt 0 ] && printf 'ROLE libeufin %s %s\n' "$n_libeufin" "$rss_libeufin" +[ "$n_taler_merchant" -gt 0 ] && printf 'ROLE taler-merchant %s %s\n' "$n_taler_merchant" "$rss_taler_merchant" +[ "$n_taler_exchange" -gt 0 ] && printf 'ROLE taler-exchange %s %s\n' "$n_taler_exchange" "$rss_taler_exchange" +[ "$n_nginx" -gt 0 ] && printf 'ROLE nginx %s %s\n' "$n_nginx" "$rss_nginx" +[ "$n_java" -gt 0 ] && printf 'ROLE java %s %s\n' "$n_java" "$rss_java" +[ "$n_other" -gt 0 ] && printf 'ROLE other %s %s\n' "$n_other" "$rss_other" +""" + o = podman_sh(name, script, t=25) + by = {} + n = None + rss_kb = None + for line in o.splitlines(): + p = line.split() + if not p: + continue + if p[0] == "TOTAL" and len(p) >= 3: + try: + n = int(p[1]) + rss_kb = int(p[2]) + except Exception: + pass + elif p[0] == "ROLE" and len(p) >= 4: + try: + by[p[1]] = {"n": int(p[2]), "rss_b": int(p[3]) * 1024} + except Exception: + pass + if n is None: + nn = sh(f"podman exec {name} sh -c 'ps -e --no-headers 2>/dev/null | wc -l'").strip() + n = int(nn) if nn.isdigit() else None + return { + "proc_total": n, + "rss_total_b": (rss_kb * 1024) if rss_kb is not None else None, + "by_role": by, + } + +def dbs(name): + o = sh( + f"podman exec {name} su -s /bin/bash postgres -c " + + json.dumps( + "psql -Atc \"SELECT datname||'|'||pg_database_size(datname)||'|'||pg_size_pretty(pg_database_size(datname)) " + "FROM pg_database WHERE datistemplate=false ORDER BY 1\"" + ) + ) + out=[] + for line in o.splitlines(): + p=line.strip().split("|") + if len(p)>=3: + try: out.append({"name":p[0],"size_b":int(p[1]),"size_pretty":p[2]}) + except Exception: pass + du=sh(f"podman exec {name} sh -c 'du -sb /var/lib/postgresql 2>/dev/null | cut -f1'").strip() + pretty=sh(f"podman exec {name} sh -c 'du -sh /var/lib/postgresql 2>/dev/null | cut -f1'").strip() + return {"databases": out, "pgdata_bytes": int(du) if du.isdigit() else None, "pgdata_pretty": pretty or None} + +components={} +for role,cname in CTRS: + if not running(cname): + components[role]={"container":cname,"running":False} + continue + components[role]={ + "container":cname,"running":True, + "podman_stats":stats(cname), + "processes":procs(cname), + "databases":dbs(cname), + } + +host_ps=sh("ps -eo comm=") +host_counts={k:sum(1 for line in host_ps.splitlines() if k in line.lower()) + for k in ("postgres","nginx","caddy","podman","conmon","pasta")} +print(json.dumps({ + "ok": True, + "ts": time.strftime("%Y-%m-%dT%H:%M:%S%z"), + "host": { + "hostname": sh("hostname").strip(), + "loadavg": loadavg(), + "nproc": os.cpu_count(), + "memory": meminfo(), + "disk_io": disk_io(), + "process_counts": host_counts, + }, + "taler": components, +})) +PY +) + # Prefer stagepaysan (stage-lfp), else koopa SSH, else local podman only on LOCAL_STACK. + if [ -n "${_m_ssh}" ] && type mon_ssh_ok >/dev/null 2>&1 && mon_ssh_ok "${_m_ssh}"; then + raw=$(printf '%s' "$remote_py" | with_timeout "${METRICS_LOAD_SSH_TIMEOUT:-90}" \ + ssh "${SSH_BASE_OPTS[@]}" "${_m_ssh}" python3 - 2>/dev/null || true) + elif [ "${SKIP_SSH:-0}" != "1" ] && type koopa_ssh_ok >/dev/null 2>&1 && koopa_ssh_ok; then + if type koopa_ssh_python >/dev/null 2>&1; then + raw=$(printf '%s' "$remote_py" | koopa_ssh_python "${METRICS_LOAD_SSH_TIMEOUT:-90}" 2>/dev/null || true) + else + raw=$(printf '%s' "$remote_py" | with_timeout "${METRICS_LOAD_SSH_TIMEOUT:-90}" \ + ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" python3 - 2>/dev/null || true) + fi + elif [ "${LOCAL_STACK:-0}" = "1" ] && command -v podman >/dev/null 2>&1; then + raw=$(printf '%s' "$remote_py" | python3 - 2>/dev/null || true) + else + printf '%s\n' "{\"ok\":false,\"reason\":\"no-ssh-for-load\",\"label\":\"$label\"}" >"$out" + return 0 + fi + if [ -z "$raw" ]; then + printf '%s\n' "{\"ok\":false,\"reason\":\"probe-failed\",\"label\":\"$label\",\"ssh\":\"${_m_ssh:-${KOOPA_SSH:-?}}\"}" >"$out" + return 1 + fi + printf '%s\n' "$raw" | python3 -c ' +import sys,json,re +t=sys.stdin.read(); obj=None +for m in re.finditer(r"\{", t): + try: + obj=json.loads(t[m.start():]) + if isinstance(obj, dict) and (obj.get("host") or obj.get("taler") or obj.get("ok") is False): + break + except Exception: + obj=None +if not obj: obj={"ok":False,"reason":"parse"} +obj["label"]=sys.argv[1] +json.dump(obj, open(sys.argv[2],"w"), indent=2) +' "$label" "$out" +} + +# One-line human summaries (no leading indent) for info()/ok() detail fields. +metrics_load_lines() { + local f="$1" + python3 - "$f" <<'PY' +import json, sys +try: + d = json.load(open(sys.argv[1])) +except Exception as e: + print(f"unreadable: {e}") + raise SystemExit(0) +if not d.get("ok"): + print(f"unavailable: {d.get('reason', 'n/a')} ssh={d.get('ssh', '')}".strip()) + raise SystemExit(0) +h = d.get("host") or {} +mem = h.get("memory") or {} +la = h.get("loadavg") or [] + +def gi(b): + if b is None: + return "?" + return f"{b/1024/1024/1024:.2f}GiB" + +def mi(b): + if b is None: + return "?" + return f"{b/1024/1024:.0f}MiB" + +la_s = " ".join(f"{x:.2f}" for x in la) if la else "?" +tot = mem.get("mem_total_b") +avail = mem.get("mem_available_b") +used = (tot - avail) if (tot is not None and avail is not None) else None +print( + f"host loadavg=[{la_s}] nproc={h.get('nproc')} " + f"mem_used={gi(used)} avail={gi(avail)} total={gi(tot)}" +) +pc = h.get("process_counts") or {} +if pc: + bits = [f"{k}={v}" for k, v in sorted(pc.items()) if v] + if bits: + print("host procs " + " ".join(bits)) +for role in ("bank", "exchange", "merchant"): + c = (d.get("taler") or {}).get(role) or {} + if not c: + continue + if not c.get("running"): + print(f"{role} DOWN ({c.get('container')})") + continue + pr = c.get("processes") or {} + st = c.get("podman_stats") or {} + roles = pr.get("by_role") or {} + role_bits = [] + for k in ("libeufin", "taler-exchange", "taler-merchant", "postgres", "nginx", "java"): + if k in roles: + role_bits.append(f"{k}={mi(roles[k].get('rss_b'))}") + dbs = c.get("databases") or {} + db_s = ",".join( + f"{x.get('name')}={x.get('size_pretty')}" + for x in (dbs.get("databases") or [])[:5] + ) + # podman MemUsage often reports 0B under pasta/cgroup — prefer /proc RSS + mem_u = st.get("mem_usage") or "" + mem_bit = "" + if mem_u and not str(mem_u).startswith("0B"): + mem_bit = f" podman_mem={mem_u}" + cpu = st.get("cpu_pct") or "?" + blk = st.get("block_io") or "" + blk_bit = "" + if blk and blk not in ("0B / 0B", "0B/0B", "-- / --"): + blk_bit = f" block={blk}" + line = ( + f"{role} rss={gi(pr.get('rss_total_b'))} procs={pr.get('proc_total')} " + f"cpu={cpu}{mem_bit}{blk_bit}" + ) + if role_bits: + line += " | " + " ".join(role_bits) + if db_s: + line += f" | db:{db_s}" + if dbs.get("pgdata_pretty"): + line += f" pgdata={dbs.get('pgdata_pretty')}" + print(line) +PY +} + +# Snapshot load and emit as monitoring info lines (withdraw/pay phase markers). +# $1=json path $2=label (e.g. after-withdraw) +metrics_report_load() { + local out="$1" label="$2" + local line rc=0 + if [ "${METRICS_LOAD}" = "0" ]; then + info "load ${label}" "skipped (METRICS_LOAD=0)" + return 0 + fi + if [ "${SKIP_SSH:-0}" = "1" ] && ! command -v podman >/dev/null 2>&1; then + info "load ${label}" "skipped (no SSH / no local podman)" + return 0 + fi + if ! metrics_taler_load "$out" "$label"; then + warn "load ${label}" "probe failed via ${KOOPA_SSH:-?} — try KOOPA_SSH=koopa-external" + return 1 + fi + while IFS= read -r line; do + [ -n "$line" ] || continue + case "$line" in + unavailable:*|unreadable:*) + warn "load ${label}" "$line" + rc=1 + ;; + *) + info "load ${label}" "$line" + ;; + esac + done < <(metrics_load_lines "$out") + return "$rc" +} + +# Human one-screen summary of a load JSON +metrics_print_load() { + local f="$1" title="${2:-load}" + python3 - "$f" "$title" <<'PY' +import json,sys +try: + d=json.load(open(sys.argv[1])) +except Exception as e: + print(f" ({sys.argv[2]}: unreadable {e})") + raise SystemExit +if not d.get("ok"): + print(f" ({sys.argv[2]}: {d.get('reason','n/a')})") + raise SystemExit +h=d.get("host") or {} +mem=h.get("memory") or {} +la=h.get("loadavg") or [] +def gi(b): + if b is None: return "?" + return f"{b/1024/1024/1024:.2f} GiB" +print(f" host loadavg {la} nproc={h.get('nproc')} mem_avail={gi(mem.get('mem_available_b'))}/{gi(mem.get('mem_total_b'))}") +dio=h.get("disk_io") or {} +if dio: + print(f" host disk Δsectors read={dio.get('sectors_read')} written={dio.get('sectors_written')} (~write {gi(dio.get('approx_write_bytes'))})") +for role in ("bank","exchange","merchant"): + c=(d.get("taler") or {}).get(role) or {} + if not c: + continue + if not c.get("running"): + print(f" {role:8} DOWN ({c.get('container')})") + continue + pr=c.get("processes") or {} + rss=pr.get("rss_total_b") + n=pr.get("proc_total") + roles=pr.get("by_role") or {} + bits=[] + for k in ("libeufin","taler-exchange","taler-merchant","postgres","nginx"): + if k in roles: + bits.append(f"{k}:n={roles[k].get('n')} rss={gi(roles[k].get('rss_b'))}") + dbs=c.get("databases") or {} + db_s=", ".join(f"{x.get('name')}={x.get('size_pretty')}" for x in (dbs.get("databases") or [])[:6]) + pg=dbs.get("pgdata_pretty") or "" + st=c.get("podman_stats") or {} + print(f" {role:8} procs={n} rss={gi(rss)} cpu={st.get('cpu_pct','?')} block={st.get('block_io','?')}") + if bits: + print(f" " + " ".join(bits)) + if db_s or pg: + print(f" db: {db_s}" + (f" pgdata={pg}" if pg else "")) +PY +} + +# Diff two load snaps: highlight RAM/proc/DB growth for taler roles +metrics_print_load_delta() { + local before="$1" after="$2" + python3 - "$before" "$after" <<'PY' +import json,sys +def load(p): + try: return json.load(open(p)) + except Exception: return {} +b,a=load(sys.argv[1]),load(sys.argv[2]) +if not b.get("ok") or not a.get("ok"): + print(" (load delta unavailable)") + raise SystemExit +def gi(x): + if x is None: return None + return x/1024/1024/1024 +print(" --- delta (after − before) ---") +# host load +bla=(b.get("host") or {}).get("loadavg") or [0,0,0] +ala=(a.get("host") or {}).get("loadavg") or [0,0,0] +if bla and ala: + print(f" loadavg1 {bla[0]:.2f} → {ala[0]:.2f} (Δ {ala[0]-bla[0]:+.2f})") +bm=(b.get("host") or {}).get("memory") or {} +am=(a.get("host") or {}).get("memory") or {} +if bm.get("mem_available_b") is not None and am.get("mem_available_b") is not None: + print(f" mem_avail {gi(bm['mem_available_b']):.2f} → {gi(am['mem_available_b']):.2f} GiB (Δ {gi(am['mem_available_b'])-gi(bm['mem_available_b']):+.3f} GiB)") +bd=(b.get("host") or {}).get("disk_io") or {} +ad=(a.get("host") or {}).get("disk_io") or {} +if bd.get("sectors_written") is not None and ad.get("sectors_written") is not None: + dw=(ad["sectors_written"]-bd["sectors_written"])*512 + dr=(ad["sectors_read"]-bd["sectors_read"])*512 + print(f" disk I/O write≈{dw/1024/1024:.1f} MiB read≈{dr/1024/1024:.1f} MiB (during phase)") +for role in ("bank","exchange","merchant"): + bc=(b.get("taler") or {}).get(role) or {} + ac=(a.get("taler") or {}).get(role) or {} + if not ac.get("running"): + continue + br=(bc.get("processes") or {}).get("rss_total_b") + ar=(ac.get("processes") or {}).get("rss_total_b") + bn=(bc.get("processes") or {}).get("proc_total") + an=(ac.get("processes") or {}).get("proc_total") + line=f" {role:8}" + if br is not None and ar is not None: + line+=f" rss {gi(br):.3f}→{gi(ar):.3f} GiB (Δ{gi(ar)-gi(br):+.3f})" + if bn is not None and an is not None: + line+=f" procs {bn}→{an} (Δ{an-bn:+d})" + # DB sizes + def dbmap(c): + m={} + for x in ((c.get("databases") or {}).get("databases") or []): + m[x.get("name")]=x.get("size_b") + return m + bdb,adb=dbmap(bc),dbmap(ac) + dbits=[] + for name in sorted(set(bdb)|set(adb)): + bb,aa=bdb.get(name),adb.get(name) + if bb is not None and aa is not None and aa!=bb: + dbits.append(f"{name} {aa-bb:+d}B") + elif aa is not None and bb is None: + dbits.append(f"{name}={aa}B") + if dbits: + line+=" dbΔ["+", ".join(dbits)+"]" + print(line) +PY +} + +# Record a successful withdraw/pay amount for end statistics (one amount per line). +# $1=withdrawn|spent $2=amount (CUR:n) +metrics_record_flow() { + local kind="$1" amt="$2" + [ -n "$amt" ] || return 0 + mkdir -p "${METRICS_DIR}" 2>/dev/null || true + case "$kind" in + withdrawn|withdraw) printf '%s\n' "$amt" >>"${METRICS_DIR}/flow-withdrawn.txt" ;; + spent|pay) printf '%s\n' "$amt" >>"${METRICS_DIR}/flow-spent.txt" ;; + *) return 1 ;; + esac +} + +# Overall end-of-run statistics block +# Files under METRICS_DIR: +# coins-final.json, coins-history.tsv, perf-summary.json, +# load-before/after.json, flow-withdrawn.txt, flow-spent.txt, +# optional ladder TSV via METRICS_WITHDRAW_TSV / METRICS_PAY_TSV +metrics_print_overall() { + local title="${1:-overall statistics}" + section "statistics · $title" + metrics_print_final_stats || true + [ -n "${METRICS_EXTRA_LINES:-}" ] && printf '%s\n' "$METRICS_EXTRA_LINES" +} + +# Rich final dashboard: coins, withdrawn/spent, performance, tendencies +metrics_print_final_stats() { + python3 - \ + "${METRICS_DIR:-/tmp}" \ + "${ALT_UNITS_FILE:-}" \ + "${METRICS_WITHDRAW_TSV:-}" \ + "${METRICS_PAY_TSV:-}" \ + "${CUR:-GOA}" \ + <<'PY' +import csv, json, os, sys +from decimal import Decimal, InvalidOperation, ROUND_HALF_UP +from pathlib import Path + +mdir = Path(sys.argv[1]) +alt_path = sys.argv[2] or "" +wd_tsv = sys.argv[3] or "" +pay_tsv = sys.argv[4] or "" +cur = sys.argv[5] or "GOA" + +alt = {} +if alt_path and Path(alt_path).is_file(): + try: + alt = json.load(open(alt_path)) + except Exception: + alt = {} +if not alt: + alt = {"0": cur} + +def D(x, default=Decimal(0)): + try: + return Decimal(str(x)) + except Exception: + return default + +def parse_amt(s): + s = str(s or "").strip() + if not s: + return cur, Decimal(0) + if ":" in s: + c, v = s.split(":", 1) + try: + return c, Decimal(v) + except InvalidOperation: + return c, Decimal(0) + try: + return cur, Decimal(s) + except InvalidOperation: + return cur, Decimal(0) + +def fmt_num(v: Decimal) -> str: + if v == v.to_integral(): + return format(int(v), "d") + s = format(v.quantize(Decimal("0.00000001"), rounding=ROUND_HALF_UP), "f") + return s.rstrip("0").rstrip(".") + +def fmt_amt(c, v: Decimal) -> str: + return "%s:%s" % (c, fmt_num(v)) + +def fmt_alt(c, v: Decimal) -> str: + base_s = fmt_amt(c, v) + base_name = alt.get("0") or c + if v == 0: + return "0 %s (%s)" % (base_name, base_s) + scales = [] + for k, name in alt.items(): + try: + scales.append((int(k), str(name))) + except Exception: + pass + scales.sort(key=lambda x: -x[0]) + absval = abs(v) + for sc, name in scales: + unit = Decimal(10) ** sc + coeff = absval / unit + if coeff >= 1: + if sc == 0: + return "%s %s" % (fmt_num(v), name) + return "%s %s (%s)" % (fmt_num(coeff if v >= 0 else -coeff), name, base_s) + return "%s %s (%s)" % (fmt_num(v), base_name, base_s) + +def sum_amount_file(path: Path): + total = Decimal(0) + ccy = cur + n = 0 + if not path.is_file(): + return ccy, total, n + for line in path.read_text().splitlines(): + line = line.strip() + if not line: + continue + c, v = parse_amt(line) + ccy = c + total += v + n += 1 + return ccy, total, n + +def sum_tsv_amounts(path, amount_col="amount", status_ok=None): + total = Decimal(0) + ccy = cur + n = 0 + if not path or not Path(path).is_file(): + return ccy, total, n + with open(path, newline="") as f: + for row in csv.DictReader(f, delimiter="\t"): + st = (row.get("status") or "").upper() + if status_ok is not None and st not in status_ok: + continue + c, v = parse_amt(row.get(amount_col) or "") + if v <= 0: + continue + ccy = c + total += v + n += 1 + return ccy, total, n + +def trend_label(delta, eps=Decimal("0.01")): + if delta > eps: + return "↑ rising" + if delta < -eps: + return "↓ falling" + return "→ stable" + +def half_trend(values): + """Compare avg of second half vs first half of a list of numbers.""" + xs = [float(x) for x in values if x is not None] + if len(xs) < 4: + if len(xs) < 2: + return "n/a (few samples)", None, None + a, b = xs[0], xs[-1] + return trend_label(Decimal(str(b - a))), a, b + mid = len(xs) // 2 + first = sum(xs[:mid]) / mid + second = sum(xs[mid:]) / (len(xs) - mid) + d = second - first + lab = "↑ slowing (worse)" if d > 0.05 * max(abs(first), 1) else ( + "↓ faster (better)" if d < -0.05 * max(abs(first), 1) else "→ steady" + ) + return lab, first, second + +print("") +print("╔══════════════════════════════════════════════════════════════╗") +print("║ FINAL STATISTICS ║") +print("╚══════════════════════════════════════════════════════════════╝") + +# --- coins snapshot --- +coins = {} +cf = mdir / "coins-final.json" +if cf.is_file(): + try: + coins = json.load(open(cf)) + except Exception: + coins = {} + +print("") +print("── Coins (wallet) ─────────────────────────────────────────────") +if coins.get("ok"): + circ_n = coins.get("in_circulation") or 0 + spent_n = coins.get("spent") or 0 + total_n = coins.get("total_coins") or 0 + amt_alt = coins.get("amount_in_circulation_alt") or coins.get("amount_in_circulation_s") or "?" + spent_alt = coins.get("amount_spent_alt") or coins.get("amount_spent_s") or "0" + print(" total coins %s" % total_n) + print(" in circulation %s coins · %s" % (circ_n, amt_alt)) + print(" spent (in dump) %s coins · %s" % (spent_n, spent_alt)) + circ = coins.get("by_denom_circulation") or [] + if circ: + print(" denoms in circ:") + for x in circ: + print( + " · %s × %s = %s" + % ( + x.get("denom_alt") or x.get("denom"), + x.get("count"), + x.get("amount_alt") or x.get("amount"), + ) + ) + spent = coins.get("by_denom_spent") or [] + if spent: + print(" denoms spent:") + for x in spent: + print( + " · %s × %s = %s" + % ( + x.get("denom_alt") or x.get("denom"), + x.get("count"), + x.get("amount_alt") or x.get("amount"), + ) + ) +else: + print(" (no coins-final.json — run with wallet snaps)") + +# --- withdrawn / spent flows --- +print("") +print("── Money flow (this run) ──────────────────────────────────────") +# Prefer ladder TSV if provided; else flow-*.txt from e2e +wd_ok = {"OK", "OK_BANK", "ZERO_REJECT", "ZERO_SKIP", "SKIP_DENOM", "CEILING_REJECT"} +pay_ok = {"OK", "ZERO_SKIP", "CEILING_SKIP", "CEILING_REJECT"} +c1, wd_sum, wd_n = sum_tsv_amounts(wd_tsv, "amount", None) +# count successful withdraws by status prefix OK or soft +if wd_tsv and Path(wd_tsv).is_file(): + c1, wd_sum, wd_n = Decimal(0), Decimal(0), 0 + ccy = cur + with open(wd_tsv, newline="") as f: + for row in csv.DictReader(f, delimiter="\t"): + st = (row.get("status") or "").upper() + # count value when mint/settle produced money in wallet + if st in ("OK", "OK_BANK"): + c, v = parse_amt(row.get("amount")) + ccy, wd_sum, wd_n = c, wd_sum + v, wd_n + 1 + c1 = ccy +else: + c1, wd_sum, wd_n = sum_amount_file(mdir / "flow-withdrawn.txt") + +if pay_tsv and Path(pay_tsv).is_file(): + c2, pay_sum, pay_n = Decimal(0), Decimal(0), 0 + ccy = cur + with open(pay_tsv, newline="") as f: + for row in csv.DictReader(f, delimiter="\t"): + st = (row.get("status") or "").upper() + if st == "OK": + c, v = parse_amt(row.get("amount")) + ccy, pay_sum, pay_n = c, pay_sum + v, pay_n + 1 + c2 = ccy +else: + c2, pay_sum, pay_n = sum_amount_file(mdir / "flow-spent.txt") + +print(" withdrawn %s events · %s" % (wd_n, fmt_alt(c1, wd_sum))) +print(" spent (paid) %s events · %s" % (pay_n, fmt_alt(c2, pay_sum))) +net = wd_sum - pay_sum +print(" net (wd − spent) %s %s" % (fmt_alt(c1, net), trend_label(net))) +if coins.get("ok"): + # circulation amount vs net flow + circ_map = coins.get("amount_in_circulation") or {} + circ_v = Decimal(0) + for _c, vs in circ_map.items(): + circ_v += D(vs) + print(" wallet circ now %s" % fmt_alt(cur, circ_v)) + print(" residual check wallet_circ vs net: Δ %s" + % fmt_alt(cur, circ_v - net)) + +# --- coin history tendency --- +print("") +print("── Tendency · coins over run ─────────────────────────────────") +hist = mdir / "coins-history.tsv" +if hist.is_file() and hist.stat().st_size > 0: + rows = list(csv.DictReader(open(hist), delimiter="\t")) + if rows: + def circ_amt(row): + # amount_circ column may be "GOA:20" or alt form — try numeric from total/in_circ + a = row.get("amount_circ") or "0" + if ":" in a and " " not in a: + return parse_amt(a)[1] + # try extract GOA: from parentheses + import re + m = re.search(r"\(([^)]+:[0-9.]+)\)", a) + if m: + return parse_amt(m.group(1))[1] + return D(row.get("in_circ") or 0) + + first, last = rows[0], rows[-1] + c0, c1_ = D(first.get("in_circ") or 0), D(last.get("in_circ") or 0) + a0, a1 = circ_amt(first), circ_amt(last) + print(" samples %d snaps (%s → %s)" + % (len(rows), first.get("label"), last.get("label"))) + print(" coins in circ %s → %s (Δ %+d) %s" + % (fmt_num(c0), fmt_num(c1_), int(c1_ - c0), trend_label(c1_ - c0))) + print(" amount in circ %s → %s %s" + % (fmt_alt(cur, a0), fmt_alt(cur, a1), trend_label(a1 - a0))) + # simple linear slope on coin count + if len(rows) >= 3: + ys = [float(D(r.get("in_circ") or 0)) for r in rows] + n = len(ys) + xs = list(range(n)) + xm, ym = sum(xs) / n, sum(ys) / n + num = sum((x - xm) * (y - ym) for x, y in zip(xs, ys)) + den = sum((x - xm) ** 2 for x in xs) or 1 + slope = num / den + print(" trend slope %+.3f coins/snap %s" + % (slope, "↑ accumulating" if slope > 0.05 else ("↓ draining" if slope < -0.05 else "→ flat"))) + print(" history (label · in_circ · amount):") + for r in rows[-12:]: # last 12 + print(" · %-28s circ=%-4s %s" + % (r.get("label", "?")[:28], r.get("in_circ"), r.get("amount_circ"))) + if len(rows) > 12: + print(" · … (%d earlier snaps)" % (len(rows) - 12)) + else: + print(" (empty history)") +else: + print(" (no coins-history.tsv)") + +# --- performance --- +print("") +print("── Performance indicators ───────────────────────────────────") +perf = {} +pf = mdir / "perf-summary.json" +if pf.is_file(): + try: + perf = json.load(open(pf)) + except Exception: + perf = {} + +def print_bucket(name, v): + if not isinstance(v, dict) or not v.get("n"): + return + print( + " %-14s n=%s min=%sms p50=%sms avg=%sms max=%sms" + % (name, v.get("n"), v.get("min_ms"), v.get("p50_ms"), v.get("avg_ms"), v.get("max_ms")) + ) + +if perf: + for k, v in perf.items(): + print_bucket(k, v) +else: + print(" (no perf-summary.json)") + +# per-rung timing tendency from ladder TSV +for label, path, cols in ( + ("withdraw rungs", wd_tsv, ("ms_total", "ms_mint", "ms_settle")), + ("pay rungs", pay_tsv, ("ms_total", "ms_order", "ms_handle", "ms_settle")), +): + if not path or not Path(path).is_file(): + continue + with open(path, newline="") as f: + rrows = list(csv.DictReader(f, delimiter="\t")) + if len(rrows) < 2: + continue + print(" tendency · %s:" % label) + for col in cols: + vals = [] + for r in rrows: + try: + vals.append(int(r.get(col) or 0)) + except Exception: + pass + vals = [v for v in vals if v > 0] + if len(vals) < 2: + continue + lab, a, b = half_trend(vals) + if a is None: + print(" %-12s %s" % (col, lab)) + else: + print(" %-12s first-half avg=%.0fms → second-half avg=%.0fms %s" + % (col, a, b, lab)) + +# --- load --- +print("") +print("── Host / container load ────────────────────────────────────") +lb, la = mdir / "load-before.json", mdir / "load-after.json" + +def load_brief(path, tag): + if not path.is_file(): + print(" %s: (missing)" % tag) + return None + try: + d = json.load(open(path)) + except Exception: + print(" %s: (unreadable)" % tag) + return None + if not d.get("ok"): + print(" %s: %s" % (tag, d.get("reason", "?"))) + return d + h = d.get("host") or {} + mem = h.get("memory") or {} + la_ = h.get("loadavg") or [] + avail = mem.get("mem_available_b") + tot = mem.get("mem_total_b") + def gi(b): + return "?" if b is None else "%.2f GiB" % (b / 1024 / 1024 / 1024) + used = (tot - avail) if (tot is not None and avail is not None) else None + print(" %s loadavg=%s mem_used=%s / %s" + % (tag, la_, gi(used), gi(tot))) + for role in ("bank", "exchange", "merchant"): + c = (d.get("taler") or {}).get(role) or {} + if not c.get("running"): + continue + pr = c.get("processes") or {} + st = c.get("podman_stats") or {} + rss = pr.get("rss_total_b") + rss_s = "?" if rss is None else "%.2f GiB" % (rss / 1024 / 1024 / 1024) + print(" %-8s rss=%s cpu=%s procs=%s" + % (role, rss_s, st.get("cpu_pct") or "?", pr.get("proc_total"))) + return d + +db = load_brief(lb, "before") +da = load_brief(la, "after") +if db and da and db.get("ok") and da.get("ok"): + bla = (db.get("host") or {}).get("loadavg") or [0] + ala = (da.get("host") or {}).get("loadavg") or [0] + if bla and ala: + dload = float(ala[0]) - float(bla[0]) + print(" loadavg1 tendency %.2f → %.2f (Δ %+.2f) %s" + % (float(bla[0]), float(ala[0]), dload, + "↑ higher load" if dload > 0.1 else ("↓ lower load" if dload < -0.1 else "→ steady"))) + bm = (db.get("host") or {}).get("memory") or {} + am = (da.get("host") or {}).get("memory") or {} + if bm.get("mem_available_b") is not None and am.get("mem_available_b") is not None: + dmem = (am["mem_available_b"] - bm["mem_available_b"]) / 1024 / 1024 / 1024 + print(" mem_avail tendency %.2f → %.2f GiB (Δ %+.3f) %s" + % (bm["mem_available_b"] / 1024**3, am["mem_available_b"] / 1024**3, dmem, + "↑ more free" if dmem > 0.05 else ("↓ less free" if dmem < -0.05 else "→ steady"))) + +print("") +print("──────────────────────────────────────────────────────────────") +PY +} diff --git a/secrets.env.example b/secrets.env.example new file mode 100644 index 0000000..87ec88c --- /dev/null +++ b/secrets.env.example @@ -0,0 +1,87 @@ +# taler-monitoring — local secrets template +# +# Copy to secrets.env (gitignored) and fill values, OR export the same vars: +# cp secrets.env.example secrets.env +# $EDITOR secrets.env +# +# Never commit secrets.env. Prefer file paths under koopa-admin-secrets +# for long-term storage; this file is for laptop one-shots / overrides. +# +# Load order: env already set > secrets.env > SECRETS_ROOT files > SSH koopa + +# --- Preferred: point at secrets tree (no passwords in this file) --- +# Sibling of admin-log on this machine: +# KOOPA_ADMIN_SECRETS=$HOME/src/koopa/koopa-admin-secrets +# Or host-root only: +# SECRETS_ROOT=$HOME/src/koopa/koopa-admin-secrets/koopa/host-root + +# --- Or paste passwords / tokens (local GOA e2e) --- +# Bank admin (libeufin-bank admin) — funds / creates accounts in e2e +# E2E_BANK_ADMIN_PASS= +# BANK_ADMIN_PASS= # alias, same meaning + +# Merchant instance token for private orders (goa-demo-cp4zqk on hacktivism) +# Full "secret-token:…" or raw secret (Bearer prefix added by e2e) +# E2E_MERCHANT_TOKEN= +# MERCHANT_TOKEN= +# MERCHANT_INSTANCE_PASSWORD= + +# --- FrancPaysan STAGE (TESTPAYSAN) e2e --- +# Default phases for -d stage.lefrancpaysan.ch: urls + inside + versions + e2e +# inside uses francpaysan-stage-user (stagepaysan); stats from public HTTPS. +# Admin credit (required for ATM withdraw ladder on stage): +# E2E_BANK_ADMIN_PASS=… # or auto-load via SSH francpaysan-host +# FRANCPAYSAN_SSH=francpaysan-host +# FRANCPAYSAN_SECRETS=$HOME/francpaysan-secrets +# Public shop pays use E2E_USE_TEMPLATES=1 (default on TESTPAYSAN) — no merchant token. + +# Optional overrides +# MERCHANT_INSTANCE=goa-demo-cp4zqk +# EXPECT_CURRENCY=GOA +# E2E_TIMEOUT=340 +# PROGRESS_TOTAL=120 +# PROGRESS_SHOW_EVERY=8 +# PROGRESS_OFF=1 + +# --- Merchant SPA /webui/ fingerprints (urls phase, group www.webui) --- +# EXPECT_WEBUI_VERSION=1.6.11 +# EXPECT_WEBUI_OVERLAY=selfbuild-v1.6.11 +# WEBUI_OVERLAY_DENY=c778af04|master-11340 +# CHECK_WEBUI_SPA=0 + +# --- auth401 phase (WebUI signup/login/pwchange/admin + case matrix) --- +# Default FULL=1 creates throwaway MIX instance + runs all groups. +# Stage also loads durable mon401 for extra durable.* checks: +# AUTH401_FULL=1 +# AUTH401_CONTINUE=1 # optional: hard prereqs soft; full ERROR list always via EXIT trap +# CONTINUE_ON_ERROR=1 # alias for AUTH401_CONTINUE +# (auth401 no longer uses set -e — silent mid-run abort fixed) +# AUTH401_CREATE=1 +# AUTH401_PREFER_DURABLE=0 # 1 = only mon401, skip throwaway signup +# AUTH401_INSTANCE=mon401 +# AUTH401_PASSWORD= +# AUTH401_DURABLE_ID=mon401 +# AUTH401_CREATE_PASSWORD=Mon401-Test-Pass! +# AUTH401_SKIP_IF_MFA=1 +# AUTH401_BEARER= # default-instance-token (not admin) +# AUTH401_ADMIN_TOKEN= # required for admin-create group +# AUTH401_ADMIN_PASSWORD= # or login as AUTH401_ADMIN_USER=admin +# AUTH401_ADMIN_USER=admin +# FRANCPAYSAN_SECRETS=$HOME/francpaysan-secrets + +# --- Ladder (GOA withdraw ladder · explorer pool) --- +# Prefer SECRETS_ROOT / KOOPA_ADMIN_SECRETS (bank-explorer-password.txt). +# Overrides only if needed: +# EXP_USER=explorer +# EXP_PW= +# EXP_PW_FILE=$HOME/src/koopa/koopa-admin-secrets/koopa/host-root/taler-bank/bank-explorer-password.txt +# CLI_JS=$HOME/taler/src/taler-typescript-core/packages/taler-wallet-cli/bin/taler-wallet-cli.mjs + +# Values that typically appear on a full stack (for orientation only): +# bank-admin-password.txt +# bank-explorer-password.txt +# bank-exchange-password.txt +# bank-goa-demo-cp4zqk-password.txt +# merchant-goa-demo-cp4zqk-password.txt +# exchange-attribute-encryption.secret.conf +# (see ../SECRETS.md and koopa-admin-secrets/MAP.md) diff --git a/site-gen/README.md b/site-gen/README.md new file mode 100644 index 0000000..8d51e10 --- /dev/null +++ b/site-gen/README.md @@ -0,0 +1,81 @@ +# monitoring site-gen + +Build **console-style HTML** from `taler-monitoring` runs and stage them for +**host Caddy** (`/monitoring`, `/monitoring_err`). + +Same **global reporting defaults** as host-agents (`run-host-report.sh`): + +- **`RUN_TIMEOUT=600`** on each suite run +- **line-buffered** logs (continuous write) +- **always** write HTML (first run / empty logs too) +- Forgejo **commit link** in the page header + +Applies to all nine fronts (GOA + FP stage + FP prod) in the default `SITES` list. + +| Host | Role | +|------|------| +| **firecuda-external** | optional runner (public DNS) | +| **koopa-external** | deploy staging + Caddy (root service) | +| **host-agents** | primary: GOA + FP on-box (`../host-agent/`) | + +No secrets in `settings.conf` — only hosts, paths, site list. + +## firecuda-external (outside-only) — **disabled for now** + +**firecuda-external could** run public-only monitoring (`SKIP_SSH=1`, phases +`urls versions`) on a schedule (launchd every 4h) and write HTML under +`~/var/taler-monitoring-sites-work/html/`. Scripts for that still live here +(`install-firecuda-timer.sh`, `run-on-firecuda.sh`, plist template). + +**Current policy: do not run a real launchd job on firecuda.** +Primary GOA monitoring is **koopa host-agent** (`host-agent/`, user `hernani`, +local `podman exec`). Re-enable firecuda only if you explicitly want a second, +outside-only view. + +```bash +# optional one-shot (no timer): +# ssh firecuda-external '~/taler-monitoring-site-gen/site-gen/run-on-firecuda.sh' + +# if a timer was installed earlier, ensure it is off: +# ssh firecuda-external 'launchctl list | grep taler-monitoring || echo off' +``` + +Disabled plists may remain as `*.plist.disabled-*` under `~/Library/LaunchAgents/`. + +## Quick start (laptop) + +```bash +cd site-gen +cp settings.conf.example settings.conf # optional +chmod +x generate-monitoring-sites.sh deploy-monitoring-sites.sh console_to_html.py + +ONLY_HOSTS="bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch" \ + SKIP_SSH=1 RUNNER_SSH= \ + ./generate-monitoring-sites.sh + +./deploy-monitoring-sites.sh +``` + +Then on koopa **as root**: see **[ROOT-ON-KOOPA.md](./ROOT-ON-KOOPA.md)**. + +## URL rules + +| Path | When | +|------|------| +| `/monitoring_err/` | always: full console log + error list on top (CONTINUE_ON_ERROR run) | +| `/monitoring/` | **clean** full log if last strict run exit 0; else **stub** linking to `/monitoring_err/` | + +Footer links **Forgejo commit** of the admin-log tree used for that run. + +## Layout + +```text +site-gen/ + settings.conf.example + generate-monitoring-sites.sh + deploy-monitoring-sites.sh + console_to_html.py + caddy-monitoring-handles.snippet + ROOT-ON-KOOPA.md + README.md +``` diff --git a/site-gen/ROOT-ON-KOOPA.md b/site-gen/ROOT-ON-KOOPA.md new file mode 100644 index 0000000..604b997 --- /dev/null +++ b/site-gen/ROOT-ON-KOOPA.md @@ -0,0 +1,92 @@ +# Root on koopa (hacktivism monitoring sites) + +Host Caddy runs as **root/systemd** (`caddy` user). Static monitoring HTML is +**not** put into Taler containers — only host paths + Caddy `handle`. + +After `generate-monitoring-sites.sh` + `deploy-monitoring-sites.sh` (as hernani), +files live in: + +```text +/home/hernani/monitoring-sites-staging//monitoring/index.html +/home/hernani/monitoring-sites-staging//monitoring_err/index.html +``` + +## One-shot as root (copy + Caddy) + +```bash +# on koopa (koopa-external), as root: + +# 1) web root +install -d -o caddy -g caddy -m 755 /var/www/monitoring-sites +rsync -a --delete /home/hernani/monitoring-sites-staging/ /var/www/monitoring-sites/ +chown -R caddy:caddy /var/www/monitoring-sites + +# 2) Caddyfile — merge handles from admin-log mirror, then: +# (either edit /etc/caddy/Caddyfile by hand using snippet below, +# or copy full mirror after review) +# +# install -m 644 /home/hernani/koopa-admin-log/configs/caddy/Caddyfile /etc/caddy/Caddyfile +# # or: ~/koopa-caddy/apply.sh after syncing Caddyfile into ~/koopa-caddy/ + +caddy validate --config /etc/caddy/Caddyfile +systemctl reload caddy +systemctl is-active caddy +``` + +## Snippet per hacktivism site (inside each `*.hacktivism.ch { }` block) + +Place **before** the catch-all `reverse_proxy` (same idea as `/intro*`): + +```caddy + # Public taler-monitoring console HTML (static; host only) + handle /monitoring_err* { + root * /var/www/monitoring-sites/{host} + rewrite * /monitoring_err{uri} + # uri is /monitoring_err or /monitoring_err/ → serve directory index + file_server + } + handle /monitoring* { + root * /var/www/monitoring-sites/{host} + file_server + } +``` + +Simpler layout (recommended): path on disk matches URL under host root: + +```text +/var/www/monitoring-sites/bank.hacktivism.ch/monitoring/index.html +/var/www/monitoring-sites/bank.hacktivism.ch/monitoring_err/index.html +``` + +```caddy + handle_path /monitoring_err/* { + root * /var/www/monitoring-sites/{host}/monitoring_err + file_server + } + handle /monitoring_err { + redir /monitoring_err/ 302 + } + handle_path /monitoring/* { + root * /var/www/monitoring-sites/{host}/monitoring + file_server + } + handle /monitoring { + redir /monitoring/ 302 + } +``` + +Canonical snippet is also in `caddy-monitoring-handles.snippet` and applied +in `configs/caddy/Caddyfile` for the three GOA hosts. + +## Check + +```bash +curl -sS -o /dev/null -w '%{http_code}\n' https://bank.hacktivism.ch/monitoring/ +curl -sS -o /dev/null -w '%{http_code}\n' https://bank.hacktivism.ch/monitoring_err/ +curl -sS https://bank.hacktivism.ch/monitoring_err/ | head +``` + +## Not needed as root + +- Running `taler-monitoring` (use **firecuda-external** / laptop) +- Writing into podman Taler containers diff --git a/site-gen/caddy-monitoring-handles.snippet b/site-gen/caddy-monitoring-handles.snippet new file mode 100644 index 0000000..1c09531 --- /dev/null +++ b/site-gen/caddy-monitoring-handles.snippet @@ -0,0 +1,57 @@ +# Host Caddy on koopa (root service). Paste BEFORE catch-all reverse_proxy. +# +# Disk layout: +# /var/www/monitoring-sites/{host}/monitoring/index.html +# /var/www/monitoring-sites/{host}/monitoring_err/index.html +# /var/www/monitoring-sites/taler.hacktivism.ch/taler-monitoring-surface*/index.html +# /var/www/monitoring-sites/taler.hacktivism.ch/taler-monitoring-aptdeploy*/index.html +# +# /monitoring* → bank + exchange + taler.hacktivism.ch +# /taler-monitoring-surface* → taler.hacktivism.ch only +# /taler-monitoring-aptdeploy* → taler.hacktivism.ch only + +# --- only inside taler.hacktivism.ch { ... } --- + handle /taler-monitoring-surface_err { + redir /taler-monitoring-surface_err/ 302 + } + handle /taler-monitoring-surface_err/ { + root * /var/www/monitoring-sites/taler.hacktivism.ch/taler-monitoring-surface_err + file_server + } + handle /taler-monitoring-surface { + redir /taler-monitoring-surface/ 302 + } + handle /taler-monitoring-surface/ { + root * /var/www/monitoring-sites/taler.hacktivism.ch/taler-monitoring-surface + file_server + } + handle /taler-monitoring-aptdeploy_err { + redir /taler-monitoring-aptdeploy_err/ 302 + } + handle /taler-monitoring-aptdeploy_err/ { + root * /var/www/monitoring-sites/taler.hacktivism.ch/taler-monitoring-aptdeploy_err + file_server + } + handle /taler-monitoring-aptdeploy { + redir /taler-monitoring-aptdeploy/ 302 + } + handle /taler-monitoring-aptdeploy/ { + root * /var/www/monitoring-sites/taler.hacktivism.ch/taler-monitoring-aptdeploy + file_server + } + +# --- bank + exchange + taler (each site block) --- + handle /monitoring_err { + redir /monitoring_err/ 302 + } + handle /monitoring_err/ { + root * /var/www/monitoring-sites/{host}/monitoring_err + file_server + } + handle /monitoring { + redir /monitoring/ 302 + } + handle /monitoring/ { + root * /var/www/monitoring-sites/{host}/monitoring + file_server + } diff --git a/site-gen/com.hacktivism.taler-monitoring-sites.plist b/site-gen/com.hacktivism.taler-monitoring-sites.plist new file mode 100644 index 0000000..0b6f68b --- /dev/null +++ b/site-gen/com.hacktivism.taler-monitoring-sites.plist @@ -0,0 +1,34 @@ + + + + + + Label + com.hacktivism.taler-monitoring-sites + ProgramArguments + + /bin/bash + __SITE_GEN__/run-on-firecuda.sh + + StartInterval + 14400 + RunAtLoad + + StandardOutPath + __HOME__/Library/Logs/taler-monitoring-sites/launchd.out.log + StandardErrorPath + __HOME__/Library/Logs/taler-monitoring-sites/launchd.err.log + EnvironmentVariables + + PATH + /opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin + SKIP_SSH + 1 + CONTINUE_ON_ERROR + 1 + + + diff --git a/site-gen/console_to_html.py b/site-gen/console_to_html.py new file mode 100755 index 0000000..75db298 --- /dev/null +++ b/site-gen/console_to_html.py @@ -0,0 +1,935 @@ +#!/usr/bin/env python3 +"""Convert taler-monitoring console log to a console-feel HTML page.""" +from __future__ import annotations + +import argparse +import html +import re +from datetime import datetime, timezone +from pathlib import Path + +# ANSI strip +ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") +ERR_BULLET_RE = re.compile(r"^\s*[•·*-]\s+(?P.+)$") +TID_RE = re.compile(r"\b(?P[a-z0-9_.-]+-\d+)\b", re.I) + + +def strip_ansi(s: str) -> str: + return ANSI_RE.sub("", s) + + +def classify(line: str) -> str: + u = line.upper() + # section / summary headers before badge matching (they contain the word ERROR) + if "SUMMARY" in u or "--- ERRORS" in u or "ERRORS ·" in u or "ERRORS (" in u: + return "meta" + if "numbered checks" in line.lower() or "totals:" in line.lower(): + return "meta" + if "BLOCKER" in u or "┌ BLOCKER" in u: + return "blocker" + if "ERROR" in u or "┌ ERROR" in u: + return "error" + if "WARN" in u or "┌ WARN" in u: + return "warn" + if "INFO" in u or "┌ INFO" in u: + return "info" + if " OK" in u or "[OK" in u or "┌ OK" in u or u.strip().startswith("OK"): + return "ok" + if u.strip().startswith("-- ") or u.strip().startswith("=="): + return "section" + return "plain" + + +def is_run_timeout_text(text: str) -> bool: + u = text.upper() + return "RUN.TIMEOUT" in u or "RUN_TIMEOUT" in u or "RUN TIMEOUT" in u + + +def is_summary_error_line(ln: str) -> bool: + """Skip summary/header lines that contain ERROR but are not checks.""" + low = ln.lower() + if "failed — see" in low or "failed - see" in low: + return True + if "errors · failed" in low or "errors (failed" in low: + return True + if "--- errors" in low: + return True + if "┌ summary" in low or "[ summary" in low: + return True + if "totals:" in low: + return True + # header-ish RUN_TIMEOUT= without error badge + if is_run_timeout_text(ln) and "ERROR" not in ln.upper() and "run.timeout" not in low: + return True + return False + + +def extract_errors(lines: list[str]) -> list[str]: + errs: list[str] = [] + timeout_detail = "" + in_block = False + for ln in lines: + if "--- ERRORS" in ln or "ERRORS (failed" in ln or "RUN TIMEOUT · extraordinary" in ln: + in_block = True + continue + if in_block: + if ln.strip().startswith("---") or "[ SUMMARY" in ln or "totals:" in ln: + in_block = False + continue + m = ERR_BULLET_RE.match(ln) + if m: + body = m.group("body").strip() + if is_run_timeout_text(body): + timeout_detail = body + else: + errs.append(body) + elif "ERROR" in ln.upper() and ln.strip() and not is_run_timeout_text(ln): + errs.append(ln.strip()) + if re.search(r"\bERROR\b", ln, re.I) and "--- ERRORS" not in ln: + if is_run_timeout_text(ln): + if "run.timeout-01" in ln.lower() or "RUN_TIMEOUT exceeded" in ln: + timeout_detail = re.sub(r"\s+", " ", strip_ansi(ln)).strip() + continue + if is_summary_error_line(ln): + continue + body = re.sub(r"^.*\bERROR\b\s*\]?\s*", "", ln, flags=re.I).strip(" ·") + if body and body not in errs and "failed — see" not in body.lower(): + if re.search( + r"#\d+|www\.|e2e\.|auth401\.|versions\.|inside\.|surface\.|aptdeploy\.", + body, + ): + errs.append(body) + seen: set[str] = set() + out: list[str] = [] + for e in errs: + if e in seen or is_run_timeout_text(e): + continue + seen.add(e) + out.append(e) + if timeout_detail or any( + is_run_timeout_text(l) and ("ERROR" in l.upper() or "run.timeout" in l.lower()) + for l in lines + ): + # only if real timeout ERROR badge, not mere RUN_TIMEOUT= header + if timeout_detail or any( + "run.timeout-01" in l.lower() or "RUN_TIMEOUT exceeded" in l for l in lines + ): + canon = timeout_detail or "run.timeout-01 [run] RUN_TIMEOUT exceeded" + out = [canon] + out + return out + + +def count_status(lines: list[str]) -> tuple[int, int, int | None, int | None]: + """Return (error_count, warn_count, first_error_idx, first_warn_idx).""" + n_err = 0 + n_warn = 0 + first_err: int | None = None + first_warn: int | None = None + for i, ln in enumerate(lines): + kind = classify(ln) + if kind in ("error", "blocker"): + if is_summary_error_line(ln): + continue + n_err += 1 + if first_err is None: + first_err = i + elif kind == "warn": + n_warn += 1 + if first_warn is None: + first_warn = i + return n_err, n_warn, first_err, first_warn + + +def slug_error(i: int, text: str) -> str: + if is_run_timeout_text(text): + return "err-run-timeout" + m = TID_RE.search(text) + if m: + return "err-" + re.sub(r"[^a-z0-9_-]+", "-", m.group("tid").lower()) + return f"err-{i:03d}" + + +def render_line(ln: str, err_slugs: dict[str, str], extra_ids: list[str] | None = None) -> str: + kind = classify(ln) + esc = html.escape(ln) + for tid, slug in err_slugs.items(): + if tid in ln: + esc = esc.replace( + html.escape(tid), + f'{html.escape(tid)}', + ) + id_attr = "" + if extra_ids: + id_attr = f' id="{" ".join(extra_ids)}"' # invalid multi-id; use first only + # HTML allows one id — join carefully + id_attr = f' id="{html.escape(extra_ids[0])}"' + if len(extra_ids) > 1: + # secondary anchors as empty spans prepended + spans = "".join(f'' for x in extra_ids[1:]) + return f'{spans}{esc}\n' + return f'{esc}\n' + + +def status_level(n_err: int, n_warn: int) -> str: + if n_err > 0: + return "red" + if n_warn > 0: + return "yellow" + return "green" + + +def extract_phases(log_text: str) -> str: + """Best-effort phases string from host-agent / taler-monitoring log header.""" + for pat in ( + r"phases=([a-z0-9 _.-]+)", + r"· phases=([a-z0-9 _.-]+)", + r"phases\s{2,}([a-z0-9 _.-]+)", + r"^\s*phases\s+([a-z0-9 _.-]+)\s*$", + ): + m = re.search(pat, log_text, re.I | re.M) + if m: + return re.sub(r"\s+", " ", m.group(1)).strip(" ·") + return "" + + +def monitoring_scope( + page_label: str, + hostname: str, + log_text: str = "", +) -> dict[str, object]: + """ + Human-readable description of what this monitoring page covers. + Returned keys: title, summary, items (list[str]), kind + """ + label = (page_label or "monitoring").lower().replace("_", "-") + phases = extract_phases(log_text) + host = hostname or "?" + + if "surface" in label: + return { + "kind": "surface", + "title": "Remote ecosystem surface inventory", + "summary": "Public catalog hosts · DNS / TCP / HTTPS / TLS · Server version · CVE", + "items": [ + "Outside-in only: no SSH and no podman exec on remote targets", + "Catalog (surface-catalog.conf): taler.net, demo/test, gnunet.org, " + "taler-ops.ch, taler-systems.com, deb.taler.net, ftp.gnu.org, …", + "Per host: DNS resolve, ICMP (info), open ports, protocol probes, " + "TLS cert expiry, Server-header software version, optional CVE check", + f"HTML published only on taler.hacktivism.ch " + f"(this page host: {host})", + "Timer: taler-monitoring-surface.timer (hourly)", + ], + } + + if "aptdeploy" in label or "apt-deploy" in label or "apt_src" in label: + return { + "kind": "aptdeploy", + "title": "apt-src merchant deploy tests (podman on koopa)", + "summary": "4 containers · fresh + upgrade tracks · trixie & trixie-testing", + "items": [ + "koopa-taler-deploy-test-apt-src-trixie (fresh, suite trixie)", + "koopa-taler-deploy-test-apt-src-trixie-testing (fresh, trixie-testing)", + "koopa-taler-deploy-test-apt-src-trixie-upgrade (upgrade track)", + "koopa-taler-deploy-test-apt-src-trixie-testing-upgrade (upgrade track)", + "Checks: packages, taler-merchant-httpd --version / ldd, systemd unit, " + "optional local /config probe — not public nginx/HTTPS", + f"HTML published only on taler.hacktivism.ch (this page host: {host})", + "Timer: taler-monitoring-aptdeploy.timer (4h)", + ], + } + + # Default: stack /monitoring pages (bank, exchange, merchant) + phase_txt = phases or "urls · inside · versions" + role = "stack" + if host.startswith("bank."): + role = "bank" + focus = "Libeufin bank public HTTPS + container inside checks" + elif host.startswith("exchange.") or host.startswith("stage.exchange."): + role = "exchange" + focus = "Exchange public HTTPS + container inside checks" + elif host.startswith("taler.") or "merchant" in host or host.startswith("monnaie."): + role = "merchant" + focus = "Merchant backend / SPA public HTTPS + container inside checks" + else: + focus = "Taler stack public + inside checks" + + items = [ + f"Focus: {focus}", + f"Phases this run: {phase_txt}", + "Typical: public URLs (HTTPS, QR, perf), inside (podman/ssh), package versions", + f"HTML host: {host}", + "Timer: taler-monitoring-hacktivism (path + 4h) · /monitoring on bank, exchange, taler", + ] + if phases: + items.insert(2, f"Log header phases= {phases}") + + return { + "kind": role, + "title": f"Stack monitoring · {host}", + "summary": f"{phase_txt} · public + inside", + "items": items, + } + + +def sticky_bar_html( + *, + generated_at: str, + generated_iso: str, + n_err: int, + n_warn: int, + hostname: str, + page_label: str, + mode: str, + commit_html: str, + suite_path: str, + link_other: str | None, + first_err_href: str | None, + first_warn_href: str | None, + scope: dict[str, object] | None = None, +) -> str: + level = status_level(n_err, n_warn) + if level == "red": + status_txt = "ERRORS" + elif level == "yellow": + status_txt = "WARNINGS" + else: + status_txt = "OK" + + if first_err_href and n_err: + err_stat = ( + f'{n_err} error{"s" if n_err != 1 else ""}' + ) + else: + err_stat = f'{n_err} errors' + + if first_warn_href and n_warn: + warn_stat = ( + f'{n_warn} warning{"s" if n_warn != 1 else ""}' + ) + else: + warn_stat = f'{n_warn} warnings' + + other = "" + if link_other: + other = ( + f'' + f"{html.escape(link_other)}" + ) + + if scope is None: + scope = monitoring_scope(page_label, hostname, "") + scope_title = html.escape(str(scope.get("title") or "Monitoring")) + scope_summary = html.escape(str(scope.get("summary") or "")) + scope_kind = html.escape(str(scope.get("kind") or "monitoring")) + lis = "\n".join( + f"
  • {html.escape(str(it))}
    • " for it in (scope.get("items") or []) + ) + suite_bit = ( + f" · {html.escape(suite_path)}" if suite_path else "" + ) + + # Compact sticky row always visible; "Was geprüft wird" expands inside sticky-bar + return f""" + +""" + + +STICKY_CSS = """ +.sticky-bar { + position: sticky; top: 0; z-index: 100; + border-bottom: 2px solid var(--border); + padding: 10px 14px 8px; + backdrop-filter: blur(8px); + -webkit-backdrop-filter: blur(8px); + box-shadow: 0 2px 12px #0008; +} +.sticky-bar.sticky-green { + background: linear-gradient(180deg, #0d1f14f2, #0a1410ee); + border-bottom-color: #1f6b3a; +} +.sticky-bar.sticky-yellow { + background: linear-gradient(180deg, #2a2210f2, #1a160cee); + border-bottom-color: #a68b2d; +} +.sticky-bar.sticky-red { + background: linear-gradient(180deg, #2a1010f2, #1a0808ee); + border-bottom-color: #a33; +} +.sticky-bar-row { + display: flex; flex-wrap: wrap; align-items: center; gap: 6px 10px; + max-width: 1200px; +} +.sticky-bar-row.secondary { + margin-top: 8px; font-size: 12px; color: var(--dim); +} +.sticky-bar-row.secondary a { color: var(--info); text-decoration: none; } +.sticky-bar-row.secondary a:hover { text-decoration: underline; } +.status-pill { + display: inline-block; font-size: 11px; font-weight: 800; + letter-spacing: 0.06em; padding: 2px 10px; border-radius: 3px; + border: 1px solid; +} +.status-pill.sticky-green { color: var(--ok); border-color: #1f6b3a; background: #0a1a10; } +.status-pill.sticky-yellow { color: var(--warn); border-color: #a68b2d; background: #1a1608; } +.status-pill.sticky-red { color: var(--err); border-color: #a33; background: #1a0a0a; } +.host { font-weight: 600; color: #eee; font-size: 13px; } +.sep { color: var(--dim); } +.stat { + font-weight: 700; font-size: 12px; text-decoration: none; + padding: 1px 8px; border-radius: 3px; border: 1px solid transparent; +} +.stat.err { color: var(--err); border-color: #522; background: #1a0a0a; } +.stat.warn { color: var(--warn); border-color: #664; background: #1a1608; } +.stat.muted { opacity: 0.55; font-weight: 600; } +a.stat:hover { text-decoration: underline; filter: brightness(1.15); } +.generated { color: var(--dim); font-size: 12px; } +.generated time { color: #bbb; } +.generated .age { color: #888; margin-left: 4px; } +.generated .age:not(:empty)::before { content: "("; } +.generated .age:not(:empty)::after { content: ")"; } +.badge { + display: inline-block; padding: 1px 8px; border-radius: 3px; + font-size: 11px; font-weight: 700; margin-right: 4px; + border: 1px solid var(--border); +} +.badge.mode-err { color: var(--err); border-color: #522; background: #1a0a0a; } +.badge.mode-ok { color: var(--ok); border-color: #143; background: #0a1a10; } +.badge.mode-redirect { color: var(--warn); border-color: #664; background: #1a1608; } +/* collapsible block inside sticky-bar */ +.scope-toggle { + margin-left: auto; + display: inline-flex; + align-items: center; + gap: 8px; + max-width: min(42rem, 100%); + cursor: pointer; + font: inherit; + font-size: 12px; + color: #ddd; + background: #0006; + border: 1px solid #444; + border-radius: 4px; + padding: 4px 10px; + text-align: left; +} +.scope-toggle:hover { border-color: #888; background: #111a; color: #fff; } +.scope-toggle:focus-visible { outline: 2px solid var(--info); outline-offset: 2px; } +.scope-toggle[aria-expanded="true"] { + border-color: #777; + background: #141414; +} +.scope-chevron { + display: inline-block; + width: 0; height: 0; + border-left: 5px solid #aaa; + border-top: 4px solid transparent; + border-bottom: 4px solid transparent; + transition: transform 0.12s ease; + flex-shrink: 0; +} +.scope-toggle[aria-expanded="true"] .scope-chevron { + transform: rotate(90deg); + border-left-color: #eee; +} +.scope-toggle-label { + font-weight: 700; + letter-spacing: 0.03em; + text-transform: uppercase; + font-size: 11px; + white-space: nowrap; +} +.scope-toggle-hint { + color: #888; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + min-width: 0; +} +.sticky-scope-panel { + margin-top: 8px; + max-width: 1200px; + border: 1px solid #333; + border-radius: 4px; + background: #080808e6; + padding: 8px 10px 10px; +} +.sticky-scope-panel[hidden] { display: none !important; } +.scope-body { padding: 4px 4px 2px 6px; } +.scope-title { + margin: 4px 0 4px; + font-size: 13px; + font-weight: 600; + color: #eee; +} +.scope-lead { + margin: 0 0 6px; + font-size: 12px; + color: #999; +} +.scope-list { + margin: 0; + padding-left: 1.15em; + color: #b0b0b0; + font-size: 12px; + line-height: 1.5; +} +.scope-list li { margin: 3px 0; } +#first-error, #first-warn, +[id^="err-"][id$="-line"] { + scroll-margin-top: 120px; +} +@media (max-width: 720px) { + .scope-toggle { margin-left: 0; width: 100%; } + .scope-toggle-hint { display: none; } +} +""" + +STICKY_JS = """ + +""" + + +def build_html( + *, + title: str, + hostname: str, + mode: str, + log_text: str, + commit: str, + commit_url: str, + suite_path: str, + generated_at: str, + generated_iso: str, + link_other: str | None, + page_label: str = "monitoring", +) -> str: + raw_lines = [strip_ansi(l).rstrip("\n") for l in log_text.splitlines()] + while raw_lines and not raw_lines[-1].strip(): + raw_lines.pop() + + n_err, n_warn, first_err_i, first_warn_i = count_status(raw_lines) + errors = extract_errors(raw_lines) + + err_slugs: dict[str, str] = {} + for i, e in enumerate(errors, 1): + slug = slug_error(i, e) + err_slugs[e] = slug + m = TID_RE.search(e) + if m: + err_slugs[m.group("tid")] = slug + + tid_to_slug = { + k: v for k, v in err_slugs.items() if re.match(r"^[a-z0-9_.-]+-\d+$", k, re.I) + } + + # Build body with anchors for first error/warn + per-error slugs + body_lines: list[str] = [] + claimed_err: set[str] = set() + for idx, ln in enumerate(raw_lines): + extra: list[str] = [] + if first_err_i is not None and idx == first_err_i: + extra.append("first-error") + if first_warn_i is not None and idx == first_warn_i: + extra.append("first-warn") + # per-error jump targets (all modes) + if classify(ln) in ("error", "blocker") and not is_summary_error_line(ln): + for i, e in enumerate(errors, 1): + slug = slug_error(i, e) + if slug in claimed_err: + continue + tid_m = TID_RE.search(e) + if is_run_timeout_text(e): + if "run.timeout-01" in ln and "ERROR" in ln.upper(): + extra.append(f"{slug}-line") + claimed_err.add(slug) + break + continue + key = tid_m.group(1) if tid_m else e[:40] + if key and key in ln: + extra.append(f"{slug}-line") + claimed_err.add(slug) + break + body_lines.append(render_line(ln, tid_to_slug, extra or None)) + + err_nav = "" + if errors and mode == "err": + items = [] + timeout_items = [] + for i, e in enumerate(errors, 1): + slug = slug_error(i, e) + li = ( + f'
  • ' + f'{html.escape(e)}
    • ' + ) + if is_run_timeout_text(e): + timeout_items.append(li) + else: + items.append(li) + if timeout_items: + err_nav += ( + '
    \n' + "

    EXTRAORDINARY · RUN TIMEOUT

    \n" + "

    Whole-run wall clock exceeded. " + 'Jump to detail →

    \n' + '
      \n' + + "\n".join(timeout_items[:1]) + + "\n
    \n
    \n" + ) + rest = items if timeout_items else (timeout_items + items) + if rest or not timeout_items: + err_nav += ( + '
    \n' + f"

    ERRORS ({len(errors)}) — jump to detail

    \n" + '
      \n' + + "\n".join(rest if rest else items) + + "\n
    \n
    \n" + ) + + commit_short = commit[:12] if commit else "unknown" + commit_html = ( + f'{html.escape(commit_short)}' + if commit_url + else html.escape(commit_short) + ) + + first_err_href = "#first-error" if first_err_i is not None else None + first_warn_href = "#first-warn" if first_warn_i is not None else None + scope = monitoring_scope(page_label, hostname, log_text) + + bar = sticky_bar_html( + generated_at=generated_at, + generated_iso=generated_iso, + n_err=n_err, + n_warn=n_warn, + hostname=hostname, + page_label=page_label, + mode=mode, + commit_html=commit_html, + suite_path=suite_path, + link_other=link_other, + first_err_href=first_err_href, + first_warn_href=first_warn_href, + scope=scope, + ) + + return f""" + + + + + +{html.escape(title)} + + + +{bar} +
    +{err_nav} +
    +{"".join(body_lines)} +
    +
    + Console-style render of taler-monitoring output. + Sticky bar: green = clean · yellow = warnings · red = errors. + Commit pins the exact tree used for this run. +
    +
    +{STICKY_JS} + + +""" + + +def build_redirect_html( + *, + hostname: str, + commit: str, + commit_url: str, + path_err: str = "/monitoring_err/", + page_label: str = "monitoring", + generated_at: str = "", + generated_iso: str = "", + n_err: int = 0, + n_warn: int = 0, + log_text: str = "", +) -> str: + commit_short = commit[:12] if commit else "unknown" + link = path_err if path_err.endswith("/") else path_err + "/" + c = ( + f'{html.escape(commit_short)}' + if commit_url + else html.escape(commit_short) + ) + if not generated_at: + now = datetime.now(timezone.utc) + generated_iso = now.strftime("%Y-%m-%dT%H:%M:%SZ") + generated_at = now.strftime("%Y-%m-%d %H:%M:%SZ") + # Redirect pages exist only when the run failed → treat as red if n_err unknown + if n_err <= 0: + n_err = max(n_err, 1) + scope = monitoring_scope(page_label, hostname, log_text) + bar = sticky_bar_html( + generated_at=generated_at, + generated_iso=generated_iso, + n_err=n_err, + n_warn=n_warn, + hostname=hostname, + page_label=page_label, + mode="redirect", + commit_html=c, + suite_path="", + link_other=link, + first_err_href=link, + first_warn_href=None, + scope=scope, + ) + return f""" + + + + + + +{html.escape(page_label)} · errors · {html.escape(hostname)} + + + +{bar} +
    +

    {html.escape(page_label)} has failures.

    +

    See {html.escape(link)} for the full console log, + error index, and sticky status bar.

    +

    generated {html.escape(generated_at)} · source {c}

    +
    +{STICKY_JS} + + +""" + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--log", required=True, type=Path) + ap.add_argument("--out", required=True, type=Path) + ap.add_argument("--hostname", required=True) + ap.add_argument("--mode", choices=("ok", "err", "redirect"), required=True) + ap.add_argument("--commit", default="") + ap.add_argument("--commit-url", default="") + ap.add_argument("--suite-path", default=".") + ap.add_argument("--title", default="") + ap.add_argument("--link-other", default="") + ap.add_argument( + "--path-err", + default="/monitoring_err/", + help="URL path for error page (redirect target)", + ) + ap.add_argument( + "--page-label", + default="monitoring", + help="Short name shown in titles/redirect box", + ) + args = ap.parse_args() + + log_text = args.log.read_text(errors="replace") if args.log.is_file() else "" + now = datetime.now(timezone.utc) + generated_iso = now.strftime("%Y-%m-%dT%H:%M:%SZ") + generated_at = now.strftime("%Y-%m-%d %H:%M:%SZ") + title = args.title or f"{args.page_label} {args.mode} · {args.hostname}" + + raw_for_counts = [strip_ansi(l).rstrip("\n") for l in log_text.splitlines()] + n_err, n_warn, _, _ = count_status(raw_for_counts) + + if args.mode == "redirect": + html_out = build_redirect_html( + hostname=args.hostname, + commit=args.commit, + commit_url=args.commit_url, + path_err=args.path_err, + page_label=args.page_label, + generated_at=generated_at, + generated_iso=generated_iso, + n_err=n_err, + n_warn=n_warn, + log_text=log_text, + ) + else: + html_out = build_html( + title=title, + hostname=args.hostname, + mode=args.mode, + log_text=log_text, + commit=args.commit, + commit_url=args.commit_url, + suite_path=args.suite_path, + generated_at=generated_at, + generated_iso=generated_iso, + link_other=args.link_other or None, + page_label=args.page_label, + ) + + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(html_out, encoding="utf-8") + print(f"wrote {args.out} (errors={n_err} warnings={n_warn} level={status_level(n_err, n_warn)})") + + +if __name__ == "__main__": + main() diff --git a/site-gen/deploy-monitoring-sites.sh b/site-gen/deploy-monitoring-sites.sh new file mode 100755 index 0000000..c5718e1 --- /dev/null +++ b/site-gen/deploy-monitoring-sites.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# deploy-monitoring-sites.sh — rsync generated HTML to DEPLOY_WWW_ROOT on koopa. +# Does NOT reload Caddy (needs root — see ROOT-ON-KOOPA.md). +set -uo pipefail + +ROOT=$(cd "$(dirname "$0")" && pwd) +SETTINGS="${SITE_GEN_SETTINGS:-$ROOT/settings.conf}" +[ -f "$SETTINGS" ] || SETTINGS="$ROOT/settings.conf.example" +# shellcheck disable=SC1090 +set -a +source <(grep -v '^\s*#' "$SETTINGS" | grep -v '^\s*$' | sed 's/\r$//') +set +a + +WORK_ROOT="${WORK_ROOT:-/tmp/taler-monitoring-sites-work}" +HTML_DIR="$WORK_ROOT/html" +DEPLOY_SSH="${DEPLOY_SSH:-koopa-external}" +DEPLOY_WWW_ROOT="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" + +if [ ! -d "$HTML_DIR" ]; then + echo "no HTML_DIR $HTML_DIR — run generate-monitoring-sites.sh first" >&2 + exit 1 +fi + +echo "deploy $HTML_DIR → ${DEPLOY_SSH}:${DEPLOY_WWW_ROOT}/" +# Prefer writing to a hernani-owned staging dir if /var/www not writable +STAGE_REMOTE="${DEPLOY_STAGING:-/home/hernani/monitoring-sites-staging}" + +ssh -o BatchMode=yes -o ConnectTimeout=20 "$DEPLOY_SSH" \ + "mkdir -p '$STAGE_REMOTE' '$DEPLOY_WWW_ROOT' 2>/dev/null || mkdir -p '$STAGE_REMOTE'" + +rsync -az --delete "$HTML_DIR/" "${DEPLOY_SSH}:${STAGE_REMOTE}/" +echo "staged on $DEPLOY_SSH:$STAGE_REMOTE" +echo +echo "If $DEPLOY_WWW_ROOT is root-owned, as root on koopa:" +echo " mkdir -p $DEPLOY_WWW_ROOT" +echo " rsync -a --delete $STAGE_REMOTE/ $DEPLOY_WWW_ROOT/" +echo " chown -R caddy:caddy $DEPLOY_WWW_ROOT # or root:caddy — match other www" +echo " # then Caddyfile handles + reload — see ROOT-ON-KOOPA.md" +echo +# try direct rsync to DEPLOY_WWW_ROOT if writable +if ssh -o BatchMode=yes "$DEPLOY_SSH" "test -w '$DEPLOY_WWW_ROOT' 2>/dev/null || test -w \"\$(dirname '$DEPLOY_WWW_ROOT')\""; then + rsync -az --delete "$HTML_DIR/" "${DEPLOY_SSH}:${DEPLOY_WWW_ROOT}/" && \ + echo "also synced directly to $DEPLOY_WWW_ROOT" +fi diff --git a/site-gen/generate-monitoring-sites.sh b/site-gen/generate-monitoring-sites.sh new file mode 100755 index 0000000..f9e344e --- /dev/null +++ b/site-gen/generate-monitoring-sites.sh @@ -0,0 +1,273 @@ +#!/usr/bin/env bash +# generate-monitoring-sites.sh — run taler-monitoring, build console HTML for +# /monitoring and /monitoring_err (static files for host Caddy later). +# +# Usage: +# ./generate-monitoring-sites.sh # all SITES (or ONLY_HOSTS) +# ONLY_HOSTS="bank.hacktivism.ch taler.hacktivism.ch" ./generate-monitoring-sites.sh +# ./generate-monitoring-sites.sh --dry-run # HTML from empty/missing logs only +# ./generate-monitoring-sites.sh --skip-run # only convert existing logs +# +# Settings: settings.conf (from settings.conf.example). No secrets in settings. +# Secrets for e2e: ../secrets.env (optional; default phases are urls versions). +# +set -uo pipefail + +export PYTHONUNBUFFERED=1 +# Global reporting defaults (same as host-agent run-host-report.sh) +: "${RUN_TIMEOUT:=600}" +export RUN_TIMEOUT + +ROOT=$(cd "$(dirname "$0")" && pwd) +SETTINGS="${SITE_GEN_SETTINGS:-$ROOT/settings.conf}" +EXAMPLE="$ROOT/settings.conf.example" + +if [ ! -f "$SETTINGS" ]; then + if [ -f "$EXAMPLE" ]; then + echo "note: no $SETTINGS — using example defaults (copy to settings.conf to customize)" + SETTINGS="$EXAMPLE" + else + echo "missing settings: $EXAMPLE" >&2 + exit 2 + fi +fi + +# Load KEY=value lines; allow SITES already set in environment to win. +# Multiline SITES in the conf file: use SITES_FILE= or env SITES= instead. +while IFS= read -r _line || [ -n "$_line" ]; do + _line=${_line%%#*} + _line=$(echo "$_line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + [ -z "$_line" ] && continue + case "$_line" in + SITES=*|SITES_FILE=*) continue ;; # handled below / env + *=*) + _k=${_line%%=*} + _v=${_line#*=} + # do not override pre-set env + if [ -z "${!_k+x}" ] 2>/dev/null || [ -z "${!_k}" ]; then + export "$_k=$_v" + fi + ;; + esac +done < "$SETTINGS" + +# SITES from dedicated file or keep env; else parse heredoc-style from example list +if [ -n "${SITES_FILE:-}" ] && [ -f "$SITES_FILE" ]; then + SITES=$(grep -v '^\s*#' "$SITES_FILE" | grep -v '^\s*$') + export SITES +elif [ -z "${SITES:-}" ]; then + # default 9 fronts + SITES="bank.hacktivism.ch|hacktivism.ch +exchange.hacktivism.ch|hacktivism.ch +taler.hacktivism.ch|hacktivism.ch +stage.bank.lefrancpaysan.ch|stage.lefrancpaysan.ch +stage.exchange.lefrancpaysan.ch|stage.lefrancpaysan.ch +stage.monnaie.lefrancpaysan.ch|stage.lefrancpaysan.ch +bank.lefrancpaysan.ch|lefrancpaysan.ch +exchange.lefrancpaysan.ch|lefrancpaysan.ch +monnaie.lefrancpaysan.ch|lefrancpaysan.ch" + export SITES +fi + +SKIP_RUN=0 +DRY=0 +while [ $# -gt 0 ]; do + case "$1" in + --skip-run) SKIP_RUN=1; shift ;; + --dry-run) DRY=1; SKIP_RUN=1; shift ;; + -h|--help) sed -n '1,20p' "$0"; exit 0 ;; + *) echo "unknown arg: $1" >&2; exit 2 ;; + esac +done + +MONITORING_ROOT=$(cd "$ROOT/${MONITORING_ROOT:-..}" && pwd) +WORK_ROOT="${WORK_ROOT:-/tmp/taler-monitoring-sites-work}" +LOG_DIR="$WORK_ROOT/logs" +HTML_DIR="$WORK_ROOT/html" +mkdir -p "$LOG_DIR" "$HTML_DIR" + +# commit of monitoring suite used for this run +if git -C "$MONITORING_ROOT/../.." rev-parse HEAD >/dev/null 2>&1; then + REPO_ROOT=$(cd "$MONITORING_ROOT/../.." && pwd) +elif git -C "$MONITORING_ROOT" rev-parse HEAD >/dev/null 2>&1; then + REPO_ROOT=$(cd "$MONITORING_ROOT" && pwd) +else + REPO_ROOT=$(cd "$ROOT/../../.." && pwd) +fi +COMMIT=$(git -C "$REPO_ROOT" rev-parse HEAD 2>/dev/null || echo unknown) +COMMIT_SHORT=$(git -C "$REPO_ROOT" rev-parse --short=12 HEAD 2>/dev/null || echo unknown) +SOURCE_REPO_WEB="${SOURCE_REPO_WEB:-https://git.hacktivism.ch/hernani/taler-monitoring}" +TMPL="${SOURCE_COMMIT_URL_TMPL:-\{repo\}/src/commit/\{commit\}}" +COMMIT_URL=$(printf '%s' "$TMPL" | sed "s|{repo}|$SOURCE_REPO_WEB|g; s|{commit}|$COMMIT|g") +SUITE_PATH="${SOURCE_SUITE_PATH:-.}" + +# Outside-only public checks (no SSH into koopa/stage containers) +PHASES_ERR="${PHASES_ERR:-urls versions}" +PHASES_OK="${PHASES_OK:-urls versions}" +CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" +SKIP_SSH="${SKIP_SSH:-1}" +# Drop SSH phases if someone passes "all" / inside / server / e2e by mistake +filter_phases_outside() { + local out=() p + for p in "$@"; do + case "$p" in + inside|server|e2e|ladder|goa-ladder|auth401) continue ;; + all) out+=(urls versions);; + *) out+=("$p") ;; + esac + done + printf '%s\n' "${out[@]}" +} + +run_mon() { + local domain="$1" phases="$2" logfile="$3" cont="$4" + # shellcheck disable=SC2206 + local ph=( $phases ) + mapfile -t ph < <(filter_phases_outside "${ph[@]}") + [ "${#ph[@]}" -gt 0 ] || ph=(urls versions) + + : >"$logfile" + echo "+ -d $domain · phases=${ph[*]} · CONTINUE=$cont · SKIP_SSH=$SKIP_SSH · RUN_TIMEOUT=$RUN_TIMEOUT" + + if [ -n "${RUNNER_SSH:-}" ]; then + echo "+ via ssh $RUNNER_SSH" + rsync -az --delete \ + --exclude secrets.env \ + --exclude 'android-test/artifacts' \ + --exclude '.git' \ + "$MONITORING_ROOT/" \ + "${RUNNER_SSH}:${RUNNER_REMOTE_WORKDIR}/" \ + || return 1 + # shellcheck disable=SC2029 + if command -v stdbuf >/dev/null 2>&1; then + ssh -o BatchMode=yes -o ConnectTimeout=30 "$RUNNER_SSH" \ + "cd '${RUNNER_REMOTE_WORKDIR}' && chmod +x taler-monitoring.sh check_*.sh 2>/dev/null; \ + export CONTINUE_ON_ERROR='$cont' AUTH401_CONTINUE='$cont' SKIP_SSH='$SKIP_SSH' RUN_TIMEOUT='$RUN_TIMEOUT'; \ + stdbuf -oL -eL ./taler-monitoring.sh -d '$domain' ${ph[*]}" \ + 2>&1 | stdbuf -oL -eL tee -a "$logfile" + else + ssh -o BatchMode=yes -o ConnectTimeout=30 "$RUNNER_SSH" \ + "cd '${RUNNER_REMOTE_WORKDIR}' && chmod +x taler-monitoring.sh check_*.sh 2>/dev/null; \ + export CONTINUE_ON_ERROR='$cont' AUTH401_CONTINUE='$cont' SKIP_SSH='$SKIP_SSH' RUN_TIMEOUT='$RUN_TIMEOUT'; \ + ./taler-monitoring.sh -d '$domain' ${ph[*]}" \ + 2>&1 | tee -a "$logfile" + fi + return "${PIPESTATUS[0]}" + fi + + if command -v stdbuf >/dev/null 2>&1; then + ( cd "$MONITORING_ROOT" && \ + CONTINUE_ON_ERROR="$cont" AUTH401_CONTINUE="$cont" SKIP_SSH="$SKIP_SSH" \ + RUN_TIMEOUT="$RUN_TIMEOUT" \ + stdbuf -oL -eL ./taler-monitoring.sh -d "$domain" "${ph[@]}" ) \ + 2>&1 | stdbuf -oL -eL tee -a "$logfile" + else + ( cd "$MONITORING_ROOT" && \ + CONTINUE_ON_ERROR="$cont" AUTH401_CONTINUE="$cont" SKIP_SSH="$SKIP_SSH" \ + RUN_TIMEOUT="$RUN_TIMEOUT" \ + ./taler-monitoring.sh -d "$domain" "${ph[@]}" ) \ + 2>&1 | tee -a "$logfile" + fi + return "${PIPESTATUS[0]}" +} + +htmlify() { + local host="$1" mode="$2" log="$3" out="$4" other="$5" + mkdir -p "$(dirname "$out")" + if [ ! -f "$ROOT/console_to_html.py" ]; then + echo "WARN: console_to_html.py missing — bootstrap $out" + { + echo "$host $mode" + echo "

    $host · $mode

    commit $COMMIT_SHORT

    " + echo "
    "
+      tail -n 100 "$log" 2>/dev/null | sed 's/&/\&/g;s/"
+    } >"$out"
+    return 0
+  fi
+  python3 "$ROOT/console_to_html.py" \
+    --log "$log" \
+    --out "$out" \
+    --hostname "$host" \
+    --mode "$mode" \
+    --commit "$COMMIT" \
+    --commit-url "$COMMIT_URL" \
+    --suite-path "$SUITE_PATH" \
+    --link-other "$other"
+}
+
+# Parse SITES
+mapfile -t SITE_LINES < <(printf '%s\n' "$SITES" | sed '/^\s*$/d' | sed 's/#.*//')
+ONLY="${ONLY_HOSTS:-}"
+
+for line in "${SITE_LINES[@]}"; do
+  line=$(echo "$line" | tr -d ' \t')
+  [ -z "$line" ] && continue
+  host=${line%%|*}
+  domain=${line#*|}
+  if [ -n "$ONLY" ]; then
+    echo " $ONLY " | grep -q " $host " || continue
+  fi
+
+  echo "======== $host  (domain=$domain)  commit=$COMMIT_SHORT ========"
+  base_html="$HTML_DIR/$host"
+  mkdir -p "$base_html/monitoring" "$base_html/monitoring_err"
+  log_err="$LOG_DIR/${host}.err.log"
+  log_ok="$LOG_DIR/${host}.ok.log"
+  ec_err=1
+  ec_ok=1
+
+  if [ "$SKIP_RUN" != "1" ]; then
+    run_mon "$domain" "$PHASES_ERR" "$log_err" "$CONTINUE_ON_ERROR"
+    ec_err=$?
+    # ok-run without continue (strict)
+    run_mon "$domain" "$PHASES_OK" "$log_ok" "0"
+    ec_ok=$?
+  else
+    [ -f "$log_err" ] || : >"$log_err"
+    [ -f "$log_ok" ] || : >"$log_ok"
+    # infer exit from log if present
+    if grep -q '\[ ERROR \]' "$log_err" 2>/dev/null || grep -q 'ERROR' "$log_err" 2>/dev/null; then
+      ec_err=1
+    else
+      ec_err=0
+    fi
+    ec_ok=$ec_err
+  fi
+
+  # Always write pages (first run / empty logs too)
+  [ -f "$log_err" ] || : >"$log_err"
+  [ -f "$log_ok" ] || : >"$log_ok"
+
+  htmlify "$host" "err" "$log_err" \
+    "$base_html/monitoring_err/index.html" "/monitoring/"
+
+  if [ "$ec_ok" -eq 0 ] && [ "$ec_err" -eq 0 ]; then
+    htmlify "$host" "ok" "$log_ok" \
+      "$base_html/monitoring/index.html" "/monitoring_err/"
+    echo "  → /monitoring (clean) + /monitoring_err"
+  else
+    htmlify "$host" "redirect" "$log_err" \
+      "$base_html/monitoring/index.html" "/monitoring_err/"
+    echo "  → /monitoring (redirect stub) + /monitoring_err (errors)  ec_err=$ec_err ec_ok=$ec_ok"
+  fi
+
+  # first-run safety
+  if [ ! -f "$base_html/monitoring/index.html" ]; then
+    htmlify "$host" "err" "$log_err" \
+      "$base_html/monitoring/index.html" "/monitoring_err/"
+  fi
+
+  printf '%s\n' "$COMMIT" >"$base_html/COMMIT"
+  printf '%s\n' "$COMMIT_URL" >"$base_html/COMMIT_URL"
+  printf 'host=%s domain=%s ec_err=%s ec_ok=%s commit=%s run_timeout=%s\n' \
+    "$host" "$domain" "$ec_err" "$ec_ok" "$COMMIT" "$RUN_TIMEOUT" | tee "$base_html/STATUS.txt"
+done
+
+echo
+echo "HTML under $HTML_DIR"
+echo "commit $COMMIT_SHORT  $COMMIT_URL"
+echo "RUN_TIMEOUT=${RUN_TIMEOUT}s (global; 0=unlimited)"
+echo "Deploy (as hernani, may need sudo for /var/www):"
+echo "  ./deploy-monitoring-sites.sh"
+echo "Root on koopa: see ROOT-ON-KOOPA.md"
diff --git a/site-gen/install-firecuda-timer.sh b/site-gen/install-firecuda-timer.sh
new file mode 100755
index 0000000..55052e9
--- /dev/null
+++ b/site-gen/install-firecuda-timer.sh
@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+# install-firecuda-timer.sh — OPTIONAL: rsync suite to firecuda + launchd every 4h
+#
+# POLICY (2026-07): firecuda launchd is **disabled**. Prefer koopa host-agent.
+# This script remains for documentation / future re-enable only.
+# Do not run unless you intentionally want a second outside-only scheduler.
+#
+#   ./install-firecuda-timer.sh
+#   ./install-firecuda-timer.sh --run-now
+#
+set -euo pipefail
+
+ROOT=$(cd "$(dirname "$0")" && pwd)
+MON_ROOT=$(cd "$ROOT/.." && pwd)
+FIRECUDA="${FIRECUDA_SSH:-firecuda-external}"
+REMOTE_BASE="${FIRECUDA_INSTALL_DIR:-taler-monitoring-site-gen}"
+RUN_NOW=0
+[ "${1:-}" = "--run-now" ] && RUN_NOW=1
+
+echo "install → $FIRECUDA:~/$REMOTE_BASE (outside-only, every 4h)"
+
+ssh -o BatchMode=yes -o ConnectTimeout=20 "$FIRECUDA" \
+  "mkdir -p \"\$HOME/$REMOTE_BASE\" \"\$HOME/Library/Logs/taler-monitoring-sites\" \"\$HOME/var/taler-monitoring-sites-work\" \"\$HOME/Library/LaunchAgents\""
+
+# Sync full taler-monitoring suite (no secrets.env)
+rsync -az --delete \
+  --exclude secrets.env \
+  --exclude 'android-test/artifacts' \
+  --exclude '.git' \
+  --exclude 'site-gen/settings.conf' \
+  "$MON_ROOT/" \
+  "${FIRECUDA}:$REMOTE_BASE/"
+
+# settings on firecuda: outside only, run locally on firecuda, deploy to koopa
+ssh -o BatchMode=yes "$FIRECUDA" "cat > \"\$HOME/$REMOTE_BASE/site-gen/settings.conf\" <<'EOF'
+# auto-installed by install-firecuda-timer.sh — no secrets
+RUNNER_SSH=
+SKIP_SSH=1
+CONTINUE_ON_ERROR=1
+PHASES_ERR=urls versions
+PHASES_OK=urls versions
+DEPLOY_SSH=koopa-external
+DEPLOY_STAGING=/home/hernani/monitoring-sites-staging
+WORK_ROOT=
+MONITORING_ROOT=..
+SOURCE_REPO_WEB=https://git.hacktivism.ch/hernani/taler-monitoring
+SOURCE_COMMIT_URL_TMPL={repo}/src/commit/{commit}
+SOURCE_SUITE_PATH=.
+EOF
+# WORK_ROOT expanded on remote
+sed -i.bak \"s|^WORK_ROOT=.*|WORK_ROOT=\$HOME/var/taler-monitoring-sites-work|\" \"\$HOME/$REMOTE_BASE/site-gen/settings.conf\" 2>/dev/null || \
+  sed -i '' \"s|^WORK_ROOT=.*|WORK_ROOT=\$HOME/var/taler-monitoring-sites-work|\" \"\$HOME/$REMOTE_BASE/site-gen/settings.conf\"
+"
+
+# launchd plist with absolute paths
+ssh -o BatchMode=yes "$FIRECUDA" bash -s < "\$PLIST_DST"
+chmod +x "\$SITE_GEN"/*.sh "\$SITE_GEN"/console_to_html.py 2>/dev/null || true
+chmod +x "\$SITE_GEN/run-on-firecuda.sh"
+launchctl bootout "gui/\$(id -u)/com.hacktivism.taler-monitoring-sites" 2>/dev/null || true
+launchctl unload "\$PLIST_DST" 2>/dev/null || true
+launchctl bootstrap "gui/\$(id -u)" "\$PLIST_DST" 2>/dev/null || launchctl load "\$PLIST_DST"
+launchctl enable "gui/\$(id -u)/com.hacktivism.taler-monitoring-sites" 2>/dev/null || true
+launchctl print "gui/\$(id -u)/com.hacktivism.taler-monitoring-sites" 2>/dev/null | head -20 || launchctl list | grep taler-monitoring || true
+echo "OK: launchd com.hacktivism.taler-monitoring-sites (StartInterval=14400)"
+REMOTE
+
+if [ "$RUN_NOW" = "1" ]; then
+  echo "run once now…"
+  ssh -o BatchMode=yes "$FIRECUDA" "\"\$HOME/$REMOTE_BASE/site-gen/run-on-firecuda.sh\"" || true
+fi
+
+echo "done. logs: $FIRECUDA:~/Library/Logs/taler-monitoring-sites/"
+echo "manual: ssh $FIRECUDA '~/taler-monitoring-site-gen/site-gen/run-on-firecuda.sh'"
diff --git a/site-gen/pull-firecuda-to-koopa.sh b/site-gen/pull-firecuda-to-koopa.sh
new file mode 100755
index 0000000..b1b10ec
--- /dev/null
+++ b/site-gen/pull-firecuda-to-koopa.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# pull-firecuda-to-koopa.sh — copy HTML from firecuda work dir → koopa staging
+# Run on laptop (hernani) where both SSH aliases work.
+set -euo pipefail
+FIRECUDA="${FIRECUDA_SSH:-firecuda-external}"
+KOOPA="${DEPLOY_SSH:-koopa-external}"
+REMOTE_HTML="${FIRECUDA_HTML:-var/taler-monitoring-sites-work/html/}"
+STAGE="${DEPLOY_STAGING:-/home/hernani/monitoring-sites-staging}"
+
+echo "firecuda:$REMOTE_HTML → $KOOPA:$STAGE"
+ssh -o BatchMode=yes -o ConnectTimeout=20 "$KOOPA" "mkdir -p '$STAGE'"
+rsync -az --delete \
+  "${FIRECUDA}:${REMOTE_HTML}" \
+  "${KOOPA}:${STAGE}/"
+echo "OK staged. Root: rsync to /var/www/monitoring-sites (see ROOT-ON-KOOPA.md)"
diff --git a/site-gen/run-on-firecuda.sh b/site-gen/run-on-firecuda.sh
new file mode 100755
index 0000000..6f77de6
--- /dev/null
+++ b/site-gen/run-on-firecuda.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# run-on-firecuda.sh — entrypoint for launchd/cron on firecuda-external.
+# Outside-only monitoring (SKIP_SSH=1), all 9 sites, then stage HTML to koopa.
+#
+# Installed path on firecuda (default):
+#   ~/taler-monitoring-site-gen/site-gen/run-on-firecuda.sh
+#
+set -uo pipefail
+
+export PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:${PATH:-}"
+
+ROOT=$(cd "$(dirname "$0")" && pwd)
+LOG_DIR="${FIRECUDA_LOG_DIR:-$HOME/Library/Logs/taler-monitoring-sites}"
+mkdir -p "$LOG_DIR"
+STAMP=$(date +%Y%m%d-%H%M%S)
+LOG="$LOG_DIR/run-${STAMP}.log"
+exec >>"$LOG" 2>&1
+
+echo "======== $(date -Iseconds 2>/dev/null || date) firecuda monitoring sites ========"
+
+# Always: public DNS only, no SSH checks into stacks
+export SKIP_SSH=1
+export CONTINUE_ON_ERROR=1
+export AUTH401_CONTINUE=1
+export RUNNER_SSH=
+export PHASES_ERR="${PHASES_ERR:-urls versions}"
+export PHASES_OK="${PHASES_OK:-urls versions}"
+export WORK_ROOT="${WORK_ROOT:-$HOME/var/taler-monitoring-sites-work}"
+export MONITORING_ROOT="${MONITORING_ROOT:-$ROOT/..}"
+export DEPLOY_SSH="${DEPLOY_SSH:-koopa-external}"
+export DEPLOY_STAGING="${DEPLOY_STAGING:-/home/hernani/monitoring-sites-staging}"
+
+# Prefer full 9-site list from settings or defaults inside generate script
+unset ONLY_HOSTS
+
+cd "$ROOT" || exit 1
+chmod +x generate-monitoring-sites.sh deploy-monitoring-sites.sh console_to_html.py 2>/dev/null || true
+
+./generate-monitoring-sites.sh
+ec=$?
+
+# HTML stays on firecuda under WORK_ROOT/html.
+# Optional push to koopa if this host can SSH there (often only from laptop):
+if [ -n "${DEPLOY_SSH:-}" ] && [ "${DEPLOY_FROM_FIRECUDA:-0}" = "1" ]; then
+  if [ -x ./deploy-monitoring-sites.sh ]; then
+    ./deploy-monitoring-sites.sh || echo "WARN: deploy staging failed"
+  fi
+else
+  echo "note: HTML at ${WORK_ROOT}/html — pull from laptop:"
+  echo "  rsync -az firecuda-external:var/taler-monitoring-sites-work/html/ \\"
+  echo "    koopa-external:monitoring-sites-staging/"
+  echo "  # or: ./pull-firecuda-to-koopa.sh"
+fi
+
+# keep last 20 logs
+ls -1t "$LOG_DIR"/run-*.log 2>/dev/null | tail -n +21 | xargs rm -f 2>/dev/null || true
+
+echo "======== done ec=$ec ========"
+exit "$ec"
diff --git a/site-gen/settings.conf.example b/site-gen/settings.conf.example
new file mode 100644
index 0000000..f70959b
--- /dev/null
+++ b/site-gen/settings.conf.example
@@ -0,0 +1,55 @@
+# monitoring site-gen — non-secret settings (safe as example)
+# Copy:  cp settings.conf.example settings.conf
+# Never put passwords/tokens here.
+
+# On firecuda the suite runs *locally* (RUNNER_SSH empty). From a laptop you
+# can set RUNNER_SSH=firecuda-external to drive a one-off remote run instead.
+RUNNER_SSH=
+RUNNER_SSH_FALLBACKS=
+RUNNER_REMOTE_WORKDIR=/tmp/taler-monitoring-site-gen
+MONITORING_ROOT=..
+
+# Outside-only: never SSH into bank/exchange/merchant containers
+SKIP_SSH=1
+
+# Deploy staging on koopa (host Caddy later — root copies to /var/www)
+DEPLOY_SSH=koopa-external
+DEPLOY_SSH_FALLBACKS=koopa
+# Static files for Caddy file_server (created per hostname)
+DEPLOY_WWW_ROOT=/var/www/monitoring-sites
+DEPLOY_SUDO=sudo
+CADDY_CONFIG=/etc/caddy/Caddyfile
+CADDY_MIRROR_REL=configs/caddy/Caddyfile
+# Working tree on deploy host for apply (optional)
+KOOPA_CADDY_DIR=/home/hernani/koopa-caddy
+
+# Source link for HTML footer (Forgejo)
+SOURCE_REPO_WEB=https://git.hacktivism.ch/hernani/taler-monitoring
+SOURCE_COMMIT_URL_TMPL={repo}/src/commit/{commit}
+SOURCE_SUITE_PATH=.
+
+CONTINUE_ON_ERROR=1
+PHASES_ERR=urls versions
+PHASES_OK=urls versions
+SKIP_E2E=1
+
+# Local work dir for logs + HTML
+WORK_ROOT=/tmp/taler-monitoring-sites-work
+
+# Sites: hostname|domain_profile  (9 fronts)
+# Start with hacktivism; stage + prod LFP follow
+SITES="
+bank.hacktivism.ch|hacktivism.ch
+exchange.hacktivism.ch|hacktivism.ch
+taler.hacktivism.ch|hacktivism.ch
+stage.bank.lefrancpaysan.ch|stage.lefrancpaysan.ch
+stage.exchange.lefrancpaysan.ch|stage.lefrancpaysan.ch
+stage.monnaie.lefrancpaysan.ch|stage.lefrancpaysan.ch
+bank.lefrancpaysan.ch|lefrancpaysan.ch
+exchange.lefrancpaysan.ch|lefrancpaysan.ch
+monnaie.lefrancpaysan.ch|lefrancpaysan.ch
+"
+
+# Only process this subset (space-separated hostnames). Empty = all SITES.
+# Example for first deploy: ONLY_HOSTS=bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch
+ONLY_HOSTS=
diff --git a/surface-catalog.conf b/surface-catalog.conf
new file mode 100644
index 0000000..16d5755
--- /dev/null
+++ b/surface-catalog.conf
@@ -0,0 +1,52 @@
+# surface-catalog.conf — known public Taler / GNUnet / Taler Systems hosts
+# Format (whitespace-separated; # comments):
+#   host   ports   expect_proto   label
+# expect_proto: https | http | ssh | tcp | any
+# ports: comma-separated (empty = defaults for proto)
+#
+# This list is expanded at runtime (DNS discovery, cert SANs, -d domain profile).
+# Phase `surface` is NOT in default phases — run explicitly:
+#   ./taler-monitoring.sh surface
+#   ./taler-monitoring.sh -d hacktivism.ch surface
+
+# --- taler.net ecosystem ---
+www.taler.net                 443     https   taler-www
+docs.taler.net                443     https   taler-docs
+git.taler.net                 443,22  https   taler-git
+bugs.taler.net                443     https   taler-bugs
+lists.taler.net               443     https   taler-lists
+tutorials.taler.net           443     https   taler-tutorials
+bank.demo.taler.net           443     https   demo-bank
+exchange.demo.taler.net       443     https   demo-exchange
+backend.demo.taler.net        443     https   demo-merchant
+shop.demo.taler.net           443     https   demo-shop
+bank.test.taler.net           443     https   test-bank
+exchange.test.taler.net       443     https   test-exchange
+backend.test.taler.net        443     https   test-merchant
+
+# --- taler-ops / TOPS ---
+www.taler-ops.ch              443     https   tops-www
+taler-ops.ch                  443     https   tops-apex
+bank.taler-ops.ch             443     https   tops-bank
+exchange.taler-ops.ch         443     https   tops-exchange
+my.taler-ops.ch               443     https   tops-merchant
+stage.taler-ops.ch            443     https   tops-stage-www
+stage.my.taler-ops.ch         443     https   tops-stage-merchant
+exchange.stage.taler-ops.ch   443     https   tops-stage-exchange
+nexus.stage.taler-ops.ch      443     https   tops-stage-nexus
+shops.taler-ops.ch            443     https   tops-shops
+mattermost.taler.net          443     https   mattermost
+
+# --- gnunet.org ---
+www.gnunet.org                443,80  https   gnunet-www
+gnunet.org                    443,80  https   gnunet-apex
+git.gnunet.org                443,22  https   gnunet-git
+bugs.gnunet.org               443     https   gnunet-bugs
+
+# --- taler-systems.com ---
+www.taler-systems.com         443     https   tsa-www
+taler-systems.com             443     https   tsa-apex
+
+# --- common infra hostnames (remote only; may 404/redirect) ---
+deb.taler.net                 443     https   deb-taler
+ftp.gnu.org                   443     https   gnu-ftp
diff --git a/taler-monitoring.sh b/taler-monitoring.sh
new file mode 100755
index 0000000..557c7a0
--- /dev/null
+++ b/taler-monitoring.sh
@@ -0,0 +1,477 @@
+#!/usr/bin/env bash
+# taler-monitoring — public URL / stack checks for a Taler domain
+#
+#   ./taler-monitoring.sh                          # local GOA (urls + inside + e2e)
+#   ./taler-monitoring.sh -d taler.net urls         # public demo, no SSH
+#   ./taler-monitoring.sh --domain taler-ops.ch     # public ops, no SSH
+#   TALER_DOMAIN=demo.taler.net ./taler-monitoring.sh urls
+#
+# Tags: [OK] [INFO] [WARN] [ERROR] [BLOCKER]
+# Exit 0 only if every selected phase exits 0.
+
+set -euo pipefail
+ROOT=$(cd "$(dirname "$0")" && pwd)
+
+# Line-buffered stdout/stderr so redirected logs (tee/host-agent) flush each line.
+if [ "${_TALER_MON_STDBUF:-0}" != "1" ] && command -v stdbuf >/dev/null 2>&1; then
+  export _TALER_MON_STDBUF=1
+  exec stdbuf -oL -eL bash "$0" "$@"
+fi
+
+usage() {
+  cat <<'EOF'
+taler-monitoring — bank / exchange / merchant checks
+
+Usage:
+  ./taler-monitoring.sh [options] [phases...]
+
+Phases:
+  urls     public HTTPS (no SSH)          ← default with --domain
+           (includes merchant /webui/ SPA fingerprints: version.txt, overlay, assets)
+  inside   container status via SSH       (local stack only)
+  versions taler packages vs deb.taler.net trixie + repo availability
+  sanity   public + optional server
+  server   server-side only (SSH)
+  e2e      withdraw + pay (small amounts; remote aborts on login/KYC)
+  ladder   withdraw/pay amount ladder (GOA ceiling or stage TESTPAYSAN max_wire)
+  auth401  merchant Basic-auth / case matrix (HTTP 401 paths; may create throwaway instance)
+  aptdeploy  koopa podman apt-src smoke: taler-merchant in
+             koopa-taler-deploy-test-apt-src-trixie{,-testing}
+  surface  REMOTE-ONLY public inventory (NOT in default/all/full):
+             ecosystem hosts (taler.net, gnunet.org, taler-systems.com, mattermost, …)
+             or -d DOMAIN → that domain’s surface; port/protocol/TLS/CVE (OSV)
+  all      urls + inside + versions + sanity + e2e  (SSH phases only on koopa)
+  full     all + server + ladder + auth401  (maximum; long-running, needs secrets)
+           NOTE: surface is never included in all/full — pass it explicitly
+
+Options:
+  -d, --domain DOMAIN   load profile from domains.conf (bank/exchange/merchant)
+                        presets: koopa | hacktivism.ch | taler.net | taler-ops.ch
+                                 | my.taler-ops.ch | stage.taler-ops.ch
+                        unknown domain → heuristic hosts (see README)
+                        local (koopa) allows SSH; others public-only
+  --bank URL            bank base (hostname or https://…) — overrides profile
+  --exchange URL        exchange base — overrides profile
+  --merchant URL        merchant-backend base — overrides profile
+  --currency CODE       expected currency (GOA, KUDOS, CHF, …); empty = report only
+  --no-probe            do not probe alternate hosts (unknown domains only)
+  -h, --help
+
+Add a stack: edit domains.conf (name + bank + exchange + merchant + currency).
+
+Examples:
+  ./taler-monitoring.sh -d taler.net
+  ./taler-monitoring.sh -d taler-ops.ch urls
+  ./taler-monitoring.sh -d my.taler-ops.ch urls
+  ./taler-monitoring.sh -d taler-ops.ch --merchant https://my.taler-ops.ch urls
+  ./taler-monitoring.sh -d demo.taler.net --currency KUDOS
+  ./taler-monitoring.sh --exchange https://exchange.taler-ops.ch \
+      --merchant https://my.taler-ops.ch --bank https://bank.taler-ops.ch urls
+
+Env (same meaning):
+  TALER_DOMAIN  BANK_PUBLIC  EXCHANGE_PUBLIC  MERCHANT_PUBLIC  EXPECT_CURRENCY
+  TALER_DOMAINS_CONF  SKIP_SSH=1
+  NO_COLOR=1 / CLICOLOR=0   disable green/yellow/red tags (default: coloured)
+  SKIP_SSH=1    NO_COLOR=1
+  RUN_TIMEOUT=600   whole-run wall clock seconds (default 600; 0=unlimited)
+                    long phases (ladder/full/e2e) need a higher value or 0
+  DISK_WARN_USED_PCT=85  disk free: WARN when used ≥ this %
+  DISK_ERR_USED_PCT=95   disk free: ERROR when used ≥ this % (or avail=0 / 100%)
+  PERF_WARN_MS  PERF_FAIL_MS   (urls latency; default 8000 / 20000)
+  QR_CHECK=0    skip QR form + qrencode/zbarimg (urls phase)
+  QR_ECC=M      qrencode ECC level (default M)
+  KOOPA_SSH  KOOPA_SSH_FALLBACKS   (default koopa → koopa-external)
+  METRICS_LOAD=0   skip host/container RAM/CPU probes (e2e/ladder/inside)
+  EXPECT_WEBUI_VERSION=1.6.11   pin merchant /webui/version.txt (urls)
+  EXPECT_WEBUI_OVERLAY=selfbuild  require substr in version-overlay.txt
+  WEBUI_OVERLAY_DENY='master-11340'  fail if overlay matches regex
+  CHECK_WEBUI_SPA=0             skip SPA fingerprint block in urls
+  AUTH401_*                     see check_auth401.sh / secrets.env.example
+  AUTH401_CONTINUE=1 / CONTINUE_ON_ERROR=1
+                        auth401: run all groups, collect every ERROR (no mid-run halt)
+  APT_DEPLOY_SKIP=1     skip aptdeploy phase
+  APT_DEPLOY_CONTAINERS="name:suite …"  default trixie + trixie-testing deploy-test pods
+  SURFACE_CVE=0         disable CVE queries in surface phase
+  SURFACE_CVE_LEVEL=warn|error  bare Server-header versions default warn
+                        (Debian package versions default error)
+  SURFACE_CATALOG=path  override surface-catalog.conf
+
+Full load (GOA / hacktivism):
+  ./taler-monitoring.sh -d hacktivism.ch full
+  # or explicit:
+  ./taler-monitoring.sh -d hacktivism.ch urls inside versions sanity server e2e ladder auth401
+  # hacktivism host-agent also runs aptdeploy (apt-src merchant containers on koopa)
+
+Remote surface / ecosystem (explicit only — never default):
+  ./taler-monitoring.sh surface
+  ./taler-monitoring.sh -d hacktivism.ch surface
+  ./taler-monitoring.sh -d lefrancpaysan.ch surface
+
+SPA pin after selfbuild:
+  EXPECT_WEBUI_VERSION=1.6.11 EXPECT_WEBUI_OVERLAY=selfbuild-v1.6.11 \\
+    WEBUI_OVERLAY_DENY='c778af04|master-11340' \\
+    ./taler-monitoring.sh -d hacktivism.ch urls
+EOF
+}
+
+# shellcheck source=lib.sh
+source "$ROOT/lib.sh"
+# Optional gitignored secrets.env next to this suite (or MONITORING_SECRETS_ENV=)
+load_monitoring_secrets_env 2>/dev/null || true
+
+# Persist #NNN + progress across check_*.sh child processes
+TALER_MON_STATE=$(mktemp "${TMPDIR:-/tmp}/taler-mon-state.XXXXXX")
+export TALER_MON_STATE
+trap 'rm -f "${TALER_MON_STATE:-}"' EXIT
+
+PHASES=()
+DOMAIN_SET=0
+BANK_OVERRIDE=""
+EXCHANGE_OVERRIDE=""
+MERCHANT_OVERRIDE=""
+CURRENCY_OVERRIDE=""
+NO_PROBE=0
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    -h|--help) usage; exit 0 ;;
+    -d|--domain)
+      [ $# -ge 2 ] || { echo "missing arg for $1" >&2; exit 2; }
+      apply_taler_domain "$2"
+      DOMAIN_SET=1
+      TALER_DOMAIN_FROM_CLI=1
+      shift 2
+      ;;
+    --bank)
+      [ $# -ge 2 ] || { echo "missing arg for $1" >&2; exit 2; }
+      BANK_OVERRIDE="${2%/}"; shift 2
+      ;;
+    --exchange)
+      [ $# -ge 2 ] || { echo "missing arg for $1" >&2; exit 2; }
+      EXCHANGE_OVERRIDE="${2%/}"; shift 2
+      ;;
+    --merchant)
+      [ $# -ge 2 ] || { echo "missing arg for $1" >&2; exit 2; }
+      MERCHANT_OVERRIDE="${2%/}"; shift 2
+      ;;
+    --currency)
+      [ $# -ge 2 ] || { echo "missing arg for $1" >&2; exit 2; }
+      CURRENCY_OVERRIDE="$2"; shift 2
+      ;;
+    --no-probe) NO_PROBE=1; shift ;;
+    urls|inside|versions|sanity|server|e2e|ladder|goa-ladder|auth401|aptdeploy|apt-deploy|apt_src|surface|ecosystem|all|full) PHASES+=("$1"); shift ;;
+    *)
+      # bare domain shorthand: ./taler-monitoring.sh taler.net
+      if [[ "$1" == *.* && "$1" != *://* && "$1" != -* ]]; then
+        apply_taler_domain "$1"
+        DOMAIN_SET=1
+        TALER_DOMAIN_FROM_CLI=1
+        shift
+      else
+        echo "Unknown: $1" >&2; usage >&2; exit 2
+      fi
+      ;;
+  esac
+done
+
+if [ "$NO_PROBE" = "1" ]; then
+  TALER_DOMAIN_PROBE=0
+fi
+if [ -n "$CURRENCY_OVERRIDE" ]; then
+  EXPECT_CURRENCY="$CURRENCY_OVERRIDE"
+fi
+if [ -n "$BANK_OVERRIDE" ]; then BANK_PUBLIC="$BANK_OVERRIDE"; fi
+if [ -n "$EXCHANGE_OVERRIDE" ]; then EXCHANGE_PUBLIC="$EXCHANGE_OVERRIDE"; fi
+if [ -n "$MERCHANT_OVERRIDE" ]; then MERCHANT_PUBLIC="$MERCHANT_OVERRIDE"; fi
+
+# Only koopa may use SSH. Remote domains: public + optional e2e (no SSH).
+if [ "${LOCAL_STACK:-1}" != "1" ]; then
+  SKIP_SSH=1
+fi
+
+# Remote: no SSH; ATM withdraw ladder set in check_e2e (smaller notes)
+if [ "${LOCAL_STACK}" != "1" ]; then
+  E2E_FAKE_INCOMING=0
+  E2E_REMOTE=1
+  : "${E2E_WITHDRAW_VALUES:=10 20 50}"
+  : "${E2E_PAY_VALUES:=0.01 0.05 0.1 1}"
+fi
+
+# Export so check_*.sh (re-source lib) see the same targets via env
+export TALER_DOMAIN BANK_PUBLIC EXCHANGE_PUBLIC MERCHANT_PUBLIC
+export EXPECT_CURRENCY SKIP_SSH LOCAL_STACK TALER_DOMAIN_PROBE CHECK_LANDING
+export TALER_DOMAIN_FROM_CLI="${TALER_DOMAIN_FROM_CLI:-0}"
+export DOMAIN_SET="${DOMAIN_SET:-0}"
+export WITHDRAW_AMT PAY_AMT CREDIT_AMT MERCHANT_INSTANCE
+export E2E_FAKE_INCOMING E2E_REMOTE E2E_VARIABLE E2E_ATM_MAX
+export E2E_WITHDRAW_VALUES E2E_PAY_VALUES E2E_USE_TEMPLATES E2E_TEMPLATE_MAP
+export PAIVANA_PUBLIC E2E_PAIVANA E2E_PAIVANA_TEMPLATE E2E_PAIVANA_AMOUNT E2E_PAIVANA_INSTANCE
+export INSIDE_PROFILE INSIDE_SSH
+export INSIDE_BANK_CTR INSIDE_EXCHANGE_CTR INSIDE_MERCHANT_CTR
+export INSIDE_BANK_PORT INSIDE_EXCHANGE_PORT INSIDE_MERCHANT_PORT
+export INSIDE_DNS_BANK INSIDE_DNS_EXCHANGE INSIDE_DNS_MERCHANT
+# Ladder: withdraw then pay — 0 + random mids + max-1 + max (see check_goa_ladder.sh).
+# Defaults so set -u export is safe when vars were never set by caller.
+: "${LADDER_STEPS:=23}"
+: "${LADDER_MIN_AMOUNT:=0.000001}"
+: "${LADDER_CONFIRM_POLLS:=40}"
+: "${LADDER_MAX_RUNGS:=99}"
+: "${LADDER_TIMEOUT_S:=3600}"
+: "${LADDER_REPORT_DIR:=}"
+: "${LADDER_SETTLE_ROUNDS:=18}"
+: "${LADDER_SETTLE_SLEEP:=2}"
+: "${LADDER_MAX_AMOUNT:=4503599627370496}"
+: "${LADDER_INCLUDE_ZERO:=1}"
+: "${LADDER_INCLUDE_MAX:=1}"
+: "${LADDER_HIGH_FROM:=1000000}"
+: "${LADDER_HIGH_RUNGS:=12}"
+: "${LADDER_LOAD:=1}"
+: "${LADDER_PAY:=1}"
+: "${LADDER_WITHDRAW_SCALE:=1.5}"
+: "${LADDER_PAY_SETTLE_ROUNDS:=6}"
+export LADDER_STEPS LADDER_MIN_AMOUNT LADDER_CONFIRM_POLLS LADDER_MAX_RUNGS LADDER_TIMEOUT_S LADDER_REPORT_DIR
+export LADDER_SETTLE_ROUNDS LADDER_SETTLE_SLEEP EXP_PW_FILE EXP_USER
+export LADDER_MAX_AMOUNT LADDER_INCLUDE_ZERO LADDER_INCLUDE_MAX
+export LADDER_HIGH_FROM LADDER_HIGH_RUNGS LADDER_LOAD
+export LADDER_PAY LADDER_WITHDRAW_SCALE LADDER_PAY_SETTLE_ROUNDS
+export TALER_DOMAIN_APPLIED=1
+# Optional: auth401 and other phases may honor CONTINUE_ON_ERROR
+export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-}"
+export AUTH401_CONTINUE="${AUTH401_CONTINUE:-${CONTINUE_ON_ERROR:-}}"
+
+# Default phases
+# - local GOA (koopa): urls + inside + versions + e2e
+# - FrancPaysan STAGE: urls + inside (stagepaysan) + versions + e2e
+# - other remote: urls only (pass e2e/inside explicitly)
+if [ "${#PHASES[@]}" -eq 0 ]; then
+  if [ "${LOCAL_STACK}" = "1" ]; then
+    PHASES=(urls inside versions e2e)
+  elif [ "${EXPECT_CURRENCY:-}" = "TESTPAYSAN" ] \
+    || [[ "${TALER_DOMAIN:-}" == stage.*lefrancpaysan* ]] \
+    || [[ "${TALER_DOMAIN:-}" == *stage.lefrancpaysan* ]]; then
+    # Like hacktivism breadth: public urls + inside (stagepaysan) + versions + e2e
+    PHASES=(urls inside versions e2e)
+  else
+    PHASES=(urls)
+  fi
+fi
+
+OUT_PHASES=()
+for p in "${PHASES[@]}"; do
+  if [ "$p" = "all" ]; then
+    if [ "${LOCAL_STACK}" = "1" ]; then
+      OUT_PHASES+=(urls inside versions sanity e2e)
+    elif [ -n "${INSIDE_SSH:-}" ] || [ "${INSIDE_PROFILE:-}" = "stage-lfp" ]; then
+      OUT_PHASES+=(urls inside versions e2e)
+    else
+      # remote without inside SSH: public + optional e2e
+      OUT_PHASES+=(urls versions e2e)
+    fi
+  elif [ "$p" = "full" ]; then
+    # Maximum coverage. Long. Needs secrets for e2e/ladder/auth401.
+    if [ "${LOCAL_STACK}" = "1" ]; then
+      OUT_PHASES+=(urls inside versions sanity server e2e ladder auth401)
+    elif [ -n "${INSIDE_SSH:-}" ] || [ "${INSIDE_PROFILE:-}" = "stage-lfp" ]; then
+      OUT_PHASES+=(urls inside versions e2e ladder auth401)
+    else
+      OUT_PHASES+=(urls versions e2e auth401)
+    fi
+  else
+    OUT_PHASES+=("$p")
+  fi
+done
+PHASES=()
+seen=" "
+for p in "${OUT_PHASES[@]}"; do
+  # server = koopa-only. inside allowed on koopa OR stagepaysan (INSIDE_SSH).
+  case "$p" in
+    server)
+      if [ "${LOCAL_STACK}" != "1" ]; then
+        echo "[INFO]  skip phase 'server' (koopa only)" >&2
+        continue
+      fi
+      ;;
+    inside)
+      if [ "${LOCAL_STACK}" != "1" ] && [ -z "${INSIDE_SSH:-}" ] && [ "${INSIDE_PROFILE:-}" != "stage-lfp" ]; then
+        echo "[INFO]  skip phase 'inside' (no INSIDE_SSH / not local)" >&2
+        continue
+      fi
+      ;;
+  esac
+  case "$seen" in *" $p "*) ;; *) PHASES+=("$p"); seen="$seen$p " ;; esac
+done
+
+if [ "${#PHASES[@]}" -eq 0 ]; then
+  PHASES=(urls)
+fi
+
+# Rough expected check counts for progress % (override: PROGRESS_TOTAL=N).
+# Prefer slightly high — lib.sh rebalances if still short (never shows done>total).
+if [ "${PROGRESS_TOTAL:-0}" = "0" ] || [ -z "${PROGRESS_TOTAL:-}" ]; then
+  _pt=0
+  for p in "${PHASES[@]}"; do
+    case "$p" in
+      urls)     _pt=$((_pt + 95)) ;;   # +qr + webui SPA fingerprints
+      inside)   _pt=$((_pt + 30)) ;;
+      versions) _pt=$((_pt + 25)) ;;
+      sanity)   _pt=$((_pt + 35)) ;;
+      server)   _pt=$((_pt + 20)) ;;
+      # e2e: ATM ladder emits many INFO lines (coins before/after each note)
+      e2e)      _pt=$((_pt + 240)) ;;
+      ladder|goa-ladder) _pt=$((_pt + 120)) ;;
+      auth401)  _pt=$((_pt + 70)) ;;
+      aptdeploy) _pt=$((_pt + 20)) ;;
+      surface|ecosystem) _pt=$((_pt + 80)) ;;
+    esac
+  done
+  set_progress_total "$_pt"
+  unset _pt
+fi
+
+# Whole-run wall clock (default 10 min). Override: RUN_TIMEOUT=0 (unlimited).
+: "${RUN_TIMEOUT:=600}"
+export RUN_TIMEOUT
+MON_T0=$(date +%s)
+export MON_T0
+RUN_TIMED_OUT=0
+RUN_TIMEOUT_AT_PHASE=""
+RUN_SKIPPED_PHASES=()
+
+mon_seconds_left() {
+  if [ "${RUN_TIMEOUT:-0}" -eq 0 ]; then
+    printf '%s' "999999"
+    return 0
+  fi
+  local now left
+  now=$(date +%s)
+  left=$((RUN_TIMEOUT - (now - MON_T0)))
+  [ "$left" -lt 0 ] && left=0
+  printf '%s' "$left"
+}
+
+# Run one phase script under remaining RUN_TIMEOUT budget.
+# Exit 124 (timeout utility) → mark RUN_TIMED_OUT.
+run_phase() {
+  local phase="$1" script="$2" left rc
+  left=$(mon_seconds_left)
+  if [ "${RUN_TIMEOUT:-0}" -gt 0 ] && [ "$left" -le 0 ]; then
+    RUN_TIMED_OUT=1
+    RUN_TIMEOUT_AT_PHASE="${RUN_TIMEOUT_AT_PHASE:-$phase}"
+    return 1
+  fi
+  if [ "${RUN_TIMEOUT:-0}" -eq 0 ]; then
+    "$script"
+    return $?
+  fi
+  # Keep at least 2s so timeout(1) can start the child.
+  [ "$left" -lt 2 ] && left=2
+  set +e
+  with_timeout "$left" "$script"
+  rc=$?
+  set -e
+  if [ "$rc" -eq 124 ]; then
+    RUN_TIMED_OUT=1
+    RUN_TIMEOUT_AT_PHASE="$phase"
+    return 1
+  fi
+  return "$rc"
+}
+
+# How inside/versions will reach containers (host-podman vs ssh)
+_INSIDE_ACCESS_HINT=ssh
+if [ "${INSIDE_PODMAN:-0}" = "1" ] || [ "${INSIDE_MODE:-}" = "local-podman" ]; then
+  _INSIDE_ACCESS_HINT=host-podman
+elif [ "${SKIP_SSH:-0}" = "1" ]; then
+  _INSIDE_ACCESS_HINT=skipped
+elif [ "${LOCAL_STACK:-0}" = "1" ] && command -v podman >/dev/null 2>&1 \
+  && podman ps --format '{{.Names}}' 2>/dev/null | grep -qE 'taler-hacktivism'; then
+  _INSIDE_ACCESS_HINT=host-podman
+fi
+
+printf 'target domain=%s\n' "${TALER_DOMAIN}"
+printf '  bank     %s\n' "$BANK_PUBLIC"
+printf '  exchange %s\n' "$EXCHANGE_PUBLIC"
+printf '  merchant %s\n' "$MERCHANT_PUBLIC"
+printf '  currency expect=%s\n' "${EXPECT_CURRENCY:-any}"
+printf '  phases   %s\n' "${PHASES[*]}"
+printf '  flags    LOCAL_STACK=%s SKIP_SSH=%s INSIDE_PODMAN=%s INSIDE_MODE=%s\n' \
+  "${LOCAL_STACK:-}" "${SKIP_SSH:-0}" "${INSIDE_PODMAN:-0}" "${INSIDE_MODE:-}"
+printf '  flags    KOOPA_SSH=%s INSIDE_SSH=%s INSIDE_PROFILE=%s\n' \
+  "${KOOPA_SSH:-}" "${INSIDE_SSH:-}" "${INSIDE_PROFILE:-}"
+printf '  access   inside/versions → %s' "$_INSIDE_ACCESS_HINT"
+case "$_INSIDE_ACCESS_HINT" in
+  host-podman) printf ' (podman exec on this host; IDs inside.host-*)\n' ;;
+  ssh)         printf ' (SSH then podman; IDs inside.ssh-*)\n' ;;
+  skipped)     printf ' (SKIP_SSH=1)\n' ;;
+  *)           printf '\n' ;;
+esac
+if [ "${RUN_TIMEOUT:-0}" -eq 0 ]; then
+  printf '  run_timeout unlimited (RUN_TIMEOUT=0)\n'
+else
+  printf '  run_timeout %ss wall clock (RUN_TIMEOUT=; 0=unlimited)\n' "$RUN_TIMEOUT"
+fi
+printf '  progress total≈%s (set PROGRESS_TOTAL= to override; PROGRESS_OFF=1 to hide)\n' "${PROGRESS_TOTAL:-0}"
+unset _INSIDE_ACCESS_HINT
+
+chmod +x "$ROOT"/check_*.sh 2>/dev/null || true
+
+ec=0
+for p in "${PHASES[@]}"; do
+  if [ "$RUN_TIMED_OUT" = "1" ]; then
+    RUN_SKIPPED_PHASES+=("$p")
+    continue
+  fi
+  left=$(mon_seconds_left)
+  if [ "${RUN_TIMEOUT:-0}" -gt 0 ] && [ "$left" -le 0 ]; then
+    RUN_TIMED_OUT=1
+    RUN_TIMEOUT_AT_PHASE="${RUN_TIMEOUT_AT_PHASE:-$p}"
+    RUN_SKIPPED_PHASES+=("$p")
+    continue
+  fi
+  case "$p" in
+    urls)     run_phase urls     "$ROOT/check_urls.sh"     || ec=1 ;;
+    inside)   run_phase inside   "$ROOT/check_inside.sh"   || ec=1 ;;
+    versions) run_phase versions "$ROOT/check_versions.sh" || ec=1 ;;
+    sanity)   run_phase sanity   "$ROOT/check_sanity.sh"   || ec=1 ;;
+    server)   run_phase server   "$ROOT/check_server.sh"   || ec=1 ;;
+    e2e)      run_phase e2e      "$ROOT/check_e2e.sh"      || ec=1 ;;
+    ladder|goa-ladder) run_phase ladder "$ROOT/check_goa_ladder.sh" || ec=1 ;;
+    auth401)  run_phase auth401  "$ROOT/check_auth401.sh"  || ec=1 ;;
+    aptdeploy|apt-deploy|apt_src) run_phase aptdeploy "$ROOT/check_apt_deploy.sh" || ec=1 ;;
+    surface|ecosystem) run_phase surface "$ROOT/check_surface.sh" || ec=1 ;;
+  esac
+done
+
+# Extraordinary run-budget failure: always report at end; HTML links top → here.
+if [ "$RUN_TIMED_OUT" = "1" ]; then
+  ec=1
+  elapsed=$(( $(date +%s) - MON_T0 ))
+  skipped="${RUN_SKIPPED_PHASES[*]:-}"
+  # Stable id for jump links (console HTML + err-top banner)
+  printf '\n'
+  printf '╔══════════════════════════════════════════════════════════╗\n'
+  printf '║ RUN TIMEOUT · extraordinary (see jump target below)     ║\n'
+  printf '╚══════════════════════════════════════════════════════════╝\n'
+  printf '┌ ERROR  ┐ #run.timeout-01  RUN_TIMEOUT exceeded · budget %ss · elapsed %ss\n' \
+    "$RUN_TIMEOUT" "$elapsed"
+  printf '  detail: wall-clock limit hit'
+  if [ -n "${RUN_TIMEOUT_AT_PHASE:-}" ]; then
+    printf ' during/after phase "%s"' "$RUN_TIMEOUT_AT_PHASE"
+  fi
+  if [ -n "$skipped" ]; then
+    printf ' · skipped: %s' "$skipped"
+  fi
+  printf '\n'
+  printf '  id=run.timeout-01\n'
+  printf '  hint: raise RUN_TIMEOUT= (seconds) or set RUN_TIMEOUT=0 for unlimited\n'
+  printf -- '┌ RUN TIMEOUT · extraordinary ┐\n'
+  printf -- '  • run.timeout-01 [run] RUN_TIMEOUT=%ss exceeded (elapsed %ss)%s%s\n' \
+    "$RUN_TIMEOUT" "$elapsed" \
+    "${RUN_TIMEOUT_AT_PHASE:+ · phase $RUN_TIMEOUT_AT_PHASE}" \
+    "${skipped:+ · skipped $skipped}"
+  printf -- '--- ERRORS (failed checks) ---\n'
+  printf -- '  • run.timeout-01 [run] RUN_TIMEOUT=%ss exceeded (elapsed %ss)\n' \
+    "$RUN_TIMEOUT" "$elapsed"
+fi
+
+exit "$ec"

From 334594c423e98be4fc9e008f180e30bb51fe2ecf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= 
Date: Sat, 18 Jul 2026 14:57:14 +0200
Subject: [PATCH 002/101] i18n: English default, French for FrancPaysan (v1.1)

---
 README.md                              |   4 +
 VERSION                                |   2 +-
 VERSIONS.md                            |  30 ++-
 host-agent/env.example                 |   1 +
 host-agent/env/francpaysan.env.example |   3 +
 host-agent/env/hacktivism.env.example  |   2 +
 host-agent/env/stagepaysan.env.example |   3 +
 host-agent/run-host-report.sh          |  12 +
 i18n.sh                                | 101 +++++++++
 lib.sh                                 |  61 +++--
 site-gen/console_to_html.py            | 295 +++++++++++++++++++------
 site-gen/settings.conf                 |  51 +++++
 taler-monitoring.sh                    |   4 +
 13 files changed, 483 insertions(+), 86 deletions(-)
 create mode 100644 i18n.sh
 create mode 100644 site-gen/settings.conf

diff --git a/README.md b/README.md
index b8125f0..63871da 100644
--- a/README.md
+++ b/README.md
@@ -360,3 +360,7 @@ LADDER_LOAD=0 ./taler-monitoring.sh ladder
 - SSH `koopa` or `koopa-external` (inside / load / sanity server bits)
 - secrets under `koopa-admin-secrets/...` for e2e
 - `taler-wallet-cli` for e2e
+
+## Languages (i18n)
+
+Default **English**. FrancPaysan stacks use **French** (`TALER_MON_LANG=fr`, auto for `*lefrancpaysan*`). See `VERSIONS.md` and `i18n.sh`.
diff --git a/VERSION b/VERSION
index afaf360..1cc5f65 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.0.0
\ No newline at end of file
+1.1.0
\ No newline at end of file
diff --git a/VERSIONS.md b/VERSIONS.md
index baf7929..c53eac5 100644
--- a/VERSIONS.md
+++ b/VERSIONS.md
@@ -1,14 +1,36 @@
 # VERSIONS — taler-monitoring
 
+Release history for the standalone **taler-monitoring** suite  
+(https://git.hacktivism.ch/hernani/taler-monitoring).
+
 | Tag | Date (UTC) | Notes |
 |-----|------------|--------|
-| **v1.0** | 2026-07-18 | Initial standalone release. Split from `koopa-admin-log` at `scripts/taler-monitoring/` (tip after monitoring host-agent/site-gen split commits). |
+| **v1.1** | 2026-07-18 | **i18n**: default UI/console language **English**; **French** for FrancPaysan (`TALER_MON_LANG=fr`, auto on `*lefrancpaysan*`). Sticky bar “what this monitors” EN/FR; badge/summary chrome + message translation via `i18n.sh`. |
+| **v1.0** | 2026-07-18 | Initial standalone release. Split from `koopa-admin-log` (`scripts/taler-monitoring/`). Host-agent HTML, surface, aptdeploy, sticky bar. |
 
-Source: https://git.hacktivism.ch/hernani/taler-monitoring
+## Language (v1.1+)
+
+| Context | Language |
+|---------|----------|
+| Default / GOA / hacktivism | **English** |
+| FrancPaysan prod + stage (`*lefrancpaysan*`) | **French** (auto or `TALER_MON_LANG=fr`) |
+
+```bash
+# force
+TALER_MON_LANG=fr ./taler-monitoring.sh -d lefrancpaysan.ch urls
+TALER_MON_LANG=en ./taler-monitoring.sh -d lefrancpaysan.ch urls
+```
+
+Host-agent env examples:
+- `host-agent/env/francpaysan.env.example` → `TALER_MON_LANG=fr`
+- `host-agent/env/stagepaysan.env.example` → `TALER_MON_LANG=fr`
+- `host-agent/env/hacktivism.env.example` → `TALER_MON_LANG=en`
+
+## Pin a release
 
 ```bash
 git clone https://git.hacktivism.ch/hernani/taler-monitoring.git ~/src/taler-monitoring
-cd ~/src/taler-monitoring && git checkout v1.0   # or stay on main
+cd ~/src/taler-monitoring && git checkout v1.1
 ```
 
-Default host-agent tracks `origin/main` (`SUITE_UPDATE_MODE=reset`). Pin with `SUITE_GIT_REF=v1.0` in `~/.config/taler-monitoring/env`.
+Default host-agent tracks `origin/main`. Pin with `SUITE_GIT_REF=v1.1` in `~/.config/taler-monitoring/env`.
diff --git a/host-agent/env.example b/host-agent/env.example
index b7aec8b..2826f83 100644
--- a/host-agent/env.example
+++ b/host-agent/env.example
@@ -24,3 +24,4 @@
 # HTML_OUT=$HOME/monitoring-sites-staging
 # MON_HOSTS="bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch"
 # SOURCE_REPO_WEB=https://git.hacktivism.ch/hernani/taler-monitoring
+# TALER_MON_LANG=en   # en default; fr for FrancPaysan (auto if domain *lefrancpaysan*)
diff --git a/host-agent/env/francpaysan.env.example b/host-agent/env/francpaysan.env.example
index d3132d8..2f5019d 100644
--- a/host-agent/env/francpaysan.env.example
+++ b/host-agent/env/francpaysan.env.example
@@ -22,3 +22,6 @@ SUITE_UPDATE_MODE=reset
 SOURCE_REPO_WEB=https://git.hacktivism.ch/hernani/taler-monitoring
 
 # DEPLOY_WWW_ROOT=/var/www/monitoring-sites
+
+# UI language for console + HTML sticky bar (en|fr)
+TALER_MON_LANG=fr
diff --git a/host-agent/env/hacktivism.env.example b/host-agent/env/hacktivism.env.example
index 32c8e00..ebdcb8f 100644
--- a/host-agent/env/hacktivism.env.example
+++ b/host-agent/env/hacktivism.env.example
@@ -22,3 +22,5 @@ SUITE_UPDATE_MODE=reset
 SOURCE_REPO_WEB=https://git.hacktivism.ch/hernani/taler-monitoring
 
 DEPLOY_WWW_ROOT=/var/www/monitoring-sites
+
+TALER_MON_LANG=en
diff --git a/host-agent/env/stagepaysan.env.example b/host-agent/env/stagepaysan.env.example
index e21180a..f1510f2 100644
--- a/host-agent/env/stagepaysan.env.example
+++ b/host-agent/env/stagepaysan.env.example
@@ -25,3 +25,6 @@ SOURCE_REPO_WEB=https://git.hacktivism.ch/hernani/taler-monitoring
 
 # Optional public docroot if this user can write it
 # DEPLOY_WWW_ROOT=/var/www/monitoring-sites
+
+# UI language for console + HTML sticky bar (en|fr)
+TALER_MON_LANG=fr
diff --git a/host-agent/run-host-report.sh b/host-agent/run-host-report.sh
index c38cdcf..16857c1 100755
--- a/host-agent/run-host-report.sh
+++ b/host-agent/run-host-report.sh
@@ -72,6 +72,15 @@ export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}"
 export AUTH401_CONTINUE="${AUTH401_CONTINUE:-${CONTINUE_ON_ERROR}}"
 export RUN_TIMEOUT="${RUN_TIMEOUT:-600}"
 export TALER_DOMAIN="${TALER_DOMAIN:-hacktivism.ch}"
+# UI language: en default; fr for FrancPaysan (override with TALER_MON_LANG)
+if [ -z "${TALER_MON_LANG:-}" ]; then
+  case "$TALER_DOMAIN" in
+    *lefrancpaysan*|*francpaysan*) export TALER_MON_LANG=fr ;;
+    *) export TALER_MON_LANG=en ;;
+  esac
+fi
+export TALER_MON_LANG
+
 PHASES="${PHASES:-urls inside versions}"
 HTML_BASE="${HTML_OUT:-$HOME/monitoring-sites-staging}"
 MON_HOSTS="${MON_HOSTS:-}"
@@ -285,6 +294,7 @@ htmlify_host() {
 
   if [ -n "$SITE_GEN" ] && [ -f "$SITE_GEN/console_to_html.py" ]; then
     python3 "$SITE_GEN/console_to_html.py" \
+      --lang "${TALER_MON_LANG:-en}" \
       --log "$LOG" \
       --out "$mon_err" \
       --hostname "$host" \
@@ -297,6 +307,7 @@ htmlify_host() {
       --link-other "$HTML_URL_OK"
     if [ "$ec" -eq 0 ]; then
       python3 "$SITE_GEN/console_to_html.py" \
+      --lang "${TALER_MON_LANG:-en}" \
         --log "$LOG" \
         --out "$mon" \
         --hostname "$host" \
@@ -311,6 +322,7 @@ htmlify_host() {
       echo "html $host → ${HTML_URL_OK} only (clean · ${COMMIT_SHORT:-?})"
     else
       python3 "$SITE_GEN/console_to_html.py" \
+      --lang "${TALER_MON_LANG:-en}" \
         --log "$LOG" \
         --out "$mon" \
         --hostname "$host" \
diff --git a/i18n.sh b/i18n.sh
new file mode 100644
index 0000000..45dd1b7
--- /dev/null
+++ b/i18n.sh
@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+# i18n.sh — language for taler-monitoring console + HTML chrome
+#
+# TALER_MON_LANG=en|fr   (default en)
+# Auto: domains *lefrancpaysan* → fr unless TALER_MON_LANG is set explicitly.
+
+: "${TALER_MON_LANG:=}"
+: "${TALER_MON_LANG_SET:=0}"
+
+i18n_init() {
+  if [ -n "${TALER_MON_LANG:-}" ]; then
+    TALER_MON_LANG_SET=1
+  fi
+  if [ "${TALER_MON_LANG_SET}" != "1" ]; then
+    case "${TALER_DOMAIN:-}" in
+      *lefrancpaysan*|*francpaysan*)
+        TALER_MON_LANG=fr
+        ;;
+      *)
+        TALER_MON_LANG=en
+        ;;
+    esac
+  fi
+  case "${TALER_MON_LANG}" in
+    fr|FR|fra|french) TALER_MON_LANG=fr ;;
+    *) TALER_MON_LANG=en ;;
+  esac
+  export TALER_MON_LANG
+}
+
+i18n_tag() {
+  local t="$1"
+  if [ "${TALER_MON_LANG:-en}" != "fr" ]; then
+    printf '%s' "$t"
+    return 0
+  fi
+  case "$t" in
+    OK) printf 'OK' ;;
+    ERROR) printf 'ERREUR' ;;
+    WARN) printf 'AVERT' ;;
+    INFO) printf 'INFO' ;;
+    BLOCKER) printf 'BLOCAGE' ;;
+    BLOCK) printf 'BLOC' ;;
+    SUMMARY) printf 'RESUME' ;;
+    PROG|PROGRESS) printf 'PROG' ;;
+    *) printf '%s' "$t" ;;
+  esac
+}
+
+i18n_text() {
+  local s="$*"
+  if [ "${TALER_MON_LANG:-en}" != "fr" ] || [ -z "$s" ]; then
+    printf '%s' "$s"
+    return 0
+  fi
+  case "$s" in
+    "numbered checks this run:") printf 'controles numerotes de cette execution :' ;;
+    "failed — see list above"|"failed - see list above") printf 'echec — voir la liste ci-dessus' ;;
+    "BLOCKERS · pay/withdraw cannot finish") printf 'BLOCAGES · paiement/retrait impossible' ;;
+    "BLOCKERS (pay/withdraw cannot finish)") printf 'BLOCAGES (paiement/retrait impossible)' ;;
+    "ERRORS · failed checks") printf 'ERREURS · controles en echec' ;;
+    "ERRORS (failed checks)") printf 'ERREURS (controles en echec)' ;;
+    "pay/withdraw cannot finish") printf 'paiement/retrait impossible a terminer' ;;
+    "SUMMARY") printf 'RESUME' ;;
+    "totals:") printf 'totaux :' ;;
+    "container") printf 'conteneur' ;;
+    "disk") printf 'disque' ;;
+    "package") printf 'paquet' ;;
+    "version") printf 'version' ;;
+    "reachability") printf 'accessibilite' ;;
+    "server-header") printf 'en-tete Server' ;;
+    "inventory") printf 'inventaire' ;;
+    "flags") printf 'options' ;;
+    "target domain") printf 'domaine cible' ;;
+    "currency") printf 'devise' ;;
+    "phases") printf 'phases' ;;
+    "access") printf 'acces' ;;
+    *)
+      local out="$s"
+      out=${out//shared library missing/bibliotheque partagee manquante}
+      out=${out//shared library error/erreur de bibliotheque partagee}
+      out=${out//not active/non actif}
+      out=${out//after_start/apres_demarrage}
+      out=${out//catalogued but not reachable/catalogue mais inaccessible}
+      out=${out//does not resolve/ne resout pas}
+      out=${out//protocol confirmed/protocole confirme}
+      out=${out//catalogued service OK/service catalogue OK}
+      out=${out//from Server header/depuis en-tete Server}
+      out=${out//OSV clean/OSV sans vulnerabilite}
+      out=${out//actionable vuln/vuln. actionnable}
+      out=${out//header version · not Debian pkg/version en-tete · pas paquet Debian}
+      out=${out//RUN_TIMEOUT exceeded/RUN_TIMEOUT depasse}
+      out=${out//public HTTPS/HTTPS public}
+      out=${out//container status/etat du conteneur}
+      out=${out//remote-only/uniquement distant}
+      out=${out//no SSH/sans SSH}
+      out=${out//failed — see list above/echec — voir la liste ci-dessus}
+      printf '%s' "$out"
+      ;;
+  esac
+}
diff --git a/lib.sh b/lib.sh
index 8455c6b..27bbca0 100755
--- a/lib.sh
+++ b/lib.sh
@@ -2,6 +2,18 @@
 # Shared helpers for taler-monitoring (laptop or koopa).
 
 # Default stack = GOA / hacktivism (overridden by TALER_DOMAIN / --domain)
+# i18n (en default; fr for FrancPaysan — see i18n.sh / TALER_MON_LANG)
+_I18N_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+if [ -f "$_I18N_DIR/i18n.sh" ]; then
+  # shellcheck source=i18n.sh
+  source "$_I18N_DIR/i18n.sh"
+  i18n_init
+else
+  i18n_tag() { printf '%s' "$1"; }
+  i18n_text() { printf '%s' "$*"; }
+  i18n_init() { :; }
+fi
+
 : "${TALER_DOMAIN:=hacktivism.ch}"
 : "${BANK_PUBLIC:=https://bank.hacktivism.ch}"
 : "${EXCHANGE_PUBLIC:=https://exchange.hacktivism.ch}"
@@ -114,6 +126,11 @@ set_taler_stack() {
   BANK_PUBLIC=${BANK_PUBLIC%/}
   EXCHANGE_PUBLIC=${EXCHANGE_PUBLIC%/}
   MERCHANT_PUBLIC=${MERCHANT_PUBLIC%/}
+  # re-evaluate language after domain profile (FrancPaysan → fr)
+  if [ "${TALER_MON_LANG_SET:-0}" != "1" ]; then
+    TALER_MON_LANG=
+    i18n_init
+  fi
 }
 
 # Load first matching profile from domains.conf.
@@ -265,6 +282,11 @@ apply_taler_domain() {
   BANK_PUBLIC=${BANK_PUBLIC%/}
   EXCHANGE_PUBLIC=${EXCHANGE_PUBLIC%/}
   MERCHANT_PUBLIC=${MERCHANT_PUBLIC%/}
+  # re-evaluate language after domain profile (FrancPaysan → fr)
+  if [ "${TALER_MON_LANG_SET:-0}" != "1" ]; then
+    TALER_MON_LANG=
+    i18n_init
+  fi
 
   # Koopa SSH only for LOCAL_STACK=1. Stage LFP uses INSIDE_SSH (stagepaysan) separately.
   if [ "${LOCAL_STACK}" != "1" ]; then
@@ -679,6 +701,10 @@ _fmt_badge() {
 # $1=badge style  $2=tag text  $3=label colour  $4=label  $5=detail
 _msg_line() {
   local badge="$1" tag="$2" lcol="$3" label="$4" detail="${5:-}"
+  # i18n: tags + free text (en default; fr for FrancPaysan)
+  tag=$(i18n_tag "$tag")
+  label=$(i18n_text "$label")
+  [ -n "$detail" ] && detail=$(i18n_text "$detail")
   if [ -n "$detail" ]; then
     printf -- '%s %s%s%s%s %s·%s %s%s%s\n' \
       "$(_fmt_badge "$badge" "$tag")" "$(_fmt_tid)" "$lcol" "$label" "$N" "$D" "$N" "$D" "$detail" "$N"
@@ -749,7 +775,8 @@ blocker() {
 }
 section() {
   # Boxed section header (3 lines when colour on)
-  local title="$*"
+  local title
+  title=$(i18n_text "$*")
   local w=${#title}
   [ "$w" -lt 24 ] && w=24
   [ "$w" -gt 56 ] && w=56
@@ -783,13 +810,13 @@ summary() {
     _progress_bar_line "$PROGRESS_DONE" "$PROGRESS_TOTAL"
   fi
   if [ "$GLOBAL_N" -gt 0 ]; then
-    printf -- '%s numbered checks this run: #001…#%03d%s\n' "$D" "$GLOBAL_N" "$N"
+    printf -- '%s %s #001…#%03d%s\n' "$D" "$(i18n_text 'numbered checks this run:')" "$GLOBAL_N" "$N"
   fi
   if [ "$blk_n" -gt 0 ]; then
     if [ "${BOX:-0}" = "1" ]; then
-      printf -- '%s┌ BLOCKERS · pay/withdraw cannot finish ┐%s\n' "$BG_BLK" "$N"
+      printf -- '%s┌ %s ┐%s\n' "$BG_BLK" "$(i18n_text 'BLOCKERS · pay/withdraw cannot finish')" "$N"
     else
-      printf -- '%s--- BLOCKERS (pay/withdraw cannot finish) ---%s\n' "$M" "$N"
+      printf -- '%s--- %s ---%s\n' "$M" "$(i18n_text 'BLOCKERS (pay/withdraw cannot finish)')" "$N"
     fi
     local b
     for b in "${BLOCKERS[@]}"; do
@@ -798,9 +825,9 @@ summary() {
   fi
   if [ "${#ERRORS[@]}" -gt 0 ] && [ "$blk_n" -lt "${#ERRORS[@]}" ]; then
     if [ "${BOX:-0}" = "1" ]; then
-      printf -- '%s┌ ERRORS · failed checks ┐%s\n' "$BG_ERR" "$N"
+      printf -- '%s┌ %s ┐%s\n' "$BG_ERR" "$(i18n_text 'ERRORS · failed checks')" "$N"
     else
-      printf -- '%s--- ERRORS (failed checks) ---%s\n' "$R" "$N"
+      printf -- '%s--- %s ---%s\n' "$R" "$(i18n_text 'ERRORS (failed checks)')" "$N"
     fi
     local e
     for e in "${ERRORS[@]}"; do
@@ -811,9 +838,11 @@ summary() {
 
   # Coloured totals — same severity badges as check lines (always; NO_COLOR blanks ANSI)
   if [ "${BOX:-0}" = "1" ]; then
-    printf -- '\n%s┌ SUMMARY ┐%s\n' "$BG_SEC" "$N"
+    printf -- '\n%s┌ %s ┐%s\n' "$BG_SEC" "$(i18n_tag SUMMARY)" "$N"
     _sum_badge() { # $1=bg $2=tag $3=fg $4=n
-      printf -- '  %s┌ %-5s┐%s %s%4d%s\n' "$1" "$2" "$N" "$3" "$4" "$N"
+      local _tg
+      _tg=$(i18n_tag "$2")
+      printf -- '  %s┌ %-5s┐%s %s%4d%s\n' "$1" "$_tg" "$N" "$3" "$4" "$N"
     }
     _sum_badge "$BG_OK" "OK" "$G" "$PASS_N"
     [ "$FAIL_N" -gt 0 ] && _sum_badge "$BG_ERR" "ERROR" "$R" "$FAIL_N"
@@ -821,12 +850,12 @@ summary() {
     [ "$INFO_N" -gt 0 ] && _sum_badge "$BG_INFO" "INFO" "$C" "$INFO_N"
     [ "$blk_n" -gt 0 ] && _sum_badge "$BG_BLK" "BLOCK" "$M" "$blk_n"
   else
-    printf -- '\n[ SUMMARY ]\n'
-    printf -- '  [ OK    ] %s%4d%s\n' "$G" "$PASS_N" "$N"
-    [ "$FAIL_N" -gt 0 ] && printf -- '  [ ERROR ] %s%4d%s\n' "$R" "$FAIL_N" "$N"
-    [ "$WARN_N" -gt 0 ] && printf -- '  [ WARN  ] %s%4d%s\n' "$Y" "$WARN_N" "$N"
-    [ "$INFO_N" -gt 0 ] && printf -- '  [ INFO  ] %s%4d%s\n' "$C" "$INFO_N" "$N"
-    [ "$blk_n" -gt 0 ] && printf -- '  [ BLOCK ] %s%4d%s\n' "$M" "$blk_n" "$N"
+    printf -- '\n[ %s ]\n' "$(i18n_tag SUMMARY)"
+    printf -- '  [ %-5s ] %s%4d%s\n' "$(i18n_tag OK)" "$G" "$PASS_N" "$N"
+    [ "$FAIL_N" -gt 0 ] && printf -- '  [ %-5s ] %s%4d%s\n' "$(i18n_tag ERROR)" "$R" "$FAIL_N" "$N"
+    [ "$WARN_N" -gt 0 ] && printf -- '  [ %-5s ] %s%4d%s\n' "$(i18n_tag WARN)" "$Y" "$WARN_N" "$N"
+    [ "$INFO_N" -gt 0 ] && printf -- '  [ %-5s ] %s%4d%s\n' "$(i18n_tag INFO)" "$C" "$INFO_N" "$N"
+    [ "$blk_n" -gt 0 ] && printf -- '  [ %-5s ] %s%4d%s\n' "$(i18n_tag BLOCK)" "$M" "$blk_n" "$N"
   fi
   # one-line rollup (bold label, severity-coloured counts)
   printf -- '  %stotals:%s %s%d OK%s' "$B" "$N" "$G" "$PASS_N" "$N"
@@ -850,9 +879,9 @@ summary() {
     fi
   else
     if [ "${BOX:-0}" = "1" ]; then
-      printf -- '  %s┌ ERROR ┐%s %sfailed — see list above%s\n' "$BG_ERR" "$N" "$R" "$N"
+      printf -- '  %s┌ ERROR ┐%s %s"$(i18n_text "failed — see list above")"%s\n' "$BG_ERR" "$N" "$R" "$N"
     else
-      printf -- '  %s[ ERROR ] failed — see list above%s\n' "$R" "$N"
+      printf -- '  %s[ ERROR ] "$(i18n_text "failed — see list above")"%s\n' "$R" "$N"
     fi
   fi
   [ "$FAIL_N" -eq 0 ]
diff --git a/site-gen/console_to_html.py b/site-gen/console_to_html.py
index 75db298..4c4927f 100755
--- a/site-gen/console_to_html.py
+++ b/site-gen/console_to_html.py
@@ -8,6 +8,129 @@ import re
 from datetime import datetime, timezone
 from pathlib import Path
 
+def ui_lang(explicit: str = "") -> str:
+    """en (default) or fr. Env: TALER_MON_LANG; auto fr for lefrancpaysan hosts."""
+    import os
+    lang = (explicit or os.environ.get("TALER_MON_LANG") or "").strip().lower()
+    if lang in ("fr", "fra", "french"):
+        return "fr"
+    if lang in ("en", "eng", "english"):
+        return "en"
+    return "en"
+
+
+def ui(lang: str, key: str, **kwargs) -> str:
+    """UI chrome strings for sticky bar (English default; French for FrancPaysan)."""
+    en = {
+        "errors": "ERRORS",
+        "warnings": "WARNINGS",
+        "ok": "OK",
+        "error_one": "{n} error",
+        "error_many": "{n} errors",
+        "warn_one": "{n} warning",
+        "warn_many": "{n} warnings",
+        "jump_first_error": "Jump to first error",
+        "jump_first_warn": "Jump to first warning",
+        "generated": "generated",
+        "source": "source",
+        "what_monitors": "What this monitors",
+        "what_monitors_title": "Expand: what this page checks",
+        "surface_title": "Remote ecosystem surface inventory",
+        "surface_summary": "Public catalog hosts · DNS / TCP / HTTPS / TLS · Server version · CVE",
+        "surface_i1": "Outside-in only: no SSH and no podman exec on remote targets",
+        "surface_i2": "Catalog (surface-catalog.conf): taler.net, demo/test, gnunet.org, taler-ops.ch, taler-systems.com, deb.taler.net, ftp.gnu.org, …",
+        "surface_i3": "Per host: DNS resolve, ICMP (info), open ports, protocol probes, TLS cert expiry, Server-header software version, optional CVE check",
+        "surface_i4": "HTML published only on taler.hacktivism.ch (this page host: {host})",
+        "surface_i5": "Timer: taler-monitoring-surface.timer (hourly)",
+        "apt_title": "apt-src merchant deploy tests (podman on koopa)",
+        "apt_summary": "4 containers · fresh + upgrade tracks · trixie & trixie-testing",
+        "apt_i1": "koopa-taler-deploy-test-apt-src-trixie (fresh, suite trixie)",
+        "apt_i2": "koopa-taler-deploy-test-apt-src-trixie-testing (fresh, trixie-testing)",
+        "apt_i3": "koopa-taler-deploy-test-apt-src-trixie-upgrade (upgrade track)",
+        "apt_i4": "koopa-taler-deploy-test-apt-src-trixie-testing-upgrade (upgrade track)",
+        "apt_i5": "Checks: packages, taler-merchant-httpd --version / ldd, systemd unit, optional local /config probe — not public nginx/HTTPS",
+        "apt_i6": "HTML published only on taler.hacktivism.ch (this page host: {host})",
+        "apt_i7": "Timer: taler-monitoring-aptdeploy.timer (4h)",
+        "focus_bank": "Libeufin bank public HTTPS + container inside checks",
+        "focus_exchange": "Exchange public HTTPS + container inside checks",
+        "focus_merchant": "Merchant backend / SPA public HTTPS + container inside checks",
+        "focus_stack": "Taler stack public + inside checks",
+        "stack_title": "Stack monitoring · {host}",
+        "stack_summary": "{phases} · public + inside",
+        "stack_i1": "Focus: {focus}",
+        "stack_i2": "Phases this run: {phases}",
+        "stack_i3": "Typical: public URLs (HTTPS, QR, perf), inside (podman/ssh), package versions",
+        "stack_i4": "HTML host: {host}",
+        "stack_i5": "Timer: host-agent path + 4h · /monitoring on bank, exchange, merchant fronts",
+        "stack_i_phases": "Log header phases= {phases}",
+        "footer": "Console-style render of taler-monitoring output. Sticky bar: green = clean · yellow = warnings · red = errors. Commit pins the exact tree used for this run.",
+        "redirect_fail": "has failures.",
+        "redirect_see": "See {link} for the full console log, error index, and sticky status bar.",
+        "ago_s": "{n}s ago",
+        "ago_m": "{n}m ago",
+        "ago_h": "{n}h ago",
+        "ago_d": "{n}d ago",
+        "age_title": "page age · updates every second",
+    }
+    fr = {
+        "errors": "ERREURS",
+        "warnings": "AVERTISSEMENTS",
+        "ok": "OK",
+        "error_one": "{n} erreur",
+        "error_many": "{n} erreurs",
+        "warn_one": "{n} avertissement",
+        "warn_many": "{n} avertissements",
+        "jump_first_error": "Aller à la première erreur",
+        "jump_first_warn": "Aller au premier avertissement",
+        "generated": "généré",
+        "source": "source",
+        "what_monitors": "Ce que cette page contrôle",
+        "what_monitors_title": "Déplier : ce que cette page vérifie",
+        "surface_title": "Inventaire de surface de l'écosystème (distant)",
+        "surface_summary": "Hôtes du catalogue public · DNS / TCP / HTTPS / TLS · version serveur · CVE",
+        "surface_i1": "Uniquement de l'extérieur : pas de SSH ni de podman exec sur les cibles distantes",
+        "surface_i2": "Catalogue (surface-catalog.conf) : taler.net, demo/test, gnunet.org, taler-ops.ch, taler-systems.com, deb.taler.net, ftp.gnu.org, …",
+        "surface_i3": "Par hôte : résolution DNS, ICMP (info), ports ouverts, sondes de protocole, expiration du certificat TLS, version logicielle (en-tête Server), CVE optionnel",
+        "surface_i4": "HTML publié uniquement sur taler.hacktivism.ch (hôte de cette page : {host})",
+        "surface_i5": "Minuterie : taler-monitoring-surface.timer (horaire)",
+        "apt_title": "Tests de déploiement apt-src marchand (podman sur koopa)",
+        "apt_summary": "4 conteneurs · pistes fresh + upgrade · trixie et trixie-testing",
+        "apt_i1": "koopa-taler-deploy-test-apt-src-trixie (fresh, suite trixie)",
+        "apt_i2": "koopa-taler-deploy-test-apt-src-trixie-testing (fresh, trixie-testing)",
+        "apt_i3": "koopa-taler-deploy-test-apt-src-trixie-upgrade (piste upgrade)",
+        "apt_i4": "koopa-taler-deploy-test-apt-src-trixie-testing-upgrade (piste upgrade)",
+        "apt_i5": "Contrôles : paquets, taler-merchant-httpd --version / ldd, unité systemd, sonde locale /config optionnelle — pas de nginx/HTTPS public",
+        "apt_i6": "HTML publié uniquement sur taler.hacktivism.ch (hôte de cette page : {host})",
+        "apt_i7": "Minuterie : taler-monitoring-aptdeploy.timer (4 h)",
+        "focus_bank": "Banque Libeufin HTTPS public + contrôles inside conteneur",
+        "focus_exchange": "Exchange HTTPS public + contrôles inside conteneur",
+        "focus_merchant": "Backend marchand / SPA HTTPS public + contrôles inside conteneur",
+        "focus_stack": "Pile Taler public + inside",
+        "stack_title": "Surveillance de pile · {host}",
+        "stack_summary": "{phases} · public + inside",
+        "stack_i1": "Focus : {focus}",
+        "stack_i2": "Phases de cette exécution : {phases}",
+        "stack_i3": "Typique : URL publiques (HTTPS, QR, perf), inside (podman/ssh), versions des paquets",
+        "stack_i4": "Hôte HTML : {host}",
+        "stack_i5": "Minuterie : host-agent path + 4 h · /monitoring sur bank, exchange, fronts marchands",
+        "stack_i_phases": "En-tête de journal phases= {phases}",
+        "footer": "Rendu console de taler-monitoring. Barre collante : vert = OK · jaune = avertissements · rouge = erreurs. Le commit fixe l'arbre exact de cette exécution.",
+        "redirect_fail": "a des échecs.",
+        "redirect_see": "Voir {link} pour le journal console complet, l'index d'erreurs et la barre de statut.",
+        "ago_s": "il y a {n}s",
+        "ago_m": "il y a {n} min",
+        "ago_h": "il y a {n} h",
+        "ago_d": "il y a {n} j",
+        "age_title": "âge de la page · mise à jour chaque seconde",
+    }
+    table = fr if lang == "fr" else en
+    s = table.get(key) or en.get(key) or key
+    try:
+        return s.format(**kwargs)
+    except Exception:
+        return s
+
+
 # ANSI strip
 ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
 ERR_BULLET_RE = re.compile(r"^\s*[•·*-]\s+(?P.+)$")
@@ -192,82 +315,84 @@ def extract_phases(log_text: str) -> str:
     return ""
 
 
+
 def monitoring_scope(
     page_label: str,
     hostname: str,
     log_text: str = "",
+    lang: str = "en",
 ) -> dict[str, object]:
-    """
-    Human-readable description of what this monitoring page covers.
-    Returned keys: title, summary, items (list[str]), kind
-    """
+    """Human-readable description of what this page monitors (en default; fr for FP)."""
     label = (page_label or "monitoring").lower().replace("_", "-")
     phases = extract_phases(log_text)
     host = hostname or "?"
+    # Auto-fr for FrancPaysan hosts if lang not forced
+    if lang == "en" and re.search(r"lefrancpaysan|francpaysan", host + " " + log_text, re.I):
+        import os
+        if not (os.environ.get("TALER_MON_LANG") or "").strip():
+            lang = "fr"
 
     if "surface" in label:
         return {
             "kind": "surface",
-            "title": "Remote ecosystem surface inventory",
-            "summary": "Public catalog hosts · DNS / TCP / HTTPS / TLS · Server version · CVE",
+            "lang": lang,
+            "title": ui(lang, "surface_title"),
+            "summary": ui(lang, "surface_summary"),
             "items": [
-                "Outside-in only: no SSH and no podman exec on remote targets",
-                "Catalog (surface-catalog.conf): taler.net, demo/test, gnunet.org, "
-                "taler-ops.ch, taler-systems.com, deb.taler.net, ftp.gnu.org, …",
-                "Per host: DNS resolve, ICMP (info), open ports, protocol probes, "
-                "TLS cert expiry, Server-header software version, optional CVE check",
-                f"HTML published only on taler.hacktivism.ch "
-                f"(this page host: {host})",
-                "Timer: taler-monitoring-surface.timer (hourly)",
+                ui(lang, "surface_i1"),
+                ui(lang, "surface_i2"),
+                ui(lang, "surface_i3"),
+                ui(lang, "surface_i4", host=host),
+                ui(lang, "surface_i5"),
             ],
         }
 
     if "aptdeploy" in label or "apt-deploy" in label or "apt_src" in label:
         return {
             "kind": "aptdeploy",
-            "title": "apt-src merchant deploy tests (podman on koopa)",
-            "summary": "4 containers · fresh + upgrade tracks · trixie & trixie-testing",
+            "lang": lang,
+            "title": ui(lang, "apt_title"),
+            "summary": ui(lang, "apt_summary"),
             "items": [
-                "koopa-taler-deploy-test-apt-src-trixie (fresh, suite trixie)",
-                "koopa-taler-deploy-test-apt-src-trixie-testing (fresh, trixie-testing)",
-                "koopa-taler-deploy-test-apt-src-trixie-upgrade (upgrade track)",
-                "koopa-taler-deploy-test-apt-src-trixie-testing-upgrade (upgrade track)",
-                "Checks: packages, taler-merchant-httpd --version / ldd, systemd unit, "
-                "optional local /config probe — not public nginx/HTTPS",
-                f"HTML published only on taler.hacktivism.ch (this page host: {host})",
-                "Timer: taler-monitoring-aptdeploy.timer (4h)",
+                ui(lang, "apt_i1"),
+                ui(lang, "apt_i2"),
+                ui(lang, "apt_i3"),
+                ui(lang, "apt_i4"),
+                ui(lang, "apt_i5"),
+                ui(lang, "apt_i6", host=host),
+                ui(lang, "apt_i7"),
             ],
         }
 
-    # Default: stack /monitoring pages (bank, exchange, merchant)
     phase_txt = phases or "urls · inside · versions"
     role = "stack"
-    if host.startswith("bank."):
+    if host.startswith("bank.") or "bank." in host:
         role = "bank"
-        focus = "Libeufin bank public HTTPS + container inside checks"
-    elif host.startswith("exchange.") or host.startswith("stage.exchange."):
+        focus = ui(lang, "focus_bank")
+    elif "exchange." in host:
         role = "exchange"
-        focus = "Exchange public HTTPS + container inside checks"
-    elif host.startswith("taler.") or "merchant" in host or host.startswith("monnaie."):
+        focus = ui(lang, "focus_exchange")
+    elif host.startswith("taler.") or "merchant" in host or "monnaie." in host:
         role = "merchant"
-        focus = "Merchant backend / SPA public HTTPS + container inside checks"
+        focus = ui(lang, "focus_merchant")
     else:
-        focus = "Taler stack public + inside checks"
+        focus = ui(lang, "focus_stack")
 
     items = [
-        f"Focus: {focus}",
-        f"Phases this run: {phase_txt}",
-        "Typical: public URLs (HTTPS, QR, perf), inside (podman/ssh), package versions",
-        f"HTML host: {host}",
-        "Timer: taler-monitoring-hacktivism (path + 4h) · /monitoring on bank, exchange, taler",
+        ui(lang, "stack_i1", focus=focus),
+        ui(lang, "stack_i2", phases=phase_txt),
+        ui(lang, "stack_i3"),
+        ui(lang, "stack_i4", host=host),
+        ui(lang, "stack_i5"),
     ]
     if phases:
-        items.insert(2, f"Log header phases= {phases}")
+        items.insert(2, ui(lang, "stack_i_phases", phases=phases))
 
     return {
         "kind": role,
-        "title": f"Stack monitoring · {host}",
-        "summary": f"{phase_txt} · public + inside",
+        "lang": lang,
+        "title": ui(lang, "stack_title", host=host),
+        "summary": ui(lang, "stack_summary", phases=phase_txt),
         "items": items,
     }
 
@@ -289,28 +414,41 @@ def sticky_bar_html(
     scope: dict[str, object] | None = None,
 ) -> str:
     level = status_level(n_err, n_warn)
-    if level == "red":
-        status_txt = "ERRORS"
-    elif level == "yellow":
-        status_txt = "WARNINGS"
+    lang = "en"
+    if scope and scope.get("lang"):
+        lang = str(scope.get("lang"))
     else:
-        status_txt = "OK"
+        lang = ui_lang()
+        # auto-fr for FP hostnames
+        if lang == "en" and re.search(r"lefrancpaysan|francpaysan", hostname or "", re.I):
+            lang = "fr"
+
+    if level == "red":
+        status_txt = ui(lang, "errors")
+    elif level == "yellow":
+        status_txt = ui(lang, "warnings")
+    else:
+        status_txt = ui(lang, "ok")
 
     if first_err_href and n_err:
+        err_lbl = ui(lang, "error_one", n=n_err) if n_err == 1 else ui(lang, "error_many", n=n_err)
         err_stat = (
             f'{n_err} error{"s" if n_err != 1 else ""}'
+            f'title="{html.escape(ui(lang, "jump_first_error"))}">{html.escape(err_lbl)}'
         )
     else:
-        err_stat = f'{n_err} errors'
+        err_lbl = ui(lang, "error_many", n=n_err)
+        err_stat = f'{html.escape(err_lbl)}'
 
     if first_warn_href and n_warn:
+        warn_lbl = ui(lang, "warn_one", n=n_warn) if n_warn == 1 else ui(lang, "warn_many", n=n_warn)
         warn_stat = (
             f'{n_warn} warning{"s" if n_warn != 1 else ""}'
+            f'title="{html.escape(ui(lang, "jump_first_warn"))}">{html.escape(warn_lbl)}'
         )
     else:
-        warn_stat = f'{n_warn} warnings'
+        warn_lbl = ui(lang, "warn_many", n=n_warn)
+        warn_stat = f'{html.escape(warn_lbl)}'
 
     other = ""
     if link_other:
@@ -320,7 +458,7 @@ def sticky_bar_html(
         )
 
     if scope is None:
-        scope = monitoring_scope(page_label, hostname, "")
+        scope = monitoring_scope(page_label, hostname, "", lang=ui_lang())
     scope_title = html.escape(str(scope.get("title") or "Monitoring"))
     scope_summary = html.escape(str(scope.get("summary") or ""))
     scope_kind = html.escape(str(scope.get("kind") or "monitoring"))
@@ -347,21 +485,21 @@ def sticky_bar_html(
     
-      generated 
+      {html.escape(ui(lang, "generated"))} 
       
     
     
   
   \n'
 
@@ -1175,38 +1182,83 @@ body.filter-warn .filter-chip { border-color: #a68b2d; background: #1a1608; colo
   padding: 1px 8px;
 }
 .filter-clear:hover { border-color: #aaa; background: #222; }
-/* Console filter: sticky + overviews stay; match lines + dimmed neighbours (v1.13.11) */
-body.filter-error #mon-console .line:not(.error):not(.blocker):not(.filter-ctx) { display: none !important; }
-body.filter-warn #mon-console .line:not(.warn):not(.filter-ctx) { display: none !important; }
+/* Console filter: sticky + overviews stay; match lines + dimmed neighbours (v1.13.11+)
+   v1.15.2: stamp out black voids — hide blanks, zero-size non-matches, compact console */
+body.filter-error #mon-console,
+body.filter-warn #mon-console {
+  padding: 6px 8px;
+  min-height: 0;
+  display: flex;
+  flex-direction: column;
+  gap: 1px;
+}
+body.filter-error #mon-console .line:not(.error):not(.blocker):not(.filter-ctx),
+body.filter-warn #mon-console .line:not(.warn):not(.filter-ctx) {
+  display: none !important;
+  height: 0 !important;
+  margin: 0 !important;
+  padding: 0 !important;
+  border: none !important;
+  overflow: hidden !important;
+  line-height: 0 !important;
+}
+/* empty / whitespace-only lines never take space in filter mode */
+body.filter-error #mon-console .line:empty,
+body.filter-warn #mon-console .line:empty,
+body.filter-error #mon-console .line.line-blank,
+body.filter-warn #mon-console .line.line-blank {
+  display: none !important;
+}
 /* Context lines: grayed / muted so the warn|error still stands out */
 body.filter-error #mon-console .line.filter-ctx,
 body.filter-warn #mon-console .line.filter-ctx {
-  opacity: 0.48;
-  color: #7a7a7a !important;
-  filter: grayscale(0.9);
+  opacity: 0.55;
+  color: #8a8a8a !important;
+  filter: grayscale(0.85);
+  margin: 0 !important;
+  padding: 1px 4px !important;
+  border: none !important;
+  background: transparent !important;
 }
 body.filter-error #mon-console .line.filter-ctx a.jump,
 body.filter-warn #mon-console .line.filter-ctx a.jump {
   color: #6a6a6a !important;
 }
-/* Visual break when filter jumps across a gap in the log */
+/* Match lines stay tight */
+body.filter-error #mon-console .line.error,
+body.filter-error #mon-console .line.blocker,
+body.filter-warn #mon-console .line.warn {
+  margin: 1px 0 !important;
+  padding: 3px 6px !important;
+}
+/* Compact ellipsis between distant clusters (not a black slab) */
 body.filter-error #mon-console .filter-gap,
 body.filter-warn #mon-console .filter-gap {
   display: block;
-  color: #555;
+  color: #666;
   text-align: center;
-  padding: 6px 0;
-  margin: 4px 0;
-  letter-spacing: 0.35em;
-  font-size: 11px;
+  padding: 0;
+  margin: 2px 0;
+  letter-spacing: 0.25em;
+  font-size: 10px;
+  line-height: 1.2;
   user-select: none;
-  border-top: 1px dashed #333;
-  border-bottom: 1px dashed #333;
+  border: none;
+  background: transparent;
+  height: auto;
 }
 body:not(.filter-error):not(.filter-warn) #mon-console .filter-gap { display: none !important; }
 body.filter-error .env-context,
 body.filter-warn .env-context { display: none !important; }
 body.filter-warn .err-top { display: none; }
+/* secondary jump anchors: never affect layout */
+#mon-console .anchor-only {
+  display: inline;
+  width: 0;
+  height: 0;
+  overflow: hidden;
+  position: absolute;
+}
 /* v1.12.0: collapsible monitoring env context (default closed) */
 .env-context {
   margin: 0 0 12px;
@@ -1436,14 +1488,14 @@ STICKY_JS = """
   }
 
   // v1.11.0: click error/warn stats → filter log; sticky + overviews stay
-  // v1.13.11: keep a few neighbour lines (dimmed) so phase/group context stays readable
+  // v1.13.11: dimmed neighbour context; v1.15.2: stamp out black voids, compact layout
   var bar = document.getElementById("status-bar");
   var chip = document.getElementById("filter-chip");
   var chipLabel = document.getElementById("filter-chip-label");
   var clearBtn = document.getElementById("filter-clear");
   var consoleEl = document.getElementById("mon-console");
-  // lines above + below each match (same count either side)
-  var FILTER_CTX = 3;
+  // useful neighbours each side (skip blanks / SUMMARY chrome / pure box frames)
+  var FILTER_CTX = 2;
 
   function currentFilter() {
     if (document.body.classList.contains("filter-error")) return "error";
@@ -1459,11 +1511,13 @@ STICKY_JS = """
     consoleEl.querySelectorAll(".filter-gap").forEach(function (el) {
       el.parentNode && el.parentNode.removeChild(el);
     });
+    document.body.classList.remove("filter-compact");
   }
 
   function markFilterContext(mode) {
     clearFilterContext();
     if (!consoleEl || !mode) return;
+    document.body.classList.add("filter-compact");
     var lines = Array.prototype.slice.call(consoleEl.querySelectorAll(".line"));
     if (!lines.length) return;
     function isMatch(el) {
@@ -1477,19 +1531,30 @@ STICKY_JS = """
       var t = (el.textContent || "").replace(/\u00a0/g, " ").trim();
       return t.length === 0;
     }
-    // 0 = hide, 1 = match, 2 = context
+    function isLowValue(el) {
+      if (isBlankLine(el)) return true;
+      if (el.classList.contains("sum-header") || el.classList.contains("sum-ok")
+          || el.classList.contains("sum-warn") || el.classList.contains("sum-err")
+          || el.classList.contains("sum-info") || el.classList.contains("sum-block")
+          || el.classList.contains("sum-totals") || el.classList.contains("sum-verdict-ok")
+          || el.classList.contains("sum-verdict-warn") || el.classList.contains("sum-verdict-err")) {
+        return true;
+      }
+      var t = (el.textContent || "").replace(/\u00a0/g, " ").trim();
+      if (/^[╔╗╚╝║═\-\s│┌┐└┘]+$/.test(t)) return true;
+      return false;
+    }
     var keep = new Array(lines.length);
     var i, j, k, n, idx;
     for (i = 0; i < lines.length; i++) keep[i] = 0;
     for (i = 0; i < lines.length; i++) {
       if (!isMatch(lines[i])) continue;
       keep[i] = 1;
-      // ±FILTER_CTX *non-empty* neighbours (skip blank lines so filter is not full of voids)
       n = 0;
       idx = i - 1;
       while (idx >= 0 && n < FILTER_CTX) {
         if (keep[idx] === 1) { idx--; continue; }
-        if (isBlankLine(lines[idx])) { idx--; continue; }
+        if (isLowValue(lines[idx])) { idx--; continue; }
         keep[idx] = 2;
         n++;
         idx--;
@@ -1498,7 +1563,7 @@ STICKY_JS = """
       idx = i + 1;
       while (idx < lines.length && n < FILTER_CTX) {
         if (keep[idx] === 1) { idx++; continue; }
-        if (isBlankLine(lines[idx])) { idx++; continue; }
+        if (isLowValue(lines[idx])) { idx++; continue; }
         keep[idx] = 2;
         n++;
         idx++;
@@ -1506,10 +1571,11 @@ STICKY_JS = """
     }
     for (j = 0; j < lines.length; j++) {
       if (keep[j] === 2) lines[j].classList.add("filter-ctx");
-      // never show pure blank lines in filter mode
-      if (isBlankLine(lines[j]) && keep[j] !== 1) keep[j] = 0;
+      if (isBlankLine(lines[j])) {
+        keep[j] = 0;
+        lines[j].classList.add("line-blank");
+      }
     }
-    // dashed gap between non-contiguous visible clusters
     var prevVisible = -2;
     for (k = 0; k < lines.length; k++) {
       if (keep[k] === 0) continue;

From ae8a72a5f3cdc3d2a551fdb0dde9c73f5936c2d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= 
Date: Sun, 19 Jul 2026 04:45:12 +0200
Subject: [PATCH 063/101] release 1.15.3: auto-French console for lefrancpaysan
 domains

Do not treat a non-empty TALER_MON_LANG from env as explicit language.
Domain *lefrancpaysan* selects fr unless TALER_MON_LANG_SET=1 (--lang).
---
 VERSION     |  2 +-
 VERSIONS.md |  1 +
 i18n.sh     | 21 ++++++++++++++-------
 3 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/VERSION b/VERSION
index 42cf067..f2380cc 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.15.2
+1.15.3
diff --git a/VERSIONS.md b/VERSIONS.md
index 470fe10..b900ef4 100644
--- a/VERSIONS.md
+++ b/VERSIONS.md
@@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre
 
 | Tag | Date (UTC) | Notes |
 |-----|------------|--------|
+| **v1.15.3** | 2026-07-19 | **Bugfix:** console language auto-**fr** for `*lefrancpaysan*` domains again — env `TALER_MON_LANG=en` no longer locks English; only `--lang` / `TALER_MON_LANG_SET=1` is explicit. |
 | **v1.15.2** | 2026-07-19 | **Bugfix:** warn/error filter — drop blank log lines from HTML; compact console (no black voids above/mid/below); context skips SUMMARY chrome & pure box frames; tighter filter-gap. |
 | **v1.15.1** | 2026-07-19 | **Bugfix:** mon HTML — phase `section()` boxes no longer painted as SUMMARY (were blue/green `sum-header`); only real SUMMARY rows get solid severity backgrounds (OK/WARN/ERROR/INFO). |
 | **v1.15.0** | 2026-07-19 | **Feature:** CHF **fake-incoming amount ladder** in `devtesting` (`DEVTESTING_LADDER=1`, default **23** log-spaced rungs **CHF:0.01 → CHF:50000**); synthetic geniban debit funds each rung; baseline plumbing probe kept. |
diff --git a/i18n.sh b/i18n.sh
index 5ca1d18..1b0c90d 100644
--- a/i18n.sh
+++ b/i18n.sh
@@ -1,23 +1,29 @@
 #!/usr/bin/env bash
 # i18n.sh — language for taler-monitoring console + HTML chrome
 #
-# TALER_MON_LANG=en|fr   (default en)
-# Auto: domains *lefrancpaysan* → fr unless TALER_MON_LANG is set explicitly.
+# TALER_MON_LANG=en|fr
+# TALER_MON_LANG_SET=1  → language is explicit (--lang or host-agent wrapper); do not override.
+# Auto (SET≠1): domain *lefrancpaysan* / *francpaysan* → fr, else keep env or en.
+#
+# Important: a bare TALER_MON_LANG=en in ~/.config/taler-monitoring/env must NOT lock
+# language for FP domains (that used to force English on stage.monnaie…).
 
 : "${TALER_MON_LANG:=}"
 : "${TALER_MON_LANG_SET:=0}"
 
 i18n_init() {
-  if [ -n "${TALER_MON_LANG:-}" ]; then
-    TALER_MON_LANG_SET=1
-  fi
-  if [ "${TALER_MON_LANG_SET}" != "1" ]; then
+  # Only TALER_MON_LANG_SET=1 is "explicit". Do not promote non-empty TALER_MON_LANG
+  # to SET=1 — env files often ship LANG=en for the default GOA workstation.
+  if [ "${TALER_MON_LANG_SET:-0}" != "1" ]; then
     case "${TALER_DOMAIN:-}" in
       *lefrancpaysan*|*francpaysan*)
         TALER_MON_LANG=fr
         ;;
       *)
-        TALER_MON_LANG=en
+        # Non-FP: keep pre-set env value if any, else English
+        if [ -z "${TALER_MON_LANG:-}" ]; then
+          TALER_MON_LANG=en
+        fi
         ;;
     esac
   fi
@@ -26,6 +32,7 @@ i18n_init() {
     *) TALER_MON_LANG=en ;;
   esac
   export TALER_MON_LANG
+  export TALER_MON_LANG_SET
 }
 
 i18n_tag() {

From b4f5f060ccb07bfe405b82d7515cfe67b4bee975 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= 
Date: Sun, 19 Jul 2026 04:51:42 +0200
Subject: [PATCH 064/101] release 1.15.4: French sticky titles and SUMMARY
 verdict strings

Localize monitoring host label, stack summary (public + interne), and
SUMMARY all-clear / warnings-only / totals for FR console and HTML.
---
 VERSION                     |  2 +-
 VERSIONS.md                 |  1 +
 i18n.sh                     |  5 +++++
 lib.sh                      | 18 ++++++++-------
 site-gen/console_to_html.py | 44 +++++++++++++++++++++++++++++--------
 5 files changed, 52 insertions(+), 18 deletions(-)

diff --git a/VERSION b/VERSION
index f2380cc..e34208c 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.15.3
+1.15.4
diff --git a/VERSIONS.md b/VERSIONS.md
index b900ef4..ee61b4c 100644
--- a/VERSIONS.md
+++ b/VERSIONS.md
@@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre
 
 | Tag | Date (UTC) | Notes |
 |-----|------------|--------|
+| **v1.15.4** | 2026-07-19 | **Bugfix:** French sticky titles — page label *surveillance*, stack summary *public + interne*, SUMMARY verdict/totals via i18n (`tout est clair`, `totaux :`). |
 | **v1.15.3** | 2026-07-19 | **Bugfix:** console language auto-**fr** for `*lefrancpaysan*` domains again — env `TALER_MON_LANG=en` no longer locks English; only `--lang` / `TALER_MON_LANG_SET=1` is explicit. |
 | **v1.15.2** | 2026-07-19 | **Bugfix:** warn/error filter — drop blank log lines from HTML; compact console (no black voids above/mid/below); context skips SUMMARY chrome & pure box frames; tighter filter-gap. |
 | **v1.15.1** | 2026-07-19 | **Bugfix:** mon HTML — phase `section()` boxes no longer painted as SUMMARY (were blue/green `sum-header`); only real SUMMARY rows get solid severity backgrounds (OK/WARN/ERROR/INFO). |
diff --git a/i18n.sh b/i18n.sh
index 1b0c90d..a5a2285 100644
--- a/i18n.sh
+++ b/i18n.sh
@@ -73,6 +73,11 @@ i18n_text() {
     "pay/withdraw cannot finish") printf 'paiement/retrait impossible a terminer' ;;
     "SUMMARY") printf 'RESUME' ;;
     "totals:") printf 'totaux :' ;;
+    "all clear") printf 'tout est clair' ;;
+    "warnings only (no hard fail)") printf 'avertissements seulement (pas d'\''echec dur)' ;;
+    "monitoring") printf 'surveillance' ;;
+    "surface") printf 'surface' ;;
+    "aptdeploy") printf 'aptdeploy' ;;
     "container") printf 'conteneur' ;;
     "disk") printf 'disque' ;;
     "package") printf 'paquet' ;;
diff --git a/lib.sh b/lib.sh
index d9ca60c..4e7f137 100755
--- a/lib.sh
+++ b/lib.sh
@@ -944,7 +944,7 @@ _summary_totals_box() {
     [ "$blk_n" -gt 0 ] && _sum_row "$ROW_BLK" "$BG_BLK" "BLOCK" "$blk_n" "$M"
 
     # totals rollup on tinted bar
-    printf -- '  %s %s totals: %s%d OK%s' "$ROW_TOT" "$B" "$G$B" "$pass_n" "$N$ROW_TOT"
+    printf -- '  %s %s %s %s%d OK%s' "$ROW_TOT" "$B" "$(i18n_text 'totals:')" "$G$B" "$pass_n" "$N$ROW_TOT"
     [ "$fail_n" -gt 0 ] && printf -- ' %s·%s %s%d ERROR%s' "$D" "$N$ROW_TOT" "$R$B" "$fail_n" "$N$ROW_TOT"
     [ "$warn_n" -gt 0 ] && printf -- ' %s·%s %s%d WARN%s' "$D" "$N$ROW_TOT" "$Y$B" "$warn_n" "$N$ROW_TOT"
     [ "$info_n" -gt 0 ] && printf -- ' %s·%s %s%d INFO%s' "$D" "$N$ROW_TOT" "$C$B" "$info_n" "$N$ROW_TOT"
@@ -957,28 +957,30 @@ _summary_totals_box() {
     [ "$warn_n" -gt 0 ] && printf -- '  [ %-5s ] %s%4d%s\n' "$(i18n_tag WARN)" "$Y$B" "$warn_n" "$N"
     [ "$info_n" -gt 0 ] && printf -- '  [ %-5s ] %s%4d%s\n' "$(i18n_tag INFO)" "$C$B" "$info_n" "$N"
     [ "$blk_n" -gt 0 ] && printf -- '  [ %-5s ] %s%4d%s\n' "$(i18n_tag BLOCK)" "$M$B" "$blk_n" "$N"
-    printf -- '  %stotals:%s %s%d OK%s' "$B" "$N" "$G$B" "$pass_n" "$N"
+    printf -- '  %s%s%s %s%d OK%s' "$B" "$(i18n_text 'totals:')" "$N" "$G$B" "$pass_n" "$N"
     [ "$fail_n" -gt 0 ] && printf -- ' · %s%d ERROR%s' "$R$B" "$fail_n" "$N"
     [ "$warn_n" -gt 0 ] && printf -- ' · %s%d WARN%s' "$Y$B" "$warn_n" "$N"
     [ "$info_n" -gt 0 ] && printf -- ' · %s%d INFO%s' "$C$B" "$info_n" "$N"
     [ "$blk_n" -gt 0 ] && printf -- ' · %s%d BLOCKER%s' "$M$B" "$blk_n" "$N"
     printf '\n'
   fi
-  # overall verdict — bold label, filled badge
+  # overall verdict — bold label, filled badge (i18n for FR console/HTML)
+  local _ok_msg _warn_msg _fail_msg
+  _ok_msg=$(i18n_text 'all clear')
+  _warn_msg=$(i18n_text 'warnings only (no hard fail)')
   if [ "$fail_n" -eq 0 ] && [ "$blk_n" -eq 0 ] && [ "$warn_n" -eq 0 ]; then
     if [ "${BOX:-0}" = "1" ]; then
-      printf -- '  %s┌ OK    ┐%s %s%sall clear%s\n' "$BG_OK" "$N" "$G" "$B" "$N"
+      printf -- '  %s┌ OK    ┐%s %s%s%s%s\n' "$BG_OK" "$N" "$G" "$B" "$_ok_msg" "$N"
     else
-      printf -- '  %s[ OK    ] %sall clear%s\n' "$G" "$B" "$N"
+      printf -- '  %s[ OK    ] %s%s%s\n' "$G" "$B" "$_ok_msg" "$N"
     fi
   elif [ "$fail_n" -eq 0 ] && [ "$blk_n" -eq 0 ]; then
     if [ "${BOX:-0}" = "1" ]; then
-      printf -- '  %s┌ WARN  ┐%s %s%swarnings only (no hard fail)%s\n' "$BG_WARN" "$N" "$Y" "$B" "$N"
+      printf -- '  %s┌ WARN  ┐%s %s%s%s%s\n' "$BG_WARN" "$N" "$Y" "$B" "$_warn_msg" "$N"
     else
-      printf -- '  %s[ WARN  ] %swarnings only (no hard fail)%s\n' "$Y" "$B" "$N"
+      printf -- '  %s[ WARN  ] %s%s%s\n' "$Y" "$B" "$_warn_msg" "$N"
     fi
   else
-    local _fail_msg
     _fail_msg=$(i18n_text 'failed — see list above')
     if [ "${BOX:-0}" = "1" ]; then
       printf -- '  %s┌ ERROR ┐%s %s%s%s%s\n' "$BG_ERR" "$N" "$R" "$B" "$_fail_msg" "$N"
diff --git a/site-gen/console_to_html.py b/site-gen/console_to_html.py
index 89dd4ee..0874c1e 100755
--- a/site-gen/console_to_html.py
+++ b/site-gen/console_to_html.py
@@ -114,6 +114,12 @@ def ui(lang: str, key: str, **kwargs) -> str:
         "stack_i4": "HTML host: {host}",
         "stack_i5": "Timer: host-agent path + 4h · /monitoring on bank, exchange, merchant fronts",
         "stack_i_phases": "Log header phases= {phases}",
+        "page_label_monitoring": "monitoring",
+        "page_label_surface": "surface",
+        "page_label_aptdeploy": "aptdeploy",
+        "mode_ok": "OK",
+        "mode_err": "ERR",
+        "mode_redirect": "REDIR",
         "pages_title": "Monitoring pages (suite HTML)",
         "pages_summary": "Public sticky-bar reports · phase monpages · part of taler-monitoring",
         "pages_i1": "Suite name taler-monitoring ≠ URL. Only three public families (bare + slash each):",
@@ -200,14 +206,20 @@ def ui(lang: str, key: str, **kwargs) -> str:
         "focus_merchant": "Backend marchand / SPA HTTPS public + contrôles inside conteneur",
         "focus_stack": "Pile Taler public + inside",
         "stack_title": "Surveillance de pile · {host}",
-        "stack_summary": "{phases} · public + inside",
+        "stack_summary": "{phases} · public + interne",
         "stack_i1": "Focus : {focus}",
         "stack_i2": "Phases de cette exécution : {phases}",
         "stack_i3": "Typique : URL publiques (HTTPS, QR, perf), inside (podman/ssh), versions des paquets",
         "stack_i4": "Hôte HTML : {host}",
         "stack_i5": "Minuterie : host-agent path + 4 h · /monitoring sur bank, exchange, fronts marchands",
         "stack_i_phases": "En-tête de journal phases= {phases}",
-                "pages_title": "Pages de monitoring (HTML de la suite)",
+        "page_label_monitoring": "surveillance",
+        "page_label_surface": "surface",
+        "page_label_aptdeploy": "aptdeploy",
+        "mode_ok": "OK",
+        "mode_err": "ERR",
+        "mode_redirect": "REDIR",
+        "pages_title": "Pages de monitoring (HTML de la suite)",
         "pages_summary": "Rapports publics sticky-bar · phase monpages · partie de taler-monitoring",
         "pages_i1": "Nom de suite taler-monitoring ≠ URL. Trois familles publiques seulement (sans et avec /) :",
         "pages_i2": "1) /monitoring(+_err) · 2) /taler-monitoring-surface(+_err) · 3) /taler-monitoring-aptdeploy(+_err) sur taler.hacktivism.ch",
@@ -288,14 +300,14 @@ def classify(line: str) -> str:
         return "meta"
     if "numbered checks" in low:
         return "meta"
-    if "totals:" in low:
+    if "totals:" in low or "totaux :" in low or low.strip().startswith("totaux"):
         return "sum-totals"
     # SUMMARY verdict chrome (not a check) — keep distinct for HTML boxes
-    if "warnings only" in low:
+    if "warnings only" in low or "avertissements seulement" in low:
         return "sum-verdict-warn"
-    if "all clear" in low:
+    if "all clear" in low or "tout est clair" in low:
         return "sum-verdict-ok"
-    if "failed — see" in low or "failed - see" in low:
+    if "failed — see" in low or "failed - see" in low or "echec — voir" in low or "échec — voir" in low:
         return "sum-verdict-err"
     # host-agent tees converter "wrote … (errors=N …)" into the same log
     if line.lstrip().lower().startswith("wrote ") and "errors=" in low:
@@ -443,8 +455,12 @@ def is_summary_error_line(ln: str) -> bool:
         return True
     if "warnings only" in low or "all clear" in low:
         return True
+    if "avertissements seulement" in low or "tout est clair" in low:
+        return True
+    if "totaux" in low and not is_numbered_check_line(ln):
+        return True
     # SUMMARY count rows without #NNN / tid (e.g. "  [ ERROR ]    3")
-    if re.search(r"[\[\u250c]\s*(ERROR|WARN|OK|INFO|BLOCK)", ln, re.I) and not is_numbered_check_line(
+    if re.search(r"[\[\u250c]\s*(ERROR|WARN|OK|INFO|BLOCK|ERREUR|AVERT)", ln, re.I) and not is_numbered_check_line(
         ln
     ):
         return True
@@ -1066,6 +1082,16 @@ def sticky_bar_html(
             version_html = f'{html.escape(ver)}'
     else:
         version_html = ""
+    # Localize page label for sticky host title (monitoring → surveillance in FR)
+    pl = (page_label or "monitoring").lower().replace("_", "-")
+    if "surface" in pl:
+        pl_disp = ui(lang, "page_label_surface")
+    elif "aptdeploy" in pl or "apt-deploy" in pl:
+        pl_disp = ui(lang, "page_label_aptdeploy")
+    else:
+        pl_disp = ui(lang, "page_label_monitoring")
+    mode_key = f"mode_{mode}" if mode in ("ok", "err", "redirect") else ""
+    mode_disp = ui(lang, mode_key) if mode_key and ui(lang, mode_key) != mode_key else mode.upper()
     # Compact sticky row always visible; "Was geprüft wird" expands inside sticky-bar
     return f"""