From 84588f2648be31c9996132991642f74a6928a373 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 06:06:41 +0200 Subject: [PATCH 01/29] docs: mattermost help lists all client floors --- taler-monitoring.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taler-monitoring.sh b/taler-monitoring.sh index 13df412..9445a53 100755 --- a/taler-monitoring.sh +++ b/taler-monitoring.sh @@ -100,7 +100,7 @@ Phases: surface REMOTE-ONLY public inventory (NOT in default/all/full): ecosystem hosts (taler.net, gnunet.org, taler-systems.com, mattermost, …) or -d DOMAIN → that domain’s surface; port/protocol/TLS/CVE (OSV) - mattermost Mattermost chat health (default mattermost.taler.net; SPA + ping + mobile server min ≥10.11) + mattermost Mattermost chat health (SPA + ping + client floors: mobile/desktop/support ≥10.11) mail MX/SMTP/IMAP for Taler mail (firefly, pixel/TSA, catalogued domains) monpages public monitoring HTML via FQDN (obligatory ERROR; GOA full inventory, FP only FP) uses MON_HOSTS + HTML_URL_OK (same as host-agent); v1.3.1+ From 7b3a345b7de1f2b2b3021dfc7a39315b9fec7ff1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 08:45:44 +0200 Subject: [PATCH 02/29] release 1.18.1: FR mon badges + stage-lfp host-podman + suite keep mode --- VERSION | 2 +- VERSIONS.md | 1 + check_inside.sh | 88 +++++++++++++++++++++++++++++-------- host-agent/update-suite.sh | 61 +++++++++++++++---------- site-gen/console_to_html.py | 60 +++++++++++++++++++------ 5 files changed, 154 insertions(+), 58 deletions(-) diff --git a/VERSION b/VERSION index 84cc529..ec6d649 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.18.0 +1.18.1 diff --git a/VERSIONS.md b/VERSIONS.md index b8fb8d4..bb3d16d 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| +| **v1.18.1** | 2026-07-19 | **Bugfix:** (1) mon HTML sticky/err counts honor French badges **ERREUR** / **AVERT** (were plain → 0 errors on FP pages); (2) `stage-lfp` + `INSIDE_PODMAN=1` uses **host-podman** (no empty `INSIDE_SSH` fail); (3) reaffirm filter JS has no box-glyph/`\\-` regex (charCodeAt only — kills SyntaxWarning “geheimsprache” noise). | | **v1.18.0** | 2026-07-19 | **Feature:** mattermost — client compatibility matrix: **mobile**, **desktop**, **vendor support** floors (default ≥10.11.0, ERROR) + **web** WARN on EOL; env `MATTERMOST_DESKTOP/SUPPORT_MIN_SERVER`, `MATTERMOST_CLIENT_CHECK`. | | **v1.17.0** | 2026-07-19 | **Feature:** mattermost phase — check **server version** (`X-Version-Id`) against **mobile store min** (default **≥10.11.0**); ERROR when too old (newest Android/iOS apps break); env `MATTERMOST_MOBILE_MIN_SERVER` / `MATTERMOST_MOBILE_CHECK=0`. | | **v1.16.0** | 2026-07-19 | **Feature:** mon sticky — blue **info** + green **ok** filter badges; suite **version tag under status pill** (not blue badge); filters `#filter-info` / `#filter-ok`. | diff --git a/check_inside.sh b/check_inside.sh index 08d3d5d..38f145f 100755 --- a/check_inside.sh +++ b/check_inside.sh @@ -29,12 +29,14 @@ if [ -z "$PROFILE" ]; then fi fi -# Resolve host-podman vs ssh before first check ID +# Resolve host-podman vs ssh before first check ID. +# stage-lfp on the FP host itself uses local podman (INSIDE_PODMAN=1 / +# INSIDE_MODE=local-podman); laptop → stage still uses INSIDE_SSH. _use_local_podman=0 -if [ "$PROFILE" != "stage-lfp" ]; then - if [ "${INSIDE_PODMAN:-0}" = "1" ] || [ "${INSIDE_MODE:-}" = "local-podman" ]; then - _use_local_podman=1 - elif command -v podman >/dev/null 2>&1 \ +if [ "${INSIDE_PODMAN:-0}" = "1" ] || [ "${INSIDE_MODE:-}" = "local-podman" ]; then + _use_local_podman=1 +elif [ "$PROFILE" != "stage-lfp" ]; then + if command -v podman >/dev/null 2>&1 \ && podman ps --format '{{.Names}}' 2>/dev/null | grep -qE 'taler-hacktivism'; then _use_local_podman=1 fi @@ -52,7 +54,7 @@ section "inside · collect (${PROFILE} · access=${INSIDE_ACCESS})" info "flags" "INSIDE_ACCESS=${INSIDE_ACCESS} INSIDE_PODMAN=${INSIDE_PODMAN:-0} INSIDE_MODE=${INSIDE_MODE:-} LOCAL_STACK=${LOCAL_STACK:-0} SKIP_SSH=${SKIP_SSH:-0} KOOPA_SSH=${KOOPA_SSH:-} INSIDE_SSH=${INSIDE_SSH:-}" # --------------------------------------------------------------------------- -# stage-lfp: low-priv stagepaysan on the FP stage host (INSIDE_SSH) +# stage-lfp: stagepaysan podman (host-local or SSH INSIDE_SSH from laptop) # --------------------------------------------------------------------------- if [ "$PROFILE" = "stage-lfp" ]; then SSH_HOST="${INSIDE_SSH:-}" @@ -69,15 +71,23 @@ if [ "$PROFILE" = "stage-lfp" ]; then STAGE_SSH_T="${INSIDE_SSH_TIMEOUT:-${SSH_CMD_TIMEOUT:-24}}" if [ "${STAGE_SSH_T}" -lt 24 ] 2>/dev/null; then STAGE_SSH_T=24; fi - if ! mon_ssh_ok "$SSH_HOST"; then - err "ssh" "cannot reach ${SSH_HOST} (stagepaysan low-priv) — set INSIDE_SSH= or SKIP" + if [ "$_use_local_podman" = "1" ]; then + if ! command -v podman >/dev/null 2>&1; then + err "host" "INSIDE_PODMAN/host-podman but podman missing" + summary + exit 1 + fi + ok "host→container" "podman exec on this host (INSIDE_ACCESS=host-podman · no SSH)" + elif ! mon_ssh_ok "$SSH_HOST"; then + err "ssh" "cannot reach ${SSH_HOST:-?} (stagepaysan low-priv) — set INSIDE_SSH= or INSIDE_PODMAN=1" summary exit 1 + else + ok "ssh ${SSH_HOST}" "stagepaysan (podman, no sudo)" fi - ok "ssh ${SSH_HOST}" "stagepaysan (podman, no sudo)" - # Inject names/ports into remote (ssh bash -s does not inherit local env). - RAW=$( + # Inject names/ports (ssh bash -s does not inherit local env; local bash does). + _stage_lfp_script() { { printf 'BANK_CTR=%q; EX_CTR=%q; MER_CTR=%q\n' "$BANK_CTR" "$EX_CTR" "$MER_CTR" printf 'BANK_PORT=%q; EX_PORT=%q; MER_PORT=%q\n' "$BANK_PORT" "$EX_PORT" "$MER_PORT" @@ -173,11 +183,20 @@ fi echo DONE REMOTE - } | mon_ssh_bash "$SSH_HOST" "${STAGE_SSH_T}" || true - ) + } + } + if [ "$_use_local_podman" = "1" ]; then + RAW=$(_stage_lfp_script | bash || true) + else + RAW=$(_stage_lfp_script | mon_ssh_bash "$SSH_HOST" "${STAGE_SSH_T}" || true) + fi if [ -z "$RAW" ] || ! echo "$RAW" | grep -q '^E|'; then - err "ssh" "stage remote timed out or empty (cap ${STAGE_SSH_T}s · host=${SSH_HOST})" + if [ "$_use_local_podman" = "1" ]; then + err "host" "stage collect timed out or empty (local podman)" + else + err "ssh" "stage remote timed out or empty (cap ${STAGE_SSH_T}s · host=${SSH_HOST})" + fi summary exit 1 fi @@ -189,7 +208,7 @@ REMOTE IFS='|' read -r _ comp level key detail <<<"$line" case "$comp" in bank|exchange|merchant|caddy) _g="$comp" ;; - *) _g="ssh" ;; + *) _g="$INSIDE_ACCESS" ;; esac if [ "$_g" != "$_last_inside_grp" ]; then set_group "$_g" @@ -242,7 +261,29 @@ REMOTE # Host load as stagepaysan (no container RSS from koopa metrics) set_group load section "inside · load (stagepaysan host)" - LOAD_LINE=$(mon_ssh_bash "$SSH_HOST" 8 <<'EOF' || true + _load_py() { + python3 - <<'PY' +import os +la=os.getloadavg() +print("loadavg=%.2f,%.2f,%.2f" % la) +try: + with open("/proc/meminfo") as f: + d={} + for line in f: + k,v=line.split(":")[0], line.split(":")[1].strip().split()[0] + d[k]=int(v) + total=d.get("MemTotal",0)/1024/1024 + avail=d.get("MemAvailable",0)/1024/1024 + used=total-avail + print("mem_used=%.2fGiB avail=%.2fGiB total=%.2fGiB" % (used, avail, total)) +except Exception: + print("mem=?") +PY + } + if [ "$_use_local_podman" = "1" ]; then + LOAD_LINE=$(_load_py || true) + else + LOAD_LINE=$(mon_ssh_bash "$SSH_HOST" 8 <<'EOF' || true python3 - <<'PY' import os la=os.getloadavg() @@ -262,6 +303,7 @@ except Exception: PY EOF ) + fi if [ -n "$LOAD_LINE" ]; then info "stage host" "$(echo "$LOAD_LINE" | tr '\n' ' ')" else @@ -270,7 +312,8 @@ EOF set_group disk section "inside · disk free space (stage host + containers)" - _disk_raw=$(mon_ssh_bash "$SSH_HOST" "${STAGE_SSH_T:-24}" <<'DISK' || true + _disk_script() { + cat <<'DISK' set +e echo "###HOST###" df -Pk / /var /home /tmp /mnt/data 2>/dev/null || df -Pk @@ -280,9 +323,16 @@ for c in $(podman ps --format '{{.Names}}' 2>/dev/null); do podman exec "$c" df -Pk / /var /tmp 2>/dev/null || podman exec "$c" df -Pk 2>/dev/null done DISK -) + } + if [ "$_use_local_podman" = "1" ]; then + _disk_raw=$(_disk_script | bash || true) + _disk_label="host" + else + _disk_raw=$(_disk_script | mon_ssh_bash "$SSH_HOST" "${STAGE_SSH_T:-24}" || true) + _disk_label="ssh:${SSH_HOST}" + fi _host_df=$(printf '%s\n' "$_disk_raw" | sed -n '/^###HOST###$/,/^###CTRS###$/p' | sed '1d;$d') - mon_disk_check_remote_text "ssh:${SSH_HOST}" "$_host_df" || true + mon_disk_check_remote_text "$_disk_label" "$_host_df" || true _ctr=""; _buf="" while IFS= read -r _line || [ -n "$_line" ]; do case "$_line" in diff --git a/host-agent/update-suite.sh b/host-agent/update-suite.sh index e93aa11..c24f7ce 100755 --- a/host-agent/update-suite.sh +++ b/host-agent/update-suite.sh @@ -8,7 +8,8 @@ # SUITE_GIT_URL default https://git.hacktivism.ch/hernani/taler-monitoring.git # SUITE_GIT_REF default main # SUITE_DIR default ~/src/taler-monitoring -# SUITE_UPDATE_MODE reset (default) | pull +# SUITE_UPDATE_MODE reset (default) | pull | keep +# keep = no fetch/reset (local deploy / unpushed fixes) # SUITE_UPDATE_STRICT 1 (default) = fetch/reset failure aborts the agent run # 0 = warn and continue on old tree (emergency only) # @@ -55,29 +56,41 @@ _before_ver=$(git describe --tags --exact-match 2>/dev/null \ || git describe --tags --abbrev=0 2>/dev/null \ || echo unknown) -echo "fetch origin ($SUITE_GIT_REF) + tags …" -if ! git fetch --tags --force --prune origin 2>&1; then - _upd_die "git fetch failed — suite would stay at ${_before} (${_before_ver})" - # fall through only if STRICT=0 -else - case "$SUITE_UPDATE_MODE" in - reset) - if ! git checkout -B "$SUITE_GIT_REF" "origin/$SUITE_GIT_REF" 2>/dev/null \ - && ! git checkout -B "$SUITE_GIT_REF" FETCH_HEAD; then - _upd_die "git checkout $SUITE_GIT_REF failed" - fi - if ! git reset --hard "origin/$SUITE_GIT_REF" 2>/dev/null \ - && ! git reset --hard FETCH_HEAD; then - _upd_die "git reset --hard origin/$SUITE_GIT_REF failed" - fi - ;; - *) - if ! git pull --ff-only origin "$SUITE_GIT_REF" 2>&1; then - _upd_die "git pull --ff-only failed" - fi - ;; - esac -fi +case "$SUITE_UPDATE_MODE" in + keep|skip|off|none) + echo "suite update skipped (SUITE_UPDATE_MODE=$SUITE_UPDATE_MODE) — keep local tree ${_before} (${_before_ver})" + ;; + *) + echo "fetch origin ($SUITE_GIT_REF) + tags …" + if ! git fetch --tags --force --prune origin 2>&1; then + _upd_die "git fetch failed — suite would stay at ${_before} (${_before_ver})" + # fall through only if STRICT=0 + else + case "$SUITE_UPDATE_MODE" in + reset) + if ! git checkout -B "$SUITE_GIT_REF" "origin/$SUITE_GIT_REF" 2>/dev/null \ + && ! git checkout -B "$SUITE_GIT_REF" FETCH_HEAD; then + _upd_die "git checkout $SUITE_GIT_REF failed" + fi + if ! git reset --hard "origin/$SUITE_GIT_REF" 2>/dev/null \ + && ! git reset --hard FETCH_HEAD; then + _upd_die "git reset --hard origin/$SUITE_GIT_REF failed" + fi + ;; + pull) + if ! git pull --ff-only origin "$SUITE_GIT_REF" 2>&1; then + _upd_die "git pull --ff-only failed" + fi + ;; + *) + if ! git pull --ff-only origin "$SUITE_GIT_REF" 2>&1; then + _upd_die "git pull --ff-only failed (mode=$SUITE_UPDATE_MODE)" + fi + ;; + esac + fi + ;; +esac COMMIT=$(git rev-parse HEAD) COMMIT_SHORT=$(git rev-parse --short=12 HEAD) diff --git a/site-gen/console_to_html.py b/site-gen/console_to_html.py index 7ec303f..4aab08d 100755 --- a/site-gen/console_to_html.py +++ b/site-gen/console_to_html.py @@ -323,7 +323,15 @@ def classify(line: str) -> str: # Double-line box frames from section() → section (no sum-* paint) if re.match(r"^\s*[╔╚].*═", line) or re.match(r"^\s*║\s", line): return "section" - if "--- ERRORS" in u or "ERRORS ·" in u or "ERRORS (" in u: + # EN + FR section headers (i18n_tag ERROR→ERREUR, ERRORS→ERREURS) + if ( + "--- ERRORS" in u + or "--- ERREURS" in u + or "ERRORS ·" in u + or "ERREURS ·" in u + or "ERRORS (" in u + or "ERREURS (" in u + ): return "meta" if "numbered checks" in low: return "meta" @@ -350,14 +358,15 @@ def classify(line: str) -> str: return "sum-block" return "blocker" # Only real check badges (not free-text "ERROR" inside commit messages, - # and not SUMMARY count rows like " [ ERROR ] 3") - if re.search(r"┌\s*ERROR\b|\[\s*ERROR\b", u): + # and not SUMMARY count rows like " [ ERROR ] 3"). + # French console (i18n_tag): ERROR→ERREUR, WARN→AVERT (v1.18.1). + if re.search(r"┌\s*(?:ERROR|ERREUR)\b|\[\s*(?:ERROR|ERREUR)\b", u): if not is_numbered_check_line(line): return "sum-err" return "error" if re.search(r"\bERROR monpages\b", line, re.I): return "error" - if re.search(r"┌\s*WARN\b|\[\s*WARN\b", u): + if re.search(r"┌\s*(?:WARN|AVERT)\b|\[\s*(?:WARN|AVERT)\b", u): if not is_numbered_check_line(line): return "sum-warn" return "warn" @@ -508,11 +517,25 @@ def extract_errors(lines: list[str]) -> list[str]: timeout_detail = "" in_block = False for ln in lines: - if "--- ERRORS" in ln or "ERRORS (failed" in ln or "RUN TIMEOUT · extraordinary" in ln: + u = ln.upper() + if ( + "--- ERRORS" in u + or "--- ERREURS" in u + or "ERRORS (FAILED" in u + or "ERREURS (CONTROLES" in u + or "ERREURS (FAILED" in u + or "RUN TIMEOUT · EXTRAORDINARY" in u + ): in_block = True continue if in_block: - if ln.strip().startswith("---") or "[ SUMMARY" in ln or "totals:" in ln: + if ( + ln.strip().startswith("---") + or "[ SUMMARY" in ln + or "[ RESUME" in u + or "totals:" in ln.lower() + or "totaux :" in ln.lower() + ): in_block = False continue m = ERR_BULLET_RE.match(ln) @@ -522,10 +545,15 @@ def extract_errors(lines: list[str]) -> list[str]: timeout_detail = body else: errs.append(body) - elif "ERROR" in ln.upper() and ln.strip() and not is_run_timeout_text(ln): + elif ("ERROR" in u or "ERREUR" in u) and ln.strip() and not is_run_timeout_text(ln): errs.append(ln.strip()) # Only real check badges (not INFO text mentioning thresholds like "… ERROR (0=disable)") - if re.search(r"┌\s*ERROR\b|\[\s*ERROR\b", ln, re.I) and "--- ERRORS" not in ln: + # FR: i18n_tag ERROR→ERREUR (v1.18.1) + if ( + re.search(r"┌\s*(?:ERROR|ERREUR)\b|\[\s*(?:ERROR|ERREUR)\b", ln, re.I) + and "--- ERRORS" not in u + and "--- ERREURS" not in u + ): if is_run_timeout_text(ln): if "run.timeout-01" in ln.lower() or "RUN_TIMEOUT exceeded" in ln: timeout_detail = re.sub(r"\s+", " ", strip_ansi(ln)).strip() @@ -534,8 +562,10 @@ def extract_errors(lines: list[str]) -> list[str]: continue if classify(ln) not in ("error", "blocker"): continue - body = re.sub(r"^.*\bERROR\b\s*\]?\s*", "", ln, flags=re.I).strip(" ·") - if body and body not in errs and "failed — see" not in body.lower(): + body = re.sub( + r"^.*\b(?:ERROR|ERREUR)\b\s*[\]┐]?\s*", "", ln, flags=re.I + ).strip(" ·") + if body and body not in errs and "failed — see" not in body.lower() and "echec — voir" not in body.lower(): if re.search( r"#\d+|www\.|e2e\.|auth401\.|versions\.|inside\.|surface\.|aptdeploy\.", body, @@ -1699,13 +1729,15 @@ STICKY_JS = """ return true; } var t = (el.textContent || "").replace(/\u00a0/g, " ").trim(); - // Pure box-drawing / whitespace chrome (no literal box glyphs in source — - // those used to appear as "weird chars" in Python SyntaxWarning output). + // Skip pure chrome lines (spaces/tabs/newlines/hyphen + Unicode box-drawing). + // Do NOT use a JS character-class regex with box glyphs here: when this + // script is embedded in a Python string, Python emits SyntaxWarning for + // sequences like \\- and the glyphs show up as noise in mon logs. if (!t.length) return true; for (var bi = 0; bi < t.length; bi++) { var bc = t.charCodeAt(bi); - if (bc === 32 || bc === 9 || bc === 10 || bc === 13 || bc === 0x2d) continue; // sp/tab/nl/cr/- - if (bc >= 0x2500 && bc <= 0x257f) continue; // box drawing + if (bc === 32 || bc === 9 || bc === 10 || bc === 13 || bc === 0x2d) continue; + if (bc >= 0x2500 && bc <= 0x257f) continue; // U+2500–U+257F box drawing return false; } return true; From 4bdb08bf3cc0b61e6d46484c0c378719fc98f8ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 09:02:55 +0200 Subject: [PATCH 03/29] release 1.18.2: refuse stale mon HTML / box-drawing generator assets --- VERSION | 2 +- VERSIONS.md | 1 + check_monitoring_pages.sh | 32 +++++++++++++++++++++ host-agent/run-host-report.sh | 54 ++++++++++++++++++++++++++--------- host-agent/update-suite.sh | 8 ++++++ 5 files changed, 83 insertions(+), 14 deletions(-) diff --git a/VERSION b/VERSION index ec6d649..b57fc72 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.18.1 +1.18.2 diff --git a/VERSIONS.md b/VERSIONS.md index bb3d16d..7344c07 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| +| **v1.18.2** | 2026-07-19 | **Bugfix:** never publish / accept **stale mon HTML assets** — refuse pre-1.15.8 box-drawing sticky JS in site-gen + generated pages; monpages ERROR on public pages still carrying that JS or teed `SyntaxWarning: invalid escape sequence`; purge `site-gen/__pycache__` after suite reset; convert with `PYTHONWARNINGS=error::SyntaxWarning`. | | **v1.18.1** | 2026-07-19 | **Bugfix:** (1) mon HTML sticky/err counts honor French badges **ERREUR** / **AVERT** (were plain → 0 errors on FP pages); (2) `stage-lfp` + `INSIDE_PODMAN=1` uses **host-podman** (no empty `INSIDE_SSH` fail); (3) reaffirm filter JS has no box-glyph/`\\-` regex (charCodeAt only — kills SyntaxWarning “geheimsprache” noise). | | **v1.18.0** | 2026-07-19 | **Feature:** mattermost — client compatibility matrix: **mobile**, **desktop**, **vendor support** floors (default ≥10.11.0, ERROR) + **web** WARN on EOL; env `MATTERMOST_DESKTOP/SUPPORT_MIN_SERVER`, `MATTERMOST_CLIENT_CHECK`. | | **v1.17.0** | 2026-07-19 | **Feature:** mattermost phase — check **server version** (`X-Version-Id`) against **mobile store min** (default **≥10.11.0**); ERROR when too old (newest Android/iOS apps break); env `MATTERMOST_MOBILE_MIN_SERVER` / `MATTERMOST_MOBILE_CHECK=0`. | diff --git a/check_monitoring_pages.sh b/check_monitoring_pages.sh index dceaad7..fea39cd 100755 --- a/check_monitoring_pages.sh +++ b/check_monitoring_pages.sh @@ -227,6 +227,25 @@ is_bootstrap_html() { && ! grep -qE 'taler-mon:top|id="mon-console"|class="console"' "$f" 2>/dev/null } +# Stale mon HTML from pre-1.15.8 generators: box-drawing char-class regex in +# sticky filter JS (Python SyntaxWarning: invalid escape sequence '\-'). +# Never accept these as "current" public assets (v1.18.2+). +is_stale_generator_html() { + local f="$1" + # Old isLowValue() JS (literal box glyphs + \- class) + if grep -qE 'if \(/\^\[[╔╗╚╝║═]' "$f" 2>/dev/null; then + return 0 + fi + # Converter noise teed into old logs and re-rendered + if grep -qiE 'SyntaxWarning: invalid escape sequence' "$f" 2>/dev/null; then + return 0 + fi + if grep -qiE 'console_to_html\.py:[0-9]+: SyntaxWarning' "$f" 2>/dev/null; then + return 0 + fi + return 1 +} + # True if body looks like our monitoring HTML (not merchant API / WP 404 / bootstrap). is_monitoring_html() { local f="$1" @@ -475,6 +494,19 @@ check_one() { return 0 fi + # Stale generator JS / SyntaxWarning in public body (pre-1.15.8 assets) + if is_stale_generator_html "$body"; then + if [ "$attempt" -lt "$max_try" ]; then + attempt=$((attempt + 1)) + sleep "$sleep_s" 2>/dev/null || sleep 1 + continue + fi + _mon_fail_or_soft "public mon page stale generator" \ + "$url -> old mon HTML (box-drawing JS regex and/or SyntaxWarning in body) — re-run host-agent on suite ≥v1.15.8 / v1.18.2; do not leave pre-update assets live" || { rm -f "$body"; return 1; } + rm -f "$body" + return 0 + fi + if ! is_monitoring_html "$body"; then if [ "$attempt" -lt "$max_try" ]; then attempt=$((attempt + 1)) diff --git a/host-agent/run-host-report.sh b/host-agent/run-host-report.sh index e26113c..e601562 100755 --- a/host-agent/run-host-report.sh +++ b/host-agent/run-host-report.sh @@ -353,13 +353,29 @@ htmlify_host() { mkdir -p "$HTML_BASE/$host/${HTML_OK_DIR}" "$HTML_BASE/$host/${HTML_ERR_DIR}" if [ -n "$SITE_GEN" ] && [ -f "$SITE_GEN/console_to_html.py" ]; then - # PYTHONWARNINGS: never let SyntaxWarning leak into the suite log (tee). - # stderr → agent log file only, not the mon console stream. + # Never convert with a pre-1.15.8 generator (old assets → SyntaxWarning + bad sticky JS). + if grep -qE 'if \(/\^\[[╔╗╚╝║═]' "$SITE_GEN/console_to_html.py" 2>/dev/null; then + echo "ERROR: refuse stale site-gen $SITE_GEN/console_to_html.py (box-drawing JS regex)" >&2 + return 1 2>/dev/null || exit 1 + fi + # SyntaxWarning = generator regression: fail convert (do not publish old-style pages). + # Other stderr still goes to agent log file only, not the mon console stream. _mon_html_py() { - PYTHONWARNINGS=ignore::SyntaxWarning \ - python3 "$SITE_GEN/console_to_html.py" "$@" 2>>"${LOG_DIR:-/tmp}/console_to_html.err" + local _out="" _i + for _i in "$@"; do + if [ "$_i" = "--out" ]; then _out=1; continue; fi + if [ "$_out" = "1" ]; then _out="$_i"; break; fi + done + PYTHONWARNINGS=error::SyntaxWarning \ + python3 "$SITE_GEN/console_to_html.py" "$@" 2>>"${LOG_DIR:-/tmp}/console_to_html.err" || return 1 + # Never leave pre-1.15.8 sticky JS in a just-written page (stale log text is filtered in converter). + if [ -n "$_out" ] && [ -f "$_out" ] && grep -qE 'if \(/\^\[[╔╗╚╝║═]' "$_out" 2>/dev/null; then + echo "ERROR: generated mon HTML still has stale box-drawing JS: $_out" >&2 + return 1 + fi + return 0 } - _mon_html_py \ + if ! _mon_html_py \ --lang "${TALER_MON_LANG:-en}" \ --log "$LOG" \ --out "$mon_err" \ @@ -373,37 +389,49 @@ htmlify_host() { --page-label "$PAGE_LABEL" \ --path-err "$HTML_URL_ERR" \ --link-other "$HTML_URL_OK" + then + echo "ERROR: mon HTML convert failed (err page) for $host — refuse publish of stale/broken assets" >&2 + return 1 2>/dev/null || exit 1 + fi if [ "$ec" -eq 0 ]; then - _mon_html_py \ - --lang "${TALER_MON_LANG:-en}" \ + if ! _mon_html_py \ + --lang "${TALER_MON_LANG:-en}" \ --log "$LOG" \ --out "$mon" \ --hostname "$host" \ --mode ok \ --commit "${COMMIT:-}" \ --commit-url "${COMMIT_URL:-}" \ - --suite-version "${SUITE_VERSION:-unknown}" \ - --suite-version-url "${SUITE_VERSION_URL:-}" \ + --suite-version "${SUITE_VERSION:-unknown}" \ + --suite-version-url "${SUITE_VERSION_URL:-}" \ --suite-path "$SUITE_PATH" \ --page-label "$PAGE_LABEL" \ --path-err "$HTML_URL_ERR" \ --link-other "" + then + echo "ERROR: mon HTML convert failed (ok page) for $host" >&2 + return 1 2>/dev/null || exit 1 + fi rm -rf "$HTML_BASE/$host/${HTML_ERR_DIR}" echo "html $host → ${HTML_URL_OK} only (clean · ${COMMIT_SHORT:-?})" else - _mon_html_py \ - --lang "${TALER_MON_LANG:-en}" \ + if ! _mon_html_py \ + --lang "${TALER_MON_LANG:-en}" \ --log "$LOG" \ --out "$mon" \ --hostname "$host" \ --mode redirect \ --commit "${COMMIT:-}" \ --commit-url "${COMMIT_URL:-}" \ - --suite-version "${SUITE_VERSION:-unknown}" \ - --suite-version-url "${SUITE_VERSION_URL:-}" \ + --suite-version "${SUITE_VERSION:-unknown}" \ + --suite-version-url "${SUITE_VERSION_URL:-}" \ --suite-path "$SUITE_PATH" \ --page-label "$PAGE_LABEL" \ --path-err "$HTML_URL_ERR" + then + echo "ERROR: mon HTML convert failed (redirect page) for $host" >&2 + return 1 2>/dev/null || exit 1 + fi echo "html $host → ${HTML_URL_OK} stub + ${HTML_URL_ERR} (ec=$ec · ${COMMIT_SHORT:-?})" fi else diff --git a/host-agent/update-suite.sh b/host-agent/update-suite.sh index c24f7ce..9c5f3ea 100755 --- a/host-agent/update-suite.sh +++ b/host-agent/update-suite.sh @@ -142,4 +142,12 @@ fi ln -sfn "$SUITE_DIR" "$HOME/taler-monitoring" 2>/dev/null || true chmod +x taler-monitoring.sh check_*.sh host-agent/*.sh site-gen/*.sh site-gen/*.py 2>/dev/null || true +# Drop bytecode so a stale site-gen never wins over the tree we just reset to. +rm -rf "$SUITE_DIR/site-gen/__pycache__" 2>/dev/null || true +# Refuse known-bad pre-1.15.8 generator (box-drawing JS regex → SyntaxWarning noise) +if [ -f "$SUITE_DIR/site-gen/console_to_html.py" ] \ + && grep -qE 'if \(/\^\[[╔╗╚╝║═]' "$SUITE_DIR/site-gen/console_to_html.py" 2>/dev/null; then + _upd_die "site-gen/console_to_html.py still has pre-1.15.8 box-drawing JS regex — refuse stale generator" + return 1 2>/dev/null || exit 1 +fi return 0 2>/dev/null || true From efb9047a678a6a95b8789cd5c70af601f3d12483 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 09:06:06 +0200 Subject: [PATCH 04/29] release 1.18.3: Bank withdraw wording (was ATM for CHF-style note amounts) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename user-facing e2e "ATM withdraw" to "Bank withdraw". The ladder amounts (20/50/100/200/…) were labeled ATM because they match typical CHF cash-note sizes (Bankomat), not because anything hits a physical ATM. Keep E2E_ATM_MAX as alias of E2E_BANK_WITHDRAW_MAX; keep e2e.atm-* group ids. --- CLI-AUTOMATION-NOTES.md | 4 +-- README.md | 8 ++--- TESTS.md | 6 ++-- VERSION | 2 +- VERSIONS.md | 1 + check_e2e.sh | 80 +++++++++++++++++++++-------------------- check_urls.sh | 2 +- lib.sh | 4 +-- taler-monitoring.sh | 6 ++-- 9 files changed, 58 insertions(+), 55 deletions(-) diff --git a/CLI-AUTOMATION-NOTES.md b/CLI-AUTOMATION-NOTES.md index 937fed0..b6ac18f 100644 --- a/CLI-AUTOMATION-NOTES.md +++ b/CLI-AUTOMATION-NOTES.md @@ -66,7 +66,7 @@ Legend: | **Seen** | Android endless spinner; GOA `bank.hacktivism.ch` withdraw; fixed in app branch `fix/bank-withdraw-auto-exchange` | | **Area** | **core** / **cli** | | **Problem** | Withdraw URI names an exchange the wallet has never added → stuck loading / empty exchange list. CLI requires explicit `exchanges add` + `update` + `accept-tos` every fresh DB. | -| **Workaround** | e2e always adds exchange before ATM ladder; Android needs ensureExchange-style fix. | +| **Workaround** | e2e always adds exchange before bank withdraw ladder; Android needs ensureExchange-style fix. | | **Wanted** | CLI/core: on `accept-uri` withdraw, **auto-add** exchange from URI/bank details (`allowCompletion`), then surface ToS if needed in one command. | --- @@ -87,7 +87,7 @@ Legend: | | | |--|--| -| **Seen** | e2e uses bank admin + ATM create; ladder/gui-chain reimplement explorer mint in Python | +| **Seen** | e2e uses bank admin + bank withdraw create; ladder/gui-chain reimplement explorer mint in Python | | **Area** | **cli** / **ops** | | **Problem** | Automation always hand-rolls Basic auth → token → `POST …/withdrawals` → `taler_withdraw_uri`. Easy to get wrong (auth header, amount currency). | | **Workaround** | Shared Python in `goa-chain` / ladder / e2e; secrets files. | diff --git a/README.md b/README.md index 81802c9..d5c156e 100644 --- a/README.md +++ b/README.md @@ -262,7 +262,7 @@ Other domains: never SSH. Optional **e2e** aborts cleanly on login/KYC. | | Local (koopa) | Remote | |--|---------------|--------| -| **ATM withdraw** | 20 · 50 · 100 · 200 · **4200** (paivana) | 10 · 20 · 50 | +| **Bank withdraw** | 20 · 50 · 100 · 200 · **4200** (paivana) | 10 · 20 · 50 | | **Pay ladder** | 0.01 … 10 | 0.01 … 1 | | **Paivana** | HTTP paywall + template pay **GOA:4200** (`goa-shop` / `paivana`) | skipped | @@ -338,8 +338,8 @@ Amounts use exchange **`alt_unit_names`** (Kilo-GOA, Mega-GOA, …) with the bas **GOA value in parentheses**: ```text -[INFO] coins after-ATM — amount_circ=2 Kilo-GOA (GOA:2000) -[INFO] coins after-ATM — denoms_in_circ 1 Kilo-GOA (GOA:1000)×2 (=2 Kilo-GOA (GOA:2000)) +[INFO] coins after-bank-wd — amount_circ=2 Kilo-GOA (GOA:2000) +[INFO] coins after-bank-wd — denoms_in_circ 1 Kilo-GOA (GOA:1000)×2 (=2 Kilo-GOA (GOA:2000)) == ladder · rung 12 GOA:1000000 · 1 Mega-GOA (GOA:1000000) == ``` @@ -370,7 +370,7 @@ snapshotted via SSH whenever the stack is under load: | Phase | When | |-------|------| | **inside** | end of container status | -| **e2e** | before work · after ATM withdraws · after payments · after shop · e2e-end (+ delta) | +| **e2e** | before work · after bank withdraws · after payments · after shop · e2e-end (+ delta) | | **ladder** | before first rung · after last rung (+ delta + timing overall) | ```bash diff --git a/TESTS.md b/TESTS.md index c88506e..2917e4c 100644 --- a/TESTS.md +++ b/TESTS.md @@ -97,10 +97,10 @@ SSH: `KOOPA_SSH` (default `koopa`), then `KOOPA_SSH_FALLBACKS` when LAN is unrea | Group | Checks | |-------|--------| | **e2e.prereq-** | wallet-cli, currency, budgets, secrets, public reachability | -| **e2e.load-** | host/container load snapshots (before / after ATM / after pay) | +| **e2e.load-** | host/container load snapshots (before / after bank withdraw / after pay) | | **e2e.bank-** | account, credit, withdraw ops | | **e2e.wallet-** | exchange + ToS, wallet setup | -| **e2e.atm-** | ATM withdraw ladder rungs | +| **e2e.atm-** (group id; UI says Bank withdraw) | bank withdraw ladder rungs | | **e2e.settle-** | wallet settlement wait / spendable balance | | **e2e.pay-** | variable payments | | **e2e.shop-** | GOA shop product catalog pays | @@ -109,7 +109,7 @@ SSH: `KOOPA_SSH` (default `koopa`), then `KOOPA_SSH_FALLBACKS` when LAN is unrea | **e2e.report-** | final tallies | When filing an issue, quote the full id + label, e.g. -`e2e.atm-03 ATM withdraw 5 GOA · bank confirmed, wallet empty`. +`e2e.atm-03 Bank withdraw 5 GOA · bank confirmed, wallet empty`. --- diff --git a/VERSION b/VERSION index b57fc72..b9fb27a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.18.2 +1.18.3 diff --git a/VERSIONS.md b/VERSIONS.md index 7344c07..e188d79 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| +| **v1.18.3** | 2026-07-19 | **Bugfix / wording:** e2e user-facing **ATM withdraw** → **Bank withdraw** (amounts still CHF-style cash-note ladder 20/50/100/200/… — “ATM” was only a metaphor for those note sizes, not a physical ATM). Env: `E2E_BANK_WITHDRAW_MAX` (legacy alias `E2E_ATM_MAX`). Internal group id `e2e.atm-*` kept for check-id stability. | | **v1.18.2** | 2026-07-19 | **Bugfix:** never publish / accept **stale mon HTML assets** — refuse pre-1.15.8 box-drawing sticky JS in site-gen + generated pages; monpages ERROR on public pages still carrying that JS or teed `SyntaxWarning: invalid escape sequence`; purge `site-gen/__pycache__` after suite reset; convert with `PYTHONWARNINGS=error::SyntaxWarning`. | | **v1.18.1** | 2026-07-19 | **Bugfix:** (1) mon HTML sticky/err counts honor French badges **ERREUR** / **AVERT** (were plain → 0 errors on FP pages); (2) `stage-lfp` + `INSIDE_PODMAN=1` uses **host-podman** (no empty `INSIDE_SSH` fail); (3) reaffirm filter JS has no box-glyph/`\\-` regex (charCodeAt only — kills SyntaxWarning “geheimsprache” noise). | | **v1.18.0** | 2026-07-19 | **Feature:** mattermost — client compatibility matrix: **mobile**, **desktop**, **vendor support** floors (default ≥10.11.0, ERROR) + **web** WARN on EOL; env `MATTERMOST_DESKTOP/SUPPORT_MIN_SERVER`, `MATTERMOST_CLIENT_CHECK`. | diff --git a/check_e2e.sh b/check_e2e.sh index f5f1aec..f0f9b2f 100755 --- a/check_e2e.sh +++ b/check_e2e.sh @@ -8,7 +8,7 @@ source "$ROOT/lib.sh" # shellcheck source=metrics.sh source "$ROOT/metrics.sh" -: "${E2E_TIMEOUT:=180}" # ATM ladder + wirewatch lag needs room (was 55 — too tight) +: "${E2E_TIMEOUT:=180}" # Bank withdraw ladder + wirewatch lag needs room (was 55 — too tight) : "${E2E_PAY_SECS:=22}" # hard reserve for handle-uri + settle (do not starve pay) : "${E2E_SETTLE_ROUNDS:=18}" : "${E2E_SETTLE_SLEEP:=3}" @@ -226,18 +226,20 @@ except Exception: print("")' 2>/dev/null || true) # --------------------------------------------------------------------------- # Variable amounts -# Withdraw = typical ATM notes only (Bankomat): 20 / 50 / 100 / 200 … +# Withdraw = typical bank cash-note amounts (CHF-style: was called ATM): 20 / 50 / 100 / 200 … # Pay = several smaller purchase amounts # Env: # E2E_WITHDRAW_VALUES="20 50 100" # bare numbers → prefixed with CUR # E2E_PAY_VALUES="0.01 0.05 0.5 1 2 5" # E2E_VARIABLE=0 # single fixed WITHDRAW_AMT / PAY_AMT -# E2E_ATM_MAX=N # try at most N ATM withdraws (budget) +# E2E_BANK_WITHDRAW_MAX=N # try at most N bank withdraws (budget) +# E2E_ATM_MAX=N # legacy alias of E2E_BANK_WITHDRAW_MAX # --------------------------------------------------------------------------- : "${E2E_VARIABLE:=1}" -: "${E2E_ATM_MAX:=5}" +: "${E2E_BANK_WITHDRAW_MAX:=${E2E_ATM_MAX:-5}}" +E2E_ATM_MAX="${E2E_ATM_MAX:-$E2E_BANK_WITHDRAW_MAX}" # legacy alias if [ "$E2E_REMOTE" = "1" ]; then - # remote: still ATM-shaped; TESTPAYSAN needs larger notes for shop pays + # remote: still bank cash-note shaped; TESTPAYSAN needs larger notes for shop pays if [ "$CUR" = "TESTPAYSAN" ]; then : "${E2E_WITHDRAW_VALUES:=20 50}" # farmer shop template face values (oeufs-6=5, fromage=8.5, jus=6) @@ -247,7 +249,7 @@ if [ "$E2E_REMOTE" = "1" ]; then : "${E2E_PAY_VALUES:=0.01 0.05 0.1 1}" fi else - # local GOA: classic ATM notes + 4200 for paivana paywall template + # local GOA: classic bank cash-note amounts + 4200 for paivana paywall template : "${E2E_WITHDRAW_VALUES:=20 50 100 200 4200}" : "${E2E_PAY_VALUES:=0.01 0.05 0.1 0.5 1 2 5 10}" fi @@ -284,7 +286,7 @@ if [ "$E2E_VARIABLE" != "1" ]; then fi fi -# Credit = sum of planned ATM withdraws + small buffer for fees +# Credit = sum of planned Bank withdraws + small buffer for fees CREDIT_AMT=$(python3 -c ' import sys cur=sys.argv[1] @@ -301,7 +303,7 @@ print("%s:%g" % (cur, s + buf)) WITHDRAW_AMT=$(printf '%s' "$WITHDRAW_LIST" | awk '{print $1}') PAY_AMT=$(printf '%s' "$PAY_LIST" | awk '{print $1}') -# More wall time when running ATM ladder +# More wall time when running Bank withdraw ladder if [ "$E2E_VARIABLE" = "1" ]; then n_w=$(printf '%s' "$WITHDRAW_LIST" | wc -w | tr -d ' ') n_p=$(printf '%s' "$PAY_LIST" | wc -w | tr -d ' ') @@ -309,9 +311,9 @@ if [ "$E2E_VARIABLE" = "1" ]; then if [ "${E2E_TIMEOUT}" -lt "$need" ]; then E2E_TIMEOUT=$need fi - # cap ATM attempts but always keep CUR:4200 if listed (paivana) + # cap bank withdraw attempts but always keep CUR:4200 if listed (paivana) WITHDRAW_LIST=$( - printf '%s' "$WITHDRAW_LIST" | tr ' ' '\n' | awk -v max="$E2E_ATM_MAX" -v big="${CUR}:4200" ' + printf '%s' "$WITHDRAW_LIST" | tr ' ' '\n' | awk -v max="$E2E_BANK_WITHDRAW_MAX" -v big="${CUR}:4200" ' NF==0 { next } $0==big { has=1; next } { small[++n]=$0 } @@ -413,7 +415,7 @@ else fi info "e2e mode" "$([ "$E2E_REMOTE" = "1" ] && echo "remote/public domain (no SSH)" || echo "local koopa stack")" info "currency" "$CUR" -info "ATM withdraw ladder" "$WITHDRAW_LIST (credit $CREDIT_AMT)" +info "Bank withdraw ladder" "$WITHDRAW_LIST (credit $CREDIT_AMT)" info "pay ladder" "$PAY_LIST" info "e2e budget" "${E2E_TIMEOUT}s" @@ -863,7 +865,7 @@ else warn "wallet accept ToS" "$(tail -c 80 "$SCRATCH/ex-tos.out" | tr '\n' ' ')" fi fi -# Empty wallet baseline before ATM / pay load +# Empty wallet baseline before bank withdraw / pay load metrics_report_coins "wallet-ready" || true wd_status() { @@ -901,15 +903,15 @@ fake_incoming_speedup() { fi } -# One ATM-style withdraw: create → accept → select → confirm → settle +# One bank withdraw: create → accept → select → confirm → settle # returns 0 on wallet balance increase, 1 on soft fail (ladder continues) e2e_one_withdraw() { WITHDRAW_AMT="$1" local tag tag=$(printf '%s' "$WITHDRAW_AMT" | tr '.:' '__') - section "e2e · ATM withdraw $WITHDRAW_AMT · $(format_amount_alt "$WITHDRAW_AMT")" - e2e_over && { warn "ATM withdraw" "budget exhausted — skip $WITHDRAW_AMT"; return 1; } - metrics_report_coins "before-ATM-${tag}" || true + section "e2e · Bank withdraw $WITHDRAW_AMT · $(format_amount_alt "$WITHDRAW_AMT")" + e2e_over && { warn "Bank withdraw" "budget exhausted — skip $WITHDRAW_AMT"; return 1; } + metrics_report_coins "before-bank-wd-${tag}" || true curl -sS -m 15 -o "$SCRATCH/wd-$tag.json" -H "Authorization: Bearer $UT" \ -H 'Content-Type: application/json' \ @@ -919,10 +921,10 @@ e2e_one_withdraw() { URI=$(python3 -c 'import json;d=json.load(open("'"$SCRATCH"'/wd-'"$tag"'.json"));print(d.get("taler_withdraw_uri") or "")' 2>/dev/null || true) URI_CLEAN=$(python3 -c 'import json;u=json.load(open("'"$SCRATCH"'/wd-'"$tag"'.json")).get("taler_withdraw_uri") or "";print(u.replace(":443/","/"))' 2>/dev/null || true) if [ -z "$WID" ] || [ -z "$URI_CLEAN" ]; then - warn "ATM withdraw $WITHDRAW_AMT" "create failed — $(head -c 80 "$SCRATCH/wd-$tag.json" | tr '\n' ' ')" + warn "Bank withdraw $WITHDRAW_AMT" "create failed — $(head -c 80 "$SCRATCH/wd-$tag.json" | tr '\n' ' ')" return 1 fi - ok "ATM withdrawal created $WITHDRAW_AMT ($WID)" + ok "Bank withdrawal created $WITHDRAW_AMT ($WID)" cp "$SCRATCH/wd-$tag.json" "$SCRATCH/wd.json" USE_URI="$URI" @@ -933,7 +935,7 @@ e2e_one_withdraw() { && wcli withdraw accept-uri --exchange "${EXCHANGE_PUBLIC}/" "$URI_CLEAN" >"$SCRATCH/accept-$tag.out" 2>&1; then ok "wallet accept $WITHDRAW_AMT (no :443)" else - warn "ATM withdraw $WITHDRAW_AMT" "accept-uri failed — $(tail -c 100 "$SCRATCH/accept-$tag.out" | tr '\n' ' ')" + warn "Bank withdraw $WITHDRAW_AMT" "accept-uri failed — $(tail -c 100 "$SCRATCH/accept-$tag.out" | tr '\n' ' ')" return 1 fi cp "$SCRATCH/accept-$tag.out" "$SCRATCH/accept.out" @@ -975,7 +977,7 @@ else: fi fi if [ "$st" != "selected" ] && [ "$st" != "confirmed" ]; then - warn "ATM withdraw $WITHDRAW_AMT" "not selected (status=${st:-?})" + warn "Bank withdraw $WITHDRAW_AMT" "not selected (status=${st:-?})" return 1 fi if [ "$st" != "confirmed" ]; then @@ -984,32 +986,32 @@ else: "$BANK/accounts/${USER}/withdrawals/${WID}/confirm") case "$code" in 200|204) ok "bank confirm $WITHDRAW_AMT" ;; - *) warn "ATM withdraw $WITHDRAW_AMT" "confirm HTTP $code"; return 1 ;; + *) warn "Bank withdraw $WITHDRAW_AMT" "confirm HTTP $code"; return 1 ;; esac fi fake_incoming_speedup - # Short per-ATM settle: poll balance + bank transfer_done only (never run-until-done) + # Short per-withdraw settle: poll balance + bank transfer_done only (never run-until-done) local ok_bal=0 r av xfer for r in 1 2 3 4 5 6 8 10; do wcli_bal_snap "$SCRATCH/bal-$tag.out" || wcli balance >"$SCRATCH/bal-$tag.out" 2>&1 || true av=$(wallet_avail_num "$SCRATCH/bal-$tag.out") if python3 -c "import sys; sys.exit(0 if float(sys.argv[1]) > 0 else 1)" "$av" 2>/dev/null; then ok_bal=1 - ok "wallet funded after ATM $WITHDRAW_AMT" "avail=${CUR}:${av}" + ok "wallet funded after bank withdraw $WITHDRAW_AMT" "avail=${CUR}:${av}" info "balance" "$(fmt_bal "$SCRATCH/bal-$tag.out")" break fi xfer=$(curl -sS -m 8 "$BANK/taler-integration/withdrawal-operation/${WID}" 2>/dev/null \ | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("transfer_done"), d.get("status"))' 2>/dev/null || echo "?") if echo "$xfer" | grep -qi True; then - ok "ATM $WITHDRAW_AMT bank transfer_done" "wallet avail=${CUR}:${av} ($xfer) — no run-until-done" + ok "Bank withdraw $WITHDRAW_AMT bank transfer_done" "wallet avail=${CUR}:${av} ($xfer) — no run-until-done" ok_bal=1 break fi sleep 2 done - metrics_report_coins "after-ATM-${tag}" || true + metrics_report_coins "after-bank-wd-${tag}" || true if [ "$ok_bal" = "1" ]; then metrics_record_flow withdrawn "$WITHDRAW_AMT" || true return 0 @@ -1017,10 +1019,10 @@ else: # Bank side often already confirmed — treat as timing lag, not hard fail st=$(wd_status) if [ "$st" = "confirmed" ]; then - warn "ATM withdraw $WITHDRAW_AMT" "bank confirmed; wallet not funded yet (settlement timing — will recheck later)" + warn "Bank withdraw $WITHDRAW_AMT" "bank confirmed; wallet not funded yet (settlement timing — will recheck later)" return 2 fi - warn "ATM withdraw $WITHDRAW_AMT" "no wallet balance yet (status=${st:-?})" + warn "Bank withdraw $WITHDRAW_AMT" "no wallet balance yet (status=${st:-?})" return 1 } @@ -1336,7 +1338,7 @@ sys.exit(1) # --------------------------------------------------------------------------- set_group atm -section "e2e · ATM withdraw ladder" +section "e2e · Bank withdraw ladder" # --------------------------------------------------------------------------- WITHDRAW_OK=0 WITHDRAW_OK_N=0 @@ -1344,7 +1346,7 @@ WITHDRAW_LAG_N=0 WITHDRAW_FAIL_N=0 WITHDRAW_REPORT="" for WITHDRAW_AMT in $WITHDRAW_LIST; do - e2e_over && { warn "ATM ladder" "time budget low — stopping more ATM withdraws (not a protocol error)"; break; } + e2e_over && { warn "Bank withdraw ladder" "time budget low — stopping more Bank withdraws (not a protocol error)"; break; } set +e e2e_one_withdraw "$WITHDRAW_AMT" wc=$? @@ -1365,26 +1367,26 @@ for WITHDRAW_AMT in $WITHDRAW_LIST; do ;; esac done -info "ATM withdraw summary" "$WITHDRAW_REPORT (ok=$WITHDRAW_OK_N lag=$WITHDRAW_LAG_N fail=$WITHDRAW_FAIL_N)" +info "Bank withdraw summary" "$WITHDRAW_REPORT (ok=$WITHDRAW_OK_N lag=$WITHDRAW_LAG_N fail=$WITHDRAW_FAIL_N)" set_group load -section "e2e · load snapshot (after ATM withdraws)" +section "e2e · load snapshot (after Bank withdraws)" metrics_report_load "${METRICS_DIR}/load-after-withdraw.json" "after-withdraw" || true cp -f "${METRICS_DIR}/load-after-withdraw.json" "${METRICS_DIR}/load-after.json" 2>/dev/null || true -metrics_report_coins "after-ATM-ladder" || true +metrics_report_coins "after-bank-wd-ladder" || true # Settlement catch-up: bank may have confirmed while wallet was still empty set_group settle section "e2e · wallet settlement (timing)" if ! wait_wallet_balance 0 "${E2E_SETTLE_ROUNDS}" "${E2E_SETTLE_SLEEP}"; then if [ "$WITHDRAW_LAG_N" -gt 0 ] || [ "$WITHDRAW_OK_N" -gt 0 ]; then - warn "settlement timing" "ATM path reached bank confirm (lag=$WITHDRAW_LAG_N ok=$WITHDRAW_OK_N) but wallet empty after wait — TIME lag, not protocol error" + warn "settlement timing" "bank withdraw path reached bank confirm (lag=$WITHDRAW_LAG_N ok=$WITHDRAW_OK_N) but wallet empty after wait — TIME lag, not protocol error" fi fi av_now=$(wallet_avail_num) if python3 -c "import sys; sys.exit(0 if float(sys.argv[1]) > 0 else 1)" "$av_now" 2>/dev/null; then WITHDRAW_OK=1 if [ "$WITHDRAW_OK_N" = "0" ]; then - warn "settlement timing" "coins arrived after ATM ladder (${CUR}:${av_now}) — earlier 'FAIL' was timing lag" + warn "settlement timing" "coins arrived after Bank withdraw ladder (${CUR}:${av_now}) — earlier 'FAIL' was timing lag" WITHDRAW_REPORT="${WITHDRAW_REPORT} → late-OK avail=${CUR}:${av_now}" fi ok "spendable balance for payments" "${CUR}:${av_now}" @@ -1403,7 +1405,7 @@ else WITHDRAW_OK=1 warn "settlement timing" "coins present after dig wait (${CUR}:${av_now}) — timing, not blocker" else - blocker "withdraw-settle" "no spendable ${CUR} after ATM ladder + settle wait ($WITHDRAW_LIST)" + blocker "withdraw-settle" "no spendable ${CUR} after Bank withdraw ladder + settle wait ($WITHDRAW_LIST)" section "e2e · report" info "WITHDRAW" "$WITHDRAW_REPORT — no coins after extended wait" info "PAY" "SKIPPED (no spendable balance after settle wait)" @@ -1596,7 +1598,7 @@ set_group paivana section "e2e · paivana paywall (template GOA:4200)" # --------------------------------------------------------------------------- # Public paywall: https://paivana.hacktivism.ch → 302 to merchant template "paivana". -# Requires spendable ≥ 4200 GOA (ATM ladder includes GOA:4200 by default). +# Requires spendable ≥ 4200 GOA (Bank withdraw ladder includes GOA:4200 by default). : "${PAIVANA_PUBLIC:=https://paivana.hacktivism.ch}" : "${E2E_PAIVANA:=1}" : "${E2E_PAIVANA_TEMPLATE:=paivana}" @@ -1630,11 +1632,11 @@ else ;; esac - # Ensure wallet can cover 4200 (extra ATM if ladder did not fund enough) + # Ensure wallet can cover 4200 (extra bank withdraw if ladder did not fund enough) pay_need=$(python3 -c 'import sys; print(float(sys.argv[1].split(":",1)[-1]))' "$E2E_PAIVANA_AMOUNT" 2>/dev/null || echo 4200) av_now=$(wallet_avail_num) if ! python3 -c "import sys; sys.exit(0 if float(sys.argv[1]) + 1e-9 >= float(sys.argv[2]) else 1)" "$av_now" "$pay_need" 2>/dev/null; then - info "paivana" "avail ${CUR}:${av_now} < ${pay_need} — ATM withdraw ${E2E_PAIVANA_AMOUNT}" + info "paivana" "avail ${CUR}:${av_now} < ${pay_need} — Bank withdraw ${E2E_PAIVANA_AMOUNT}" e2e_over && warn "paivana" "budget low before 4200 withdraw" set +e e2e_one_withdraw "$E2E_PAIVANA_AMOUNT" @@ -1687,7 +1689,7 @@ fi set_group report section "e2e · report" info "user" "$USER" -info "ATM withdraws" "$WITHDRAW_REPORT" +info "Bank withdraws" "$WITHDRAW_REPORT" info "payments" "$PAY_REPORT" info "shop" "${SHOP_REPORT:-(n/a)}" info "paivana" "${PAIVANA_REPORT:-(n/a)}" diff --git a/check_urls.sh b/check_urls.sh index aa2522e..472aba5 100755 --- a/check_urls.sh +++ b/check_urls.sh @@ -1316,7 +1316,7 @@ if [ "${LOCAL_STACK:-1}" = "1" ] || [ -n "${BANK_PUBLIC:-}" ]; then fi ;; 404|405|501) - # GOA bank landing mints demo-withdraw.json; stage TESTPAYSAN uses ATM/wallet guide only + # GOA bank landing mints demo-withdraw.json; stage TESTPAYSAN uses bank withdraw/wallet guide only if [ "${EXPECT_CURRENCY:-}" = "GOA" ] || [ "${LOCAL_STACK:-0}" = "1" ]; then fail "landing bank demo-withdraw" "HTTP $dw_code (want 200)" else diff --git a/lib.sh b/lib.sh index 4e7f137..07ee15a 100755 --- a/lib.sh +++ b/lib.sh @@ -62,9 +62,9 @@ fi # Merchant portal /private API: instance ids must be lowercase or you get odd 401s. # (Display names may use capitals; path segment /instances/{id}/ must not.) MERCHANT_INSTANCE=$(printf '%s' "${MERCHANT_INSTANCE}" | tr '[:upper:]' '[:lower:]') -: "${WITHDRAW_AMT:=GOA:20}" # single-shot fallback; e2e ladder uses ATM notes +: "${WITHDRAW_AMT:=GOA:20}" # single-shot fallback; e2e ladder uses bank cash-note amounts : "${PAY_AMT:=GOA:0.01}" -: "${CREDIT_AMT:=GOA:4700}" # covers ATM ladder 20+50+100+200+4200 (paivana) +: "${CREDIT_AMT:=GOA:4700}" # covers bank withdraw ladder 20+50+100+200+4200 (paivana) : "${TIMEOUT:=12}" : "${E2E_TIMEOUT:=55}" # whole e2e budget; skip rest when exceeded (e2e raises as needed) : "${E2E_PAY_SECS:=22}" # dedicated seconds for pay handle-uri (avoid Alarm clock) diff --git a/taler-monitoring.sh b/taler-monitoring.sh index 9445a53..4e08116 100755 --- a/taler-monitoring.sh +++ b/taler-monitoring.sh @@ -293,7 +293,7 @@ if [ "${LOCAL_STACK:-1}" != "1" ]; then SKIP_SSH=1 fi -# Remote: no SSH; ATM withdraw ladder set in check_e2e (smaller notes) +# Remote: no SSH; bank withdraw ladder set in check_e2e (smaller notes) if [ "${LOCAL_STACK}" != "1" ]; then E2E_FAKE_INCOMING=0 E2E_REMOTE=1 @@ -307,7 +307,7 @@ export EXPECT_CURRENCY SKIP_SSH LOCAL_STACK TALER_DOMAIN_PROBE CHECK_LANDING export TALER_DOMAIN_FROM_CLI="${TALER_DOMAIN_FROM_CLI:-0}" export DOMAIN_SET="${DOMAIN_SET:-0}" export WITHDRAW_AMT PAY_AMT CREDIT_AMT MERCHANT_INSTANCE -export E2E_FAKE_INCOMING E2E_REMOTE E2E_VARIABLE E2E_ATM_MAX +export E2E_FAKE_INCOMING E2E_REMOTE E2E_VARIABLE E2E_ATM_MAX E2E_BANK_WITHDRAW_MAX export E2E_WITHDRAW_VALUES E2E_PAY_VALUES E2E_USE_TEMPLATES E2E_TEMPLATE_MAP export PAIVANA_PUBLIC E2E_PAIVANA E2E_PAIVANA_TEMPLATE E2E_PAIVANA_AMOUNT E2E_PAIVANA_INSTANCE export INSIDE_PROFILE INSIDE_SSH @@ -431,7 +431,7 @@ _progress_estimate_phase() { versions) n=35 ;; sanity) n=35 ;; server) n=20 ;; - # e2e: ATM ladder emits many INFO lines (coins before/after each note) + # e2e: bank withdraw ladder emits many INFO lines (coins before/after each note) e2e) n=240 ;; ladder|goa-ladder) n=120 ;; auth401) n=70 ;; From a7b92e11c43366252fb760c177b20d5d5502a136 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 09:06:31 +0200 Subject: [PATCH 05/29] =?UTF-8?q?release=201.18.4:=20e2e=20group=20atm=20?= =?UTF-8?q?=E2=86=92=20bankwd=20(Bank=20withdraw=20IDs)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finish ATM rename: set_group bankwd so check IDs are e2e.bankwd-* and match user-facing Bank withdraw. E2E_ATM_MAX remains a legacy env alias. --- TESTS.md | 6 +++--- VERSION | 2 +- VERSIONS.md | 1 + check_e2e.sh | 2 +- lib.sh | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/TESTS.md b/TESTS.md index 2917e4c..3d2e865 100644 --- a/TESTS.md +++ b/TESTS.md @@ -28,7 +28,7 @@ Every check line has a **global** run number and a **grouped** id: | **ladder** | `check_goa_ladder.sh` | `plan` `load` `withdraw` `pay` `report` | **Why groups:** flat `www-001`…`www-080` was hard to talk about. -`www.bank-04` / `e2e.atm-02` pin the failure to a logical block. +`www.bank-04` / `e2e.bankwd-02` pin the failure to a logical block. Usage in scripts: @@ -100,7 +100,7 @@ SSH: `KOOPA_SSH` (default `koopa`), then `KOOPA_SSH_FALLBACKS` when LAN is unrea | **e2e.load-** | host/container load snapshots (before / after bank withdraw / after pay) | | **e2e.bank-** | account, credit, withdraw ops | | **e2e.wallet-** | exchange + ToS, wallet setup | -| **e2e.atm-** (group id; UI says Bank withdraw) | bank withdraw ladder rungs | +| ****e2e.bankwd-** | bank withdraw ladder rungs | | **e2e.settle-** | wallet settlement wait / spendable balance | | **e2e.pay-** | variable payments | | **e2e.shop-** | GOA shop product catalog pays | @@ -109,7 +109,7 @@ SSH: `KOOPA_SSH` (default `koopa`), then `KOOPA_SSH_FALLBACKS` when LAN is unrea | **e2e.report-** | final tallies | When filing an issue, quote the full id + label, e.g. -`e2e.atm-03 Bank withdraw 5 GOA · bank confirmed, wallet empty`. +`e2e.bankwd-03 Bank withdraw 5 GOA · bank confirmed, wallet empty`. --- diff --git a/VERSION b/VERSION index b9fb27a..a67b05e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.18.3 +1.18.4 diff --git a/VERSIONS.md b/VERSIONS.md index e188d79..056e0b4 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| +| **v1.18.4** | 2026-07-19 | **Bugfix:** e2e group id **atm → bankwd** (`e2e.bankwd-*`); finish ATM wording cleanup so mon logs match **Bank withdraw** (no leftover “atm” phase label). Env `E2E_ATM_MAX` still accepted as alias. | | **v1.18.3** | 2026-07-19 | **Bugfix / wording:** e2e user-facing **ATM withdraw** → **Bank withdraw** (amounts still CHF-style cash-note ladder 20/50/100/200/… — “ATM” was only a metaphor for those note sizes, not a physical ATM). Env: `E2E_BANK_WITHDRAW_MAX` (legacy alias `E2E_ATM_MAX`). Internal group id `e2e.atm-*` kept for check-id stability. | | **v1.18.2** | 2026-07-19 | **Bugfix:** never publish / accept **stale mon HTML assets** — refuse pre-1.15.8 box-drawing sticky JS in site-gen + generated pages; monpages ERROR on public pages still carrying that JS or teed `SyntaxWarning: invalid escape sequence`; purge `site-gen/__pycache__` after suite reset; convert with `PYTHONWARNINGS=error::SyntaxWarning`. | | **v1.18.1** | 2026-07-19 | **Bugfix:** (1) mon HTML sticky/err counts honor French badges **ERREUR** / **AVERT** (were plain → 0 errors on FP pages); (2) `stage-lfp` + `INSIDE_PODMAN=1` uses **host-podman** (no empty `INSIDE_SSH` fail); (3) reaffirm filter JS has no box-glyph/`\\-` regex (charCodeAt only — kills SyntaxWarning “geheimsprache” noise). | diff --git a/check_e2e.sh b/check_e2e.sh index f0f9b2f..1eea677 100755 --- a/check_e2e.sh +++ b/check_e2e.sh @@ -1337,7 +1337,7 @@ sys.exit(1) } # --------------------------------------------------------------------------- -set_group atm +set_group bankwd section "e2e · Bank withdraw ladder" # --------------------------------------------------------------------------- WITHDRAW_OK=0 diff --git a/lib.sh b/lib.sh index 07ee15a..16a9a0e 100755 --- a/lib.sh +++ b/lib.sh @@ -595,7 +595,7 @@ set_area() { } # Start a logical sub-group. Short stable names: # exchange bank merchant perf stats landing paivana -# prereq wallet atm settle pay shop dig load report +# prereq wallet bankwd settle pay shop dig load report # ssh caddy outside inside compare withdraw # Counter continues if the same group is re-entered later in the area. # Prints a compact group chip so log + issue text map cleanly (www.bank-03). From c83c0a522f57ab19803c2320cbeaddba9372aa56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 09:06:40 +0200 Subject: [PATCH 06/29] =?UTF-8?q?release=201.18.5:=20TESTS.md=20e2e=20grou?= =?UTF-8?q?p=20list=20atm=20=E2=86=92=20bankwd?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- TESTS.md | 2 +- VERSION | 2 +- VERSIONS.md | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/TESTS.md b/TESTS.md index 3d2e865..f161ed8 100644 --- a/TESTS.md +++ b/TESTS.md @@ -24,7 +24,7 @@ Every check line has a **global** run number and a **grouped** id: | **surface** | `check_surface.sh` | ecosystem inventory + **nmap** OS fingerprint; mail/mattermost content reported via surface page | | **sanity** | `check_sanity.sh` | `bank` `exchange` `merchant` | | **server** | `check_server.sh` | (flat `server-NN` or host groups) | -| **e2e** | `check_e2e.sh` | `prereq` `load` `bank` `wallet` `atm` `settle` `pay` `shop` `paivana` `dig` `report` | +| **e2e** | `check_e2e.sh` | `prereq` `load` `bank` `wallet` `bankwd` `settle` `pay` `shop` `paivana` `dig` `report` | | **ladder** | `check_goa_ladder.sh` | `plan` `load` `withdraw` `pay` `report` | **Why groups:** flat `www-001`…`www-080` was hard to talk about. diff --git a/VERSION b/VERSION index a67b05e..8e8b0a9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.18.4 +1.18.5 diff --git a/VERSIONS.md b/VERSIONS.md index 056e0b4..db48aa9 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| +| **v1.18.5** | 2026-07-19 | **Bugfix:** TESTS.md e2e group list `atm` → `bankwd` (missed in 1.18.4). | | **v1.18.4** | 2026-07-19 | **Bugfix:** e2e group id **atm → bankwd** (`e2e.bankwd-*`); finish ATM wording cleanup so mon logs match **Bank withdraw** (no leftover “atm” phase label). Env `E2E_ATM_MAX` still accepted as alias. | | **v1.18.3** | 2026-07-19 | **Bugfix / wording:** e2e user-facing **ATM withdraw** → **Bank withdraw** (amounts still CHF-style cash-note ladder 20/50/100/200/… — “ATM” was only a metaphor for those note sizes, not a physical ATM). Env: `E2E_BANK_WITHDRAW_MAX` (legacy alias `E2E_ATM_MAX`). Internal group id `e2e.atm-*` kept for check-id stability. | | **v1.18.2** | 2026-07-19 | **Bugfix:** never publish / accept **stale mon HTML assets** — refuse pre-1.15.8 box-drawing sticky JS in site-gen + generated pages; monpages ERROR on public pages still carrying that JS or teed `SyntaxWarning: invalid escape sequence`; purge `site-gen/__pycache__` after suite reset; convert with `PYTHONWARNINGS=error::SyntaxWarning`. | From db87092ace6b8fdecdc37e2e0d0c0c7c20d2f58a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 09:06:54 +0200 Subject: [PATCH 07/29] release 1.18.6: fix TESTS.md e2e.bankwd table markdown --- TESTS.md | 2 +- VERSION | 2 +- VERSIONS.md | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/TESTS.md b/TESTS.md index f161ed8..5c0f76a 100644 --- a/TESTS.md +++ b/TESTS.md @@ -100,7 +100,7 @@ SSH: `KOOPA_SSH` (default `koopa`), then `KOOPA_SSH_FALLBACKS` when LAN is unrea | **e2e.load-** | host/container load snapshots (before / after bank withdraw / after pay) | | **e2e.bank-** | account, credit, withdraw ops | | **e2e.wallet-** | exchange + ToS, wallet setup | -| ****e2e.bankwd-** | bank withdraw ladder rungs | +| **e2e.bankwd-** | Bank withdraw ladder rungs | | **e2e.settle-** | wallet settlement wait / spendable balance | | **e2e.pay-** | variable payments | | **e2e.shop-** | GOA shop product catalog pays | diff --git a/VERSION b/VERSION index 8e8b0a9..04a8bc2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.18.5 +1.18.6 diff --git a/VERSIONS.md b/VERSIONS.md index db48aa9..4643d72 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| +| **v1.18.6** | 2026-07-19 | **Bugfix:** TESTS.md table cell for `e2e.bankwd-` markdown typo from rename. | | **v1.18.5** | 2026-07-19 | **Bugfix:** TESTS.md e2e group list `atm` → `bankwd` (missed in 1.18.4). | | **v1.18.4** | 2026-07-19 | **Bugfix:** e2e group id **atm → bankwd** (`e2e.bankwd-*`); finish ATM wording cleanup so mon logs match **Bank withdraw** (no leftover “atm” phase label). Env `E2E_ATM_MAX` still accepted as alias. | | **v1.18.3** | 2026-07-19 | **Bugfix / wording:** e2e user-facing **ATM withdraw** → **Bank withdraw** (amounts still CHF-style cash-note ladder 20/50/100/200/… — “ATM” was only a metaphor for those note sizes, not a physical ATM). Env: `E2E_BANK_WITHDRAW_MAX` (legacy alias `E2E_ATM_MAX`). Internal group id `e2e.atm-*` kept for check-id stability. | From 29bc0c659b76c7649b3fd1085fd38da85be31b8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 09:42:06 +0200 Subject: [PATCH 08/29] release 1.18.7: fix TSA mail catalog + FP stage e2e defaults Mail/surface catalogs used non-resolving anastasis.taler-systems.com; point to mail.anastasis.lu (pixel) and anastasis.lu. Stage mon defaults include e2e with francpaysan-secrets + wallet-cli paths. --- VERSION | 2 +- VERSIONS.md | 1 + check_mail.sh | 3 ++- host-agent/env/stagepaysan.env.example | 29 +++++++++++++++++++------- host-agent/run-fp-stage-monitoring.sh | 4 ++-- host-agent/run-surface-monitoring.sh | 2 +- mail-catalog.conf | 18 +++++++++------- surface-catalog.conf | 5 ++++- 8 files changed, 42 insertions(+), 22 deletions(-) diff --git a/VERSION b/VERSION index 04a8bc2..d6f3a38 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.18.6 +1.18.7 diff --git a/VERSIONS.md b/VERSIONS.md index 4643d72..dd623e1 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| +| **v1.18.7** | 2026-07-19 | **Bugfix:** mail/surface catalogs — TSA/Anastasis MX is **mail.anastasis.lu** (→ pixel), not non-resolving `anastasis.taler-systems.com`; add `anastasis.lu` + pixel host probes (fixes surface_err mail.*). **Also:** FP stage host-agent defaults include **e2e** + 1800s timeout; stagepaysan.env.example wires francpaysan-secrets + wallet-cli. | | **v1.18.6** | 2026-07-19 | **Bugfix:** TESTS.md table cell for `e2e.bankwd-` markdown typo from rename. | | **v1.18.5** | 2026-07-19 | **Bugfix:** TESTS.md e2e group list `atm` → `bankwd` (missed in 1.18.4). | | **v1.18.4** | 2026-07-19 | **Bugfix:** e2e group id **atm → bankwd** (`e2e.bankwd-*`); finish ATM wording cleanup so mon logs match **Bank withdraw** (no leftover “atm” phase label). Env `E2E_ATM_MAX` still accepted as alias. | diff --git a/check_mail.sh b/check_mail.sh index 7b74ca4..7f92433 100755 --- a/check_mail.sh +++ b/check_mail.sh @@ -2,7 +2,8 @@ # check_mail.sh — outside-in mail (MX / SMTP / IMAP / SPF / DMARC) # # Catalog: mail-catalog.conf (MAIL_CATALOG=… to override) -# Covers firefly (taler.net, gnunet.org) and pixel (taler-systems.com, …). +# Covers firefly (taler.net, gnunet.org) and Anastasis/TSA +# (mail.anastasis.lu → pixel.taler-systems.com; anastasis.lu). # # Outside-only. Phase: mail # diff --git a/host-agent/env/stagepaysan.env.example b/host-agent/env/stagepaysan.env.example index e91d54e..abfcdc3 100644 --- a/host-agent/env/stagepaysan.env.example +++ b/host-agent/env/stagepaysan.env.example @@ -1,8 +1,10 @@ # Copy to: ~/.config/taler-monitoring/env (user stagepaysan) # Never inside the git tree — update-suite never overwrites this file. # -# Shared reporting generation (run-host-report.sh / run-fp-stage-monitoring.sh): -# RUN_TIMEOUT, line-buffered log, always HTML, commit link on git.hacktivism.ch +# Secrets: use francpaysan-secrets layout (operator tree or host mirror): +# $HOME/francpaysan-secrets/stage/bank-admin-password.txt +# $HOME/francpaysan-secrets/stage/default-instance-token.txt +# On stage host, symlink those to /mnt/data/stagepaysan/*/secrets/… TALER_DOMAIN=stage.lefrancpaysan.ch INSIDE_PODMAN=1 @@ -10,21 +12,32 @@ INSIDE_MODE=local-podman INSIDE_PROFILE=stage-lfp LOCAL_STACK=0 CONTINUE_ON_ERROR=1 -PHASES="urls inside versions" -RUN_TIMEOUT=600 +PHASES="urls inside versions monpages e2e" +RUN_TIMEOUT=1800 +E2E_TIMEOUT=900 +E2E_USE_TEMPLATES=1 HTML_OUT=$HOME/monitoring-sites-staging MON_HOSTS="stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch" +DEPLOY_WWW_ROOT=/var/www/monitoring-sites -# Git clone of taler-monitoring (repo root, not …/taler-monitoring (repo root)) SUITE_DIR=$HOME/src/taler-monitoring SUITE_GIT_URL=https://git.hacktivism.ch/hernani/taler-monitoring.git SUITE_GIT_REF=main SUITE_UPDATE_MODE=reset SOURCE_REPO_WEB=https://git.hacktivism.ch/hernani/taler-monitoring -# Optional public docroot if this user can write it -# DEPLOY_WWW_ROOT=/var/www/monitoring-sites +FRANCPAYSAN_SECRETS=$HOME/francpaysan-secrets +WALLET_CLI=$HOME/.local/lib/taler-wallet-cli/node_modules/taler-wallet-cli/bin/taler-wallet-cli.mjs +WALLET_CLI_CANDIDATES="$HOME/.local/lib/taler-wallet-cli/node_modules/taler-wallet-cli/bin/taler-wallet-cli.mjs $HOME/.local/bin/taler-wallet-cli.mjs" + +INSIDE_BANK_CTR=stage-lfp-bank +INSIDE_EXCHANGE_CTR=stage-lfp-exchange-ansible +INSIDE_MERCHANT_CTR=stage-lfp-merchant -# UI language for console + HTML sticky bar (en|fr) TALER_MON_LANG=fr +MONPAGES_REQUIRE_PUBLIC=1 +MONPAGES_INVENTORY=job +STRICT_EXIT=1 +AGENT_LABEL=fp-stage-host-agent +STATE_NAME=taler-monitoring-stage-lfp diff --git a/host-agent/run-fp-stage-monitoring.sh b/host-agent/run-fp-stage-monitoring.sh index e8eb38c..e7f95e8 100755 --- a/host-agent/run-fp-stage-monitoring.sh +++ b/host-agent/run-fp-stage-monitoring.sh @@ -18,8 +18,8 @@ export LOCAL_STACK="${LOCAL_STACK:-0}" export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" # monpages obligatory (ERROR): only FP stage mon hosts below — never GOA inventory export MONPAGES_REQUIRE_PUBLIC="${MONPAGES_REQUIRE_PUBLIC:-1}" -export RUN_TIMEOUT="${RUN_TIMEOUT:-600}" -export PHASES="${PHASES:-urls inside versions monpages}" +export RUN_TIMEOUT="${RUN_TIMEOUT:-1800}" +export PHASES="${PHASES:-urls inside versions monpages e2e}" export MON_HOSTS="${MON_HOSTS:-stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch}" export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}" export DEPLOY_WWW_ROOT="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" diff --git a/host-agent/run-surface-monitoring.sh b/host-agent/run-surface-monitoring.sh index f9d5098..d634a7f 100755 --- a/host-agent/run-surface-monitoring.sh +++ b/host-agent/run-surface-monitoring.sh @@ -6,7 +6,7 @@ # https://taler.hacktivism.ch/taler-monitoring-surface_err/ # # This page covers remote ecosystem inventory: host/port probes, nmap OS -# fingerprint, Mattermost, mail (firefly + anastasis), package/version signals. +# fingerprint, Mattermost, mail (firefly + mail.anastasis.lu/pixel), package/version signals. # There are NO separate public pages for mail / mattermost (folded here). # Apt-src deploy tests keep their own page + timer: # run-aptdeploy-monitoring.sh → /taler-monitoring-aptdeploy(+_err)/ diff --git a/mail-catalog.conf b/mail-catalog.conf index 7a67578..f489234 100644 --- a/mail-catalog.conf +++ b/mail-catalog.conf @@ -3,16 +3,18 @@ # Format (whitespace-separated; # comments): # domain expected_mx (comma) mail_hosts (comma) smtp_ports imap_ports label # -# Only these mail planes (no hacktivism / ad-hoc MX): -# • firefly.gnunet.org — taler.net + gnunet.org -# • anastasis.taler-systems.com — Anastasis / TSA mail +# Mail planes (outside-in only — no host login): +# • firefly.gnunet.org — taler.net + gnunet.org +# • mail.anastasis.lu — Anastasis / TSA (A → pixel.taler-systems.com) +# (legacy catalog host anastasis.taler-systems.com does NOT resolve) # # Override path: MAIL_CATALOG=/path/to/file # --- firefly.gnunet.org --- -taler.net firefly.gnunet.org firefly.gnunet.org 25,465,587 143,993 firefly-taler -gnunet.org firefly.gnunet.org firefly.gnunet.org 25,465,587 143,993 firefly-gnunet +taler.net firefly.gnunet.org firefly.gnunet.org 25,465,587 143,993 firefly-taler +gnunet.org firefly.gnunet.org firefly.gnunet.org 25,465,587 143,993 firefly-gnunet -# --- anastasis.taler-systems.com --- -anastasis.taler-systems.com anastasis.taler-systems.com anastasis.taler-systems.com 25,465,587 143,993 anastasis-tsa -taler-systems.com anastasis.taler-systems.com anastasis.taler-systems.com 25,465,587 143,993 tsa-via-anastasis +# --- Anastasis / TSA (MX mail.anastasis.lu → pixel) --- +# Port 587 often closed/filtered on this plane → WARN only (see check_mail.sh). +taler-systems.com mail.anastasis.lu mail.anastasis.lu,pixel.taler-systems.com 25,465,587 143,993 tsa-anastasis +anastasis.lu mail.anastasis.lu mail.anastasis.lu 25,465,587 143,993 anastasis-lu diff --git a/surface-catalog.conf b/surface-catalog.conf index cae2a14..3caa703 100644 --- a/surface-catalog.conf +++ b/surface-catalog.conf @@ -46,7 +46,10 @@ bugs.gnunet.org 443 https gnunet-bugs # --- taler-systems.com --- www.taler-systems.com 443 https tsa-www taler-systems.com 443 https tsa-apex -anastasis.taler-systems.com 443,25,465,587,143,993 https anastasis-mail +# Anastasis / TSA mail plane (MX mail.anastasis.lu → pixel). Do NOT list +# anastasis.taler-systems.com — it does not resolve (was a catalog typo). +mail.anastasis.lu 443,25,465,587,143,993 https anastasis-mail +pixel.taler-systems.com 443,25,465,587,143,993 https pixel-mail # --- mail (also covered by phase mail; listed for surface nmap/ports) --- firefly.gnunet.org 443,25,465,587,143,993 https firefly-mail From 48d51e17be676335f11b4e5e4c48f1e141b011b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 10:56:28 +0200 Subject: [PATCH 09/29] release 1.18.8: e2e wallet helper PATH + robust wcli timeout accept-uri failed when taler-helper-sqlite3 was not on PATH; prefer timeout over perl -e alarm and report real Error/ENOENT lines. --- VERSION | 2 +- VERSIONS.md | 1 + check_e2e.sh | 113 ++++++++++++++++++++++++++++++++------------------- 3 files changed, 74 insertions(+), 42 deletions(-) diff --git a/VERSION b/VERSION index d6f3a38..1a31d39 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.18.7 +1.18.8 diff --git a/VERSIONS.md b/VERSIONS.md index dd623e1..6a89347 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| +| **v1.18.8** | 2026-07-19 | **Bugfix:** e2e wallet — require `taler-helper-sqlite3` on PATH; use absolute `node` + `timeout` instead of fragile `perl -e alarm` for wcli; clearer accept-uri errors (no raw alarm-shift noise). | | **v1.18.7** | 2026-07-19 | **Bugfix:** mail/surface catalogs — TSA/Anastasis MX is **mail.anastasis.lu** (→ pixel), not non-resolving `anastasis.taler-systems.com`; add `anastasis.lu` + pixel host probes (fixes surface_err mail.*). **Also:** FP stage host-agent defaults include **e2e** + 1800s timeout; stagepaysan.env.example wires francpaysan-secrets + wallet-cli. | | **v1.18.6** | 2026-07-19 | **Bugfix:** TESTS.md table cell for `e2e.bankwd-` markdown typo from rename. | | **v1.18.5** | 2026-07-19 | **Bugfix:** TESTS.md e2e group list `atm` → `bankwd` (missed in 1.18.4). | diff --git a/check_e2e.sh b/check_e2e.sh index 1eea677..d219c73 100755 --- a/check_e2e.sh +++ b/check_e2e.sh @@ -35,22 +35,22 @@ wcli_bal_snap() { fi fi local err="${out}.err" - if command -v perl >/dev/null 2>&1; then - if [ -n "${WSERVE_PID:-}" ] && [ -S "${WSOCK:-}" ]; then - perl -e 'alarm shift; exec @ARGV' 10 \ - node "$WALLET_CLI" --wallet-connection="$WSOCK" --no-throttle balance \ + local _node="${NODE_BIN:-$(command -v node 2>/dev/null || true)}" + [ -n "$_node" ] || { echo "(no node)" >"$out"; return 1; } + if [ -n "${WSERVE_PID:-}" ] && [ -S "${WSOCK:-}" ]; then + if command -v timeout >/dev/null 2>&1; then + timeout -k 2 10 "$_node" "$WALLET_CLI" --wallet-connection="$WSOCK" --no-throttle balance \ >"$out" 2>"$err" || true else - perl -e 'alarm shift; exec @ARGV' 10 \ - node "$WALLET_CLI" --wallet-db="$WDB" --no-throttle --skip-defaults balance \ + "$_node" "$WALLET_CLI" --wallet-connection="$WSOCK" --no-throttle balance \ >"$out" 2>"$err" || true fi else - if [ -n "${WSERVE_PID:-}" ] && [ -S "${WSOCK:-}" ]; then - node "$WALLET_CLI" --wallet-connection="$WSOCK" --no-throttle balance \ + if command -v timeout >/dev/null 2>&1; then + timeout -k 2 10 "$_node" "$WALLET_CLI" --wallet-db="$WDB" --no-throttle --skip-defaults balance \ >"$out" 2>"$err" || true else - node "$WALLET_CLI" --wallet-db="$WDB" --no-throttle --skip-defaults balance \ + "$_node" "$WALLET_CLI" --wallet-db="$WDB" --no-throttle --skip-defaults balance \ >"$out" 2>"$err" || true fi fi @@ -378,13 +378,35 @@ WALLET_CLI=$(find_wallet_cli) || { blocker "prereq" "taler-wallet-cli not found (set WALLET_CLI=)" exit 1 } +# wallet-core spawns `taler-helper-sqlite3` via PATH (not an absolute path). +# Ensure operator wallet bindir + ~/.local/bin are first (systemd mon jobs). +export PATH="${WALLET_CLI_BINDIR:+$WALLET_CLI_BINDIR:}${HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin:${PATH:-}" +NODE_BIN=$(command -v node 2>/dev/null || true) +if [ -z "$NODE_BIN" ] || [ ! -x "$NODE_BIN" ]; then + blocker "prereq" "node not found on PATH (install Node or set PATH; need node for wallet-cli)" + exit 1 +fi +if ! command -v taler-helper-sqlite3 >/dev/null 2>&1; then + for _h in \ + "${WALLET_CLI_BINDIR:+$WALLET_CLI_BINDIR/taler-helper-sqlite3}" \ + "${HOME}/.local/bin/taler-helper-sqlite3" \ + /usr/bin/taler-helper-sqlite3 + do + [ -n "$_h" ] && [ -x "$_h" ] && export PATH="$(dirname "$_h"):$PATH" && break + done +fi +if ! command -v taler-helper-sqlite3 >/dev/null 2>&1; then + blocker "prereq" "taler-helper-sqlite3 not on PATH (wallet accept-uri/pay will fail with spawn ENOENT) — install next to wallet-cli or set WALLET_CLI_BINDIR" + exit 1 +fi +ok "wallet-helper" "taler-helper-sqlite3=$(command -v taler-helper-sqlite3)" # Implementation version (--version) + wallet-core API ranges (version command) WALLET_IMPL_VER=$( - node "$WALLET_CLI" --version 2>/dev/null | head -1 | tr -d '\r' || true + "$NODE_BIN" "$WALLET_CLI" --version 2>/dev/null | head -1 | tr -d '\r' || true ) WALLET_CORE_VER=$( _wver_db=$(mktemp "${TMPDIR:-/tmp}/wver.XXXXXX.sqlite3" 2>/dev/null || echo "${TMPDIR:-/tmp}/wver-$$.sqlite3") - node "$WALLET_CLI" --wallet-db="$_wver_db" --no-throttle --skip-defaults version 2>/dev/null \ + "$NODE_BIN" "$WALLET_CLI" --wallet-db="$_wver_db" --no-throttle --skip-defaults version 2>/dev/null \ | python3 -c ' import json,re,sys raw=sys.stdin.read() @@ -577,7 +599,7 @@ start_wallet_serve() { stop_wallet_serve rm -f "$WSOCK" # serve holds the DB open; clients use --wallet-connection only - node "$WALLET_CLI" --wallet-db="$WDB" --no-throttle --skip-defaults \ + "$NODE_BIN" "$WALLET_CLI" --wallet-db="$WDB" --no-throttle --skip-defaults \ advanced serve --unix-path "$WSOCK" \ >"$SCRATCH/wallet-serve.log" 2>&1 & WSERVE_PID=$! @@ -594,6 +616,25 @@ start_wallet_serve() { return 1 } +# Run node wallet-cli with a hard wall clock. Prefer GNU/coreutils `timeout` +# (clear errors). Fall back to perl alarm only if timeout is missing. +# Always use absolute NODE_BIN + WALLET_CLI so systemd PATH quirks cannot +# turn into opaque "alarm shift; exec @ARGV" noise in accept-uri output. +_wcli_exec() { + local cap="$1" + shift + local -a cmd=( "$NODE_BIN" "$WALLET_CLI" "$@" ) + if command -v timeout >/dev/null 2>&1; then + timeout -k 2 "$cap" "${cmd[@]}" + elif command -v gtimeout >/dev/null 2>&1; then + gtimeout --kill-after=2 "$cap" "${cmd[@]}" + elif command -v perl >/dev/null 2>&1; then + perl -e 'alarm shift; exec @ARGV' "$cap" "${cmd[@]}" + else + "${cmd[@]}" + fi +} + # $1 = max seconds for this call (optional); rest = wallet-cli args # Prefer live serve socket so the shepherd stays up (alternative to run-until-done). wcli() { @@ -605,22 +646,15 @@ wcli() { e2e_over && return 124 local cap cap=$(e2e_left) - [ "$cap" -gt "$maxc" ] && cap=$maxc - [ "$cap" -lt 3 ] && return 124 + # e2e_left may print warnings on stdout — keep only first integer token + cap=$(printf '%s' "$cap" | awk '/^[0-9]+$/{print; exit} {for(i=1;i<=NF;i++) if($i+0==$i){print $i; exit}}') + [ -z "$cap" ] && cap=$maxc + [ "$cap" -gt "$maxc" ] 2>/dev/null && cap=$maxc + [ "$cap" -lt 3 ] 2>/dev/null && return 124 if [ -n "${WSERVE_PID:-}" ] && [ -S "${WSOCK:-}" ]; then - if command -v perl >/dev/null 2>&1; then - perl -e 'alarm shift; exec @ARGV' "$cap" \ - node "$WALLET_CLI" --wallet-connection="$WSOCK" --no-throttle "$@" - else - node "$WALLET_CLI" --wallet-connection="$WSOCK" --no-throttle "$@" - fi + _wcli_exec "$cap" --wallet-connection="$WSOCK" --no-throttle "$@" else - if command -v perl >/dev/null 2>&1; then - perl -e 'alarm shift; exec @ARGV' "$cap" \ - node "$WALLET_CLI" --wallet-db="$WDB" --no-throttle --skip-defaults "$@" - else - node "$WALLET_CLI" --wallet-db="$WDB" --no-throttle --skip-defaults "$@" - fi + _wcli_exec "$cap" --wallet-db="$WDB" --no-throttle --skip-defaults "$@" fi } @@ -629,9 +663,10 @@ wcli_pay() { local cap=$E2E_PAY_SECS local left left=$(e2e_left) - if [ "$left" -gt "$cap" ]; then + left=$(printf '%s' "$left" | awk '/^[0-9]+$/{print; exit} {for(i=1;i<=NF;i++) if($i+0==$i){print $i; exit}}') + if [ -n "$left" ] && [ "$left" -gt "$cap" ] 2>/dev/null; then cap=$E2E_PAY_SECS - elif [ "$left" -ge 8 ]; then + elif [ -n "$left" ] && [ "$left" -ge 8 ] 2>/dev/null; then cap=$left else # still try once with floor 12s so Alarm clock is not the blocker @@ -639,19 +674,9 @@ wcli_pay() { fi info "pay wallet cap" "${cap}s" if [ -n "${WSERVE_PID:-}" ] && [ -S "${WSOCK:-}" ]; then - if command -v perl >/dev/null 2>&1; then - perl -e 'alarm shift; exec @ARGV' "$cap" \ - node "$WALLET_CLI" --wallet-connection="$WSOCK" --no-throttle "$@" - else - node "$WALLET_CLI" --wallet-connection="$WSOCK" --no-throttle "$@" - fi + _wcli_exec "$cap" --wallet-connection="$WSOCK" --no-throttle "$@" else - if command -v perl >/dev/null 2>&1; then - perl -e 'alarm shift; exec @ARGV' "$cap" \ - node "$WALLET_CLI" --wallet-db="$WDB" --no-throttle --skip-defaults "$@" - else - node "$WALLET_CLI" --wallet-db="$WDB" --no-throttle --skip-defaults "$@" - fi + _wcli_exec "$cap" --wallet-db="$WDB" --no-throttle --skip-defaults "$@" fi } @@ -935,7 +960,13 @@ e2e_one_withdraw() { && wcli withdraw accept-uri --exchange "${EXCHANGE_PUBLIC}/" "$URI_CLEAN" >"$SCRATCH/accept-$tag.out" 2>&1; then ok "wallet accept $WITHDRAW_AMT (no :443)" else - warn "Bank withdraw $WITHDRAW_AMT" "accept-uri failed — $(tail -c 100 "$SCRATCH/accept-$tag.out" | tr '\n' ' ')" + # Prefer the real Error/ENOENT line over noise (perl -e source, stack frames) + _acc_err=$( + grep -E 'Error:|ENOENT|spawn taler-helper|code:|An error occurred' \ + "$SCRATCH/accept-$tag.out" 2>/dev/null | head -3 | tr '\n' ' ' | head -c 200 + ) + [ -z "$_acc_err" ] && _acc_err=$(tail -c 160 "$SCRATCH/accept-$tag.out" 2>/dev/null | tr '\n' ' ') + warn "Bank withdraw $WITHDRAW_AMT" "accept-uri failed — ${_acc_err:-unknown}" return 1 fi cp "$SCRATCH/accept-$tag.out" "$SCRATCH/accept.out" From 8c99e13af1ea537d2f3e1b8b7e13ac308dec5935 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 11:04:44 +0200 Subject: [PATCH 10/29] release 1.18.9: FP domains default UI language fr globally Workstation TALER_MON_LANG=en no longer locks English on *lefrancpaysan*; only --lang / TALER_MON_LANG_SET=1 overrides. host-agent + HTML chrome. --- VERSION | 2 +- VERSIONS.md | 1 + host-agent/run-host-report.sh | 26 +++++++++------ i18n.sh | 10 +++--- site-gen/console_to_html.py | 60 +++++++++++++++++++---------------- 5 files changed, 57 insertions(+), 42 deletions(-) diff --git a/VERSION b/VERSION index 1a31d39..cafc0b7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.18.8 +1.18.9 diff --git a/VERSIONS.md b/VERSIONS.md index 6a89347..9f37dd4 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| +| **v1.18.9** | 2026-07-19 | **Bugfix:** UI lang — *lefrancpaysan*/*francpaysan* always default **fr** unless `TALER_MON_LANG_SET=1` / `--lang`; workstation `TALER_MON_LANG=en` no longer locks English (host-agent + console_to_html). | | **v1.18.8** | 2026-07-19 | **Bugfix:** e2e wallet — require `taler-helper-sqlite3` on PATH; use absolute `node` + `timeout` instead of fragile `perl -e alarm` for wcli; clearer accept-uri errors (no raw alarm-shift noise). | | **v1.18.7** | 2026-07-19 | **Bugfix:** mail/surface catalogs — TSA/Anastasis MX is **mail.anastasis.lu** (→ pixel), not non-resolving `anastasis.taler-systems.com`; add `anastasis.lu` + pixel host probes (fixes surface_err mail.*). **Also:** FP stage host-agent defaults include **e2e** + 1800s timeout; stagepaysan.env.example wires francpaysan-secrets + wallet-cli. | | **v1.18.6** | 2026-07-19 | **Bugfix:** TESTS.md table cell for `e2e.bankwd-` markdown typo from rename. | diff --git a/host-agent/run-host-report.sh b/host-agent/run-host-report.sh index e601562..f86b546 100755 --- a/host-agent/run-host-report.sh +++ b/host-agent/run-host-report.sh @@ -75,19 +75,25 @@ export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" export AUTH401_CONTINUE="${AUTH401_CONTINUE:-${CONTINUE_ON_ERROR}}" export RUN_TIMEOUT="${RUN_TIMEOUT:-600}" export TALER_DOMAIN="${TALER_DOMAIN:-hacktivism.ch}" -# UI language: selectable via TALER_MON_LANG / --lang (en|fr). -# If unset, auto: FrancPaysan domains → fr, else en. -if [ "${TALER_MON_LANG_SET:-0}" != "1" ] && [ -z "${TALER_MON_LANG:-}" ]; then - case "$TALER_DOMAIN" in - *lefrancpaysan*|*francpaysan*) export TALER_MON_LANG=fr ;; - *) export TALER_MON_LANG=en ;; - esac -elif [ -n "${TALER_MON_LANG:-}" ]; then - case "$TALER_MON_LANG" in +# UI language: --lang / TALER_MON_LANG_SET=1 locks; otherwise auto by domain. +# Bare TALER_MON_LANG=en in workstation env (clementine) must NOT lock language — +# *lefrancpaysan* / *francpaysan* always default to fr when SET≠1 (global rule). +if [ "${TALER_MON_LANG_SET:-0}" = "1" ]; then + case "${TALER_MON_LANG:-en}" in fr|FR|fra|french) export TALER_MON_LANG=fr ;; *) export TALER_MON_LANG=en ;; esac - export TALER_MON_LANG_SET=1 +else + case "$TALER_DOMAIN" in + *lefrancpaysan*|*francpaysan*) export TALER_MON_LANG=fr ;; + *) + case "${TALER_MON_LANG:-}" in + fr|FR|fra|french) export TALER_MON_LANG=fr ;; + *) export TALER_MON_LANG=en ;; + esac + ;; + esac + # leave TALER_MON_LANG_SET unset/0 so suite i18n_init can re-apply per -d domain fi export TALER_MON_LANG diff --git a/i18n.sh b/i18n.sh index a5a2285..585ab16 100644 --- a/i18n.sh +++ b/i18n.sh @@ -2,11 +2,13 @@ # i18n.sh — language for taler-monitoring console + HTML chrome # # TALER_MON_LANG=en|fr -# TALER_MON_LANG_SET=1 → language is explicit (--lang or host-agent wrapper); do not override. -# Auto (SET≠1): domain *lefrancpaysan* / *francpaysan* → fr, else keep env or en. +# TALER_MON_LANG_SET=1 → language is explicit (--lang only); do not override. +# Auto (SET≠1): domain *lefrancpaysan* / *francpaysan* → fr always (global). +# else keep env or en. # -# Important: a bare TALER_MON_LANG=en in ~/.config/taler-monitoring/env must NOT lock -# language for FP domains (that used to force English on stage.monnaie…). +# Important: a bare TALER_MON_LANG=en in ~/.config/taler-monitoring/env is only the +# workstation default for non-FP domains. It must NEVER lock English on FP +# (*lefrancpaysan* / *francpaysan*) — local laptop and host-agent both follow this. : "${TALER_MON_LANG:=}" : "${TALER_MON_LANG_SET:=0}" diff --git a/site-gen/console_to_html.py b/site-gen/console_to_html.py index 4aab08d..6e6aa54 100755 --- a/site-gen/console_to_html.py +++ b/site-gen/console_to_html.py @@ -45,15 +45,35 @@ def atomic_write_text(path: Path, text: str, encoding: str = "utf-8") -> None: except OSError: pass -def ui_lang(explicit: str = "") -> str: - """en (default) or fr. Env: TALER_MON_LANG; auto fr for lefrancpaysan hosts.""" +def ui_lang(explicit: str = "", host_hint: str = "") -> str: + """en (default) or fr. + + - TALER_MON_LANG_SET=1 → honour TALER_MON_LANG / explicit only (no host auto). + - Else: any *lefrancpaysan* / *francpaysan* host or log hint → fr + (workstation env TALER_MON_LANG=en must not force English on FP domains). + - Else: explicit / env / en. + """ import os - lang = (explicit or os.environ.get("TALER_MON_LANG") or "").strip().lower() - if lang in ("fr", "fra", "french"): + + env_lang = (os.environ.get("TALER_MON_LANG") or "").strip().lower() + lang = (explicit or env_lang or "").strip().lower() + locked = (os.environ.get("TALER_MON_LANG_SET") or "").strip() == "1" + is_fp = bool( + re.search(r"lefrancpaysan|francpaysan", host_hint or "", re.I) + ) + + def _norm(v: str) -> str: + if v in ("fr", "fra", "french"): + return "fr" + if v in ("en", "eng", "english"): + return "en" + return "" + + if locked: + return _norm(lang) or "en" + if is_fp: return "fr" - if lang in ("en", "eng", "english"): - return "en" - return "en" + return _norm(lang) or "en" def ui(lang: str, key: str, **kwargs) -> str: @@ -709,11 +729,9 @@ def monitoring_scope( label = (page_label or "monitoring").lower().replace("_", "-") phases = extract_phases(log_text) host = hostname or "?" - # Auto-fr for FrancPaysan hosts if lang not forced - if lang == "en" and re.search(r"lefrancpaysan|francpaysan", host + " " + log_text, re.I): - import os - if not (os.environ.get("TALER_MON_LANG") or "").strip(): - lang = "fr" + # Resolve lang again with host/log hint (*lefrancpaysan* → fr unless LANG_SET=1) + _ex = lang if lang in ("en", "fr") else "" + lang = ui_lang(explicit=_ex, host_hint=host + " " + log_text) if "surface" in label: items = [ @@ -1079,10 +1097,7 @@ def sticky_bar_html( if scope and scope.get("lang"): lang = str(scope.get("lang")) else: - lang = ui_lang() - # auto-fr for FP hostnames - if lang == "en" and re.search(r"lefrancpaysan|francpaysan", hostname or "", re.I): - lang = "fr" + lang = ui_lang(host_hint=hostname or "") if level == "red": status_txt = ui(lang, "errors") @@ -1990,12 +2005,7 @@ def build_html( page_label or "" ).lower() if _is_apt: - _lang_board = ui_lang() - if re.search(r"lefrancpaysan|francpaysan", hostname + "\n" + log_text, re.I): - import os - - if not (os.environ.get("TALER_MON_LANG") or "").strip(): - _lang_board = "fr" + _lang_board = ui_lang(host_hint=hostname + "\n" + log_text) err_nav += aptdeploy_board_html(log_text, lang=_lang_board) if errors and mode == "err": @@ -2040,11 +2050,7 @@ def build_html( first_err_href = "#first-error" if first_err_i is not None else None first_warn_href = "#first-warn" if first_warn_i is not None else None - lang = ui_lang() - if re.search(r"lefrancpaysan|francpaysan", hostname + "\n" + log_text, re.I): - import os - if not (os.environ.get("TALER_MON_LANG") or "").strip(): - lang = "fr" + lang = ui_lang(host_hint=hostname + "\n" + log_text) scope = monitoring_scope(page_label, hostname, log_text, lang=lang) page_lang = lang From 61f49f02db1c5fc3229c4f7f5a6d94f758ab2c67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 11:07:11 +0200 Subject: [PATCH 11/29] release 1.19.0: domains.conf lang column drives UI i18n default Per-stack en|fr lives in domains.conf; load_domain_profile sets TALER_DOMAIN_LANG. FP profiles use lang=fr; workstation LANG=en no longer overrides. HTML chrome resolves lang from the same conf. --- VERSION | 2 +- VERSIONS.md | 1 + domains.conf | 52 ++++++++-------- host-agent/run-host-report.sh | 18 ++---- i18n.sh | 39 +++++++----- lib.sh | 35 ++++++++++- site-gen/console_to_html.py | 109 ++++++++++++++++++++++++++++------ 7 files changed, 181 insertions(+), 75 deletions(-) diff --git a/VERSION b/VERSION index cafc0b7..815d5ca 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.18.9 +1.19.0 diff --git a/VERSIONS.md b/VERSIONS.md index 9f37dd4..634c6ec 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| +| **v1.19.0** | 2026-07-19 | **Feature:** domains.conf **`lang`** column (en\|fr) is the global UI language default per stack; i18n + HTML read `TALER_DOMAIN_LANG` / conf (FP profiles `lang=fr`). | | **v1.18.9** | 2026-07-19 | **Bugfix:** UI lang — *lefrancpaysan*/*francpaysan* always default **fr** unless `TALER_MON_LANG_SET=1` / `--lang`; workstation `TALER_MON_LANG=en` no longer locks English (host-agent + console_to_html). | | **v1.18.8** | 2026-07-19 | **Bugfix:** e2e wallet — require `taler-helper-sqlite3` on PATH; use absolute `node` + `timeout` instead of fragile `perl -e alarm` for wcli; clearer accept-uri errors (no raw alarm-shift noise). | | **v1.18.7** | 2026-07-19 | **Bugfix:** mail/surface catalogs — TSA/Anastasis MX is **mail.anastasis.lu** (→ pixel), not non-resolving `anastasis.taler-systems.com`; add `anastasis.lu` + pixel host probes (fixes surface_err mail.*). **Also:** FP stage host-agent defaults include **e2e** + 1800s timeout; stagepaysan.env.example wires francpaysan-secrets + wallet-cli. | diff --git a/domains.conf b/domains.conf index 1165936..189b1df 100644 --- a/domains.conf +++ b/domains.conf @@ -1,48 +1,52 @@ # taler-monitoring domain profiles # # When you add a stack, declare the three public endpoints here: -# name bank exchange merchant-backend currency local landing [canonical] +# name bank exchange merchant-backend currency local landing lang [canonical] # # - name: what -d / TALER_DOMAIN matches (aliases allowed as extra lines) # - bank / exchange / merchant: hostname or full https:// URL -# - currency: GOA | KUDOS | CHF | … (empty = report only) +# - currency: GOA | KUDOS | CHF | … (empty or "-" = report only) # - local: 1 = koopa SSH/inside/e2e stack; 0 = public-only # - landing: 1 = check /intro landings + assets; 0 = skip (no GOA-style landings) +# - lang: UI language default for this profile (en|fr). Source of truth for i18n +# when TALER_MON_LANG_SET≠1. Workstation env TALER_MON_LANG=en does not +# override a profile that sets fr (e.g. *lefrancpaysan*). # - canonical: optional TALER_DOMAIN label after alias match # -# CLI overrides after load: --bank --exchange --merchant --currency +# Legacy lines without lang (old 8th field = canonical only) still parse: lang→en. +# CLI overrides: --lang / TALER_MON_LANG_SET=1 # Alternate file: TALER_DOMAINS_CONF=/path/to/domains.conf # Local GOA (koopa) — landings matter -koopa bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch GOA 1 1 hacktivism.ch -hacktivism bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch GOA 1 1 hacktivism.ch -hacktivism.ch bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch GOA 1 1 +koopa bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch GOA 1 1 en hacktivism.ch +hacktivism bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch GOA 1 1 en hacktivism.ch +hacktivism.ch bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch GOA 1 1 en # Public demo (KUDOS) — has intro-style pages; keep soft landing checks -taler.net bank.demo.taler.net exchange.demo.taler.net backend.demo.taler.net KUDOS 0 1 demo.taler.net -demo.taler.net bank.demo.taler.net exchange.demo.taler.net backend.demo.taler.net KUDOS 0 1 +taler.net bank.demo.taler.net exchange.demo.taler.net backend.demo.taler.net KUDOS 0 1 en demo.taler.net +demo.taler.net bank.demo.taler.net exchange.demo.taler.net backend.demo.taler.net KUDOS 0 1 en # Public test stack (TESTKUDOS) — no GOA-style landings -test.taler.net bank.test.taler.net exchange.test.taler.net backend.test.taler.net TESTKUDOS 0 0 +test.taler.net bank.test.taler.net exchange.test.taler.net backend.test.taler.net TESTKUDOS 0 0 en # TOPS CHF — multi-tenant merchant my.taler-ops.ch; no public GOA landings -taler-ops.ch bank.taler-ops.ch exchange.taler-ops.ch my.taler-ops.ch CHF 0 0 -my.taler-ops.ch bank.taler-ops.ch exchange.taler-ops.ch my.taler-ops.ch CHF 0 0 taler-ops.ch +taler-ops.ch bank.taler-ops.ch exchange.taler-ops.ch my.taler-ops.ch CHF 0 0 en +my.taler-ops.ch bank.taler-ops.ch exchange.taler-ops.ch my.taler-ops.ch CHF 0 0 en taler-ops.ch # Stage merchant FQDN is stage.my.taler-ops.ch (on betel); mon HTML /monitoring* stage-only -stage.taler-ops.ch bank.stage.taler-ops.ch exchange.stage.taler-ops.ch stage.my.taler-ops.ch CHF 0 0 -stage.my.taler-ops.ch bank.stage.taler-ops.ch exchange.stage.taler-ops.ch stage.my.taler-ops.ch CHF 0 0 stage.taler-ops.ch -my.stage.taler-ops.ch bank.stage.taler-ops.ch exchange.stage.taler-ops.ch stage.my.taler-ops.ch CHF 0 0 stage.taler-ops.ch +stage.taler-ops.ch bank.stage.taler-ops.ch exchange.stage.taler-ops.ch stage.my.taler-ops.ch CHF 0 0 en +stage.my.taler-ops.ch bank.stage.taler-ops.ch exchange.stage.taler-ops.ch stage.my.taler-ops.ch CHF 0 0 en stage.taler-ops.ch +my.stage.taler-ops.ch bank.stage.taler-ops.ch exchange.stage.taler-ops.ch stage.my.taler-ops.ch CHF 0 0 en stage.taler-ops.ch # Franc Paysan (Infomaniak) — merchant host is monnaie.* (not taler./backend.) -# Prod: no monetary launch yet; stage uses TESTPAYSAN (public HTTPS landings + shops) -lefrancpaysan.ch bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch - 0 0 -monnaie.lefrancpaysan.ch bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch - 0 0 lefrancpaysan.ch -bank.lefrancpaysan.ch bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch - 0 0 lefrancpaysan.ch -exchange.lefrancpaysan.ch bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch - 0 0 lefrancpaysan.ch +# UI default: French (lang=fr) for all FP prod hostnames +lefrancpaysan.ch bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch - 0 0 fr +monnaie.lefrancpaysan.ch bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch - 0 0 fr lefrancpaysan.ch +bank.lefrancpaysan.ch bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch - 0 0 fr lefrancpaysan.ch +exchange.lefrancpaysan.ch bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch - 0 0 fr lefrancpaysan.ch # Stage TESTPAYSAN — landings on / + farmer shops under stage.monnaie…/shops/ -# name bank exchange merchant currency local landing [canonical] -stage.lefrancpaysan.ch stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch TESTPAYSAN 0 1 -stage.bank.lefrancpaysan.ch stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch TESTPAYSAN 0 1 stage.lefrancpaysan.ch -stage.exchange.lefrancpaysan.ch stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch TESTPAYSAN 0 1 stage.lefrancpaysan.ch -stage.monnaie.lefrancpaysan.ch stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch TESTPAYSAN 0 1 stage.lefrancpaysan.ch +# UI default: French (lang=fr) +stage.lefrancpaysan.ch stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch TESTPAYSAN 0 1 fr +stage.bank.lefrancpaysan.ch stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch TESTPAYSAN 0 1 fr stage.lefrancpaysan.ch +stage.exchange.lefrancpaysan.ch stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch TESTPAYSAN 0 1 fr stage.lefrancpaysan.ch +stage.monnaie.lefrancpaysan.ch stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch TESTPAYSAN 0 1 fr stage.lefrancpaysan.ch diff --git a/host-agent/run-host-report.sh b/host-agent/run-host-report.sh index f86b546..c34ba5b 100755 --- a/host-agent/run-host-report.sh +++ b/host-agent/run-host-report.sh @@ -75,25 +75,15 @@ export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" export AUTH401_CONTINUE="${AUTH401_CONTINUE:-${CONTINUE_ON_ERROR}}" export RUN_TIMEOUT="${RUN_TIMEOUT:-600}" export TALER_DOMAIN="${TALER_DOMAIN:-hacktivism.ch}" -# UI language: --lang / TALER_MON_LANG_SET=1 locks; otherwise auto by domain. -# Bare TALER_MON_LANG=en in workstation env (clementine) must NOT lock language — -# *lefrancpaysan* / *francpaysan* always default to fr when SET≠1 (global rule). +# UI language: --lang / TALER_MON_LANG_SET=1 locks. +# Otherwise domains.conf column `lang` → TALER_DOMAIN_LANG → TALER_MON_LANG +# (applied in suite apply_taler_domain / i18n_init). Leave SET=0 so -d can re-apply. +# Bare workstation TALER_MON_LANG=en must not override profile lang=fr. if [ "${TALER_MON_LANG_SET:-0}" = "1" ]; then case "${TALER_MON_LANG:-en}" in fr|FR|fra|french) export TALER_MON_LANG=fr ;; *) export TALER_MON_LANG=en ;; esac -else - case "$TALER_DOMAIN" in - *lefrancpaysan*|*francpaysan*) export TALER_MON_LANG=fr ;; - *) - case "${TALER_MON_LANG:-}" in - fr|FR|fra|french) export TALER_MON_LANG=fr ;; - *) export TALER_MON_LANG=en ;; - esac - ;; - esac - # leave TALER_MON_LANG_SET unset/0 so suite i18n_init can re-apply per -d domain fi export TALER_MON_LANG diff --git a/i18n.sh b/i18n.sh index 585ab16..c1a467f 100644 --- a/i18n.sh +++ b/i18n.sh @@ -3,31 +3,37 @@ # # TALER_MON_LANG=en|fr # TALER_MON_LANG_SET=1 → language is explicit (--lang only); do not override. -# Auto (SET≠1): domain *lefrancpaysan* / *francpaysan* → fr always (global). -# else keep env or en. +# Auto (SET≠1), in order: +# 1) TALER_DOMAIN_LANG from domains.conf profile (preferred — source of truth) +# 2) fallback hostname heuristic *lefrancpaysan*/*francpaysan* → fr +# 3) else keep workstation env TALER_MON_LANG or en # -# Important: a bare TALER_MON_LANG=en in ~/.config/taler-monitoring/env is only the -# workstation default for non-FP domains. It must NEVER lock English on FP -# (*lefrancpaysan* / *francpaysan*) — local laptop and host-agent both follow this. +# domains.conf column `lang` (see suite domains.conf) is the global definition. +# Bare TALER_MON_LANG=en in ~/.config must not override a profile lang=fr. : "${TALER_MON_LANG:=}" : "${TALER_MON_LANG_SET:=0}" +: "${TALER_DOMAIN_LANG:=}" i18n_init() { # Only TALER_MON_LANG_SET=1 is "explicit". Do not promote non-empty TALER_MON_LANG # to SET=1 — env files often ship LANG=en for the default GOA workstation. if [ "${TALER_MON_LANG_SET:-0}" != "1" ]; then - case "${TALER_DOMAIN:-}" in - *lefrancpaysan*|*francpaysan*) - TALER_MON_LANG=fr - ;; - *) - # Non-FP: keep pre-set env value if any, else English - if [ -z "${TALER_MON_LANG:-}" ]; then - TALER_MON_LANG=en - fi - ;; - esac + if [ -n "${TALER_DOMAIN_LANG:-}" ]; then + TALER_MON_LANG="$TALER_DOMAIN_LANG" + else + case "${TALER_DOMAIN:-}" in + *lefrancpaysan*|*francpaysan*) + # fallback if domain not (yet) in domains.conf + TALER_MON_LANG=fr + ;; + *) + if [ -z "${TALER_MON_LANG:-}" ]; then + TALER_MON_LANG=en + fi + ;; + esac + fi fi case "${TALER_MON_LANG}" in fr|FR|fra|french) TALER_MON_LANG=fr ;; @@ -35,6 +41,7 @@ i18n_init() { esac export TALER_MON_LANG export TALER_MON_LANG_SET + export TALER_DOMAIN_LANG } i18n_tag() { diff --git a/lib.sh b/lib.sh index 16a9a0e..f8dc297 100755 --- a/lib.sh +++ b/lib.sh @@ -166,13 +166,24 @@ set_taler_stack() { fi } +# Normalize domains.conf lang token → en|fr (empty if not a lang token). +_domain_lang_token() { + case "${1:-}" in + en|EN|eng|english) printf 'en' ;; + fr|FR|fra|french) printf 'fr' ;; + *) printf '' ;; + esac +} + # Load first matching profile from domains.conf. # Fields (whitespace-separated; # comments; blank lines ignored): -# name bank exchange merchant currency local[0|1] landing[0|1] [canonical_domain] +# name bank exchange merchant currency local[0|1] landing[0|1] lang[en|fr] [canonical] +# Legacy: 8th field may be canonical only (no lang) — lang defaults to en. +# Sets TALER_DOMAIN_LANG from the profile (i18n source of truth when SET≠1). # Returns 0 if found, 1 if not. load_domain_profile() { local want="$1" conf="${TALER_DOMAINS_CONF:-}" - local line name bank exchange merchant currency local_stack landing canon + local line name bank exchange merchant currency local_stack landing canon profile_lang f8 f9 [ -n "$want" ] || return 1 [ -n "$conf" ] && [ -f "$conf" ] || return 1 @@ -189,12 +200,30 @@ load_domain_profile() { currency="$5" local_stack="${6:-0}" landing="${7:-}" - canon="${8:-$name}" + f8="${8:-}" + f9="${9:-}" + canon="$name" + profile_lang="" # Backward compat: old 7th field was canonical domain (contains a dot) if [ -n "$landing" ] && [[ "$landing" == *.* && "$landing" != "0" && "$landing" != "1" ]]; then canon="$landing" landing="" + f8="${7:-}" + f9="${8:-}" fi + if [ -n "$f8" ]; then + profile_lang=$(_domain_lang_token "$f8") + if [ -n "$profile_lang" ]; then + canon="${f9:-$name}" + else + # legacy: field 8 = canonical, optional field 9 = lang + canon="$f8" + profile_lang=$(_domain_lang_token "$f9") + fi + fi + [ -z "$profile_lang" ] && profile_lang=en + TALER_DOMAIN_LANG="$profile_lang" + export TALER_DOMAIN_LANG set_taler_stack "$bank" "$exchange" "$merchant" "$currency" "$local_stack" "$landing" "$canon" return 0 done <"$conf" diff --git a/site-gen/console_to_html.py b/site-gen/console_to_html.py index 6e6aa54..fe2077d 100755 --- a/site-gen/console_to_html.py +++ b/site-gen/console_to_html.py @@ -45,35 +45,110 @@ def atomic_write_text(path: Path, text: str, encoding: str = "utf-8") -> None: except OSError: pass +def _norm_lang(v: str) -> str: + v = (v or "").strip().lower() + if v in ("fr", "fra", "french"): + return "fr" + if v in ("en", "eng", "english"): + return "en" + return "" + + +def domain_lang_from_conf(host_hint: str = "") -> str: + """Look up UI lang from domains.conf (suite profile `lang` column). + + Matches host_hint against profile name, bank, exchange, merchant, or + canonical fields. Returns '' if no profile matches. + """ + import os + from pathlib import Path + + hint = (host_hint or "").lower() + if not hint.strip(): + return "" + + conf = os.environ.get("TALER_DOMAINS_CONF") or "" + candidates = [] + if conf: + candidates.append(Path(conf)) + # suite root next to site-gen/ + here = Path(__file__).resolve().parent + candidates.append(here.parent / "domains.conf") + # cwd / mon root + candidates.append(Path("domains.conf")) + + path = next((p for p in candidates if p.is_file()), None) + if path is None: + return "" + + best = "" + try: + text = path.read_text(encoding="utf-8", errors="replace") + except OSError: + return "" + + for raw in text.splitlines(): + line = raw.split("#", 1)[0].strip() + if not line: + continue + parts = line.split() + if len(parts) < 5: + continue + name, bank, exchange, merchant = parts[0], parts[1], parts[2], parts[3] + # fields: … local landing lang [canonical] + lang = "" + canon = name + if len(parts) >= 8: + tok = _norm_lang(parts[7]) + if tok: + lang = tok + if len(parts) >= 9: + canon = parts[8] + else: + # legacy: field 8 = canonical + canon = parts[7] + if len(parts) >= 9: + lang = _norm_lang(parts[8]) + if not lang: + lang = "en" + hosts = {name.lower(), bank.lower(), exchange.lower(), merchant.lower(), canon.lower()} + # strip scheme if present + hosts = {re.sub(r"^https?://", "", h).split("/")[0] for h in hosts} + if any(h and h in hint for h in hosts): + # prefer longer / more specific name match later — keep last match + # that appears in conf order; FP aliases all share fr. + best = lang + if name.lower() in hint or canon.lower() in hint: + return lang + return best + + def ui_lang(explicit: str = "", host_hint: str = "") -> str: """en (default) or fr. - - TALER_MON_LANG_SET=1 → honour TALER_MON_LANG / explicit only (no host auto). - - Else: any *lefrancpaysan* / *francpaysan* host or log hint → fr - (workstation env TALER_MON_LANG=en must not force English on FP domains). - - Else: explicit / env / en. + - TALER_MON_LANG_SET=1 → honour TALER_MON_LANG / explicit only. + - Else TALER_DOMAIN_LANG env (set by load_domain_profile). + - Else domains.conf `lang` for host_hint. + - Else fallback *lefrancpaysan*/*francpaysan* → fr. + - Else explicit / env / en. """ import os env_lang = (os.environ.get("TALER_MON_LANG") or "").strip().lower() + domain_lang = _norm_lang(os.environ.get("TALER_DOMAIN_LANG") or "") lang = (explicit or env_lang or "").strip().lower() locked = (os.environ.get("TALER_MON_LANG_SET") or "").strip() == "1" - is_fp = bool( - re.search(r"lefrancpaysan|francpaysan", host_hint or "", re.I) - ) - - def _norm(v: str) -> str: - if v in ("fr", "fra", "french"): - return "fr" - if v in ("en", "eng", "english"): - return "en" - return "" if locked: - return _norm(lang) or "en" - if is_fp: + return _norm_lang(lang) or "en" + if domain_lang: + return domain_lang + conf_lang = domain_lang_from_conf(host_hint) + if conf_lang: + return conf_lang + if re.search(r"lefrancpaysan|francpaysan", host_hint or "", re.I): return "fr" - return _norm(lang) or "en" + return _norm_lang(lang) or "en" def ui(lang: str, key: str, **kwargs) -> str: From 915e3611577d5b5322bf331fd7bfa5bd64cb57ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 11:10:02 +0200 Subject: [PATCH 12/29] release 1.19.1: document e2e helper PATH preflight + domains.conf lang Docs-only bugfix so operators/agents smoke taler-helper-sqlite3 on the mon PATH and set UI lang in domains.conf (lang=fr for FP). --- DEPENDENCIES.md | 17 ++++++++++++++++- README.md | 8 ++++++-- VERSION | 2 +- VERSIONS.md | 1 + 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md index c6e1d4b..8e07f59 100644 --- a/DEPENDENCIES.md +++ b/DEPENDENCIES.md @@ -41,9 +41,13 @@ command -v qrencode zbarimg | `jq` | nice-to-have; suite uses python3 for JSON | | `dig` / `bind-utils` | debug DNS; not required for default phases | | `ssh` | only for access mode **ssh** (laptop → `KOOPA_SSH` / stage `INSIDE_SSH`); not needed for **host-podman** | -| `node` / `pnpm` + wallet-cli | **e2e** / ladder only (+ secrets) | +| `node` + `taler-wallet-cli` (.mjs) | **e2e** / ladder only (+ secrets) | +| **`taler-helper-sqlite3` on PATH** | **required for e2e** — wallet-core spawns it for withdraw accept-uri / pay; missing → ENOENT mid-ladder. Put next to wallet bindir (`~/.local/bin` on stagepaysan) | +| `timeout` (coreutils) | preferred hard wall for `wcli` (suite falls back to perl only if missing) | | `secrets.env` / admin passwords | e2e only — never system packages | +**Preflight (agents / ops):** before trusting e2e on a host, smoke on the **same PATH** mon uses (`run-host-report` / systemd user): `command -v node taler-helper-sqlite3`, `node "$WALLET_CLI" --version`, short temp-DB `balance`. Do not wait for a full mon run to discover ENOENT. + ### systemd user (no extra packages) - `systemd` user session + **`loginctl enable-linger hernani`** (already **Linger=yes** on koopa) @@ -69,3 +73,14 @@ command -v qrencode zbarimg ## Franc Paysan stage host (stagepaysan) FP stage (`stagepaysan` via SSH alias `francpaysan-stage-user`): curl, python3, podman if inside, qrencode/zbar for full urls; suite at `~/src/taler-monitoring`. + +**e2e on stagepaysan** also needs user-local: + +| Path | Role | +|------|------| +| `~/.local/bin/node` | Node for wallet-cli | +| `~/.local/bin/taler-wallet-cli.mjs` (or `WALLET_CLI=`) | wallet CLI | +| `~/.local/bin/taler-helper-sqlite3` | **required** — accept-uri / coins | +| `~/francpaysan-secrets/stage/*` | bank admin + merchant token (layout like laptop `francpaysan-secrets`) | + +Host-agent PATH already puts `$HOME/.local/bin` first; still smoke-test under that PATH. diff --git a/README.md b/README.md index d5c156e..cb0700b 100644 --- a/README.md +++ b/README.md @@ -203,8 +203,9 @@ Each monitored stack must say **what is bank, exchange, and merchant-backend**. Edit **[domains.conf](./domains.conf)** (or set `TALER_DOMAINS_CONF=`): ```text -# name bank exchange merchant-backend currency local [canonical] -taler-ops.ch bank.taler-ops.ch exchange.taler-ops.ch my.taler-ops.ch CHF 0 +# name bank exchange merchant-backend currency local landing lang [canonical] +taler-ops.ch bank.taler-ops.ch exchange.taler-ops.ch my.taler-ops.ch CHF 0 0 en +stage.lefrancpaysan.ch stage.bank.… stage.exchange.… stage.monnaie.… TESTPAYSAN 0 1 fr ``` | Field | Meaning | @@ -216,8 +217,11 @@ taler-ops.ch bank.taler-ops.ch exchange.taler-ops.ch my.taler-ops.ch CHF 0 | **currency** | `GOA` / `KUDOS` / `CHF` / … | | **local** | `1` = koopa SSH stack; `0` = public only | | **landing** | `1` = check `/intro` landings + assets; `0` = skip (TOPS / mytops) | +| **lang** | UI default **`en`** or **`fr`** when `TALER_MON_LANG_SET≠1` (source of truth; FP rows use **`fr`**) | | **canonical** | optional label after alias (e.g. `my.taler-ops.ch` → `taler-ops.ch`) | +Workstation `TALER_MON_LANG=en` does **not** override a profile with `lang=fr`. Force English on FP only with `--lang en` / `TALER_MON_LANG_SET=1`. + Then: ```bash diff --git a/VERSION b/VERSION index 815d5ca..66e2ae6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.19.0 +1.19.1 diff --git a/VERSIONS.md b/VERSIONS.md index 634c6ec..8f975c7 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| +| **v1.19.1** | 2026-07-19 | **Bugfix / docs:** DEPENDENCIES — e2e requires `taler-helper-sqlite3` on mon PATH + preflight; README domains.conf documents **`lang`** column. | | **v1.19.0** | 2026-07-19 | **Feature:** domains.conf **`lang`** column (en\|fr) is the global UI language default per stack; i18n + HTML read `TALER_DOMAIN_LANG` / conf (FP profiles `lang=fr`). | | **v1.18.9** | 2026-07-19 | **Bugfix:** UI lang — *lefrancpaysan*/*francpaysan* always default **fr** unless `TALER_MON_LANG_SET=1` / `--lang`; workstation `TALER_MON_LANG=en` no longer locks English (host-agent + console_to_html). | | **v1.18.8** | 2026-07-19 | **Bugfix:** e2e wallet — require `taler-helper-sqlite3` on PATH; use absolute `node` + `timeout` instead of fragile `perl -e alarm` for wcli; clearer accept-uri errors (no raw alarm-shift noise). | From c435df9728172be83a1880e5cac9b09d3936fdc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 11:13:47 +0200 Subject: [PATCH 13/29] release 1.20.0: domains.conf locale l10n + optional German UI Add locale column (fr-CH for FP, de-CH elsewhere; ch-FR/ch-DE aliases). Introduce UI lang=de (tags + sticky) without using de as any stock default. --- README.md | 9 +- VERSION | 2 +- VERSIONS.md | 1 + domains.conf | 69 +++++------ i18n.sh | 233 ++++++++++++++++++++++++------------ lib.sh | 70 +++++++++-- site-gen/console_to_html.py | 175 +++++++++++++++++++++------ taler-monitoring.sh | 12 +- 8 files changed, 408 insertions(+), 163 deletions(-) diff --git a/README.md b/README.md index cb0700b..998c65b 100644 --- a/README.md +++ b/README.md @@ -203,9 +203,9 @@ Each monitored stack must say **what is bank, exchange, and merchant-backend**. Edit **[domains.conf](./domains.conf)** (or set `TALER_DOMAINS_CONF=`): ```text -# name bank exchange merchant-backend currency local landing lang [canonical] -taler-ops.ch bank.taler-ops.ch exchange.taler-ops.ch my.taler-ops.ch CHF 0 0 en -stage.lefrancpaysan.ch stage.bank.… stage.exchange.… stage.monnaie.… TESTPAYSAN 0 1 fr +# name bank exchange merchant currency local landing lang locale [canonical] +taler-ops.ch bank.… exchange.… my.… CHF 0 0 en de-CH +stage.lefrancpaysan.ch stage.bank.… stage.exchange.… stage.monnaie.… TESTPAYSAN 0 1 fr fr-CH ``` | Field | Meaning | @@ -217,7 +217,8 @@ stage.lefrancpaysan.ch stage.bank.… stage.exchange.… stage.monnaie.… T | **currency** | `GOA` / `KUDOS` / `CHF` / … | | **local** | `1` = koopa SSH stack; `0` = public only | | **landing** | `1` = check `/intro` landings + assets; `0` = skip (TOPS / mytops) | -| **lang** | UI default **`en`** or **`fr`** when `TALER_MON_LANG_SET≠1` (source of truth; FP rows use **`fr`**) | +| **lang** | UI language **`en`** / **`fr`** / **`de`** when `TALER_MON_LANG_SET≠1`. Stock profiles use **en** or **fr** only — **`de` is never a default** (opt-in via `--lang de` or an explicit profile edit). | +| **locale** | Regional l10n (BCP 47): **`fr-CH`** for FrancPaysan (Suisse romande); **`de-CH`** for other Swiss stacks. Aliases `ch-FR` / `ch-DE` accepted. Not the same as UI `lang`. | | **canonical** | optional label after alias (e.g. `my.taler-ops.ch` → `taler-ops.ch`) | Workstation `TALER_MON_LANG=en` does **not** override a profile with `lang=fr`. Force English on FP only with `--lang en` / `TALER_MON_LANG_SET=1`. diff --git a/VERSION b/VERSION index 66e2ae6..3989355 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.19.1 +1.20.0 diff --git a/VERSIONS.md b/VERSIONS.md index 8f975c7..76bb76b 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| +| **v1.20.0** | 2026-07-19 | **Feature:** domains.conf **`locale`** (l10n) — FP **`fr-CH`**, others **`de-CH`** (aliases ch-FR/ch-DE); UI **`lang=de`** supported (sticky + console tags) but **never** a stock profile default; env `TALER_MON_LOCALE` / `TALER_DOMAIN_LOCALE`. | | **v1.19.1** | 2026-07-19 | **Bugfix / docs:** DEPENDENCIES — e2e requires `taler-helper-sqlite3` on mon PATH + preflight; README domains.conf documents **`lang`** column. | | **v1.19.0** | 2026-07-19 | **Feature:** domains.conf **`lang`** column (en\|fr) is the global UI language default per stack; i18n + HTML read `TALER_DOMAIN_LANG` / conf (FP profiles `lang=fr`). | | **v1.18.9** | 2026-07-19 | **Bugfix:** UI lang — *lefrancpaysan*/*francpaysan* always default **fr** unless `TALER_MON_LANG_SET=1` / `--lang`; workstation `TALER_MON_LANG=en` no longer locks English (host-agent + console_to_html). | diff --git a/domains.conf b/domains.conf index 189b1df..1002e69 100644 --- a/domains.conf +++ b/domains.conf @@ -1,52 +1,53 @@ # taler-monitoring domain profiles # # When you add a stack, declare the three public endpoints here: -# name bank exchange merchant-backend currency local landing lang [canonical] +# name bank exchange merchant-backend currency local landing lang locale [canonical] # # - name: what -d / TALER_DOMAIN matches (aliases allowed as extra lines) # - bank / exchange / merchant: hostname or full https:// URL # - currency: GOA | KUDOS | CHF | … (empty or "-" = report only) # - local: 1 = koopa SSH/inside/e2e stack; 0 = public-only # - landing: 1 = check /intro landings + assets; 0 = skip (no GOA-style landings) -# - lang: UI language default for this profile (en|fr). Source of truth for i18n -# when TALER_MON_LANG_SET≠1. Workstation env TALER_MON_LANG=en does not -# override a profile that sets fr (e.g. *lefrancpaysan*). +# - lang: UI language for mon console/HTML (en|fr|de). Source of truth when +# TALER_MON_LANG_SET≠1. Workstation TALER_MON_LANG=en does not override. +# **de is supported but must not be used as a profile default** (no row +# sets lang=de unless an operator deliberately chooses it). +# - locale: BCP 47 regional locale for l10n defaults (not the same as UI lang). +# Use **fr-CH** (Suisse romande) for FrancPaysan; **de-CH** (Deutschschweiz) +# for other Swiss / default stacks. Accepts also ch-FR / ch-DE as aliases. # - canonical: optional TALER_DOMAIN label after alias match # -# Legacy lines without lang (old 8th field = canonical only) still parse: lang→en. -# CLI overrides: --lang / TALER_MON_LANG_SET=1 +# Legacy lines without locale still parse (locale → de-CH except FP heuristic). +# CLI: --lang en|fr|de · TALER_MON_LANG_SET=1 locks UI lang # Alternate file: TALER_DOMAINS_CONF=/path/to/domains.conf -# Local GOA (koopa) — landings matter -koopa bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch GOA 1 1 en hacktivism.ch -hacktivism bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch GOA 1 1 en hacktivism.ch -hacktivism.ch bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch GOA 1 1 en +# Local GOA (koopa) — landings matter · UI en · locale de-CH +koopa bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch GOA 1 1 en de-CH hacktivism.ch +hacktivism bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch GOA 1 1 en de-CH hacktivism.ch +hacktivism.ch bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch GOA 1 1 en de-CH -# Public demo (KUDOS) — has intro-style pages; keep soft landing checks -taler.net bank.demo.taler.net exchange.demo.taler.net backend.demo.taler.net KUDOS 0 1 en demo.taler.net -demo.taler.net bank.demo.taler.net exchange.demo.taler.net backend.demo.taler.net KUDOS 0 1 en +# Public demo (KUDOS) +taler.net bank.demo.taler.net exchange.demo.taler.net backend.demo.taler.net KUDOS 0 1 en de-CH demo.taler.net +demo.taler.net bank.demo.taler.net exchange.demo.taler.net backend.demo.taler.net KUDOS 0 1 en de-CH -# Public test stack (TESTKUDOS) — no GOA-style landings -test.taler.net bank.test.taler.net exchange.test.taler.net backend.test.taler.net TESTKUDOS 0 0 en +# Public test stack (TESTKUDOS) +test.taler.net bank.test.taler.net exchange.test.taler.net backend.test.taler.net TESTKUDOS 0 0 en de-CH -# TOPS CHF — multi-tenant merchant my.taler-ops.ch; no public GOA landings -taler-ops.ch bank.taler-ops.ch exchange.taler-ops.ch my.taler-ops.ch CHF 0 0 en -my.taler-ops.ch bank.taler-ops.ch exchange.taler-ops.ch my.taler-ops.ch CHF 0 0 en taler-ops.ch -# Stage merchant FQDN is stage.my.taler-ops.ch (on betel); mon HTML /monitoring* stage-only -stage.taler-ops.ch bank.stage.taler-ops.ch exchange.stage.taler-ops.ch stage.my.taler-ops.ch CHF 0 0 en -stage.my.taler-ops.ch bank.stage.taler-ops.ch exchange.stage.taler-ops.ch stage.my.taler-ops.ch CHF 0 0 en stage.taler-ops.ch -my.stage.taler-ops.ch bank.stage.taler-ops.ch exchange.stage.taler-ops.ch stage.my.taler-ops.ch CHF 0 0 en stage.taler-ops.ch +# TOPS CHF +taler-ops.ch bank.taler-ops.ch exchange.taler-ops.ch my.taler-ops.ch CHF 0 0 en de-CH +my.taler-ops.ch bank.taler-ops.ch exchange.taler-ops.ch my.taler-ops.ch CHF 0 0 en de-CH taler-ops.ch +stage.taler-ops.ch bank.stage.taler-ops.ch exchange.stage.taler-ops.ch stage.my.taler-ops.ch CHF 0 0 en de-CH +stage.my.taler-ops.ch bank.stage.taler-ops.ch exchange.stage.taler-ops.ch stage.my.taler-ops.ch CHF 0 0 en de-CH stage.taler-ops.ch +my.stage.taler-ops.ch bank.stage.taler-ops.ch exchange.stage.taler-ops.ch stage.my.taler-ops.ch CHF 0 0 en de-CH stage.taler-ops.ch -# Franc Paysan (Infomaniak) — merchant host is monnaie.* (not taler./backend.) -# UI default: French (lang=fr) for all FP prod hostnames -lefrancpaysan.ch bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch - 0 0 fr -monnaie.lefrancpaysan.ch bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch - 0 0 fr lefrancpaysan.ch -bank.lefrancpaysan.ch bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch - 0 0 fr lefrancpaysan.ch -exchange.lefrancpaysan.ch bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch - 0 0 fr lefrancpaysan.ch +# Franc Paysan — UI fr · regional locale fr-CH (Suisse romande) +lefrancpaysan.ch bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch - 0 0 fr fr-CH +monnaie.lefrancpaysan.ch bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch - 0 0 fr fr-CH lefrancpaysan.ch +bank.lefrancpaysan.ch bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch - 0 0 fr fr-CH lefrancpaysan.ch +exchange.lefrancpaysan.ch bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch - 0 0 fr fr-CH lefrancpaysan.ch -# Stage TESTPAYSAN — landings on / + farmer shops under stage.monnaie…/shops/ -# UI default: French (lang=fr) -stage.lefrancpaysan.ch stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch TESTPAYSAN 0 1 fr -stage.bank.lefrancpaysan.ch stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch TESTPAYSAN 0 1 fr stage.lefrancpaysan.ch -stage.exchange.lefrancpaysan.ch stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch TESTPAYSAN 0 1 fr stage.lefrancpaysan.ch -stage.monnaie.lefrancpaysan.ch stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch TESTPAYSAN 0 1 fr stage.lefrancpaysan.ch +# Stage TESTPAYSAN — UI fr · locale fr-CH +stage.lefrancpaysan.ch stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch TESTPAYSAN 0 1 fr fr-CH +stage.bank.lefrancpaysan.ch stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch TESTPAYSAN 0 1 fr fr-CH stage.lefrancpaysan.ch +stage.exchange.lefrancpaysan.ch stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch TESTPAYSAN 0 1 fr fr-CH stage.lefrancpaysan.ch +stage.monnaie.lefrancpaysan.ch stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch TESTPAYSAN 0 1 fr fr-CH stage.lefrancpaysan.ch diff --git a/i18n.sh b/i18n.sh index c1a467f..417a751 100644 --- a/i18n.sh +++ b/i18n.sh @@ -1,19 +1,24 @@ #!/usr/bin/env bash -# i18n.sh — language for taler-monitoring console + HTML chrome +# i18n.sh — UI language + regional locale for taler-monitoring console + HTML # -# TALER_MON_LANG=en|fr -# TALER_MON_LANG_SET=1 → language is explicit (--lang only); do not override. -# Auto (SET≠1), in order: -# 1) TALER_DOMAIN_LANG from domains.conf profile (preferred — source of truth) -# 2) fallback hostname heuristic *lefrancpaysan*/*francpaysan* → fr -# 3) else keep workstation env TALER_MON_LANG or en +# UI language (mon badges / sticky chrome): +# TALER_MON_LANG=en|fr|de +# TALER_MON_LANG_SET=1 → explicit (--lang); do not override. +# Auto (SET≠1): TALER_DOMAIN_LANG from domains.conf `lang` column. +# de is supported but never a stock profile default (no domains.conf row uses lang=de). # -# domains.conf column `lang` (see suite domains.conf) is the global definition. -# Bare TALER_MON_LANG=en in ~/.config must not override a profile lang=fr. +# Regional locale (l10n defaults, BCP 47 — not the same as UI lang): +# TALER_MON_LOCALE / TALER_DOMAIN_LOCALE e.g. fr-CH, de-CH +# From domains.conf `locale` column (fr-CH for FP, de-CH for others). +# Aliases ch-FR / ch-DE accepted and normalized to fr-CH / de-CH. +# +# domains.conf is the source of truth for both lang and locale. : "${TALER_MON_LANG:=}" : "${TALER_MON_LANG_SET:=0}" : "${TALER_DOMAIN_LANG:=}" +: "${TALER_DOMAIN_LOCALE:=}" +: "${TALER_MON_LOCALE:=}" i18n_init() { # Only TALER_MON_LANG_SET=1 is "explicit". Do not promote non-empty TALER_MON_LANG @@ -24,7 +29,6 @@ i18n_init() { else case "${TALER_DOMAIN:-}" in *lefrancpaysan*|*francpaysan*) - # fallback if domain not (yet) in domains.conf TALER_MON_LANG=fr ;; *) @@ -37,89 +41,162 @@ i18n_init() { fi case "${TALER_MON_LANG}" in fr|FR|fra|french) TALER_MON_LANG=fr ;; + de|DE|deu|ger|german|deutsch) TALER_MON_LANG=de ;; *) TALER_MON_LANG=en ;; esac + + # Regional locale from profile (or FP/other fallback) + if [ -n "${TALER_DOMAIN_LOCALE:-}" ]; then + TALER_MON_LOCALE="$TALER_DOMAIN_LOCALE" + elif [ -z "${TALER_MON_LOCALE:-}" ]; then + case "${TALER_DOMAIN:-}" in + *lefrancpaysan*|*francpaysan*) TALER_MON_LOCALE=fr-CH ;; + *) TALER_MON_LOCALE=de-CH ;; + esac + fi + # normalize aliases + case "$(printf '%s' "$TALER_MON_LOCALE" | tr '[:upper:]' '[:lower:]' | tr '_' '-')" in + fr-ch|ch-fr) TALER_MON_LOCALE=fr-CH ;; + de-ch|ch-de) TALER_MON_LOCALE=de-CH ;; + en-ch|ch-en) TALER_MON_LOCALE=en-CH ;; + esac + export TALER_MON_LANG export TALER_MON_LANG_SET export TALER_DOMAIN_LANG + export TALER_DOMAIN_LOCALE + export TALER_MON_LOCALE } i18n_tag() { local t="$1" - if [ "${TALER_MON_LANG:-en}" != "fr" ]; then - printf '%s' "$t" - return 0 - fi - case "$t" in - OK) printf 'OK' ;; - ERROR) printf 'ERREUR' ;; - WARN) printf 'AVERT' ;; - INFO) printf 'INFO' ;; - BLOCKER) printf 'BLOCAGE' ;; - BLOCK) printf 'BLOC' ;; - SUMMARY) printf 'RESUME' ;; - PROG|PROGRESS) printf 'PROG' ;; - *) printf '%s' "$t" ;; + case "${TALER_MON_LANG:-en}" in + fr) + case "$t" in + OK) printf 'OK' ;; + ERROR) printf 'ERREUR' ;; + WARN) printf 'AVERT' ;; + INFO) printf 'INFO' ;; + BLOCKER) printf 'BLOCAGE' ;; + BLOCK) printf 'BLOC' ;; + SUMMARY) printf 'RESUME' ;; + PROG|PROGRESS) printf 'PROG' ;; + *) printf '%s' "$t" ;; + esac + ;; + de) + case "$t" in + OK) printf 'OK' ;; + ERROR) printf 'FEHLER' ;; + WARN) printf 'WARNUNG' ;; + INFO) printf 'INFO' ;; + BLOCKER) printf 'BLOCKER' ;; + BLOCK) printf 'BLOCK' ;; + SUMMARY) printf 'ZUSAMMENFASSUNG' ;; + PROG|PROGRESS) printf 'PROG' ;; + *) printf '%s' "$t" ;; + esac + ;; + *) + printf '%s' "$t" + ;; esac } i18n_text() { local s="$*" - if [ "${TALER_MON_LANG:-en}" != "fr" ] || [ -z "$s" ]; then - printf '%s' "$s" - return 0 - fi - case "$s" in - "numbered checks this run:") printf 'controles numerotes de cette execution :' ;; - "numbered checks:") printf 'controles numerotes :' ;; - "(global done/total)") printf '(fait/total global)' ;; - "global checks:") printf 'controles globaux :' ;; - "failed — see list above"|"failed - see list above") printf 'echec — voir la liste ci-dessus' ;; - "BLOCKERS · pay/withdraw cannot finish") printf 'BLOCAGES · paiement/retrait impossible' ;; - "BLOCKERS (pay/withdraw cannot finish)") printf 'BLOCAGES (paiement/retrait impossible)' ;; - "ERRORS · failed checks") printf 'ERREURS · controles en echec' ;; - "ERRORS (failed checks)") printf 'ERREURS (controles en echec)' ;; - "pay/withdraw cannot finish") printf 'paiement/retrait impossible a terminer' ;; - "SUMMARY") printf 'RESUME' ;; - "totals:") printf 'totaux :' ;; - "all clear") printf 'tout est clair' ;; - "warnings only (no hard fail)") printf 'avertissements seulement (pas d'\''echec dur)' ;; - "monitoring") printf 'surveillance' ;; - "surface") printf 'surface' ;; - "aptdeploy") printf 'aptdeploy' ;; - "container") printf 'conteneur' ;; - "disk") printf 'disque' ;; - "package") printf 'paquet' ;; - "version") printf 'version' ;; - "reachability") printf 'accessibilite' ;; - "server-header") printf 'en-tete Server' ;; - "inventory") printf 'inventaire' ;; - "flags") printf 'options' ;; - "target domain") printf 'domaine cible' ;; - "currency") printf 'devise' ;; - "phases") printf 'phases' ;; - "access") printf 'acces' ;; + [ -z "$s" ] && { printf '%s' "$s"; return 0; } + case "${TALER_MON_LANG:-en}" in + fr) + case "$s" in + "numbered checks this run:") printf 'controles numerotes de cette execution :' ;; + "numbered checks:") printf 'controles numerotes :' ;; + "(global done/total)") printf '(fait/total global)' ;; + "global checks:") printf 'controles globaux :' ;; + "failed — see list above"|"failed - see list above") printf 'echec — voir la liste ci-dessus' ;; + "BLOCKERS · pay/withdraw cannot finish") printf 'BLOCAGES · paiement/retrait impossible' ;; + "BLOCKERS (pay/withdraw cannot finish)") printf 'BLOCAGES (paiement/retrait impossible)' ;; + "ERRORS · failed checks") printf 'ERREURS · controles en echec' ;; + "ERRORS (failed checks)") printf 'ERREURS (controles en echec)' ;; + "pay/withdraw cannot finish") printf 'paiement/retrait impossible a terminer' ;; + "SUMMARY") printf 'RESUME' ;; + "totals:") printf 'totaux :' ;; + "all clear") printf 'tout est clair' ;; + "warnings only (no hard fail)") printf 'avertissements seulement (pas d'\''echec dur)' ;; + "monitoring") printf 'surveillance' ;; + "surface") printf 'surface' ;; + "aptdeploy") printf 'aptdeploy' ;; + "container") printf 'conteneur' ;; + "disk") printf 'disque' ;; + "package") printf 'paquet' ;; + "version") printf 'version' ;; + "reachability") printf 'accessibilite' ;; + "server-header") printf 'en-tete Server' ;; + "inventory") printf 'inventaire' ;; + "flags") printf 'options' ;; + "target domain") printf 'domaine cible' ;; + "currency") printf 'devise' ;; + "phases") printf 'phases' ;; + "access") printf 'acces' ;; + *) + local out="$s" + out=${out//shared library missing/bibliotheque partagee manquante} + out=${out//not active/non actif} + out=${out//does not resolve/ne resout pas} + out=${out//from Server header/depuis en-tete Server} + out=${out//public HTTPS/HTTPS public} + out=${out//no SSH/sans SSH} + out=${out//failed — see list above/echec — voir la liste ci-dessus} + printf '%s' "$out" + ;; + esac + ;; + de) + case "$s" in + "numbered checks this run:") printf 'Nummerierte Pruefungen dieser Ausfuehrung :' ;; + "numbered checks:") printf 'Nummerierte Pruefungen :' ;; + "(global done/total)") printf '(fertig/gesamt global)' ;; + "global checks:") printf 'Globale Pruefungen :' ;; + "failed — see list above"|"failed - see list above") printf 'fehlgeschlagen — siehe Liste oben' ;; + "BLOCKERS · pay/withdraw cannot finish") printf 'BLOCKER · Zahlung/Abhebung nicht moeglich' ;; + "BLOCKERS (pay/withdraw cannot finish)") printf 'BLOCKER (Zahlung/Abhebung nicht moeglich)' ;; + "ERRORS · failed checks") printf 'FEHLER · fehlgeschlagene Pruefungen' ;; + "ERRORS (failed checks)") printf 'FEHLER (fehlgeschlagene Pruefungen)' ;; + "pay/withdraw cannot finish") printf 'Zahlung/Abhebung kann nicht beendet werden' ;; + "SUMMARY") printf 'ZUSAMMENFASSUNG' ;; + "totals:") printf 'Summen :' ;; + "all clear") printf 'alles in Ordnung' ;; + "warnings only (no hard fail)") printf 'nur Warnungen (kein harter Fehler)' ;; + "monitoring") printf 'Ueberwachung' ;; + "surface") printf 'Surface' ;; + "aptdeploy") printf 'aptdeploy' ;; + "container") printf 'Container' ;; + "disk") printf 'Disk' ;; + "package") printf 'Paket' ;; + "version") printf 'Version' ;; + "reachability") printf 'Erreichbarkeit' ;; + "server-header") printf 'Server-Header' ;; + "inventory") printf 'Inventar' ;; + "flags") printf 'Flags' ;; + "target domain") printf 'Zieldomaene' ;; + "currency") printf 'Waehrung' ;; + "phases") printf 'Phasen' ;; + "access") printf 'Zugang' ;; + *) + local outd="$s" + outd=${outd//shared library missing/gemeinsame Bibliothek fehlt} + outd=${outd//not active/nicht aktiv} + outd=${outd//does not resolve/loest nicht auf} + outd=${outd//from Server header/aus Server-Header} + outd=${outd//public HTTPS/oeffentliches HTTPS} + outd=${outd//no SSH/ohne SSH} + outd=${outd//failed — see list above/fehlgeschlagen — siehe Liste oben} + printf '%s' "$outd" + ;; + esac + ;; *) - local out="$s" - out=${out//shared library missing/bibliotheque partagee manquante} - out=${out//shared library error/erreur de bibliotheque partagee} - out=${out//not active/non actif} - out=${out//after_start/apres_demarrage} - out=${out//catalogued but not reachable/catalogue mais inaccessible} - out=${out//does not resolve/ne resout pas} - out=${out//protocol confirmed/protocole confirme} - out=${out//catalogued service OK/service catalogue OK} - out=${out//from Server header/depuis en-tete Server} - out=${out//OSV clean/OSV sans vulnerabilite} - out=${out//actionable vuln/vuln. actionnable} - out=${out//header version · not Debian pkg/version en-tete · pas paquet Debian} - out=${out//RUN_TIMEOUT exceeded/RUN_TIMEOUT depasse} - out=${out//public HTTPS/HTTPS public} - out=${out//container status/etat du conteneur} - out=${out//remote-only/uniquement distant} - out=${out//no SSH/sans SSH} - out=${out//failed — see list above/echec — voir la liste ci-dessus} - printf '%s' "$out" + printf '%s' "$s" ;; esac } diff --git a/lib.sh b/lib.sh index f8dc297..0f92126 100755 --- a/lib.sh +++ b/lib.sh @@ -166,24 +166,53 @@ set_taler_stack() { fi } -# Normalize domains.conf lang token → en|fr (empty if not a lang token). +# Normalize domains.conf UI lang token → en|fr|de (empty if not a lang token). +# Note: de is valid for --lang / explicit profile, but no stock profile defaults to de. _domain_lang_token() { case "${1:-}" in en|EN|eng|english) printf 'en' ;; fr|FR|fra|french) printf 'fr' ;; + de|DE|deu|ger|german|deutsch) printf 'de' ;; *) printf '' ;; esac } +# Normalize regional locale → BCP 47 (fr-CH, de-CH, en-CH, …). Empty if unknown. +# Accepts fr-CH, fr_CH, ch-FR / de-CH, ch-DE (user-friendly aliases). +_domain_locale_token() { + local t="${1:-}" + t=$(printf '%s' "$t" | tr '[:upper:]' '[:lower:]' | tr '_' '-') + case "$t" in + fr-ch|ch-fr|fr_ch) printf 'fr-CH' ;; + de-ch|ch-de|de_ch) printf 'de-CH' ;; + en-ch|ch-en|en_ch) printf 'en-CH' ;; + en-us|en-gb|fr-fr|de-de|it-ch|rm-ch) + # pass through other xx-YY + printf '%s' "$t" | awk -F- '{printf "%s-%s", tolower($1), toupper($2)}' + ;; + *) + # bare xx-YY pattern + if printf '%s' "$t" | grep -qE '^[a-z]{2}-[a-z]{2}$'; then + printf '%s' "$t" | awk -F- '{printf "%s-%s", tolower($1), toupper($2)}' + else + printf '' + fi + ;; + esac +} + # Load first matching profile from domains.conf. # Fields (whitespace-separated; # comments; blank lines ignored): -# name bank exchange merchant currency local[0|1] landing[0|1] lang[en|fr] [canonical] -# Legacy: 8th field may be canonical only (no lang) — lang defaults to en. -# Sets TALER_DOMAIN_LANG from the profile (i18n source of truth when SET≠1). +# name bank exchange merchant currency local landing lang locale [canonical] +# Legacy: +# … landing lang [canonical] → locale defaulted +# … landing [canonical] → lang=en, locale defaulted +# Sets TALER_DOMAIN_LANG + TALER_DOMAIN_LOCALE from the profile. # Returns 0 if found, 1 if not. load_domain_profile() { local want="$1" conf="${TALER_DOMAINS_CONF:-}" - local line name bank exchange merchant currency local_stack landing canon profile_lang f8 f9 + local line name bank exchange merchant currency local_stack landing canon + local profile_lang profile_locale f8 f9 f10 [ -n "$want" ] || return 1 [ -n "$conf" ] && [ -f "$conf" ] || return 1 @@ -202,28 +231,53 @@ load_domain_profile() { landing="${7:-}" f8="${8:-}" f9="${9:-}" + f10="${10:-}" canon="$name" profile_lang="" + profile_locale="" # Backward compat: old 7th field was canonical domain (contains a dot) if [ -n "$landing" ] && [[ "$landing" == *.* && "$landing" != "0" && "$landing" != "1" ]]; then canon="$landing" landing="" f8="${7:-}" f9="${8:-}" + f10="${9:-}" fi if [ -n "$f8" ]; then profile_lang=$(_domain_lang_token "$f8") if [ -n "$profile_lang" ]; then - canon="${f9:-$name}" + # modern: lang [locale] [canonical] + profile_locale=$(_domain_locale_token "$f9") + if [ -n "$profile_locale" ]; then + canon="${f10:-$name}" + else + # lang + canonical (no locale column) + if [ -n "$f9" ] && [ -z "$(_domain_lang_token "$f9")" ]; then + canon="$f9" + else + canon="${f9:-$name}" + fi + profile_locale=$(_domain_locale_token "$f10") + fi else - # legacy: field 8 = canonical, optional field 9 = lang + # legacy: field 8 = canonical only canon="$f8" profile_lang=$(_domain_lang_token "$f9") + profile_locale=$(_domain_locale_token "${f10:-$f9}") + [ -z "$profile_lang" ] && profile_lang=en fi fi [ -z "$profile_lang" ] && profile_lang=en + if [ -z "$profile_locale" ]; then + # default regional locale: FP → fr-CH, else de-CH + case "$name$canon$bank$merchant" in + *lefrancpaysan*|*francpaysan*) profile_locale=fr-CH ;; + *) profile_locale=de-CH ;; + esac + fi TALER_DOMAIN_LANG="$profile_lang" - export TALER_DOMAIN_LANG + TALER_DOMAIN_LOCALE="$profile_locale" + export TALER_DOMAIN_LANG TALER_DOMAIN_LOCALE set_taler_stack "$bank" "$exchange" "$merchant" "$currency" "$local_stack" "$landing" "$canon" return 0 done <"$conf" diff --git a/site-gen/console_to_html.py b/site-gen/console_to_html.py index fe2077d..d49c839 100755 --- a/site-gen/console_to_html.py +++ b/site-gen/console_to_html.py @@ -49,44 +49,61 @@ def _norm_lang(v: str) -> str: v = (v or "").strip().lower() if v in ("fr", "fra", "french"): return "fr" + if v in ("de", "deu", "ger", "german", "deutsch"): + return "de" if v in ("en", "eng", "english"): return "en" return "" -def domain_lang_from_conf(host_hint: str = "") -> str: - """Look up UI lang from domains.conf (suite profile `lang` column). +def _norm_locale(v: str) -> str: + """BCP 47 regional locale; aliases ch-FR/ch-DE → fr-CH/de-CH.""" + t = (v or "").strip().lower().replace("_", "-") + aliases = { + "fr-ch": "fr-CH", + "ch-fr": "fr-CH", + "de-ch": "de-CH", + "ch-de": "de-CH", + "en-ch": "en-CH", + "ch-en": "en-CH", + } + if t in aliases: + return aliases[t] + m = re.fullmatch(r"([a-z]{2})-([a-z]{2})", t) + if m: + return f"{m.group(1)}-{m.group(2).upper()}" + return "" - Matches host_hint against profile name, bank, exchange, merchant, or - canonical fields. Returns '' if no profile matches. - """ + +def _domains_conf_path() -> "object | None": import os from pathlib import Path - hint = (host_hint or "").lower() - if not hint.strip(): - return "" - conf = os.environ.get("TALER_DOMAINS_CONF") or "" candidates = [] if conf: candidates.append(Path(conf)) - # suite root next to site-gen/ here = Path(__file__).resolve().parent candidates.append(here.parent / "domains.conf") - # cwd / mon root candidates.append(Path("domains.conf")) + return next((p for p in candidates if p.is_file()), None) - path = next((p for p in candidates if p.is_file()), None) + +def domain_profile_from_conf(host_hint: str = "") -> tuple[str, str]: + """Return (lang, locale) from domains.conf for host_hint; ('','') if none.""" + hint = (host_hint or "").lower() + if not hint.strip(): + return "", "" + + path = _domains_conf_path() if path is None: - return "" - - best = "" + return "", "" try: text = path.read_text(encoding="utf-8", errors="replace") except OSError: - return "" + return "", "" + best_lang, best_loc = "", "" for raw in text.splitlines(): line = raw.split("#", 1)[0].strip() if not line: @@ -95,41 +112,62 @@ def domain_lang_from_conf(host_hint: str = "") -> str: if len(parts) < 5: continue name, bank, exchange, merchant = parts[0], parts[1], parts[2], parts[3] - # fields: … local landing lang [canonical] - lang = "" - canon = name + lang, locale, canon = "", "", name if len(parts) >= 8: tok = _norm_lang(parts[7]) if tok: lang = tok if len(parts) >= 9: - canon = parts[8] + loc = _norm_locale(parts[8]) + if loc: + locale = loc + if len(parts) >= 10: + canon = parts[9] + else: + # lang + canonical (no locale) + canon = parts[8] + if len(parts) >= 10: + locale = _norm_locale(parts[9]) else: - # legacy: field 8 = canonical canon = parts[7] if len(parts) >= 9: lang = _norm_lang(parts[8]) + if len(parts) >= 10: + locale = _norm_locale(parts[9]) + elif len(parts) >= 9: + locale = _norm_locale(parts[8]) if not lang: lang = "en" + if not locale: + blob = f"{name}{canon}{bank}{merchant}".lower() + locale = "fr-CH" if ("lefrancpaysan" in blob or "francpaysan" in blob) else "de-CH" hosts = {name.lower(), bank.lower(), exchange.lower(), merchant.lower(), canon.lower()} - # strip scheme if present hosts = {re.sub(r"^https?://", "", h).split("/")[0] for h in hosts} if any(h and h in hint for h in hosts): - # prefer longer / more specific name match later — keep last match - # that appears in conf order; FP aliases all share fr. - best = lang + best_lang, best_loc = lang, locale if name.lower() in hint or canon.lower() in hint: - return lang - return best + return lang, locale + return best_lang, best_loc + + +def domain_lang_from_conf(host_hint: str = "") -> str: + """Look up UI lang from domains.conf (suite profile `lang` column).""" + lang, _ = domain_profile_from_conf(host_hint) + return lang + + +def domain_locale_from_conf(host_hint: str = "") -> str: + """Look up regional locale from domains.conf (`locale` column).""" + _, loc = domain_profile_from_conf(host_hint) + return loc def ui_lang(explicit: str = "", host_hint: str = "") -> str: - """en (default) or fr. + """en | fr | de (de never a stock profile default). - TALER_MON_LANG_SET=1 → honour TALER_MON_LANG / explicit only. - - Else TALER_DOMAIN_LANG env (set by load_domain_profile). - - Else domains.conf `lang` for host_hint. - - Else fallback *lefrancpaysan*/*francpaysan* → fr. + - Else TALER_DOMAIN_LANG / domains.conf `lang`. + - Else fallback *lefrancpaysan* → fr. - Else explicit / env / en. """ import os @@ -151,8 +189,24 @@ def ui_lang(explicit: str = "", host_hint: str = "") -> str: return _norm_lang(lang) or "en" +def ui_locale(host_hint: str = "") -> str: + """Regional locale fr-CH / de-CH from env or domains.conf.""" + import os + + for key in ("TALER_MON_LOCALE", "TALER_DOMAIN_LOCALE"): + loc = _norm_locale(os.environ.get(key) or "") + if loc: + return loc + conf = domain_locale_from_conf(host_hint) + if conf: + return conf + if re.search(r"lefrancpaysan|francpaysan", host_hint or "", re.I): + return "fr-CH" + return "de-CH" + + def ui(lang: str, key: str, **kwargs) -> str: - """UI chrome strings for sticky bar (English default; French for FrancPaysan).""" + """UI chrome strings (en default; fr for FP; de available via --lang de, never stock default).""" en = { "errors": "ERRORS", "warnings": "WARNINGS", @@ -373,7 +427,60 @@ def ui(lang: str, key: str, **kwargs) -> str: "ago_d": "il y a {n} j", "age_title": "âge de la page · mise à jour chaque seconde", } - table = fr if lang == "fr" else en + # German sticky chrome only (full pack falls back to en for missing keys). + # Not used as any domains.conf default — operators may pass --lang de. + de = { + "errors": "FEHLER", + "warnings": "WARNUNGEN", + "ok": "OK", + "error_one": "{n} Fehler", + "error_many": "{n} Fehler", + "warn_one": "{n} Warnung", + "warn_many": "{n} Warnungen", + "jump_first_error": "Zum ersten Fehler", + "jump_first_warn": "Zur ersten Warnung", + "filter_errors": "Filter: Fehler + Kontext", + "filter_warns": "Filter: Warnungen + Kontext", + "filter_infos": "Filter: Info + Kontext", + "filter_oks": "Filter: OK + Kontext", + "filter_clear": "Filter loeschen · alle Zeilen", + "filter_active_err": "Filter: Fehler + Kontext", + "filter_active_warn": "Filter: Warnungen + Kontext", + "filter_active_info": "Filter: Info + Kontext", + "filter_active_ok": "Filter: nur OK + Kontext", + "filter_show_all": "Alle anzeigen", + "info_one": "{n} Info", + "info_many": "{n} Info", + "ok_one": "{n} ok", + "ok_many": "{n} ok", + "env_context": "Monitoring-Umgebung", + "env_context_title": "Aufklappen: Agent-Env, Suite-Pin, Domain-Hosts und Flags", + "env_context_empty": "Kein Lauf-Header im Log", + "generated": "erzeugt", + "version": "Version", + "version_tree_link": "Git-Baum dieses Laufs oeffnen (Commit)", + "suite_version_title": "taler-monitoring Suite-Version · genauer Lauf-Baum", + "source": "Quelle", + "what_monitors": "Was diese Seite ueberwacht", + "what_monitors_title": "Aufklappen: was diese Seite prueft", + "stack_title": "Stack-Monitoring · {host}", + "stack_summary": "{phases} · public + inside", + "mode_ok": "OK", + "mode_err": "ERR", + "mode_redirect": "REDIR", + "footer": "Konsolen-Darstellung von taler-monitoring. Sticky: gruen = OK · gelb = Warnungen · rot = Fehler.", + "ago_s": "vor {n}s", + "ago_m": "vor {n} Min.", + "ago_h": "vor {n} Std.", + "ago_d": "vor {n} T.", + "age_title": "Seitenalter · jede Sekunde aktualisiert", + } + if lang == "fr": + table = fr + elif lang == "de": + table = de + else: + table = en s = table.get(key) or en.get(key) or key try: return s.format(**kwargs) @@ -2489,7 +2596,7 @@ def main() -> None: ap.add_argument( "--lang", default="", - help="UI language en|fr (default: env TALER_MON_LANG or auto)", + help="UI language en|fr|de (default: domains.conf lang / env; de never stock default)", ) ap.add_argument( "--page-label", diff --git a/taler-monitoring.sh b/taler-monitoring.sh index 4e08116..35c2a99 100755 --- a/taler-monitoring.sh +++ b/taler-monitoring.sh @@ -122,7 +122,7 @@ Options: --merchant URL merchant-backend base — overrides profile --currency CODE expected currency (GOA, KUDOS, CHF, …); empty = report only --no-probe do not probe alternate hosts (unknown domains only) - --lang en|fr UI/console language (sticky bar + badges) + --lang en|fr|de UI/console language (sticky bar + badges; de available, not a profile default) overrides auto (FrancPaysan→fr, else en) --ver, --version, -V print suite version, git commit, and repo URLs -h, --help @@ -165,7 +165,8 @@ Env (same meaning): SURFACE_CVE_LEVEL=warn|error bare Server-header versions default warn (Debian package versions default error) SURFACE_CATALOG=path override surface-catalog.conf - TALER_MON_LANG=en|fr same as --lang (default en; auto fr for *lefrancpaysan*) + TALER_MON_LANG=en|fr|de same as --lang (profile lang from domains.conf; de never stock default) + TALER_MON_LOCALE=fr-CH|de-CH regional l10n from domains.conf locale column Full load (GOA / hacktivism): ./taler-monitoring.sh -d hacktivism.ch full @@ -225,12 +226,13 @@ NO_PROBE=0 while [ $# -gt 0 ]; do case "$1" in --lang) - [ $# -ge 2 ] || { echo "error: --lang needs en|fr" >&2; exit 2; } + [ $# -ge 2 ] || { echo "error: --lang needs en|fr|de" >&2; exit 2; } _lang="$2" case "$_lang" in fr|FR|fra|french) TALER_MON_LANG=fr ;; + de|DE|deu|ger|german|deutsch) TALER_MON_LANG=de ;; en|EN|eng|english) TALER_MON_LANG=en ;; - *) echo "error: unknown language '$_lang' (use en or fr)" >&2; exit 2 ;; + *) echo "error: unknown language '$_lang' (use en, fr, or de)" >&2; exit 2 ;; esac TALER_MON_LANG_SET=1 export TALER_MON_LANG TALER_MON_LANG_SET @@ -547,6 +549,8 @@ elif [ "${LOCAL_STACK:-0}" = "1" ] && command -v podman >/dev/null 2>&1 \ fi printf 'target domain=%s\n' "${TALER_DOMAIN}" +printf ' ui_lang %s\n' "${TALER_MON_LANG:-en}" +printf ' locale %s\n' "${TALER_MON_LOCALE:-${TALER_DOMAIN_LOCALE:-?}}" printf ' bank %s\n' "$BANK_PUBLIC" printf ' exchange %s\n' "$EXCHANGE_PUBLIC" printf ' merchant %s\n' "$MERCHANT_PUBLIC" From ac01e4ecfea0ffe7575635c5dcafd353d2f0af63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 13:35:40 +0200 Subject: [PATCH 14/29] docs: CLI-AUTOMATION-NOTES force-select, wallet-db, helper python (1.20.1) Document magikoopa ladder repro: empty reserve_pub scrape, pin-only plans, --wallet-db *.json rejection, taler-helper-sqlite3 needs Python >=3.11, portable wall-clock, GOA withdraw checklist. --- CLI-AUTOMATION-NOTES.md | 80 +++++++++++++++++++++++++++++++++++++---- VERSION | 2 +- VERSIONS.md | 1 + 3 files changed, 75 insertions(+), 8 deletions(-) diff --git a/CLI-AUTOMATION-NOTES.md b/CLI-AUTOMATION-NOTES.md index b6ac18f..ae15ba5 100644 --- a/CLI-AUTOMATION-NOTES.md +++ b/CLI-AUTOMATION-NOTES.md @@ -47,15 +47,25 @@ Legend: --- -## 3. `reserve_pub` / force-select is fragile (5114) +## 3. `reserve_pub` / force-select is fragile (5114 + empty scrape) | | | |--|--| -| **Seen** | ladder force-select, e2e confirm path, Android spinner issues | -| **Area** | **cli** / **core** / **bank** | -| **Problem** | Cumulative wallets re-emit old reserves; binding the wrong `reserve_pub` yields **HTTP 409 / code 5114** (“already bound”). Automation scrapes accept/tx dumps for candidates. | -| **Workaround** | Score pubs by WID/amount; try several; treat 5114 as “stale reserve” not “out of money”. | -| **Wanted** | CLI returns **the** `reserve_pub` for the just-accepted withdraw in structured JSON; `force-select` / bank-side select API documented with idempotency. | +| **Seen** | ladder force-select, e2e confirm path, Android spinner issues; **repro 2026-07-19 magikoopa** GOA ladder: `force-select skipped · no reserve_pub for this withdraw (WID=…); wallet may not have selected yet` | +| **Area** | **cli** / **core** / **bank** / **ops** | +| **Problem** | (a) Cumulative wallets re-emit old reserves; binding the wrong `reserve_pub` yields **HTTP 409 / code 5114** (“already bound”). (b) Automation scrapes accept/tx dumps for candidates — when the dump has **no** pub at all, ladder WARNs and skips force-select (soft), then often **SKIP_CONFIRM**. (c) Empty scrape is **not** the same as “bank out of money”; do not treat as balance failure. | +| **Workaround** | Score pubs by WID/amount; try several; treat 5114 as “stale reserve” not “out of money”. Refresh `transactions` before each force-select try. Soft-skip confirm if still pending after polls. Fix wallet DB / helper first if accept/tx only print FATAL (see §15–16) — otherwise scrape can never find a pub. | +| **Wanted** | CLI returns **the** `reserve_pub` for the just-accepted withdraw in structured JSON; `force-select` / bank-side select API documented with idempotency. Optional: bank integration already exposes `selected_reserve_pub` when wallet selected — prefer that over scraping wallet logs. | + +### 3a. Ladder pin amounts hide mid-range behaviour + +| | | +|--|--| +| **Seen** | `LADDER_STEPS=3` plan collapses to pins only: `GOA:0`, `max-1`, `max` (no random mid rungs) | +| **Area** | **ops** / **bank** / **cli** | +| **Problem** | **Zero** withdraw often never produces a usable `reserve_pub` / coins (wallet 7006 or no denoms). **Absolute max** often **mint HTTP 500 / code 5110** SQL `P0001` (ceiling probe — expected soft on max pin). Reproducing “real” force-select needs a **mid** amount (e.g. `GOA:1` … exchange min denom scale), not only pins. | +| **Workaround** | Use `LADDER_STEPS≥5` (or higher) so random mids exist; treat zero / absolute-max as probes. Soft-skip zero (7006) and max ceiling (5110). | +| **Wanted** | Ladder plan docs: pin vs mid semantics; CLI structured error for amount-too-large / zero. | --- @@ -191,6 +201,59 @@ Legend: --- +## 15. `--wallet-db=*.json` is rejected (“memory backend not supported”) + +| | | +|--|--| +| **Seen** | **2026-07-19 magikoopa** wallet-cli **1.6.8** / **1.6.11** bundle: any command with `--wallet-db=/tmp/foo.json` fails immediately with `Error: memory backend not supported` | +| **Area** | **cli** / **ops** | +| **Problem** | Current node host treats storage paths ending in **`.json` as unsupported memory backend** and throws before opening sqlite. Old notes / local tests that used `WALLET_DB=….json` break. Accept/transactions then never emit `reserve_pub` → ladder §3 empty-scrape WARNs. | +| **Workaround** | Use a **non-`.json` path**, e.g. `--wallet-db=$SCRATCH/wallet.sqlite3` (ladder already prefers `wallet.sqlite3`). Never pass `….json` as wallet-db. | +| **Wanted** | Clearer error: `wallet-db path must not end in .json; use a directory or .sqlite3 path`. Document legal path forms in `--help`. | + +--- + +## 16. `taler-helper-sqlite3` requires Python ≥ 3.11 on PATH + +| | | +|--|--| +| **Seen** | magikoopa: helper is a **Python** script (`#!/usr/bin/env python3`); with Apple `/usr/bin/python3` **3.9.6** → `FATAL: python version >=3.11 required but running on 3.9.6` in accept/tx logs; wallet may still exit 0 on some paths while stdout is FATAL-only | +| **Area** | **ops** / **cli** | +| **Problem** | Wallet spawns `taler-helper-sqlite3` from **PATH**. Wrong `python3` ⇒ sqlite backend fails ⇒ no real wallet state ⇒ no `reserve_pub` for force-select. | +| **Workaround** | Put **Python ≥3.11** (or 3.12+) first on PATH for mon jobs; ensure `command -v taler-helper-sqlite3` works; smoke: `taler-helper-sqlite3` starts without FATAL. Prefer Homebrew / user `python3.11+` over macOS system 3.9 for agent PATH. | +| **Wanted** | Helper shebang or wrapper that finds a suitable python; wallet preflight error if helper HELLO fails (not silent empty logs). | + +--- + +## 17. Portable wall-clock for scripts (no GNU `timeout` assumed) + +| | | +|--|--| +| **Seen** | macOS agent shells: `timeout: command not found`; e2e already tries `timeout` then `gtimeout` then `perl -e alarm` | +| **Area** | **ops** / **doc** | +| **Problem** | GNU coreutils `timeout` is not on stock macOS. Hard-wiring it breaks portable repro and CI on Darwin. | +| **Workaround** | Prefer **python3** `subprocess.Popen(…).wait(timeout=…)` for outer budgets; for wallet-cli inner calls keep e2e order: `timeout` → `gtimeout` → perl alarm. Document both. | +| **Wanted** | One suite helper `run_with_budget SECS cmd…` in `lib.sh` used by ladder/e2e/smoke. | + +--- + +## 18. How to withdraw (operator checklist, GOA laptop) + +Portable path that works when §§15–16 are fixed: + +1. **PATH:** `taler-helper-sqlite3` + **python3 ≥ 3.11** ahead of system 3.9. +2. **Wallet DB:** `WDB=$TMP/wallet.sqlite3` — **not** `*.json`. +3. **Exchange:** `exchanges add` → `update -f` → `accept-tos` for `https://exchange.hacktivism.ch/`. +4. **Bank mint (explorer):** token with explorer password from `SECRETS_ROOT` / `EXP_PW`; `POST …/accounts/explorer/withdrawals` with e.g. `{"amount":"GOA:1"}` (avoid pure **0** and absolute **max** pins for happy-path tests). +5. **Strip** `:443` from `taler_withdraw_uri`. +6. **`withdraw accept-uri --exchange URL URI`**. +7. **Bank:** poll `…/taler-integration/withdrawal-operation/{WID}` → when `selected`, confirm; if stuck `pending`, force-select with scraped `reserve_pub` (§3). +8. **Settle:** poll wallet balance / bank `transfer_done` — **no** `run-until-done` (§1). + +If step 6’s stdout is only `FATAL: python version…` or `memory backend not supported`, **stop** and fix §15–16 before debugging force-select. + +--- + ## Priority shortlist for `taler-wallet-cli` / core If only a few changes land, these unlock the most automation: @@ -200,6 +263,7 @@ If only a few changes land, these unlock the most automation: 3. **Auto-add exchange + ToS on accept-uri** (§4, §8) 4. **Canonical URIs without :443** (§5) 5. **Emit reserve_pub for last withdraw** (§3) +6. **Clear wallet-db path rules + helper preflight** (§15, §16) --- @@ -212,7 +276,9 @@ If only a few changes land, these unlock the most automation: | exchange | Explicit add/update/accept-tos | | ladder explorer | `read_secret` / EXP_PW_FILE | | stage maxima | bank `max_wire` + keys min denom | -| force-select | multi-rpub try; soft-skip confirm | +| force-select | multi-rpub try; soft-skip confirm; empty scrape → WARN not balance fail | +| wallet-db path | Prefer `*.sqlite3` (never `*.json` as db path) | +| helper / python | mon PATH must include helper + python≥3.11 | --- diff --git a/VERSION b/VERSION index 3989355..0044d6c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.20.0 +1.20.1 diff --git a/VERSIONS.md b/VERSIONS.md index 76bb76b..263ffcc 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| +| **v1.20.1** | 2026-07-19 | **Docs:** CLI-AUTOMATION-NOTES — force-select empty scrape + ladder pins (§3/3a); wallet-db must not be `*.json` (§15); `taler-helper-sqlite3` needs Python ≥3.11 (§16); portable wall-clock (§17); GOA withdraw checklist (§18). No code change. | | **v1.20.0** | 2026-07-19 | **Feature:** domains.conf **`locale`** (l10n) — FP **`fr-CH`**, others **`de-CH`** (aliases ch-FR/ch-DE); UI **`lang=de`** supported (sticky + console tags) but **never** a stock profile default; env `TALER_MON_LOCALE` / `TALER_DOMAIN_LOCALE`. | | **v1.19.1** | 2026-07-19 | **Bugfix / docs:** DEPENDENCIES — e2e requires `taler-helper-sqlite3` on mon PATH + preflight; README domains.conf documents **`lang`** column. | | **v1.19.0** | 2026-07-19 | **Feature:** domains.conf **`lang`** column (en\|fr) is the global UI language default per stack; i18n + HTML read `TALER_DOMAIN_LANG` / conf (FP profiles `lang=fr`). | From 80643e6b3fb28a28824447056b2715532873fed1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 13:45:19 +0200 Subject: [PATCH 15/29] fix: ladder force-select reserve_pub extract (1.20.2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wallet accept/tx dumps mix log lines with JSON; json.loads from the first brace always failed, and regex only scanned accept (no reservePub). Parse with JSONDecoder.raw_decode, scrape both files, score by WID/amount. Prefer Homebrew python for taler-helper-sqlite3; find_wallet_cli prefers ts-core 1.6.x. Soft-skip max-1 mint 5110/P0001 as ceiling. Verified on live GOA: mid rungs force-select HTTP 200 + confirm 204 → OK_BANK. --- CLI-AUTOMATION-NOTES.md | 8 +- VERSION | 2 +- VERSIONS.md | 1 + check_goa_ladder.sh | 174 +++++++++++++++++++++++++--------------- lib.sh | 20 ++++- 5 files changed, 135 insertions(+), 70 deletions(-) diff --git a/CLI-AUTOMATION-NOTES.md b/CLI-AUTOMATION-NOTES.md index ae15ba5..057cef2 100644 --- a/CLI-AUTOMATION-NOTES.md +++ b/CLI-AUTOMATION-NOTES.md @@ -53,8 +53,8 @@ Legend: |--|--| | **Seen** | ladder force-select, e2e confirm path, Android spinner issues; **repro 2026-07-19 magikoopa** GOA ladder: `force-select skipped · no reserve_pub for this withdraw (WID=…); wallet may not have selected yet` | | **Area** | **cli** / **core** / **bank** / **ops** | -| **Problem** | (a) Cumulative wallets re-emit old reserves; binding the wrong `reserve_pub` yields **HTTP 409 / code 5114** (“already bound”). (b) Automation scrapes accept/tx dumps for candidates — when the dump has **no** pub at all, ladder WARNs and skips force-select (soft), then often **SKIP_CONFIRM**. (c) Empty scrape is **not** the same as “bank out of money”; do not treat as balance failure. | -| **Workaround** | Score pubs by WID/amount; try several; treat 5114 as “stale reserve” not “out of money”. Refresh `transactions` before each force-select try. Soft-skip confirm if still pending after polls. Fix wallet DB / helper first if accept/tx only print FATAL (see §15–16) — otherwise scrape can never find a pub. | +| **Problem** | (a) Cumulative wallets re-emit old reserves; binding the wrong `reserve_pub` yields **HTTP 409 / code 5114** (“already bound”). (b) Automation scrapes accept/tx dumps for candidates — when the dump has **no** pub at all, ladder WARNs and skips force-select (soft), then often **SKIP_CONFIRM**. (c) Empty scrape is **not** the same as “bank out of money”; do not treat as balance failure. (d) **Mon bug (fixed v1.20.2):** extract used `json.loads(file_from_first_brace)` on wallet dumps that append log lines after JSON → **zero** objects parsed; regex only scanned accept (which has no `reservePub`) while the pub lives under `transactions[].withdrawalDetails.reservePub`. | +| **Workaround** | **Mon ≥1.20.2:** `JSONDecoder.raw_decode` + regex on accept **and** transactions; score by WID in `bankConfirmationUrl` / amount. Still: score pubs by WID/amount; try several; treat 5114 as “stale reserve” not “out of money”. Refresh `transactions` before each force-select try. Soft-skip confirm if still pending after polls. Fix wallet DB / helper first if accept/tx only print FATAL (see §15–16). | | **Wanted** | CLI returns **the** `reserve_pub` for the just-accepted withdraw in structured JSON; `force-select` / bank-side select API documented with idempotency. Optional: bank integration already exposes `selected_reserve_pub` when wallet selected — prefer that over scraping wallet logs. | ### 3a. Ladder pin amounts hide mid-range behaviour @@ -63,8 +63,8 @@ Legend: |--|--| | **Seen** | `LADDER_STEPS=3` plan collapses to pins only: `GOA:0`, `max-1`, `max` (no random mid rungs) | | **Area** | **ops** / **bank** / **cli** | -| **Problem** | **Zero** withdraw often never produces a usable `reserve_pub` / coins (wallet 7006 or no denoms). **Absolute max** often **mint HTTP 500 / code 5110** SQL `P0001` (ceiling probe — expected soft on max pin). Reproducing “real” force-select needs a **mid** amount (e.g. `GOA:1` … exchange min denom scale), not only pins. | -| **Workaround** | Use `LADDER_STEPS≥5` (or higher) so random mids exist; treat zero / absolute-max as probes. Soft-skip zero (7006) and max ceiling (5110). | +| **Problem** | **Zero** withdraw often never produces a usable `reserve_pub` / coins (wallet 7006 or no denoms). **Absolute max** and sometimes **max-1** hit **mint HTTP 500 / code 5110** SQL `P0001` (ceiling probe). Reproducing “real” force-select needs a **mid** amount (e.g. `GOA:1` … exchange min denom scale), not only pins. | +| **Workaround** | Use `LADDER_STEPS≥5` (or higher) so random mids exist; treat zero / absolute-max / max-1-5110 as probes (soft `CEILING_REJECT` in mon ≥1.20.2). | | **Wanted** | Ladder plan docs: pin vs mid semantics; CLI structured error for amount-too-large / zero. | --- diff --git a/VERSION b/VERSION index 0044d6c..769e37e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.20.1 +1.20.2 diff --git a/VERSIONS.md b/VERSIONS.md index 263ffcc..ec8e050 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| +| **v1.20.2** | 2026-07-19 | **Bugfix:** ladder force-select empty `reserve_pub` scrape — wallet dumps mix log lines + JSON so `json.loads(rest_of_file)` always failed; extract now uses `JSONDecoder.raw_decode` and regex on **both** accept + transactions (`reservePub` is in tx). Prefer Homebrew python on macOS for helper; `find_wallet_cli` prefers ts-core 1.6.x over stale system wrappers. Soft-skip max-1 mint 5110/P0001 as ceiling. Verified: mid rungs `force-select HTTP 200` + `confirm 204` → `OK_BANK`. | | **v1.20.1** | 2026-07-19 | **Docs:** CLI-AUTOMATION-NOTES — force-select empty scrape + ladder pins (§3/3a); wallet-db must not be `*.json` (§15); `taler-helper-sqlite3` needs Python ≥3.11 (§16); portable wall-clock (§17); GOA withdraw checklist (§18). No code change. | | **v1.20.0** | 2026-07-19 | **Feature:** domains.conf **`locale`** (l10n) — FP **`fr-CH`**, others **`de-CH`** (aliases ch-FR/ch-DE); UI **`lang=de`** supported (sticky + console tags) but **never** a stock profile default; env `TALER_MON_LOCALE` / `TALER_DOMAIN_LOCALE`. | | **v1.19.1** | 2026-07-19 | **Bugfix / docs:** DEPENDENCIES — e2e requires `taler-helper-sqlite3` on mon PATH + preflight; README domains.conf documents **`lang`** column. | diff --git a/check_goa_ladder.sh b/check_goa_ladder.sh index 410205f..560537d 100755 --- a/check_goa_ladder.sh +++ b/check_goa_ladder.sh @@ -171,10 +171,17 @@ ladder_over() { } wcli() { + # taler-helper-sqlite3 is Python ≥3.11; prefer Homebrew/local python on macOS + # so Apple /usr/bin/python3 3.9 does not FATAL the sqlite backend (no reservePub). + local _path="${PATH:-}" + case ":${_path}:" in + *:/opt/homebrew/bin:*) ;; + *) _path="/opt/homebrew/bin:/usr/local/bin:${_path}" ;; + esac if [ -n "${CLI_JS:-}" ] && [ -f "$CLI_JS" ]; then - node "$CLI_JS" --wallet-db="$WDB" --no-throttle "$@" + PATH="$_path" node "$CLI_JS" --wallet-db="$WDB" --no-throttle "$@" else - taler-wallet-cli --wallet-db="$WDB" --no-throttle "$@" + PATH="$_path" taler-wallet-cli --wallet-db="$WDB" --no-throttle "$@" fi } @@ -238,10 +245,18 @@ wallet_avail() { wcli balance 2>/dev/null | python3 -c ' import json,sys t=sys.stdin.read() +dec=json.JSONDecoder() i=t.find("{") if i<0: print("0"); raise SystemExit -d=json.loads(t[i:t.rfind("}")+1]) +try: + d,_=dec.raw_decode(t,i) +except Exception: + # last-resort: brace slice (may fail on trailing logs) + try: + d=json.loads(t[i:t.rfind("}")+1]) + except Exception: + print("0"); raise SystemExit cur=sys.argv[1] for b in d.get("balances") or []: a=b.get("available") or "" @@ -587,6 +602,16 @@ for AMT in "$@"; do echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" continue fi + # max-1 can also hit the same libeufin SQL ceiling (5110/P0001) on live GOA — + # treat as soft ceiling when mid rungs already passed; do not block the report. + if [ "$IS_MAX_M1_PIN" = "1" ] && echo "$note" | grep -qE '5110|P0001'; then + status="CEILING_REJECT" + note="max-1 rejected (same ceiling as absolute max): $note" + warn bank "mint $AMT rejected (max-1 ceiling)" \ + "problem: bank rejects max-1 with SQL P0001/5110 (pool ceiling). Mid-rung OK_BANK/OK still count. Ladder continues. detail: $note" + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + continue + fi err bank "mint $AMT failed" "$note" status="FAIL_MINT" STOP_REASON="$note" @@ -648,6 +673,10 @@ for AMT in "$@"; do # Collect reserve_pub candidates for *this* withdrawal (WID + amount). # Cumulative wallets re-print old reserves in accept/tx dumps — never trust a single "last" blindly. # Prints unique pubs one per line, preferred order first. + # + # Critical: wallet-cli dumps mix log lines + JSON. Never json.loads(rest_of_file) — + # trailing "Shutdown requested" lines make that always fail. Use JSONDecoder.raw_decode + # and regex on *both* accept and transactions output (reservePub lives in tx, not accept). extract_rpubs_for_wid() { python3 - "$WID" "$AMT" "$SCRATCH/accept-$tag.out" "$SCRATCH/tx-$tag.json" "$SCRATCH/used-rpubs.txt" <<'PY' import json, re, sys @@ -661,99 +690,118 @@ used = set() if Path(used_path).is_file(): used = {ln.strip() for ln in open(used_path) if ln.strip()} +amt_num = amt.split(":", 1)[-1] if amt else "" +dec = json.JSONDecoder() + def walk_collect(o, bag, ctx=None): ctx = dict(ctx or {}) if isinstance(o, dict): + # First pass: sibling scalars (reservePub + bankConfirmationUrl + amounts + # are peers under withdrawalDetails — order in JSON must not matter). for k, v in o.items(): - kl = str(k).lower() - if kl in ("withdrawal_id", "withdraw_id", "wopid", "id") and isinstance(v, str): + if not isinstance(v, str): + continue + kl = str(k).lower().replace("-", "_") + if kl in ( + "withdrawal_id", "withdraw_id", "wopid", "id", + "transactionid", "transaction_id", + ): ctx["id"] = v - if kl in ("amount", "rawamount", "instructedamount") and isinstance(v, str): + elif kl in ( + "amount", "rawamount", "instructedamount", + "amounteffective", "amountraw", "transferamount", + ): ctx["amount"] = v - if kl in ("taler_withdraw_uri", "talerwithdrawuri", "uri") and isinstance(v, str): + elif kl in ( + "taler_withdraw_uri", "talerwithdrawuri", "uri", + "bankconfirmationurl", "confirmtransferurl", + ): ctx["uri"] = v + for k, v in o.items(): + kl = str(k).lower().replace("-", "_") if kl in ("reserve_pub", "reservepub") and isinstance(v, str) and len(v) >= 40: bag.append((v, dict(ctx))) - walk_collect(v, bag, ctx) + elif not isinstance(v, (str, int, float, bool)) and v is not None: + walk_collect(v, bag, ctx) elif isinstance(o, list): for i in o: walk_collect(i, bag, ctx) -raw_pubs = [] # ordered as found -scored = [] # (score, pub) higher better -blob_all = "" -for p in paths: - try: - blob_all += open(p, errors="replace").read() + "\n" - except Exception: - pass +def score_ctx(pub, ctx, window=""): + score = 0 + cid = str(ctx.get("id") or "") + camt = str(ctx.get("amount") or "") + curi = str(ctx.get("uri") or "") + # bankConfirmationUrl / withdraw URI embeds this op's WID + if wid and wid in curi: + score += 120 + if wid and wid in cid: + score += 100 + if wid and wid in window: + score += 80 + if amt and (camt == amt or camt.endswith(amt_num)): + score += 50 + if amt and amt in window: + score += 30 + if pub in used: + score -= 200 + return score -# 1) full JSON objects in files +def iter_json_objects(text): + """Yield top-level JSON objects from mixed log+JSON wallet dumps.""" + i, n = 0, len(text) + while i < n: + if text[i] == "{": + try: + o, end = dec.raw_decode(text, i) + yield o + i = end + continue + except Exception: + pass + i += 1 + +raw_pubs = [] +scored = [] + +# 1) structured JSON via raw_decode (accept + transactions) for p in paths: try: t = open(p, errors="replace").read() except Exception: continue - # whole-file JSON - for m in re.finditer(r"\{", t): - try: - o = json.loads(t[m.start():]) - except Exception: - continue + for o in iter_json_objects(t): bag = [] walk_collect(o, bag) for pub, ctx in bag: raw_pubs.append(pub) - score = 0 - cid = str(ctx.get("id") or "") - camt = str(ctx.get("amount") or "") - curi = str(ctx.get("uri") or "") - if wid and wid in cid: - score += 100 - if wid and wid in curi: - score += 80 - if amt and (camt == amt or camt.endswith(amt.split(":", 1)[-1])): - score += 40 - if pub in used: - score -= 200 - scored.append((score, pub)) + scored.append((score_ctx(pub, ctx), pub)) -# 2) regex fallback on accept file only (more current) -try: - acc = open(paths[0], errors="replace").read() -except Exception: - acc = "" -for pat in ( - r"\"reserve_pub\"\s*:\s*\"([A-Z0-9]{40,})\"", +# 2) regex on *both* accept and tx (reservePub is normally only in transactions) +RPUB_PATS = ( r"\"reservePub\"\s*:\s*\"([A-Z0-9]{40,})\"", - r"reserve_pub[\"\s:=]+([A-Z0-9]{40,})", -): - for m in re.finditer(pat, acc, re.I): - pub = m.group(1) - raw_pubs.append(pub) - score = 10 - # proximity to WID in accept output - window = acc[max(0, m.start() - 400) : m.end() + 400] - if wid and wid in window: - score += 100 - if amt and amt in window: - score += 30 - if pub in used: - score -= 200 - scored.append((score, pub)) + r"\"reserve_pub\"\s*:\s*\"([A-Z0-9]{40,})\"", + r"reserve[_ ]?pub[\"\s:=]+([A-Z0-9]{40,})", +) +for p in paths: + try: + blob = open(p, errors="replace").read() + except Exception: + continue + for pat in RPUB_PATS: + for m in re.finditer(pat, blob, re.I): + pub = m.group(1) + raw_pubs.append(pub) + window = blob[max(0, m.start() - 600) : m.end() + 600] + scored.append((score_ctx(pub, {}, window) + 10, pub)) -# prefer high score, then later occurrence order = [] seen = set() -for score, pub in sorted(scored, key=lambda x: (-x[0],), reverse=False): - # sort by score desc: use reverse sorted - pass for score, pub in sorted(scored, key=lambda x: x[0], reverse=True): if pub in seen or pub in used: continue seen.add(pub) order.append(pub) -# append unused raw in reverse (newest-ish) for pub in reversed(raw_pubs): if pub in seen or pub in used: continue diff --git a/lib.sh b/lib.sh index 0f92126..9ced76b 100755 --- a/lib.sh +++ b/lib.sh @@ -1512,12 +1512,20 @@ find_wallet_cli() { echo "$WALLET_CLI"; return 0 fi local c cand - # WALLET_CLI / WALLET_CLI_CANDIDATES from env; then portable package paths only. + # WALLET_CLI / WALLET_CLI_CANDIDATES from env; then portable package paths. + # Prefer local ts-core / mon-env builds (schema-compatible) over stale system + # wrappers (e.g. /usr/local 1.3.x vs DB schema 29 from 1.6.x). # shellcheck disable=SC2086 for c in ${WALLET_CLI_CANDIDATES:-} \ "${HOME}/.local/bin/taler-wallet-cli.mjs" \ + "${HOME}/src/taler/taler-typescript-core/packages/taler-wallet-cli/bin/taler-wallet-cli.mjs" \ + "${HOME}/src/taler/taler-typescript-core/packages/taler-wallet-cli/dist/taler-wallet-cli.mjs" \ + "${HOME}/src/taler/taler-typescript-core/packages/taler-wallet-cli/dist/taler-wallet-cli-local.mjs" \ + "${HOME}/src/taler/taler-typescript-core/packages/taler-wallet-cli/dist/taler-wallet-cli-bundled.cjs" \ /usr/lib/taler-wallet-cli/node_modules/taler-wallet-cli/bin/taler-wallet-cli.mjs \ - /usr/share/taler-wallet-cli/bin/taler-wallet-cli.mjs + /usr/share/taler-wallet-cli/bin/taler-wallet-cli.mjs \ + /usr/local/lib/taler-wallet-cli/node_modules/taler-wallet-cli/bin/taler-wallet-cli.mjs \ + /usr/local/lib/taler-wallet-cli/node_modules/taler-wallet-cli/dist/taler-wallet-cli-bundled.cjs do [ -n "$c" ] && [ -f "$c" ] && { echo "$c"; return 0; } done @@ -1533,6 +1541,14 @@ find_wallet_cli() { c=$(readlink -f "$cand" 2>/dev/null || true) [ -n "$c" ] && [ -f "$c" ] && { echo "$c"; return 0; } fi + # macOS /usr/local wrapper often runs an older bundle; prefer its bundled.cjs only + # when no newer candidate above matched (already returned). + for c in \ + /usr/local/lib/taler-wallet-cli/node_modules/taler-wallet-cli/dist/taler-wallet-cli-bundled.cjs \ + /usr/local/lib/node_modules/taler-wallet-cli/dist/taler-wallet-cli-bundled.cjs + do + [ -f "$c" ] && { echo "$c"; return 0; } + done fi return 1 } From 32ffb6cdb178df8c5ad2ca30dab0b6dc74e664d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 14:15:56 +0200 Subject: [PATCH 16/29] release 1.21.0: amount ladder classic + max-search Generic currency ladder (check_amount_ladder.sh; goa-ladder shim). Default remains classic 23-rung withdraw/pay. New max-ladder mode hunts highest withdrawable amount with alt_unit_names in logs/report. Fixes force-select extract (external ladder_extract_rpubs.py; no stdin return SyntaxError), soft CEILING_REJECT on mint 5110/P0001, force-select 409 tries next rpub, pay SKIP_BALANCE when over available, and crash-proof report formatting. Verified live GOA: force-select 200, confirm 204, OK_BANK, report finishes without Traceback. --- CLI-AUTOMATION-NOTES.md | 14 +- README.md | 18 +- TESTS.md | 2 +- VERSION | 2 +- VERSIONS.md | 4 +- check_amount_ladder.sh | 1729 +++++++++++++++++++++++++++++++++++++++ check_goa_ladder.sh | 1405 +------------------------------ ladder_extract_rpubs.py | 163 ++++ metrics.sh | 8 +- taler-monitoring.sh | 28 +- 10 files changed, 1952 insertions(+), 1421 deletions(-) create mode 100755 check_amount_ladder.sh create mode 100755 ladder_extract_rpubs.py diff --git a/CLI-AUTOMATION-NOTES.md b/CLI-AUTOMATION-NOTES.md index 057cef2..a541d28 100644 --- a/CLI-AUTOMATION-NOTES.md +++ b/CLI-AUTOMATION-NOTES.md @@ -9,7 +9,7 @@ Companions: - Android GUI → [`android-test/GUI-AUTOMATION-NOTES.md`](./android-test/GUI-AUTOMATION-NOTES.md) - Git / suite tree → [`VERSIONS.md`](./VERSIONS.md) -Scripts: `check_e2e.sh`, `check_goa_ladder.sh`, `taler-monitoring.sh`. +Scripts: `check_e2e.sh`, `check_amount_ladder.sh` (compat `check_goa_ladder.sh`), `taler-monitoring.sh`. Legend: @@ -67,6 +67,16 @@ Legend: | **Workaround** | Use `LADDER_STEPS≥5` (or higher) so random mids exist; treat zero / absolute-max / max-1-5110 as probes (soft `CEILING_REJECT` in mon ≥1.20.2). | | **Wanted** | Ladder plan docs: pin vs mid semantics; CLI structured error for amount-too-large / zero. | +### 3b. Higher withdraw amounts often fail at mint + +| | | +|--|--| +| **Seen** | Classic GOA ladder high/random mids and ceiling pins — e.g. `GOA:747827072844319` → `FAIL_MINT` / `STOPPED … mint HTTP 500 { code :5110, … P0001 }`; same class of error on max / max-1. | +| **Area** | **bank** / **ops** | +| **Problem** | **Larger amounts** frequently hit bank/libeufin limits (HTTP 500, Taler **5110**, SQL **P0001**). Not specific to pin:max only — mid-high ladder steps fail the same way. Do not treat as wallet/`reserve_pub` when mint never succeeded. | +| **Workaround** | Expect failures above some live ceiling; use max-search / soft ceiling handling; keep mid-range rungs for real withdraw/confirm tests. | +| **Wanted** | Clear amount-too-large from bank (not opaque SQL 500); mon reports ceiling separately from hard fails. | + --- ## 4. Exchange not auto-known for bank deep links / withdraw URIs @@ -277,6 +287,8 @@ If only a few changes land, these unlock the most automation: | ladder explorer | `read_secret` / EXP_PW_FILE | | stage maxima | bank `max_wire` + keys min denom | | force-select | multi-rpub try; soft-skip confirm; empty scrape → WARN not balance fail | +| higher amounts | mint 5110/P0001 common above live ceiling (§3b); classic soft `CEILING_REJECT`; max-ladder bounds hi; report uses alt_unit_names (Tera-GOA …) | +| pay vs balance | never order more than wallet **available** (soft `SKIP_BALANCE`); pendingIncoming does not count as spendable | | wallet-db path | Prefer `*.sqlite3` (never `*.json` as db path) | | helper / python | mon PATH must include helper + python≥3.11 | diff --git a/README.md b/README.md index 998c65b..d8cf093 100644 --- a/README.md +++ b/README.md @@ -274,12 +274,14 @@ Other domains: never SSH. Optional **e2e** aborts cleanly on login/KYC. ```bash ./taler-monitoring.sh e2e ./taler-monitoring.sh -d taler.net urls e2e -# Amount ladder (withdraw 0 → … → max, then pay): same script for GOA + stage -./taler-monitoring.sh ladder # GOA / current domain +# Amount ladder (any currency): classic 0 → … → max, then optional pay +./taler-monitoring.sh ladder # classic (current domain) ./taler-monitoring.sh -d stage.lefrancpaysan.ch ladder # TESTPAYSAN maxima +# Max-search: start high, narrow toward highest withdrawable +./taler-monitoring.sh max-ladder # Stage auto-reads bank max_wire_transfer_amount (e.g. 2000) and min denom (0.01). # Explorer password: francpaysan-stage-user …/bank-explorer-password.txt (or EXP_PW=). -# Override: LADDER_MAX_AMOUNT=100 LADDER_STEPS=7 LADDER_PAY=0 … +# Override: LADDER_MODE=max LADDER_MAX_PROBES=16 LADDER_MAX_AMOUNT=100 LADDER_STEPS=7 LADDER_PAY=0 … # customize e2e: E2E_WITHDRAW_VALUES="20 50 100" E2E_PAY_VALUES="0.05 1 5" ./taler-monitoring.sh e2e E2E_VARIABLE=0 WITHDRAW_AMT=GOA:50 PAY_AMT=GOA:1 ./taler-monitoring.sh e2e # single fixed @@ -304,7 +306,7 @@ E2E_VARIABLE=0 WITHDRAW_AMT=GOA:50 PAY_AMT=GOA:1 ./taler-monitoring.sh e2e # s | **versions** | `deb.taler.net` reachable (InRelease/Packages/pool `.deb`); containers can reach it + have apt source; installed Taler packages vs **trixie** | | **sanity** | bank · exchange · merchant sections (public + server) | | **e2e** | account → credit → withdraw → wallet → confirm → order → pay | -| **ladder** | GOA withdraw: fixed **0** + low bands + **random high ranges** + fixed **max** | +| **ladder** | amount ladder: **classic** (0 + mids + max) or **max-search** (`max-ladder` / `LADDER_MODE=max`) | ```bash # package suite (default trixie on deb.taler.net) @@ -394,8 +396,12 @@ LADDER_LOAD=0 ./taler-monitoring.sh ladder | `METRICS_LOAD` | `1` | `0` = skip all host/container load probes | | `LADDER_LOAD` | `1` | `0` = skip load in ladder only | | `LADDER_PAY` | `1` | `0` = skip payment ladder after withdraws | -| `LADDER_WITHDRAW_SCALE` | `1.5` | withdraw mids / pay mids ratio (funding headroom) | -| `LADDER_STEPS` | `23` | rungs for both withdraw and pay (0 + mids + max−1 + max) | +| `LADDER_MODE` | `classic` | default **classic** 23-rung; `max` = hunt highest withdrawable (`max-ladder`) | +| `LADDER_MAX_PROBES` | `32` | max-search probe budget; `0` = until converge / timeout | +| `LADDER_HIGH_RUNGS` | `6` | max-search: first N probes random-high, then bisect | +| `LADDER_MAX_TOL_REL` | `0.001` | max-search relative lo/hi gap to stop | +| `LADDER_WITHDRAW_SCALE` | `1.5` | classic: withdraw mids / pay mids ratio (funding headroom) | +| `LADDER_STEPS` | `23` | classic: rungs for withdraw and pay (0 + mids + max−1 + max) | | `METRICS_LOAD_SSH_TIMEOUT` | `90` | seconds for remote load python | ## Needs diff --git a/TESTS.md b/TESTS.md index 5c0f76a..0c89237 100644 --- a/TESTS.md +++ b/TESTS.md @@ -25,7 +25,7 @@ Every check line has a **global** run number and a **grouped** id: | **sanity** | `check_sanity.sh` | `bank` `exchange` `merchant` | | **server** | `check_server.sh` | (flat `server-NN` or host groups) | | **e2e** | `check_e2e.sh` | `prereq` `load` `bank` `wallet` `bankwd` `settle` `pay` `shop` `paivana` `dig` `report` | -| **ladder** | `check_goa_ladder.sh` | `plan` `load` `withdraw` `pay` `report` | +| **ladder** | `check_amount_ladder.sh` (alias `check_goa_ladder.sh`) | `plan` `load` `withdraw` `pay` `report` — modes classic / max (`max-ladder`) | **Why groups:** flat `www-001`…`www-080` was hard to talk about. `www.bank-04` / `e2e.bankwd-02` pin the failure to a logical block. diff --git a/VERSION b/VERSION index 769e37e..3500250 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.20.2 +1.21.0 diff --git a/VERSIONS.md b/VERSIONS.md index ec8e050..f8d32cd 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,8 +17,8 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| -| **v1.20.2** | 2026-07-19 | **Bugfix:** ladder force-select empty `reserve_pub` scrape — wallet dumps mix log lines + JSON so `json.loads(rest_of_file)` always failed; extract now uses `JSONDecoder.raw_decode` and regex on **both** accept + transactions (`reservePub` is in tx). Prefer Homebrew python on macOS for helper; `find_wallet_cli` prefers ts-core 1.6.x over stale system wrappers. Soft-skip max-1 mint 5110/P0001 as ceiling. Verified: mid rungs `force-select HTTP 200` + `confirm 204` → `OK_BANK`. | -| **v1.20.1** | 2026-07-19 | **Docs:** CLI-AUTOMATION-NOTES — force-select empty scrape + ladder pins (§3/3a); wallet-db must not be `*.json` (§15); `taler-helper-sqlite3` needs Python ≥3.11 (§16); portable wall-clock (§17); GOA withdraw checklist (§18). No code change. | +| **v1.21.0** | 2026-07-19 | **Feature:** generic **amount ladder** (`check_amount_ladder.sh`; compat shim `check_goa_ladder.sh`). Default **classic** 23-rung withdraw (+ pay when enabled); **max-search** (`max-ladder` / `LADDER_MODE=max`) hunts highest mintable amount (probes default 32, `0`=until timeout). **Pay:** never overspend — soft `SKIP_BALANCE` if amount > wallet available. **Alt units:** report/logs show human names (e.g. `8.15 Tera-GOA (GOA:…)`) + raw. **Fixes bundled:** external `ladder_extract_rpubs.py` (no stdin SyntaxError); force-select multi-rpub + raw_decode; 409 tries next rpub; soft `CEILING_REJECT` on mid/high mint 5110/P0001; report python never aborts ladder; helper PATH python≥3.11; wallet-db not `*.json`. CLI-AUTOMATION-NOTES §3/3a/3b/§15–18. | +| **v1.20.1** | 2026-07-19 | **Docs:** CLI-AUTOMATION-NOTES — force-select empty scrape + ladder pins (§3/3a); wallet-db must not be `*.json` (§15); `taler-helper-sqlite3` needs Python ≥3.11 (§16); portable wall-clock (§17); GOA withdraw checklist (§18). No code change. *(Superseded notes folded into **v1.21.0** feature release.)* | | **v1.20.0** | 2026-07-19 | **Feature:** domains.conf **`locale`** (l10n) — FP **`fr-CH`**, others **`de-CH`** (aliases ch-FR/ch-DE); UI **`lang=de`** supported (sticky + console tags) but **never** a stock profile default; env `TALER_MON_LOCALE` / `TALER_DOMAIN_LOCALE`. | | **v1.19.1** | 2026-07-19 | **Bugfix / docs:** DEPENDENCIES — e2e requires `taler-helper-sqlite3` on mon PATH + preflight; README domains.conf documents **`lang`** column. | | **v1.19.0** | 2026-07-19 | **Feature:** domains.conf **`lang`** column (en\|fr) is the global UI language default per stack; i18n + HTML read `TALER_DOMAIN_LANG` / conf (FP profiles `lang=fr`). | diff --git a/check_amount_ladder.sh b/check_amount_ladder.sh new file mode 100755 index 0000000..9c7e2c1 --- /dev/null +++ b/check_amount_ladder.sh @@ -0,0 +1,1729 @@ +#!/usr/bin/env bash +# check_amount_ladder.sh — generic withdraw/pay amount ladder (any Taler currency) +# +# Currency-agnostic bank landing ladder for GOA, TESTPAYSAN, KUDOS, … +# (formerly check_goa_ladder.sh — shim retained for compatibility). +# +# Flow (bank landings): +# 1) GET /intro/auto-account.json → personal *account-* (balance 0) +# 2) Mint pool withdrawals as explorer + confirm when selected +# 3) wallet-cli accept-uri only (no run-until-done) +# 4) bank confirm when selected; settle = balance + transfer_done +# +# Modes (LADDER_MODE): +# classic — random strictly increasing rungs + pins 0, max-1, max (default) +# max — free max-search: start randomly high, narrow toward highest +# amount that mint+accept+confirm (+ bank transfer_done) succeeds +# +# Phase A: withdraw. Phase B: pay (classic only by default; off in max mode). +# +# Stacks auto-tune via bank /config max_wire + exchange min denom when possible. +# +# Usage: +# ./taler-monitoring.sh ladder +# ./taler-monitoring.sh max-ladder # LADDER_MODE=max +# ./taler-monitoring.sh -d stage.lefrancpaysan.ch ladder +# LADDER_MODE=max LADDER_MAX_PROBES=16 ./taler-monitoring.sh ladder +# LADDER_MAX_AMOUNT=100 LADDER_STEPS=7 ./taler-monitoring.sh -d stage.lefrancpaysan.ch ladder +# +# Env: LADDER_MODE, LADDER_STEPS, LADDER_MAX_AMOUNT, LADDER_MIN_AMOUNT, +# LADDER_MAX_PROBES, LADDER_MAX_TOL_REL, LADDER_HIGH_RUNGS, LADDER_STACK_AUTO, +# LADDER_PAY, EXP_PW / EXP_PW_FILE, CLI_JS, MERCHANT_INSTANCE, … +set -euo pipefail +ROOT=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=lib.sh +source "$ROOT/lib.sh" +# When invoked standalone (not via taler-monitoring.sh), still load secrets.env +load_monitoring_secrets_env 2>/dev/null || true + +# Area ladder.* — withdraw/pay amount ladder (any EXPECT_CURRENCY / stack) +# Groups: ladder.plan / ladder.load / ladder.withdraw / ladder.pay / ladder.report +set_area ladder +set_group plan +SECTION_T0=$(date +%s) +now_ms() { python3 -c 'import time; print(int(time.time()*1000))'; } +elapsed_ms() { + # elapsed_ms START_MS + python3 -c 'import sys; print(int(sys.argv[1]) - int(sys.argv[2]))' "$(now_ms)" "$1" +} + +: "${LADDER_TIMEOUT_S:=3600}" +: "${LADDER_SETTLE_ROUNDS:=18}" +: "${LADDER_SETTLE_SLEEP:=2}" +: "${EXP_USER:=explorer}" +# EXP_PW_FILE / EXP_PW optional overrides; otherwise read_secret + SECRETS_ROOT / stage SSH +: "${EXP_PW_FILE:=}" +: "${EXP_PW:=}" +: "${CLI_JS:=}" +# GOA libeufin amount ceiling (hacktivism). Stage auto-replaces via bank /config. +: "${LADDER_MAX_AMOUNT:=4503599627370496}" +: "${LADDER_STEPS:=23}" +: "${LADDER_LOAD:=1}" +: "${LADDER_PAY:=1}" +: "${LADDER_STACK_AUTO:=1}" +# Withdraw mids = pay mids × scale (so wallet can afford the pay ladder) +: "${LADDER_WITHDRAW_SCALE:=1.5}" +# Floor for random mids (must be ≥ smallest exchange coin; GOA min denom ≈ 0.000001) +: "${LADDER_MIN_AMOUNT:=0.000001}" +: "${LADDER_CONFIRM_POLLS:=40}" +: "${LADDER_PAY_SETTLE_ROUNDS:=6}" +: "${MERCHANT_INSTANCE:=goa-demo-cp4zqk}" +# Public variable template for stage pays (optional; fixed templates used when amount maps) +: "${LADDER_PAY_TEMPLATE:=}" +: "${LADDER_PAY_TEMPLATE_INSTANCE:=fermes-des-collines}" +# classic | max (max = find highest withdrawable; phase alias max-ladder) +: "${LADDER_MODE:=classic}" +# max-search: probe budget (0 = run until convergence / LADDER_TIMEOUT_S only) +: "${LADDER_MAX_PROBES:=32}" +: "${LADDER_MAX_TOL_REL:=0.001}" +# first N probes = random high (log-uniform); then log-bisect between lo/hi +: "${LADDER_HIGH_RUNGS:=6}" +# optional seed for reproducible max-search (empty = time-based) +: "${LADDER_MAX_SEED:=}" + +# Historic libeufin-ish absolute ceiling used as default LADDER_MAX_AMOUNT on GOA +LADDER_ABS_CEILING="4503599627370496" +GOA_LADDER_CEILING="${LADDER_ABS_CEILING}" # back-compat alias + +# Resolve wallet-cli .mjs (no hardcoded laptop path) +if [ -z "${CLI_JS}" ] || [ ! -f "${CLI_JS}" ]; then + CLI_JS=$(find_wallet_cli 2>/dev/null || true) +fi +if [ -z "${CLI_JS}" ] || [ ! -f "${CLI_JS}" ]; then + # allow PATH wrapper as last resort (wcli falls back to taler-wallet-cli) + CLI_JS="" +fi + +CUR="${EXPECT_CURRENCY:-GOA}" +BANK="${BANK_PUBLIC%/}" +EX="${EXCHANGE_PUBLIC%/}/" +MER="${MERCHANT_PUBLIC%/}" +INST="${MERCHANT_INSTANCE}" + +# --- Stack-specific ladder maxima (TESTPAYSAN stage) --- +# Uses bank max_wire_transfer_amount and exchange min denom when still on GOA defaults. +apply_ladder_stack_defaults() { + [ "${LADDER_STACK_AUTO}" = "1" ] || return 0 + if [ "${CUR}" != "TESTPAYSAN" ]; then + case "${TALER_DOMAIN:-}" in + stage.lefrancpaysan.ch|stage.bank.lefrancpaysan.ch|stage.exchange.lefrancpaysan.ch|stage.monnaie.lefrancpaysan.ch) + CUR="TESTPAYSAN" + ;; + *) return 0 ;; + esac + fi + + local bank_cfg max_wire min_denom + bank_cfg=$(curl -sS -m 12 "${BANK}/config" 2>/dev/null || true) + max_wire=$(printf '%s' "$bank_cfg" | python3 -c ' +import json,sys +try: + d=json.load(sys.stdin) +except Exception: + print("") + raise SystemExit(0) +a=d.get("max_wire_transfer_amount") or "" +print(a.split(":",1)[-1] if a else "") +' 2>/dev/null || true) + + min_denom=$(curl -sS -m 25 -H 'Accept: application/json' "${EX%/}/keys" 2>/dev/null | python3 -c ' +import json,sys +try: + d=json.load(sys.stdin) +except Exception: + print("") + raise SystemExit(0) +den=d.get("denoms") or d.get("denominations") or [] +vals=[] +for x in den if isinstance(den,list) else []: + v=x.get("value") or x.get("amount") or "" + if not v: continue + try: vals.append(float(str(v).split(":")[-1])) + except Exception: pass +print(min(vals) if vals else "") +' 2>/dev/null || true) + + # Fallback maxima if public config unreachable + [ -n "$max_wire" ] || max_wire="2000" + [ -n "$min_denom" ] || min_denom="0.01" + + # Only rewrite when caller left GOA defaults (explicit LADDER_MAX_AMOUNT=… kept) + if [ -n "$max_wire" ] && { [ "${LADDER_MAX_AMOUNT}" = "${GOA_LADDER_CEILING}" ] || [ "${LADDER_MAX_AMOUNT}" = "${LADDER_ABS_CEILING}" ] || [ "${LADDER_MAX_AMOUNT}" = "4503599627370496" ]; }; then + LADDER_MAX_AMOUNT="$max_wire" + fi + if [ -n "$min_denom" ] && { [ "${LADDER_MIN_AMOUNT}" = "0.000001" ] || [ "${LADDER_MIN_AMOUNT}" = "0.00000001" ]; }; then + LADDER_MIN_AMOUNT="$min_denom" + fi + # Slightly fewer rungs on stage (full GOA 23 still ok if user set LADDER_STEPS) + if [ "${LADDER_STEPS}" = "23" ]; then + LADDER_STEPS=15 + fi + # Stage farmer shops: private goa-demo instance does not exist + if [ "${MERCHANT_INSTANCE}" = "goa-demo-cp4zqk" ]; then + MERCHANT_INSTANCE="${LADDER_PAY_TEMPLATE_INSTANCE:-fermes-des-collines}" + INST="$MERCHANT_INSTANCE" + fi + # Prefer public templates for pays when no merchant token (set later) + : "${E2E_USE_TEMPLATES:=1}" + export E2E_USE_TEMPLATES + export LADDER_MAX_AMOUNT LADDER_MIN_AMOUNT LADDER_STEPS MERCHANT_INSTANCE + info "ladder stack" "TESTPAYSAN · max=${CUR}:${LADDER_MAX_AMOUNT} min=${CUR}:${LADDER_MIN_AMOUNT} steps=${LADDER_STEPS} (from bank max_wire / keys denoms)" +} +apply_ladder_stack_defaults + +# Normalize mode (max-ladder phase / aliases) +case "${LADDER_MODE}" in + max|MAX|max-search|maxsearch|find-max|findmax) LADDER_MODE=max ;; + classic|CLASSIC|default|plan|"" ) LADDER_MODE=classic ;; + *) + warn ladder "unknown LADDER_MODE=${LADDER_MODE} — using classic" + LADDER_MODE=classic + ;; +esac +# max mode: pay ladder off unless caller explicitly set LADDER_PAY=1 *and* LADDER_PAY_WITH_MAX=1 +if [ "$LADDER_MODE" = "max" ]; then + if [ "${LADDER_PAY_WITH_MAX:-0}" != "1" ]; then + LADDER_PAY=0 + fi +fi +export LADDER_MODE LADDER_MAX_PROBES LADDER_MAX_TOL_REL LADDER_HIGH_RUNGS LADDER_MAX_SEED LADDER_PAY + +SCRATCH=$(mktemp -d) +WDB="$SCRATCH/wallet.sqlite3" +REPORT_DIR="${LADDER_REPORT_DIR:-$SCRATCH}" +mkdir -p "$REPORT_DIR" +TSV="$REPORT_DIR/ladder-results.tsv" +PAY_TSV="$REPORT_DIR/ladder-pay-results.tsv" +JSON="$REPORT_DIR/ladder-report.json" +LOAD_BEFORE="$REPORT_DIR/load-before.json" +LOAD_AFTER="$REPORT_DIR/load-after.json" +echo -e "rung\trange\tamount\tstatus\tms_mint\tms_accept\tms_confirm\tms_settle\tms_total\twid\tnote" >"$TSV" +echo -e "rung\trange\tamount\tstatus\tms_order\tms_handle\tms_settle\tms_total\toid\tnote" >"$PAY_TSV" + +ladder_over() { + local now + now=$(date +%s) + [ $((now - SECTION_T0)) -ge "$LADDER_TIMEOUT_S" ] +} + +wcli() { + # taler-helper-sqlite3 is Python ≥3.11; prefer Homebrew/local python on macOS + # so Apple /usr/bin/python3 3.9 does not FATAL the sqlite backend (no reservePub). + local _path="${PATH:-}" + case ":${_path}:" in + *:/opt/homebrew/bin:*) ;; + *) _path="/opt/homebrew/bin:/usr/local/bin:${_path}" ;; + esac + if [ -n "${CLI_JS:-}" ] && [ -f "$CLI_JS" ]; then + PATH="$_path" node "$CLI_JS" --wallet-db="$WDB" --no-throttle "$@" + else + PATH="$_path" taler-wallet-cli --wallet-db="$WDB" --no-throttle "$@" + fi +} + +# Explorer pool password: env → file override → stack secrets → SSH +resolve_explorer_pw() { + local pw="" f="" host="" + if [ -n "${EXP_PW:-}" ]; then + printf '%s' "$EXP_PW" + return 0 + fi + if [ -n "${EXP_PW_FILE:-}" ] && [ -f "$EXP_PW_FILE" ]; then + tr -d '\n\r' <"$EXP_PW_FILE" + return 0 + fi + + # Stage TESTPAYSAN — never use GOA koopa-admin-secrets explorer pw first + if [ "${CUR}" = "TESTPAYSAN" ] \ + || [[ "${TALER_DOMAIN:-}" == stage.*lefrancpaysan* ]] \ + || [[ "${TALER_DOMAIN:-}" == *stage.lefrancpaysan* ]]; then + for f in \ + ${FRANCPAYSAN_SECRETS:+"${FRANCPAYSAN_SECRETS}/stage/bank-explorer-password.txt"} \ + "${HOME}/.config/taler-landing/stage-bank-explorer-password.txt" + do + [ -n "$f" ] && [ -f "$f" ] || continue + tr -d '\n\r' <"$f" + return 0 + done + _remote_exp="${FRANCPAYSAN_REMOTE_BANK_EXPLORER_SECRET:-}" + [ -z "$_remote_exp" ] && [ -n "${FRANCPAYSAN_REMOTE_SECRETS_ROOT:-}" ] && \ + _remote_exp="${FRANCPAYSAN_REMOTE_SECRETS_ROOT}/bank/secrets/bank-explorer-password.txt" + for host in ${INSIDE_SSH:+"$INSIDE_SSH"} ${FRANCPAYSAN_SSH:+"$FRANCPAYSAN_SSH"}; do + [ -n "$host" ] && [ -n "$_remote_exp" ] || continue + pw=$(ssh -o BatchMode=yes -o ConnectTimeout=12 "$host" \ + "tr -d '\\n\\r' <${_remote_exp} 2>/dev/null || sudo tr -d '\\n\\r' <${_remote_exp} 2>/dev/null" \ + 2>/dev/null || true) + if [ -n "$pw" ]; then + printf '%s' "$pw" + return 0 + fi + done + return 1 + fi + + # GOA / default: SECRETS_ROOT / ~/.config / koopa SSH + if pw=$(read_secret "taler-bank/bank-explorer-password.txt" 2>/dev/null) && [ -n "$pw" ]; then + printf '%s' "$pw" + return 0 + fi + for f in \ + "${SECRETS_ROOT:+${SECRETS_ROOT}/taler-bank/bank-explorer-password.txt}" \ + "${HOME}/.config/taler-landing/bank-explorer-password.txt" + do + [ -n "$f" ] && [ -f "$f" ] || continue + tr -d '\n\r' <"$f" + return 0 + done + return 1 +} + +wallet_avail() { + wcli balance 2>/dev/null | python3 -c ' +import json,sys +t=sys.stdin.read() +dec=json.JSONDecoder() +i=t.find("{") +if i<0: + print("0"); raise SystemExit +try: + d,_=dec.raw_decode(t,i) +except Exception: + # last-resort: brace slice (may fail on trailing logs) + try: + d=json.loads(t[i:t.rfind("}")+1]) + except Exception: + print("0"); raise SystemExit +cur=sys.argv[1] +for b in d.get("balances") or []: + a=b.get("available") or "" + if a.startswith(cur+":"): + print(a.split(":",1)[1]); raise SystemExit +print("0") +' "$CUR" 2>/dev/null || echo "0" +} + +# Build paired pay + withdraw ladders (same step count, shared random shape). +# Pay: [0] + log-uniform mids + [max-1] + [max] +# Wd: [0] + max(mid, mid×scale) mids + [max-1] + [max] (mids higher → enough to spend) +# Writes: $1 = withdraw list file, $2 = pay list file (space-separated CUR:amt lines as one line each) +build_ladder_pair() { + local wd_out="$1" pay_out="$2" + python3 - <<'PY' "$CUR" "${LADDER_MAX_AMOUNT}" "${LADDER_STEPS}" "${LADDER_MIN_AMOUNT}" "${LADDER_WITHDRAW_SCALE}" "$wd_out" "$pay_out" +import math, random, sys +from decimal import Decimal, ROUND_HALF_UP, ROUND_UP +from pathlib import Path + +cur = sys.argv[1] +max_amt = Decimal(sys.argv[2]) +steps = max(1, int(sys.argv[3])) +min_amt = Decimal(sys.argv[4]) +scale = Decimal(sys.argv[5]) +wd_path, pay_path = Path(sys.argv[6]), Path(sys.argv[7]) +if min_amt <= 0: + min_amt = Decimal("0.000001") +if min_amt >= max_amt: + min_amt = max_amt / Decimal(1000) +if scale < 1: + scale = Decimal(1) +max_m1 = max_amt - Decimal(1) if max_amt > 1 else max_amt + +def fmt(v: Decimal) -> str: + q = v.quantize(Decimal("0.00000001"), rounding=ROUND_HALF_UP) + if q == q.to_integral(): + return format(int(q), "d") + return format(q, "f").rstrip("0").rstrip(".") + +def quant(v: Decimal) -> Decimal: + if v >= 1: + return v.quantize(Decimal(1), rounding=ROUND_HALF_UP) + if v < min_amt: + return min_amt + return v.quantize(Decimal("0.00000001"), rounding=ROUND_UP) + +def amt(v: Decimal) -> str: + return "%s:%s" % (cur, fmt(v)) + +def make_mids(n_mid: int, hi_cap: Decimal): + if n_mid <= 0 or hi_cap <= min_amt: + return [] + lo, hi = float(min_amt), float(hi_cap) * 0.999999 + if hi <= lo: + hi = lo * 10 + cuts = sorted(math.exp(random.uniform(math.log(lo), math.log(hi))) for _ in range(n_mid)) + out, prev = [], Decimal(0) + for c in cuts: + v = quant(Decimal(str(c))) + if v <= prev: + step = min_amt if prev < 1 else max(prev * Decimal("1e-6"), Decimal(1)) + v = quant(prev + step) + if v >= hi_cap: + v = quant(hi_cap - (Decimal(1) if hi_cap > 1 else min_amt)) + if v <= prev or v >= hi_cap: + continue + out.append(v) + prev = v + return out + +# Build withdraw ladder first (full range), then pay mids = withdraw/scale +# so each pay mid is always cheaper than the matching withdraw mid. +if steps == 1: + wd_vals = [max_amt] +elif steps == 2: + wd_vals = [Decimal(0), max_amt] +else: + n_mid = max(0, steps - 3) + wd_vals = [Decimal(0)] + make_mids(n_mid, max_m1) + [max_m1, max_amt] + while len(wd_vals) > steps and len(wd_vals) > 3: + wd_vals.pop(len(wd_vals) // 2) + while len(wd_vals) < steps and len(wd_vals) >= 2: + i = max(1, len(wd_vals) - 2) + a, b = wd_vals[i - 1], wd_vals[i] + if a <= 0: + m = max(min_amt, b / 2 if b > 0 else min_amt) + else: + m = (a * b).sqrt() if a * b > 0 else (a + b) / 2 + m = quant(m) + if m <= a or m >= b: + break + wd_vals.insert(i, m) + wd_vals = wd_vals[:steps] + +pay_vals = [] +prev_p = Decimal(-1) +for i, w in enumerate(wd_vals): + is_first = i == 0 + is_last = i == len(wd_vals) - 1 + is_m1 = (not is_last) and w == max_m1 and i == len(wd_vals) - 2 + if is_first and w == 0: + pay_vals.append(Decimal(0)) + prev_p = Decimal(0) + continue + if is_last: + pay_vals.append(max_amt) + continue + if is_m1 or w == max_m1: + pay_vals.append(max_m1) + prev_p = max_m1 + continue + # pay mid = withdraw / scale (strictly less funding needed per step) + p = quant(w / scale) if scale > 0 else w + if p < min_amt and w >= min_amt: + p = min_amt + if p <= prev_p: + p = quant(prev_p + (min_amt if prev_p < 1 else Decimal(1))) + if p >= w: + # keep pay strictly below this withdraw rung when possible + p = quant(w - (Decimal(1) if w > 1 else min_amt)) if w > prev_p else prev_p + if p <= prev_p: + p = prev_p # flat ok only if stuck; still ≤ w + pay_vals.append(p) + prev_p = p + +assert len(pay_vals) == len(wd_vals) +# sanity: every non-pin pay mid ≤ matching withdraw mid +for i, (p, w) in enumerate(zip(pay_vals, wd_vals)): + if i == 0 or i >= len(wd_vals) - 2: + continue + if p > w: + pay_vals[i] = w + +wd_path.write_text(" ".join(amt(v) for v in wd_vals) + "\n") +pay_path.write_text(" ".join(amt(v) for v in pay_vals) + "\n") +print(" ".join(amt(v) for v in wd_vals)) +PY +} + +# shellcheck source=metrics.sh +source "$ROOT/metrics.sh" +METRICS_DIR="$REPORT_DIR" +ALT_UNITS_FILE="${REPORT_DIR}/alt_unit_names.json" +export METRICS_DIR CUR WDB CLI_JS ALT_UNITS_FILE +# Ladder can disable host load without killing coin metrics +if [ "${LADDER_LOAD:-1}" = "0" ]; then + METRICS_LOAD=0 + export METRICS_LOAD +fi + +if [ "${LADDER_MODE}" = "max" ]; then + section "ladder · ${CUR} max-search (find highest withdrawable)" +else + section "ladder · ${CUR} classic (0 → random → max · ${LADDER_STEPS} steps)" +fi +info "bank" "$BANK" +info "exchange" "$EX" +info "currency" "$CUR" +info "ladder mode" "${LADDER_MODE}" +# alt_unit_names for human amounts (Kilo-GOA / Mega-GOA / … + base in parentheses) +if metrics_load_alt_units "${EX%/}/config"; then + info "alt_unit_names" "from ${EX%/}/config → $ALT_UNITS_FILE" +else + warn "alt_unit_names" "using built-in SI fallback ($ALT_UNITS_FILE)" +fi +info "budget" "${LADDER_TIMEOUT_S}s" +_max_m1=$(python3 -c 'import sys; from decimal import Decimal; m=Decimal(sys.argv[1]); print(m-1 if m>1 else m)' "${LADDER_MAX_AMOUNT}" 2>/dev/null || echo "${LADDER_MAX_AMOUNT}-1") +if [ "${LADDER_MODE}" = "max" ]; then + info "max-search" "probes≤${LADDER_MAX_PROBES} high_rand=${LADDER_HIGH_RUNGS} tol=${LADDER_MAX_TOL_REL} ceiling=${CUR}:${LADDER_MAX_AMOUNT}" +else + info "steps" "${LADDER_STEPS} (0 + random≥${LADDER_MIN_AMOUNT} + max-1=${CUR}:${_max_m1} + max=${CUR}:${LADDER_MAX_AMOUNT})" + info "withdraw_scale" "${LADDER_WITHDRAW_SCALE}× pay mids (fund pay ladder)" +fi +info "pay_phase" "$([ "${LADDER_PAY}" = "1" ] && echo enabled || echo disabled)" +info "currency/domain" "${CUR} · ${TALER_DOMAIN:-?} · bank=${BANK}" + +set_group load +section "ladder · load snapshot (before withdraws)" +metrics_report_load "$LOAD_BEFORE" "ladder-start" || true +# Fresh wallet per rung — baseline empty (or last-rung DB if re-used later) +metrics_report_coins "ladder-start" || true + +if ! EXP_PW=$(resolve_explorer_pw); then + err bank "explorer password missing" \ + "set EXP_PW / EXP_PW_FILE or SECRETS_ROOT=…/koopa/host-root (taler-bank/bank-explorer-password.txt)${SECRETS_ROOT:+ · SECRETS_ROOT=${SECRETS_ROOT}}" + secrets_hint 2>/dev/null || true + exit 1 +fi +if [ -n "${SECRETS_ROOT:-}" ]; then + info "explorer secret" "resolved (SECRETS_ROOT=${SECRETS_ROOT} · stage uses stagepaysan secret first when CUR=TESTPAYSAN)" +else + info "explorer secret" "resolved via EXP_PW / EXP_PW_FILE / stage SSH / koopa" +fi +if [ -n "${CLI_JS:-}" ] && [ -f "${CLI_JS}" ]; then + info "wallet-cli" "$CLI_JS" +else + info "wallet-cli" "PATH taler-wallet-cli ($(command -v taler-wallet-cli 2>/dev/null || echo missing))" +fi + +# --- auto-account --- +t0=$(now_ms) +if ! curl -sS -m 30 -o "$SCRATCH/auto-account.json" "${BANK}/intro/auto-account.json"; then + err bank "auto-account.json unreachable" + exit 1 +fi +ms_auto=$(elapsed_ms "$t0") +if ! python3 -c 'import json; d=json.load(open("'"$SCRATCH"'/auto-account.json")); assert d.get("ok") or d.get("username")' 2>/dev/null; then + err bank "auto-account create failed" "$(head -c 120 "$SCRATCH/auto-account.json" | tr '\n' ' ')" + exit 1 +fi +ACCT_USER=$(python3 -c 'import json; print(json.load(open("'"$SCRATCH"'/auto-account.json"))["username"])') +ok "auto-account ${ACCT_USER} (${ms_auto}ms) — personal ${CUR}:0; pool=explorer" +info "auto-account password" "(see $SCRATCH/auto-account.json — not logged)" + +# --- explorer token --- +t0=$(now_ms) +TOK=$(curl -sS -m 20 -u "${EXP_USER}:${EXP_PW}" \ + -H 'Content-Type: application/json' \ + -d '{"scope":"readwrite","duration":{"d_us":3600000000}}' \ + "${BANK}/accounts/${EXP_USER}/token" \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["access_token"])') +ms_tok=$(elapsed_ms "$t0") +[ -n "$TOK" ] || { err bank "explorer token failed"; exit 1; } +ok "explorer token (${ms_tok}ms)" + +# ONE cumulative wallet for all withdraws + pays (need balance to spend). +# force-select uses *last* reserve_pub from the current accept output. +wallet_prepare() { + local label="${1:-wallet}" + WDB="$SCRATCH/wallet-${label}.sqlite3" + export WDB + if [ ! -f "$WDB" ]; then + wcli exchanges add "$EX" >"$SCRATCH/ex-add-$label.out" 2>&1 || true + wcli exchanges update "$EX" >"$SCRATCH/ex-upd-$label.out" 2>&1 || true + wcli exchanges accept-tos "$EX" >"$SCRATCH/ex-tos-$label.out" 2>&1 || true + fi +} + +t0=$(now_ms) +rm -f "$SCRATCH/wallet-main.sqlite3" +wallet_prepare "main" +ms_tos=$(elapsed_ms "$t0") +ok "wallet exchange + ToS (${ms_tos}ms) — cumulative DB for withdraw+pay (no run-until-done)" + +: "${LADDER_MAX_RUNGS:=99}" +if [ "${LADDER_MODE}" = "max" ]; then + info "ladder mode" "max-search (find highest withdrawable ${CUR})" + info "max-search params" "probes≤${LADDER_MAX_PROBES} high_rand=${LADDER_HIGH_RUNGS} tol_rel=${LADDER_MAX_TOL_REL} ceiling=${CUR}:${LADDER_MAX_AMOUNT}" + LADDER_LIST="" + PAY_LIST="" + LADDER_N=0 + set -- + printf '%s\n' "# max-search (amounts chosen adaptively)" >"$SCRATCH/ladder-plan.txt" + : >"$SCRATCH/ladder-wd-plan.txt" + : >"$SCRATCH/ladder-pay-plan.txt" +else + info "ladder mode" "classic (0 → random mids → max-1 → max)" + build_ladder_pair "$SCRATCH/ladder-wd-plan.txt" "$SCRATCH/ladder-pay-plan.txt" + LADDER_LIST=$(tr -d '\n' <"$SCRATCH/ladder-wd-plan.txt") + PAY_LIST=$(tr -d '\n' <"$SCRATCH/ladder-pay-plan.txt") + # shellcheck disable=SC2086 + set -- $LADDER_LIST + if [ "$#" -gt "$LADDER_MAX_RUNGS" ]; then + # shellcheck disable=SC2046 + set -- $(printf '%s\n' "$@" | head -n "$LADDER_MAX_RUNGS") + fi + LADDER_N=$# + info "withdraw plan" "$*" + info "withdraw plan (alt)" "$(format_amount_list_alt "$@")" + info "pay plan" "$PAY_LIST" + # shellcheck disable=SC2086 + info "pay plan (alt)" "$(format_amount_list_alt $PAY_LIST)" + printf '%s\n' "$@" >"$SCRATCH/ladder-plan.txt" + # shellcheck disable=SC2086 + printf '%s\n' $PAY_LIST >"$SCRATCH/ladder-pay-plan-lines.txt" 2>/dev/null || true +fi + +OK_N=0 +FAIL_N_L=0 +PAY_OK_N=0 +PAY_FAIL_N=0 +STOP_REASON="" +STOP_AMOUNT="" +MAX_BEST_AMT="" +MAX_LO="0" +MAX_HI="${LADDER_MAX_AMOUNT}" +declare -a RUNG_JSON=() + +# --------------------------------------------------------------------------- +# One withdraw rung. Caller sets: AMT, range_note, rung. +# Returns 0 = soft continue (ok/skip/ceiling); 1 = hard stop. +# Sets: status, AMT_NUM, WID, note, ms_*. +# --------------------------------------------------------------------------- +ladder_withdraw_rung() { + if ladder_over; then + STOP_REASON="timeout budget ${LADDER_TIMEOUT_S}s" + warn ladder "time budget exhausted" \ + "problem: LADDER_TIMEOUT_S=${LADDER_TIMEOUT_S}s reached after ${OK_N} ok rungs; remaining amounts not tried" + return 1 + fi + + # section header is printed by classic/max controller (avoids double banners) + tag=$(printf '%s' "$AMT" | tr '.:' '__') + # range_note set by caller (classic pins / max-search labels) + t_rung=$(now_ms) + ms_mint=0 ms_accept=0 ms_confirm=0 ms_settle=0 + note="" + status="FAIL" + WID="-" + FORCE_SEL_409_LOGGED=0 + + # keep cumulative main wallet + wallet_prepare "main" + + # mint from explorer pool + t0=$(now_ms) + code=$(curl -sS -m 30 -o "$SCRATCH/wd-$tag.json" -w '%{http_code}' \ + -H "Authorization: Bearer ${TOK}" \ + -H 'Content-Type: application/json' \ + -d "{\"amount\":\"${AMT}\"}" \ + "${BANK}/accounts/${EXP_USER}/withdrawals") + ms_mint=$(elapsed_ms "$t0") + WID=$(python3 -c 'import json;d=json.load(open("'"$SCRATCH"'/wd-'"$tag"'.json"));print(d.get("withdrawal_id") or "")' 2>/dev/null || true) + URI=$(python3 -c 'import json;u=json.load(open("'"$SCRATCH"'/wd-'"$tag"'.json")).get("taler_withdraw_uri") or "";print(u.replace(":443/","/"))' 2>/dev/null || true) + # numeric amount (for zero / settle / ceiling special-cases) + AMT_NUM=$(python3 -c 'import sys; print(sys.argv[1].split(":",1)[-1])' "$AMT") + IS_ZERO=0 + python3 -c 'import sys; from decimal import Decimal; sys.exit(0 if Decimal(sys.argv[1])==0 else 1)' "$AMT_NUM" 2>/dev/null && IS_ZERO=1 + IS_MAX_PIN=0 + IS_MAX_M1_PIN=0 + if [ "$AMT_NUM" = "${LADDER_MAX_AMOUNT}" ]; then + IS_MAX_PIN=1 + range_note="pin:max" + elif [ "$AMT_NUM" = "$((LADDER_MAX_AMOUNT - 1))" ] 2>/dev/null || \ + [ "$AMT_NUM" = "$(python3 -c 'print(int("'"$LADDER_MAX_AMOUNT"'")-1)')" ]; then + IS_MAX_M1_PIN=1 + range_note="pin:max-1" + fi + + if [ "$code" != "200" ] && [ "$code" != "201" ] || [ -z "$WID" ] || [ -z "$URI" ]; then + # strip quotes so STOP_REASON never breaks later shell/python argv + note="mint HTTP $code $(head -c 100 "$SCRATCH/wd-$tag.json" 2>/dev/null | tr '\n\"' ' ')" + ms_total=$(elapsed_ms "$t_rung") + if [ "$IS_ZERO" = "1" ]; then + # Probe only: bank may reject GOA:0 — record and continue ladder + status="ZERO_REJECT" + note="zero-withdraw rejected (expected possible): $note" + warn bank "mint $AMT rejected" \ + "problem: bank will not create a GOA:0 withdrawal (zero amount probe). Ladder continues. detail: $note" + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + OK_N=$((OK_N + 1)) + return 0 + fi + # Absolute max is a ceiling probe — bank often returns SQL P0001/5110; do not abort. + if [ "$IS_MAX_PIN" = "1" ]; then + status="CEILING_REJECT" + note="absolute max rejected (ceiling probe; max-1 is the hard pin): $note" + warn bank "mint $AMT rejected (ceiling)" \ + "problem: bank rejects absolute LADDER_MAX_AMOUNT (often SQL P0001/5110). max-1 rung is the last expected success. Ladder continues to report. detail: $note" + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + return 0 + fi + # max-1 can also hit the same libeufin SQL ceiling (5110/P0001) on live GOA — + # treat as soft ceiling when mid rungs already passed; do not block the report. + if [ "$IS_MAX_M1_PIN" = "1" ] && echo "$note" | grep -qE '5110|P0001'; then + status="CEILING_REJECT" + note="max-1 rejected (same ceiling as absolute max): $note" + warn bank "mint $AMT rejected (max-1 ceiling)" \ + "problem: bank rejects max-1 with SQL P0001/5110 (pool ceiling). Mid-rung OK_BANK/OK still count. Ladder continues. detail: $note" + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + return 0 + fi + # max-search: any mint reject is "too high" / ceiling — never hard-stop the hunt + if [ "${LADDER_MODE}" = "max" ]; then + status="CEILING_REJECT" + note="max-search too-high mint: $note" + warn bank "mint $AMT rejected (max-search)" \ + "problem: mint failed while hunting max withdrawable — treat as upper bound. detail: $note" + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + return 0 + fi + # Classic mid/high random rungs: same 5110/P0001 pool ceiling as pins — soft, not FAIL_MINT. + if echo "$note" | grep -qE '5110|P0001'; then + status="CEILING_REJECT" + note="amount above live mint ceiling (5110/P0001): $note" + warn bank "mint $AMT rejected (ceiling)" \ + "problem: bank mint HTTP 500/5110 SQL P0001 on high amount (not only pin:max). Ladder continues; use max-ladder to bound hi. detail: $note" + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + return 0 + fi + err bank "mint $AMT failed" "$note" + status="FAIL_MINT" + STOP_REASON="$note" + STOP_AMOUNT="$AMT" + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + FAIL_N_L=$((FAIL_N_L + 1)) + return 1 + fi + ok "mint $AMT ($WID) ${ms_mint}ms" + + before=$(wallet_avail) + + # accept + t0=$(now_ms) + if wcli withdraw accept-uri --exchange "$EX" "$URI" >"$SCRATCH/accept-$tag.out" 2>&1; then + ms_accept=$(elapsed_ms "$t0") + ok "accept-uri $AMT ${ms_accept}ms" + else + ms_accept=$(elapsed_ms "$t0") + note="accept-uri failed: $(tail -c 200 "$SCRATCH/accept-$tag.out" | tr '\n' ' ')" + ms_total=$(elapsed_ms "$t_rung") + # Wallet 7006: no denominations for this amount (0, sub-denom dust, etc.) — warn & continue + if [ "$IS_ZERO" = "1" ] || grep -qE 'code: 7006|"code"[[:space:]]*:[[:space:]]*7006|No denominations could be selected' \ + "$SCRATCH/accept-$tag.out" 2>/dev/null; then + if [ "$IS_ZERO" = "1" ]; then + status="ZERO_SKIP" + note="zero-withdraw skip (7006 / no denoms): $note" + warn wallet "accept $AMT skipped" \ + "problem: wallet code 7006 — no coin denominations for GOA:0 (zero amount cannot be withdrawn as coins). Ladder continues. detail: $note" + else + status="SKIP_DENOM" + note="skip amount (wallet 7006 no denoms): $note" + warn wallet "accept $AMT skipped" \ + "problem: wallet code 7006 — no denominations match this amount (below smallest coin or not combinable). Ladder continues with next rung. detail: $note" + fi + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + return 0 + fi + if [ "${LADDER_MODE}" = "max" ]; then + status="SKIP_ACCEPT" + warn wallet "accept $AMT skipped (max-search)" "$note" + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + return 0 + fi + err wallet "accept $AMT" "$note" + status="FAIL_ACCEPT" + STOP_REASON="$note" + STOP_AMOUNT="$AMT" + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + FAIL_N_L=$((FAIL_N_L + 1)) + return 1 + fi + + # Confirm ASAP when bank status is selected. No run-until-done (hangs on macOS/wallet). + # Server-side auto-confirm only watches landing withdraw-watch.ids — ladder must confirm itself. + bank_st() { + curl -sS -m 8 "${BANK}/taler-integration/withdrawal-operation/${WID}" 2>/dev/null \ + | python3 -c 'import json,sys; print((json.load(sys.stdin).get("status") or "").strip())' 2>/dev/null || true + } + do_confirm() { + curl -sS -m 15 -o "$SCRATCH/conf-$tag.json" -w '%{http_code}' -X POST \ + -H "Authorization: Bearer ${TOK}" -H 'Content-Type: application/json' -d '{}' \ + "${BANK}/accounts/${EXP_USER}/withdrawals/${WID}/confirm" + } + # Collect reserve_pub candidates for *this* withdrawal (WID + amount). + # Cumulative wallets re-print old reserves in accept/tx dumps — never trust a single "last" blindly. + # Prints unique pubs one per line, preferred order first. + # + # Critical: wallet-cli dumps mix log lines + JSON. Never json.loads(rest_of_file) — + # trailing "Shutdown requested" lines make that always fail. Use JSONDecoder.raw_decode + # and regex on *both* accept and transactions output (reservePub lives in tx, not accept). + extract_rpubs_for_wid() { + # External module — never feed extract via bash heredoc (refactor tools + # previously turned Python continue into return → SyntaxError line 91). + local _ext="$ROOT/ladder_extract_rpubs.py" + if [ ! -f "$_ext" ]; then + warn bank "force-select extract missing" "problem: $_ext not found" + return 0 + fi + python3 "$_ext" "$WID" "$AMT" \ + "$SCRATCH/accept-$tag.out" "$SCRATCH/tx-$tag.json" "$SCRATCH/used-rpubs.txt" + } + + mark_rpub_used() { + local p="$1" + [ -n "$p" ] || return 0 + mkdir -p "$SCRATCH" 2>/dev/null || true + grep -qxF "$p" "$SCRATCH/used-rpubs.txt" 2>/dev/null || echo "$p" >>"$SCRATCH/used-rpubs.txt" + } + + force_select_if_needed() { + local st_now="$1" + [ "$st_now" = "pending" ] || [ -z "$st_now" ] || return 0 + local rpub epayto code_fs any=0 + # refresh tx dump each try (wallet may attach reserve late) + wcli transactions >"$SCRATCH/tx-$tag.json" 2>&1 || true + epayto=$(curl -sS -m 10 "${EX%/}/keys" 2>/dev/null | python3 -c ' +import json,sys +d=json.load(sys.stdin) +acc=d.get("accounts") or [] +for a in acc: + p=a.get("payto_uri") or a.get("payto_address") or "" + if "x-taler-bank" in p or "exchange" in p: + print(p); break +else: + if acc: print(acc[0].get("payto_uri") or "") +' 2>/dev/null || true) + if [ -z "$epayto" ]; then + warn bank "force-select skipped" "problem: exchange payto empty from /keys" + return 0 + fi + # Try candidates until bank leaves pending (200/204) or we exhaust + while IFS= read -r rpub; do + [ -n "$rpub" ] || continue + any=1 + code_fs=$(curl -sS -m 12 -o "$SCRATCH/force-sel-$tag.json" -w '%{http_code}' -X POST \ + -H 'Content-Type: application/json' \ + -d "{\"reserve_pub\":\"${rpub}\",\"selected_exchange\":\"${epayto}\"}" \ + "${BANK}/taler-integration/withdrawal-operation/${WID}" 2>/dev/null || echo "000") + if [ "$code_fs" = "200" ] || [ "$code_fs" = "204" ]; then + info "force-select" "HTTP ${code_fs} rpub=${rpub:0:12}… (ok for WID ${WID:0:8})" + mark_rpub_used "$rpub" + return 0 + fi + if [ "$code_fs" = "409" ]; then + # 5114 = this reserve already bound to another op — not "out of money" + info "force-select" "HTTP 409 rpub=${rpub:0:12}… (stale/used reserve — not balance; trying next)" + mark_rpub_used "$rpub" + continue + fi + info "force-select" "HTTP ${code_fs} rpub=${rpub:0:12}… body=$(tr '\n' ' ' <"$SCRATCH/force-sel-$tag.json" 2>/dev/null | head -c 120)" + # other errors: still try next candidate + done < <(extract_rpubs_for_wid) + if [ "$any" != "1" ]; then + warn bank "force-select skipped" \ + "problem: no reserve_pub for this withdraw (WID=${WID:0:8}…); wallet may not have selected yet" + fi + } + + # No run-until-done (hangs / banned). Read transactions only for reserve_pub candidates. + wcli transactions >"$SCRATCH/tx-$tag.json" 2>&1 || true + + t0=$(now_ms) + conf_ok=0 + st="" + # Poll bank status; force-select while pending; confirm as soon as selected + for i in $(seq 1 "${LADDER_CONFIRM_POLLS}"); do + ladder_over && break + st=$(bank_st) + case "$st" in + selected) + ccode=$(do_confirm) + if [ "$ccode" = "204" ] || [ "$ccode" = "200" ]; then + conf_ok=1 + info "confirm" "HTTP $ccode after selected (poll $i)" + else + note="confirm HTTP $ccode" + fi + break + ;; + confirmed) + conf_ok=1 + break + ;; + aborted) + note="withdrawal aborted by bank" + break + ;; + esac + # force-select while pending — try alternate rpubs on 5114 (not out-of-money) + if [ "$st" = "pending" ] || [ -z "$st" ]; then + if [ "$i" -eq 1 ] || [ "$i" -eq 2 ] || [ $((i % 3)) -eq 0 ]; then + force_select_if_needed "$st" + fi + fi + sleep 0.35 + done + ms_confirm=$(elapsed_ms "$t0") + st=$(printf '%s' "${st:-}" | tr -d '\r\n' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + if [ "$conf_ok" != "1" ]; then + note="${note:-confirm timeout last=${st:-empty}}" + # Soft: not confirmed — usually stale reserve_pub (5114), NOT empty pool balance + status="SKIP_CONFIRM" + warn bank "confirm $AMT skipped" \ + "problem: bank status='${st:-empty}' after ${LADDER_CONFIRM_POLLS} polls (want selected/confirmed). Usually reserve_pub mismatch (5114), not out-of-money — mint/accept already OK. detail: $note" + ms_total=$(elapsed_ms "$t_rung") + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + return 0 + fi + ok "confirm $AMT ${ms_confirm}ms (client, on selected)" + + # settle: poll wallet balance + bank transfer_done only — never run-until-done + t0=$(now_ms) + settled=0 + xfer="?" + if [ "$IS_ZERO" = "1" ]; then + settled=1 + note="zero-amount: no coin delta expected" + else + for r in $(seq 1 "$LADDER_SETTLE_ROUNDS"); do + ladder_over && break + after=$(wallet_avail) + if python3 -c " +from decimal import Decimal +import sys +sys.exit(0 if Decimal(sys.argv[1]) > Decimal(sys.argv[2]) else 1) +" "$after" "$before" 2>/dev/null; then + settled=1 + break + fi + xfer=$(curl -sS -m 10 "${BANK}/taler-integration/withdrawal-operation/${WID}" \ + | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("transfer_done"), d.get("status"))' 2>/dev/null || echo "?") + # bank done is enough to leave settle without hanging on wallet + if echo "$xfer" | grep -qi True; then + note="bank transfer_done (no run-until-done) avail=${after} $xfer" + break + fi + sleep "$LADDER_SETTLE_SLEEP" + done + fi + ms_settle=$(elapsed_ms "$t0") + after=$(wallet_avail) + ms_total=$(elapsed_ms "$t_rung") + if [ -z "${xfer:-}" ] || [ "$xfer" = "?" ]; then + xfer=$(curl -sS -m 10 "${BANK}/taler-integration/withdrawal-operation/${WID}" \ + | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("transfer_done"), d.get("status"))' 2>/dev/null || echo "?") + fi + + if [ "$settled" = "1" ]; then + status="OK" + note="${note:-avail=${CUR}:${after}}" + ok "settle $AMT → ${CUR}:${after} (settle ${ms_settle}ms, rung ${ms_total}ms)" + OK_N=$((OK_N + 1)) + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + metrics_report_coins "ladder-r${rung}-after-${tag}" || true + metrics_record_flow withdrawn "$AMT" || true + elif echo "$xfer" | grep -qi True; then + status="OK_BANK" + note="bank transfer_done avail=${after} $xfer (no run-until-done)" + ok "settle $AMT bank transfer_done (wallet avail=${CUR}:${after}, ${ms_settle}ms)" + OK_N=$((OK_N + 1)) + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + metrics_report_coins "ladder-r${rung}-after-${tag}" || true + metrics_record_flow withdrawn "$AMT" || true + else + note="no coins / no transfer_done avail=${after} $xfer" + if [ "${LADDER_MODE}" = "max" ]; then + status="SKIP_SETTLE" + warn wallet "settle $AMT skipped (max-search)" \ + "problem: bank confirm ok but settle inconclusive — do not move max bounds. detail: $note" + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + return 0 + fi + err wallet "settle $AMT" "$note" + status="FAIL_SETTLE" + STOP_REASON="$note" + STOP_AMOUNT="$AMT" + FAIL_N_L=$((FAIL_N_L + 1)) + echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" + metrics_report_coins "ladder-r${rung}-fail-${tag}" || true + return 1 + fi + return 0 +} + +set_group withdraw + +# --- next amount for max-search (prints: AMT_NUM|range_note|action) --- +# action: probe | done +max_search_next() { + python3 - "$CUR" "$MAX_LO" "$MAX_HI" "$LADDER_MIN_AMOUNT" \ + "$LADDER_MAX_TOL_REL" "$rung" "$LADDER_HIGH_RUNGS" "$LADDER_MAX_SEED" \ + "${MAX_BEST_OK:-}" <<'PY' +import math, random, sys +from decimal import Decimal, ROUND_HALF_UP, ROUND_UP + +cur = sys.argv[1] +lo = Decimal(sys.argv[2] or "0") +hi = Decimal(sys.argv[3]) +min_amt = Decimal(sys.argv[4]) +tol = Decimal(sys.argv[5] or "0.001") +probe_i = int(sys.argv[6]) +high_n = max(1, int(sys.argv[7] or "4")) +seed = sys.argv[8].strip() +best = sys.argv[9].strip() + +if seed: + random.seed(int(seed) + probe_i) +else: + random.seed() + +if min_amt <= 0: + min_amt = Decimal("0.000001") +if hi <= min_amt: + print("0|max:done|done") + raise SystemExit(0) + +if lo > 0 and hi > lo: + gap = (hi - lo) / hi + abs_step = Decimal(1) if hi >= 1 else min_amt + if gap <= tol or (hi - lo) <= abs_step: + print("%s|max:done|done" % (best or str(lo))) + raise SystemExit(0) + +def fmt(v: Decimal) -> str: + if v >= 1: + q = v.quantize(Decimal(1), rounding=ROUND_HALF_UP) + return format(int(q), "d") + q = v.quantize(Decimal("0.00000001"), rounding=ROUND_UP) + s = format(q, "f").rstrip("0").rstrip(".") + return s or "0" + +def clamp(v: Decimal): + if v <= lo and lo > 0: + v = lo + (Decimal(1) if lo >= 1 else min_amt) + if v >= hi: + v = hi - (Decimal(1) if hi > 1 else min_amt) + if v < min_amt: + v = min_amt + if v <= 0: + v = min_amt + if v >= hi: + return None + return v + +if probe_i < high_n: + band_lo = max(lo if lo > 0 else min_amt, hi * Decimal("0.15")) + band_hi = hi * Decimal("0.999") + if band_hi <= band_lo: + band_lo = max(min_amt, hi * Decimal("0.5")) + band_hi = hi * Decimal("0.99") + flo, fhi = float(band_lo), float(band_hi) + if flo <= 0: + flo = float(min_amt) + if fhi <= flo: + fhi = flo * 2 + v = Decimal(str(math.exp(random.uniform(math.log(flo), math.log(fhi))))) + v = clamp(v) + if v is None: + print("%s|max:done|done" % (best or str(lo))) + raise SystemExit(0) + print("%s|max:high-rand|probe" % fmt(v)) + raise SystemExit(0) + +if lo <= 0: + flo = float(max(min_amt, hi * Decimal("0.05"))) + fhi = float(hi * Decimal("0.85")) + if fhi <= flo: + fhi = flo * 1.5 + v = Decimal(str(math.exp(random.uniform(math.log(flo), math.log(fhi))))) + label = "max:descent" +else: + v = (lo * hi).sqrt() + label = "max:bisect" +v = clamp(v) +if v is None: + print("%s|max:done|done" % (best or str(lo))) + raise SystemExit(0) +print("%s|%s|probe" % (fmt(v), label)) +PY +} + +if [ "${LADDER_MODE}" = "max" ]; then + section "ladder · phase A · max-search (find highest withdrawable ${CUR})" + info "max-search" "ceiling=${CUR}:${LADDER_MAX_AMOUNT} min=${CUR}:${LADDER_MIN_AMOUNT} probes≤${LADDER_MAX_PROBES} high_rand=${LADDER_HIGH_RUNGS} tol_rel=${LADDER_MAX_TOL_REL}" + info "max-search" "pay phase $([ "${LADDER_PAY}" = "1" ] && echo enabled || echo disabled)" + MAX_LO="0" + MAX_HI="${LADDER_MAX_AMOUNT}" + MAX_BEST_OK="" + MAX_BEST_AMT="" + rung=0 + # LADDER_MAX_PROBES=0 → no probe cap (stop on converge / timeout only) + while [ "${LADDER_MAX_PROBES}" = "0" ] || [ "$rung" -lt "${LADDER_MAX_PROBES}" ]; do + if ladder_over; then + STOP_REASON="timeout budget ${LADDER_TIMEOUT_S}s" + warn ladder "time budget exhausted" \ + "problem: LADDER_TIMEOUT_S=${LADDER_TIMEOUT_S}s during max-search after ${OK_N} ok; best=${MAX_BEST_AMT:-none} · $(format_amount_alt "${MAX_BEST_AMT:-${CUR}:0}")" + break + fi + _next=$(max_search_next) + _anum=$(printf '%s' "$_next" | cut -d'|' -f1) + range_note=$(printf '%s' "$_next" | cut -d'|' -f2) + _act=$(printf '%s' "$_next" | cut -d'|' -f3) + if [ "$_act" = "done" ]; then + info "max-search" "converged best=$(format_amount_alt "${MAX_BEST_AMT:-${CUR}:0}") · lo=$(format_amount_alt "${CUR}:${MAX_LO}") · hi=$(format_amount_alt "${CUR}:${MAX_HI}") · probes=${rung}" + break + fi + AMT="${CUR}:${_anum}" + if [ -n "${_LAST_MAX_AMT:-}" ] && [ "$AMT" = "$_LAST_MAX_AMT" ]; then + info "max-search" "repeat amount $(format_amount_alt "$AMT") — tightening hi" + MAX_HI=$(python3 -c 'from decimal import Decimal; a=Decimal("'"${_anum}"'"); print(a-(1 if a>=1 else Decimal("'"${LADDER_MIN_AMOUNT}"'")))') + continue + fi + _LAST_MAX_AMT="$AMT" + rung=$((rung + 1)) + section "ladder · max probe $rung $AMT · $(format_amount_alt "$AMT") [lo=$(format_amount_alt "${CUR}:${MAX_LO}") · hi=$(format_amount_alt "${CUR}:${MAX_HI}")]" + status="FAIL" + if ! ladder_withdraw_rung; then + break + fi + AMT_NUM=$(python3 -c 'import sys; print(sys.argv[1].split(":",1)[-1])' "$AMT") + case "$status" in + OK|OK_BANK) + MAX_LO="$AMT_NUM" + MAX_BEST_OK="$AMT_NUM" + MAX_BEST_AMT="$AMT" + info "max-search" "success bound lo=$(format_amount_alt "${CUR}:${MAX_LO}") (best so far)" + ;; + CEILING_REJECT|FAIL_MINT) + MAX_HI=$(python3 -c ' +from decimal import Decimal +a=Decimal("'"$AMT_NUM"'"); h=Decimal("'"$MAX_HI"'") +print(min(a,h)) +') + info "max-search" "too-high hi:=$(format_amount_alt "${CUR}:${MAX_HI}")" + ;; + SKIP_DENOM|ZERO_SKIP|ZERO_REJECT) + info "max-search" "denom-skip $(format_amount_alt "$AMT") (bounds unchanged)" + ;; + SKIP_CONFIRM|SKIP_SETTLE|SKIP_ACCEPT) + info "max-search" "inconclusive $status $(format_amount_alt "$AMT") (bounds unchanged)" + ;; + *) + info "max-search" "status=$status $(format_amount_alt "$AMT") (bounds unchanged)" + ;; + esac + if python3 -c 'from decimal import Decimal; import sys; sys.exit(0 if Decimal(sys.argv[1])>0 and Decimal(sys.argv[2])<=Decimal(sys.argv[1]) else 1)' "$MAX_LO" "$MAX_HI" 2>/dev/null; then + info "max-search" "bounds crossed lo=$(format_amount_alt "${CUR}:${MAX_LO}") · hi=$(format_amount_alt "${CUR}:${MAX_HI}") — stop" + break + fi + done + if [ -n "${MAX_BEST_AMT:-}" ]; then + ok "max-search best withdrawable $(format_amount_alt "$MAX_BEST_AMT") · lo=$(format_amount_alt "${CUR}:${MAX_LO}") · hi=$(format_amount_alt "${CUR}:${MAX_HI}") · probes=${rung}" + else + warn ladder "max-search no successful withdraw" \ + "problem: no OK/OK_BANK within ${rung} probes; hi=$(format_amount_alt "${CUR}:${MAX_HI}")" + fi + { + echo "# max-search mode" + echo "best=${MAX_BEST_AMT:-}" + echo "best_alt=$(format_amount_alt "${MAX_BEST_AMT:-${CUR}:0}")" + echo "lo=${MAX_LO}" + echo "lo_alt=$(format_amount_alt "${CUR}:${MAX_LO}")" + echo "hi=${MAX_HI}" + echo "hi_alt=$(format_amount_alt "${CUR}:${MAX_HI}")" + echo "probes=${rung}" + } >"$SCRATCH/ladder-plan.txt" + printf '%s\n' "${MAX_BEST_AMT:-${CUR}:0}" >"$SCRATCH/ladder-wd-plan.txt" + : >"$SCRATCH/ladder-pay-plan.txt" + PAY_LIST="" +else + section "ladder · phase A · withdraw (${LADDER_N} rungs)" + rung=0 + for AMT in "$@"; do + rung=$((rung + 1)) + if ladder_over; then + STOP_REASON="timeout budget ${LADDER_TIMEOUT_S}s" + warn ladder "time budget exhausted" \ + "problem: LADDER_TIMEOUT_S=${LADDER_TIMEOUT_S}s reached after ${OK_N} ok rungs; remaining amounts not tried" + break + fi + if [ "$rung" -eq 1 ]; then + range_note="pin:0" + elif [ "$rung" -eq "$LADDER_N" ]; then + range_note="pin:max" + elif [ "$rung" -eq $((LADDER_N - 1)) ] && [ "$LADDER_N" -ge 3 ]; then + range_note="pin:max-1" + else + range_note="random" + fi + section "ladder · rung $rung $AMT · $(format_amount_alt "$AMT")" + if ! ladder_withdraw_rung; then + break + fi + done +fi + + +# --------------------------------------------------------------------------- +# Phase B — payment ladder (same shape: 0 → low → … → max-1 → max) +# --------------------------------------------------------------------------- +PAY_OK_N=0 +PAY_FAIL_N=0 +if [ "${LADDER_PAY}" = "1" ] && [ -n "${PAY_LIST:-}" ] && [ "$FAIL_N_L" -eq 0 ]; then + set_group pay + section "ladder · phase B · pay" + metrics_report_coins "before-pay-ladder" || true + # Merchant secret (same as e2e) — stage often has only public shop templates + MPW="${E2E_MERCHANT_TOKEN:-${MERCHANT_TOKEN:-}}" + if [ -z "$MPW" ]; then + MPW=$(read_secret "taler-merchant/merchant-${INST}-password.txt" 2>/dev/null || true) + fi + if [ -z "$MPW" ]; then + MPW=$(read_secret "taler-merchant/merchant-goa-demo-cp4zqk-password.txt" 2>/dev/null || true) + fi + if [ -z "$MPW" ] && [ "${CUR}" = "TESTPAYSAN" ]; then + if [ -n "${FRANCPAYSAN_SECRETS:-}" ] && [ -f "${FRANCPAYSAN_SECRETS}/stage/default-instance-token.txt" ]; then + MPW=$(tr -d '\n\r' <"${FRANCPAYSAN_SECRETS}/stage/default-instance-token.txt") + fi + if [ -z "$MPW" ]; then + _remote_mer="${FRANCPAYSAN_REMOTE_MERCHANT_TOKEN:-}" + [ -z "$_remote_mer" ] && [ -n "${FRANCPAYSAN_REMOTE_SECRETS_ROOT:-}" ] && \ + _remote_mer="${FRANCPAYSAN_REMOTE_SECRETS_ROOT}/merchant/secrets/default-instance-token.txt" + for host in ${INSIDE_SSH:+"$INSIDE_SSH"} ${FRANCPAYSAN_SSH:+"$FRANCPAYSAN_SSH"}; do + [ -n "$host" ] && [ -n "$_remote_mer" ] || continue + MPW=$(ssh -o BatchMode=yes -o ConnectTimeout=12 "$host" \ + "tr -d '\\n\\r' <${_remote_mer} 2>/dev/null || true" \ + 2>/dev/null || true) + [ -n "$MPW" ] && break + done + unset _remote_mer + fi + fi + if [ -z "$MPW" ]; then + if [ "${CUR}" = "TESTPAYSAN" ]; then + # Stage: withdraw ladder is the main probe; pays need instance token or + # public templates for *exact* face values (not continuous ladder amounts). + info pay "no merchant token — skip pay ladder on TESTPAYSAN (withdraw maxima still covered)" + info pay "hint: set E2E_MERCHANT_TOKEN or use ./taler-monitoring.sh -d stage.lefrancpaysan.ch e2e for shop templates" + else + warn pay "no merchant token — skip pay ladder (set E2E_MERCHANT_TOKEN or secrets)" + fi + else + ok "merchant token" "instance ${INST}" + case "$MPW" in + secret-token:*) AUTH="Authorization: Bearer ${MPW}" ;; + *) AUTH="Authorization: Bearer secret-token:${MPW}" ;; + esac + wallet_prepare "main" + # shellcheck disable=SC2086 + set -- $PAY_LIST + PAY_N=$# + _bal0=$(wallet_avail) + info "pay rungs" "$PAY_N · $*" + info "pay budget" "wallet available ${CUR}:${_bal0} · $(format_amount_alt "${CUR}:${_bal0}") — never order more than this" + prung=0 + for PAMT in "$@"; do + prung=$((prung + 1)) + if ladder_over; then + warn pay "time budget exhausted" "after ${PAY_OK_N} ok pays" + break + fi + section "ladder · pay rung $prung $PAMT · $(format_amount_alt "$PAMT")" + ptag=$(printf '%s' "$PAMT" | tr '.:' '__') + if [ "$prung" -eq 1 ]; then + prange="pin:0" + elif [ "$prung" -eq "$PAY_N" ]; then + prange="pin:max" + elif [ "$prung" -eq $((PAY_N - 1)) ] && [ "$PAY_N" -ge 3 ]; then + prange="pin:max-1" + else + prange="random" + fi + PNUM=$(python3 -c 'import sys; print(sys.argv[1].split(":",1)[-1])' "$PAMT") + IS_PZERO=0 + python3 -c 'import sys; from decimal import Decimal; sys.exit(0 if Decimal(sys.argv[1])==0 else 1)' "$PNUM" 2>/dev/null && IS_PZERO=1 + IS_PMAX=0 + [ "$PNUM" = "${LADDER_MAX_AMOUNT}" ] && IS_PMAX=1 + t_pay=$(now_ms) + ms_order=0 ms_handle=0 ms_psettle=0 + pnote="" + pstatus="FAIL" + OID="-" + + metrics_report_coins "before-pay-${ptag}" || true + bal=$(wallet_avail) + + if [ "$IS_PZERO" = "1" ]; then + pstatus="ZERO_SKIP" + pnote="zero pay probe skipped" + warn pay "pay $PAMT skipped" "problem: zero amount order not useful. Ladder continues." + ms_total=$(elapsed_ms "$t_pay") + echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t${ms_handle}\t${ms_psettle}\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" + PAY_OK_N=$((PAY_OK_N + 1)) + continue + fi + + # Never spend more than wallet available (spendable coins only — not pendingIncoming). + # Soft-skip any over-budget rung; do not hard-fail the ladder for insufficient funds. + if ! python3 -c 'import sys; from decimal import Decimal; sys.exit(0 if Decimal(sys.argv[1])>=Decimal(sys.argv[2]) else 1)' "$bal" "$PNUM" 2>/dev/null; then + pnote="skip: would spend more than available avail=${CUR}:${bal} need=${PAMT}" + ms_total=$(elapsed_ms "$t_pay") + pstatus="SKIP_BALANCE" + warn pay "pay $PAMT skipped (balance)" \ + "problem: pay amount exceeds wallet available — never overspend. $(format_amount_alt "${CUR}:${bal}") free vs $(format_amount_alt "$PAMT"). detail: $pnote" + echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t0\t0\t0\t${ms_total}\t-\t${pnote}" >>"$PAY_TSV" + continue + fi + + t0=$(now_ms) + SUM_JSON=$(python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "ladder pay ${PAMT}" 2>/dev/null || echo '"ladder pay"') + curl -skS -m 20 -o "$SCRATCH/ord-$ptag.json" -X POST \ + -H "$AUTH" -H 'Content-Type: application/json' \ + -d "{\"order\":{\"summary\":${SUM_JSON},\"amount\":\"${PAMT}\",\"fulfillment_message\":\"ok\"},\"create_token\":true}" \ + "${MER}/instances/${INST}/private/orders" 2>"$SCRATCH/ord-$ptag.err" || true + ms_order=$(elapsed_ms "$t0") + OID=$(python3 -c 'import json,re,sys;t=open(sys.argv[1]).read() +try: print(json.loads(t).get("order_id") or "") +except Exception: + m=re.search(r"\"order_id\"\s*:\s*\"([^\"]+)\"",t); print(m.group(1) if m else "") +' "$SCRATCH/ord-$ptag.json" 2>/dev/null || true) + OTOK=$(python3 -c 'import json,re,sys;t=open(sys.argv[1]).read() +try: print(json.loads(t).get("token") or "") +except Exception: + m=re.search(r"\"token\"\s*:\s*\"([^\"]+)\"",t); print(m.group(1) if m else "") +' "$SCRATCH/ord-$ptag.json" 2>/dev/null || true) + if [ -z "$OID" ]; then + pnote="order create failed $(head -c 80 "$SCRATCH/ord-$ptag.json" 2>/dev/null | tr '\n\"' ' ')" + ms_total=$(elapsed_ms "$t_pay") + if [ "$IS_PMAX" = "1" ]; then + pstatus="CEILING_REJECT" + warn pay "pay $PAMT rejected (ceiling)" "$pnote" + echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t0\t0\t${ms_total}\t-\t${pnote}" >>"$PAY_TSV" + continue + fi + err pay "order $PAMT" "$pnote" + pstatus="FAIL_ORDER" + PAY_FAIL_N=$((PAY_FAIL_N + 1)) + FAIL_N_L=$((FAIL_N_L + 1)) + STOP_REASON="$pnote" + STOP_AMOUNT="$PAMT" + echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t0\t0\t${ms_total}\t-\t${pnote}" >>"$PAY_TSV" + break + fi + ok "order $OID ($PAMT) ${ms_order}ms" + + curl -skS -m 12 -o "$SCRATCH/ord-det-$ptag.json" -H "$AUTH" \ + "${MER}/instances/${INST}/private/orders/${OID}" 2>/dev/null || true + PAYURI=$(python3 -c 'import json,sys +try: print(json.load(open(sys.argv[1])).get("taler_pay_uri") or "") +except Exception: print("") +' "$SCRATCH/ord-det-$ptag.json" 2>/dev/null || true) + if [ -z "$PAYURI" ] && [ -n "$OTOK" ]; then + MH=$(python3 -c 'from urllib.parse import urlparse; print(urlparse("'"$MER"'").hostname or "taler.hacktivism.ch")' 2>/dev/null || echo "taler.hacktivism.ch") + PAYURI="taler://pay/${MH}/instances/${INST}/${OID}/?c=${OTOK}" + fi + PAYURI=$(printf '%s' "$PAYURI" | sed 's/:443\//\//g; s/:443?/?/g') + if [ -z "$PAYURI" ]; then + pnote="no pay URI for $OID" + ms_total=$(elapsed_ms "$t_pay") + err pay "uri $PAMT" "$pnote" + pstatus="FAIL_URI" + PAY_FAIL_N=$((PAY_FAIL_N + 1)) + FAIL_N_L=$((FAIL_N_L + 1)) + STOP_REASON="$pnote" + STOP_AMOUNT="$PAMT" + echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t0\t0\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" + break + fi + + t0=$(now_ms) + if ! wcli handle-uri --yes "$PAYURI" >"$SCRATCH/pay-$ptag.out" 2>&1; then + ms_handle=$(elapsed_ms "$t0") + pnote="handle-uri failed $(tail -c 100 "$SCRATCH/pay-$ptag.out" | tr '\n\"' ' ')" + ms_total=$(elapsed_ms "$t_pay") + if [ "$IS_PMAX" = "1" ]; then + pstatus="CEILING_REJECT" + warn pay "pay $PAMT handle failed (ceiling)" "$pnote" + echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t${ms_handle}\t0\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" + continue + fi + err pay "handle $PAMT" "$pnote" + pstatus="FAIL_HANDLE" + PAY_FAIL_N=$((PAY_FAIL_N + 1)) + FAIL_N_L=$((FAIL_N_L + 1)) + STOP_REASON="$pnote" + STOP_AMOUNT="$PAMT" + echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t${ms_handle}\t0\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" + break + fi + ms_handle=$(elapsed_ms "$t0") + ok "handle-uri $PAMT ${ms_handle}ms" + + # Short settle polls: merchant order + wallet tx only — never run-until-done + t0=$(now_ms) + settled=0 + r=0 + while [ "$r" -lt "${LADDER_PAY_SETTLE_ROUNDS}" ]; do + r=$((r + 1)) + wcli transactions >"$SCRATCH/tx-$ptag.out" 2>&1 || true + curl -skS -m 8 -o "$SCRATCH/ord-paid-$ptag.json" -H "$AUTH" \ + "${MER}/instances/${INST}/private/orders/${OID}" 2>/dev/null || true + if grep -qiE 'payment|paid|Payment' "$SCRATCH/tx-$ptag.out" 2>/dev/null \ + || python3 -c 'import json,sys +d=json.load(open(sys.argv[1])) +sys.exit(0 if d.get("paid") is True or str(d.get("order_status","")).lower()=="paid" else 1) +' "$SCRATCH/ord-paid-$ptag.json" 2>/dev/null; then + settled=1 + break + fi + sleep 0.5 + done + ms_psettle=$(elapsed_ms "$t0") + ms_total=$(elapsed_ms "$t_pay") + after=$(wallet_avail) + if [ "$settled" = "1" ]; then + pstatus="OK" + pnote="avail=${CUR}:${after}" + ok "pay settled $PAMT → bal ${CUR}:${after} (total ${ms_total}ms)" + PAY_OK_N=$((PAY_OK_N + 1)) + echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t${ms_handle}\t${ms_psettle}\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" + metrics_report_coins "after-pay-${ptag}" || true + metrics_record_flow spent "$PAMT" || true + else + pnote="not settled order=$OID avail=${CUR}:${after}" + if [ "$IS_PMAX" = "1" ]; then + pstatus="CEILING_REJECT" + warn pay "pay $PAMT not settled (ceiling)" "$pnote" + echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t${ms_handle}\t${ms_psettle}\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" + continue + fi + err pay "settle $PAMT" "$pnote" + pstatus="FAIL_SETTLE" + PAY_FAIL_N=$((PAY_FAIL_N + 1)) + FAIL_N_L=$((FAIL_N_L + 1)) + STOP_REASON="$pnote" + STOP_AMOUNT="$PAMT" + echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t${ms_handle}\t${ms_psettle}\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" + metrics_report_coins "after-pay-fail-${ptag}" || true + break + fi + done + metrics_report_coins "after-pay-ladder" || true + info "pay summary" "ok=$PAY_OK_N fail=$PAY_FAIL_N tsv=$PAY_TSV" + fi +elif [ "${LADDER_PAY}" = "1" ] && [ "$FAIL_N_L" -gt 0 ]; then + warn pay "skipped pay ladder" "withdraw phase already failed" +fi + +ms_phase=$(python3 -c 'import sys,time; print(int((time.time()-float(sys.argv[1]))*1000))' "$SECTION_T0") + +# --- report --- +set_group report +section "ladder · report" +info "auto-account" "$ACCT_USER" +info "ok_rungs" "$OK_N" +info "fail_rungs" "$FAIL_N_L" +info "pay_ok" "$PAY_OK_N" +info "pay_fail" "$PAY_FAIL_N" +info "phase_ms" "$ms_phase" +info "tsv" "$TSV" +info "pay_tsv" "$PAY_TSV" + +# speed summary via python — free-text stop reason via env (JSON " in bank errors +# used to break shell argv quoting → "syntax error near unexpected token '('") +export LADDER_REPORT_STOP_AMOUNT="${STOP_AMOUNT:-}" +export LADDER_REPORT_STOP_REASON="${STOP_REASON:-}" +export LADDER_REPORT_MODE="${LADDER_MODE:-classic}" +export LADDER_REPORT_MAX_BEST="${MAX_BEST_AMT:-}" +export LADDER_REPORT_MAX_LO="${MAX_LO:-}" +export LADDER_REPORT_MAX_HI="${MAX_HI:-}" +# Human alt_unit_names (e.g. 8.15 Tera-GOA (GOA:…)) for report + final lines +export LADDER_REPORT_MAX_BEST_ALT="$(format_amount_alt "${MAX_BEST_AMT:-${CUR}:0}" 2>/dev/null || true)" +export LADDER_REPORT_MAX_LO_ALT="$(format_amount_alt "${CUR}:${MAX_LO:-0}" 2>/dev/null || true)" +export LADDER_REPORT_MAX_HI_ALT="$(format_amount_alt "${CUR}:${MAX_HI:-0}" 2>/dev/null || true)" +export LADDER_REPORT_ALT_UNITS="${ALT_UNITS_FILE:-}" +# Report must never abort the ladder (set -e): soft-fail print/JSON only. +python3 - "$TSV" "$PAY_TSV" "$JSON" "$OK_N" "$FAIL_N_L" "$PAY_OK_N" "$PAY_FAIL_N" "$ms_phase" "$ACCT_USER" "$CUR" <<'PY' || true +import csv, json, os, sys, statistics, traceback +from decimal import Decimal, InvalidOperation, ROUND_HALF_UP + +try: + tsv, pay_tsv, jpath = sys.argv[1:4] + ok_n, fail_n, pay_ok, pay_fail, phase_ms, acct, cur = sys.argv[4:11] + stop_amt = os.environ.get("LADDER_REPORT_STOP_AMOUNT") or None + stop_reason = os.environ.get("LADDER_REPORT_STOP_REASON") or None + mode = os.environ.get("LADDER_REPORT_MODE") or "classic" + max_best = os.environ.get("LADDER_REPORT_MAX_BEST") or None + max_lo = os.environ.get("LADDER_REPORT_MAX_LO") or None + max_hi = os.environ.get("LADDER_REPORT_MAX_HI") or None + max_best_alt = os.environ.get("LADDER_REPORT_MAX_BEST_ALT") or None + max_lo_alt = os.environ.get("LADDER_REPORT_MAX_LO_ALT") or None + max_hi_alt = os.environ.get("LADDER_REPORT_MAX_HI_ALT") or None + alt_path = os.environ.get("LADDER_REPORT_ALT_UNITS") or "" + alt = {} + if alt_path: + try: + _loaded = json.load(open(alt_path)) + if isinstance(_loaded, dict): + alt = {str(k): str(v) for k, v in _loaded.items()} + except Exception: + alt = {} + if not isinstance(alt, dict) or not alt: + alt = {"0": cur or "GOA"} + elif "0" not in alt: + alt = dict(alt) + alt["0"] = cur or "GOA" + + def fmt_num(v): + try: + if v == v.to_integral(): + return format(int(v), "d") + s = format(v.quantize(Decimal("0.00000001"), rounding=ROUND_HALF_UP), "f") + return s.rstrip("0").rstrip(".") + except Exception: + return str(v) + + def format_alt(amt): + """Human alt_unit_names; never raises.""" + try: + if not amt: + return "-" + s = str(amt).strip() + try: + if ":" in s: + c, v = s.split(":", 1) + val = Decimal(v) + else: + c, val = cur, Decimal(s) + except (InvalidOperation, ValueError, TypeError): + return s + au = alt if isinstance(alt, dict) else {"0": cur or "GOA"} + base_name = au.get("0") or c or cur or "GOA" + base_s = "%s:%s" % (c, fmt_num(val)) + if val == 0: + return "0 %s (%s)" % (base_name, base_s) + scales = [] + for k, name in au.items(): + try: + scales.append((int(k), str(name))) + except Exception: + continue + scales.sort(key=lambda x: -x[0]) + absval = abs(val) + chosen = None + for sc, name in scales: + try: + unit = Decimal(10) ** sc + if unit <= 0: + continue + coeff = absval / unit + if coeff >= 1: + chosen = (sc, name, coeff if val >= 0 else -coeff) + break + except Exception: + continue + if chosen is None: + return "%s %s (%s)" % (fmt_num(val), base_name, base_s) + sc, name, coeff = chosen + if sc == 0: + return "%s %s" % (fmt_num(val), name) + return "%s %s (%s)" % (fmt_num(coeff), name, base_s) + except Exception: + return str(amt) if amt else "-" + + def as_amt(s): + try: + if s is None or s == "": + return None + s = str(s).strip() + if ":" in s: + return s + return "%s:%s" % (cur, s) + except Exception: + return None + + def load_rows(path): + rows = [] + try: + with open(path, newline="") as f: + for row in csv.DictReader(f, delimiter="\t"): + rows.append(row) + except Exception: + pass + return rows + + rows = load_rows(tsv) + prows = load_rows(pay_tsv) + + def nums(rs, key): + out = [] + for row in rs: + try: + out.append(int(row[key])) + except Exception: + pass + return out + + def stats(xs): + if not xs: + return {"n": 0} + return { + "n": len(xs), + "min_ms": min(xs), + "max_ms": max(xs), + "avg_ms": int(sum(xs) / len(xs)), + "p50_ms": int(statistics.median(xs)), + } + + _max_block = None + if mode == "max": + _best = as_amt(max_best) if max_best else None + _lo = as_amt(max_lo) if max_lo is not None and str(max_lo) != "" else None + _hi = as_amt(max_hi) if max_hi is not None and str(max_hi) != "" else None + _max_block = { + "best": _best, + "best_alt": max_best_alt or (format_alt(_best) if _best else None), + "lo": max_lo, + "lo_alt": max_lo_alt or (format_alt(_lo) if _lo else None), + "hi": max_hi, + "hi_alt": max_hi_alt or (format_alt(_hi) if _hi else None), + } + + report = { + "currency": cur, + "mode": mode, + "auto_account": acct, + "ok_rungs": int(ok_n), + "fail_rungs": int(fail_n), + "pay_ok": int(pay_ok), + "pay_fail": int(pay_fail), + "max_search": _max_block, + "stopped_at_amount": stop_amt or None, + "stop_reason": stop_reason or None, + "phase_ms": int(phase_ms), + "timing": { + "mint": stats(nums(rows, "ms_mint")), + "accept": stats(nums(rows, "ms_accept")), + "confirm": stats(nums(rows, "ms_confirm")), + "settle": stats(nums(rows, "ms_settle")), + "rung_total": stats(nums(rows, "ms_total")), + "pay_order": stats(nums(prows, "ms_order")), + "pay_handle": stats(nums(prows, "ms_handle")), + "pay_settle": stats(nums(prows, "ms_settle")), + "pay_total": stats(nums(prows, "ms_total")), + }, + "rungs": rows, + "pays": prows, + } + try: + json.dump(report, open(jpath, "w"), indent=2) + print("JSON", jpath) + except Exception as e: + print("JSON write error:", e) + + print("--- withdraw speed (ms) ---") + for k in ("mint", "accept", "confirm", "settle", "rung_total"): + v = report["timing"][k] + if v.get("n"): + print(" %s n=%s min=%s p50=%s avg=%s max=%s" % ( + k.ljust(12), v["n"], v["min_ms"], v["p50_ms"], v["avg_ms"], v["max_ms"])) + print("--- pay speed (ms) ---") + for k in ("pay_order", "pay_handle", "pay_settle", "pay_total"): + v = report["timing"][k] + if v.get("n"): + print(" %s n=%s min=%s p50=%s avg=%s max=%s" % ( + k.ljust(12), v["n"], v["min_ms"], v["p50_ms"], v["avg_ms"], v["max_ms"])) + print("--- withdraw rungs ---") + for row in rows: + try: + amt = row.get("amount") or "" + aalt = format_alt(amt) if amt else "" + print(" %2s %-18s %-14s total=%sms %s" % ( + row.get("rung"), amt, row.get("status"), row.get("ms_total"), aalt)) + except Exception as e: + print(" ? row-print-error %s" % (e,)) + print("--- pay rungs ---") + for row in prows: + try: + amt = row.get("amount") or "" + aalt = format_alt(amt) if amt else "" + print(" %2s %-18s %-14s total=%sms oid=%s %s" % ( + row.get("rung"), amt, row.get("status"), row.get("ms_total"), + row.get("oid"), aalt)) + except Exception as e: + print(" ? pay-row-print-error %s" % (e,)) + if mode == "max": + print("--- max-search ---") + try: + ba = (max_best_alt or format_alt(as_amt(max_best) or "")) if max_best else "-" + la = (max_lo_alt or format_alt(as_amt(max_lo) or "")) if max_lo is not None and str(max_lo) != "" else "-" + ha = (max_hi_alt or format_alt(as_amt(max_hi) or "")) if max_hi is not None and str(max_hi) != "" else "-" + print(" best=%s" % (max_best or "-")) + print(" best_alt=%s" % ba) + print(" lo=%s" % (max_lo if max_lo is not None else "-")) + print(" lo_alt=%s" % la) + print(" hi=%s" % (max_hi if max_hi is not None else "-")) + print(" hi_alt=%s" % ha) + except Exception as e: + print(" max-search print error: %s" % (e,)) + if stop_amt: + try: + print("STOPPED at %s · %s: %s" % (stop_amt, format_alt(stop_amt), stop_reason)) + except Exception: + print("STOPPED at %s: %s" % (stop_amt, stop_reason)) + else: + print("Completed without hard failure (or budget stop without fail).") +except Exception: + traceback.print_exc() +sys.exit(0) +PY + +wcli balance 2>&1 | tee "$REPORT_DIR/balance-final.out" | tail -20 || true + +set_group load +section "ladder · load snapshot (after withdraws)" +metrics_report_load "$LOAD_AFTER" "ladder-end" || true +if [ -f "$LOAD_BEFORE" ] && [ -f "$LOAD_AFTER" ]; then + section "metrics · ladder load delta" + metrics_print_load_delta "$LOAD_BEFORE" "$LOAD_AFTER" || true +fi +# Speed timings → metrics overall (min/p50/avg/max per phase) +if [ -f "$JSON" ]; then + python3 - "$JSON" "${METRICS_DIR}/perf-summary.json" <<'PY' 2>/dev/null || true +import json, sys +rep = json.load(open(sys.argv[1])) +out = {} +for k, v in (rep.get("timing") or {}).items(): + if isinstance(v, dict) and v.get("n"): + out[k] = v +json.dump(out, open(sys.argv[2], "w"), indent=2) +PY +fi +export METRICS_WITHDRAW_TSV="$TSV" +export METRICS_PAY_TSV="$PAY_TSV" +metrics_report_coins "ladder-end" || true +metrics_print_overall "ladder final" || true + +# Keep scratch if LADDER_REPORT_DIR set; else copy key files to /tmp +if [ -z "${LADDER_REPORT_DIR:-}" ]; then + KEEP="/tmp/amount-ladder-report-$(date +%Y%m%d-%H%M%S)" + mkdir -p "$KEEP" + cp -a "$TSV" "$PAY_TSV" "$JSON" "$SCRATCH/auto-account.json" "$SCRATCH/ladder-plan.txt" \ + "$SCRATCH/ladder-wd-plan.txt" "$SCRATCH/ladder-pay-plan.txt" \ + "$REPORT_DIR/balance-final.out" "$LOAD_BEFORE" "$LOAD_AFTER" "$KEEP/" 2>/dev/null || true + info "report_dir" "$KEEP" + echo "$KEEP" >"$SCRATCH/KEEP_PATH" +fi + +if [ "$FAIL_N_L" -gt 0 ]; then + blocker "ladder" "stopped at ${STOP_AMOUNT:-?} — ${STOP_REASON:-error}" + exit 1 +fi +if [ "$OK_N" -eq 0 ]; then + blocker "ladder" "no successful withdraw rungs" + exit 1 +fi +if [ "${LADDER_MODE}" = "max" ] && [ -n "${MAX_BEST_AMT:-}" ]; then + ok "ladder finished mode=max best=$(format_amount_alt "$MAX_BEST_AMT") · lo=$(format_amount_alt "${CUR}:${MAX_LO}") · hi=$(format_amount_alt "${CUR}:${MAX_HI}") · withdraw_ok=$OK_N phase=${ms_phase}ms" +else + ok "ladder finished mode=${LADDER_MODE} withdraw_ok=$OK_N pay_ok=$PAY_OK_N phase=${ms_phase}ms" +fi +exit 0 diff --git a/check_goa_ladder.sh b/check_goa_ladder.sh index 560537d..c32f686 100755 --- a/check_goa_ladder.sh +++ b/check_goa_ladder.sh @@ -1,1404 +1,5 @@ #!/usr/bin/env bash -# check_goa_ladder.sh — withdraw/pay amount ladder (GOA + stage TESTPAYSAN) -# -# Flow (bank landings): -# 1) GET /intro/auto-account.json → personal *account-* (balance 0) -# 2) Mint pool withdrawals as explorer + confirm when selected -# 3) wallet-cli accept-uri only (no run-until-done) -# 4) bank confirm when selected; settle = balance + transfer_done -# -# Amounts: random strictly increasing; pins 0, max-1, max. -# Phase A: withdraw ladder (cumulative wallet). Phase B: pay ladder. -# -# Stacks: -# GOA — LADDER_MAX_AMOUNT = libeufin ceiling; explorer from koopa-admin-secrets -# TESTPAYSAN stage — auto max_wire from bank /config, min denom from /keys; -# explorer from stagepaysan secrets / SSH -# -# Usage: -# ./taler-monitoring.sh ladder -# ./taler-monitoring.sh -d stage.lefrancpaysan.ch ladder -# LADDER_MAX_AMOUNT=100 LADDER_STEPS=7 ./taler-monitoring.sh -d stage.lefrancpaysan.ch ladder -# -# Env (see body): LADDER_STEPS, LADDER_MAX_AMOUNT, LADDER_MIN_AMOUNT, LADDER_STACK_AUTO, -# LADDER_PAY, EXP_PW / EXP_PW_FILE, CLI_JS, MERCHANT_INSTANCE, … -set -euo pipefail +# Compat shim — use check_amount_ladder.sh (generic currency amount ladder). +# Kept so older host-agent / docs invoking check_goa_ladder.sh still work. ROOT=$(cd "$(dirname "$0")" && pwd) -# shellcheck source=lib.sh -source "$ROOT/lib.sh" -# When invoked standalone (not via taler-monitoring.sh), still load secrets.env -load_monitoring_secrets_env 2>/dev/null || true - -# Area ladder.* — withdraw/pay amount ladder (GOA + TESTPAYSAN) -# Groups: ladder.plan / ladder.load / ladder.withdraw / ladder.pay / ladder.report -set_area ladder -set_group plan -SECTION_T0=$(date +%s) -now_ms() { python3 -c 'import time; print(int(time.time()*1000))'; } -elapsed_ms() { - # elapsed_ms START_MS - python3 -c 'import sys; print(int(sys.argv[1]) - int(sys.argv[2]))' "$(now_ms)" "$1" -} - -: "${LADDER_TIMEOUT_S:=3600}" -: "${LADDER_SETTLE_ROUNDS:=18}" -: "${LADDER_SETTLE_SLEEP:=2}" -: "${EXP_USER:=explorer}" -# EXP_PW_FILE / EXP_PW optional overrides; otherwise read_secret + SECRETS_ROOT / stage SSH -: "${EXP_PW_FILE:=}" -: "${EXP_PW:=}" -: "${CLI_JS:=}" -# GOA libeufin amount ceiling (hacktivism). Stage auto-replaces via bank /config. -: "${LADDER_MAX_AMOUNT:=4503599627370496}" -: "${LADDER_STEPS:=23}" -: "${LADDER_LOAD:=1}" -: "${LADDER_PAY:=1}" -: "${LADDER_STACK_AUTO:=1}" -# Withdraw mids = pay mids × scale (so wallet can afford the pay ladder) -: "${LADDER_WITHDRAW_SCALE:=1.5}" -# Floor for random mids (must be ≥ smallest exchange coin; GOA min denom ≈ 0.000001) -: "${LADDER_MIN_AMOUNT:=0.000001}" -: "${LADDER_CONFIRM_POLLS:=40}" -: "${LADDER_PAY_SETTLE_ROUNDS:=6}" -: "${MERCHANT_INSTANCE:=goa-demo-cp4zqk}" -# Public variable template for stage pays (optional; fixed templates used when amount maps) -: "${LADDER_PAY_TEMPLATE:=}" -: "${LADDER_PAY_TEMPLATE_INSTANCE:=fermes-des-collines}" - -GOA_LADDER_CEILING="4503599627370496" - -# Resolve wallet-cli .mjs (no hardcoded laptop path) -if [ -z "${CLI_JS}" ] || [ ! -f "${CLI_JS}" ]; then - CLI_JS=$(find_wallet_cli 2>/dev/null || true) -fi -if [ -z "${CLI_JS}" ] || [ ! -f "${CLI_JS}" ]; then - # allow PATH wrapper as last resort (wcli falls back to taler-wallet-cli) - CLI_JS="" -fi - -CUR="${EXPECT_CURRENCY:-GOA}" -BANK="${BANK_PUBLIC%/}" -EX="${EXCHANGE_PUBLIC%/}/" -MER="${MERCHANT_PUBLIC%/}" -INST="${MERCHANT_INSTANCE}" - -# --- Stack-specific ladder maxima (TESTPAYSAN stage) --- -# Uses bank max_wire_transfer_amount and exchange min denom when still on GOA defaults. -apply_ladder_stack_defaults() { - [ "${LADDER_STACK_AUTO}" = "1" ] || return 0 - if [ "${CUR}" != "TESTPAYSAN" ]; then - case "${TALER_DOMAIN:-}" in - stage.lefrancpaysan.ch|stage.bank.lefrancpaysan.ch|stage.exchange.lefrancpaysan.ch|stage.monnaie.lefrancpaysan.ch) - CUR="TESTPAYSAN" - ;; - *) return 0 ;; - esac - fi - - local bank_cfg max_wire min_denom - bank_cfg=$(curl -sS -m 12 "${BANK}/config" 2>/dev/null || true) - max_wire=$(printf '%s' "$bank_cfg" | python3 -c ' -import json,sys -try: - d=json.load(sys.stdin) -except Exception: - print("") - raise SystemExit(0) -a=d.get("max_wire_transfer_amount") or "" -print(a.split(":",1)[-1] if a else "") -' 2>/dev/null || true) - - min_denom=$(curl -sS -m 25 -H 'Accept: application/json' "${EX%/}/keys" 2>/dev/null | python3 -c ' -import json,sys -try: - d=json.load(sys.stdin) -except Exception: - print("") - raise SystemExit(0) -den=d.get("denoms") or d.get("denominations") or [] -vals=[] -for x in den if isinstance(den,list) else []: - v=x.get("value") or x.get("amount") or "" - if not v: continue - try: vals.append(float(str(v).split(":")[-1])) - except Exception: pass -print(min(vals) if vals else "") -' 2>/dev/null || true) - - # Fallback maxima if public config unreachable - [ -n "$max_wire" ] || max_wire="2000" - [ -n "$min_denom" ] || min_denom="0.01" - - # Only rewrite when caller left GOA defaults (explicit LADDER_MAX_AMOUNT=… kept) - if [ -n "$max_wire" ] && { [ "${LADDER_MAX_AMOUNT}" = "${GOA_LADDER_CEILING}" ] || [ "${LADDER_MAX_AMOUNT}" = "4503599627370496" ]; }; then - LADDER_MAX_AMOUNT="$max_wire" - fi - if [ -n "$min_denom" ] && { [ "${LADDER_MIN_AMOUNT}" = "0.000001" ] || [ "${LADDER_MIN_AMOUNT}" = "0.00000001" ]; }; then - LADDER_MIN_AMOUNT="$min_denom" - fi - # Slightly fewer rungs on stage (full GOA 23 still ok if user set LADDER_STEPS) - if [ "${LADDER_STEPS}" = "23" ]; then - LADDER_STEPS=15 - fi - # Stage farmer shops: private goa-demo instance does not exist - if [ "${MERCHANT_INSTANCE}" = "goa-demo-cp4zqk" ]; then - MERCHANT_INSTANCE="${LADDER_PAY_TEMPLATE_INSTANCE:-fermes-des-collines}" - INST="$MERCHANT_INSTANCE" - fi - # Prefer public templates for pays when no merchant token (set later) - : "${E2E_USE_TEMPLATES:=1}" - export E2E_USE_TEMPLATES - export LADDER_MAX_AMOUNT LADDER_MIN_AMOUNT LADDER_STEPS MERCHANT_INSTANCE - info "ladder stack" "TESTPAYSAN · max=${CUR}:${LADDER_MAX_AMOUNT} min=${CUR}:${LADDER_MIN_AMOUNT} steps=${LADDER_STEPS} (from bank max_wire / keys denoms)" -} -apply_ladder_stack_defaults -SCRATCH=$(mktemp -d) -WDB="$SCRATCH/wallet.sqlite3" -REPORT_DIR="${LADDER_REPORT_DIR:-$SCRATCH}" -mkdir -p "$REPORT_DIR" -TSV="$REPORT_DIR/ladder-results.tsv" -PAY_TSV="$REPORT_DIR/ladder-pay-results.tsv" -JSON="$REPORT_DIR/ladder-report.json" -LOAD_BEFORE="$REPORT_DIR/load-before.json" -LOAD_AFTER="$REPORT_DIR/load-after.json" -echo -e "rung\trange\tamount\tstatus\tms_mint\tms_accept\tms_confirm\tms_settle\tms_total\twid\tnote" >"$TSV" -echo -e "rung\trange\tamount\tstatus\tms_order\tms_handle\tms_settle\tms_total\toid\tnote" >"$PAY_TSV" - -ladder_over() { - local now - now=$(date +%s) - [ $((now - SECTION_T0)) -ge "$LADDER_TIMEOUT_S" ] -} - -wcli() { - # taler-helper-sqlite3 is Python ≥3.11; prefer Homebrew/local python on macOS - # so Apple /usr/bin/python3 3.9 does not FATAL the sqlite backend (no reservePub). - local _path="${PATH:-}" - case ":${_path}:" in - *:/opt/homebrew/bin:*) ;; - *) _path="/opt/homebrew/bin:/usr/local/bin:${_path}" ;; - esac - if [ -n "${CLI_JS:-}" ] && [ -f "$CLI_JS" ]; then - PATH="$_path" node "$CLI_JS" --wallet-db="$WDB" --no-throttle "$@" - else - PATH="$_path" taler-wallet-cli --wallet-db="$WDB" --no-throttle "$@" - fi -} - -# Explorer pool password: env → file override → stack secrets → SSH -resolve_explorer_pw() { - local pw="" f="" host="" - if [ -n "${EXP_PW:-}" ]; then - printf '%s' "$EXP_PW" - return 0 - fi - if [ -n "${EXP_PW_FILE:-}" ] && [ -f "$EXP_PW_FILE" ]; then - tr -d '\n\r' <"$EXP_PW_FILE" - return 0 - fi - - # Stage TESTPAYSAN — never use GOA koopa-admin-secrets explorer pw first - if [ "${CUR}" = "TESTPAYSAN" ] \ - || [[ "${TALER_DOMAIN:-}" == stage.*lefrancpaysan* ]] \ - || [[ "${TALER_DOMAIN:-}" == *stage.lefrancpaysan* ]]; then - for f in \ - ${FRANCPAYSAN_SECRETS:+"${FRANCPAYSAN_SECRETS}/stage/bank-explorer-password.txt"} \ - "${HOME}/.config/taler-landing/stage-bank-explorer-password.txt" - do - [ -n "$f" ] && [ -f "$f" ] || continue - tr -d '\n\r' <"$f" - return 0 - done - _remote_exp="${FRANCPAYSAN_REMOTE_BANK_EXPLORER_SECRET:-}" - [ -z "$_remote_exp" ] && [ -n "${FRANCPAYSAN_REMOTE_SECRETS_ROOT:-}" ] && \ - _remote_exp="${FRANCPAYSAN_REMOTE_SECRETS_ROOT}/bank/secrets/bank-explorer-password.txt" - for host in ${INSIDE_SSH:+"$INSIDE_SSH"} ${FRANCPAYSAN_SSH:+"$FRANCPAYSAN_SSH"}; do - [ -n "$host" ] && [ -n "$_remote_exp" ] || continue - pw=$(ssh -o BatchMode=yes -o ConnectTimeout=12 "$host" \ - "tr -d '\\n\\r' <${_remote_exp} 2>/dev/null || sudo tr -d '\\n\\r' <${_remote_exp} 2>/dev/null" \ - 2>/dev/null || true) - if [ -n "$pw" ]; then - printf '%s' "$pw" - return 0 - fi - done - return 1 - fi - - # GOA / default: SECRETS_ROOT / ~/.config / koopa SSH - if pw=$(read_secret "taler-bank/bank-explorer-password.txt" 2>/dev/null) && [ -n "$pw" ]; then - printf '%s' "$pw" - return 0 - fi - for f in \ - "${SECRETS_ROOT:+${SECRETS_ROOT}/taler-bank/bank-explorer-password.txt}" \ - "${HOME}/.config/taler-landing/bank-explorer-password.txt" - do - [ -n "$f" ] && [ -f "$f" ] || continue - tr -d '\n\r' <"$f" - return 0 - done - return 1 -} - -wallet_avail() { - wcli balance 2>/dev/null | python3 -c ' -import json,sys -t=sys.stdin.read() -dec=json.JSONDecoder() -i=t.find("{") -if i<0: - print("0"); raise SystemExit -try: - d,_=dec.raw_decode(t,i) -except Exception: - # last-resort: brace slice (may fail on trailing logs) - try: - d=json.loads(t[i:t.rfind("}")+1]) - except Exception: - print("0"); raise SystemExit -cur=sys.argv[1] -for b in d.get("balances") or []: - a=b.get("available") or "" - if a.startswith(cur+":"): - print(a.split(":",1)[1]); raise SystemExit -print("0") -' "$CUR" 2>/dev/null || echo "0" -} - -# Build paired pay + withdraw ladders (same step count, shared random shape). -# Pay: [0] + log-uniform mids + [max-1] + [max] -# Wd: [0] + max(mid, mid×scale) mids + [max-1] + [max] (mids higher → enough to spend) -# Writes: $1 = withdraw list file, $2 = pay list file (space-separated CUR:amt lines as one line each) -build_ladder_pair() { - local wd_out="$1" pay_out="$2" - python3 - <<'PY' "$CUR" "${LADDER_MAX_AMOUNT}" "${LADDER_STEPS}" "${LADDER_MIN_AMOUNT}" "${LADDER_WITHDRAW_SCALE}" "$wd_out" "$pay_out" -import math, random, sys -from decimal import Decimal, ROUND_HALF_UP, ROUND_UP -from pathlib import Path - -cur = sys.argv[1] -max_amt = Decimal(sys.argv[2]) -steps = max(1, int(sys.argv[3])) -min_amt = Decimal(sys.argv[4]) -scale = Decimal(sys.argv[5]) -wd_path, pay_path = Path(sys.argv[6]), Path(sys.argv[7]) -if min_amt <= 0: - min_amt = Decimal("0.000001") -if min_amt >= max_amt: - min_amt = max_amt / Decimal(1000) -if scale < 1: - scale = Decimal(1) -max_m1 = max_amt - Decimal(1) if max_amt > 1 else max_amt - -def fmt(v: Decimal) -> str: - q = v.quantize(Decimal("0.00000001"), rounding=ROUND_HALF_UP) - if q == q.to_integral(): - return format(int(q), "d") - return format(q, "f").rstrip("0").rstrip(".") - -def quant(v: Decimal) -> Decimal: - if v >= 1: - return v.quantize(Decimal(1), rounding=ROUND_HALF_UP) - if v < min_amt: - return min_amt - return v.quantize(Decimal("0.00000001"), rounding=ROUND_UP) - -def amt(v: Decimal) -> str: - return "%s:%s" % (cur, fmt(v)) - -def make_mids(n_mid: int, hi_cap: Decimal): - if n_mid <= 0 or hi_cap <= min_amt: - return [] - lo, hi = float(min_amt), float(hi_cap) * 0.999999 - if hi <= lo: - hi = lo * 10 - cuts = sorted(math.exp(random.uniform(math.log(lo), math.log(hi))) for _ in range(n_mid)) - out, prev = [], Decimal(0) - for c in cuts: - v = quant(Decimal(str(c))) - if v <= prev: - step = min_amt if prev < 1 else max(prev * Decimal("1e-6"), Decimal(1)) - v = quant(prev + step) - if v >= hi_cap: - v = quant(hi_cap - (Decimal(1) if hi_cap > 1 else min_amt)) - if v <= prev or v >= hi_cap: - continue - out.append(v) - prev = v - return out - -# Build withdraw ladder first (full range), then pay mids = withdraw/scale -# so each pay mid is always cheaper than the matching withdraw mid. -if steps == 1: - wd_vals = [max_amt] -elif steps == 2: - wd_vals = [Decimal(0), max_amt] -else: - n_mid = max(0, steps - 3) - wd_vals = [Decimal(0)] + make_mids(n_mid, max_m1) + [max_m1, max_amt] - while len(wd_vals) > steps and len(wd_vals) > 3: - wd_vals.pop(len(wd_vals) // 2) - while len(wd_vals) < steps and len(wd_vals) >= 2: - i = max(1, len(wd_vals) - 2) - a, b = wd_vals[i - 1], wd_vals[i] - if a <= 0: - m = max(min_amt, b / 2 if b > 0 else min_amt) - else: - m = (a * b).sqrt() if a * b > 0 else (a + b) / 2 - m = quant(m) - if m <= a or m >= b: - break - wd_vals.insert(i, m) - wd_vals = wd_vals[:steps] - -pay_vals = [] -prev_p = Decimal(-1) -for i, w in enumerate(wd_vals): - is_first = i == 0 - is_last = i == len(wd_vals) - 1 - is_m1 = (not is_last) and w == max_m1 and i == len(wd_vals) - 2 - if is_first and w == 0: - pay_vals.append(Decimal(0)) - prev_p = Decimal(0) - continue - if is_last: - pay_vals.append(max_amt) - continue - if is_m1 or w == max_m1: - pay_vals.append(max_m1) - prev_p = max_m1 - continue - # pay mid = withdraw / scale (strictly less funding needed per step) - p = quant(w / scale) if scale > 0 else w - if p < min_amt and w >= min_amt: - p = min_amt - if p <= prev_p: - p = quant(prev_p + (min_amt if prev_p < 1 else Decimal(1))) - if p >= w: - # keep pay strictly below this withdraw rung when possible - p = quant(w - (Decimal(1) if w > 1 else min_amt)) if w > prev_p else prev_p - if p <= prev_p: - p = prev_p # flat ok only if stuck; still ≤ w - pay_vals.append(p) - prev_p = p - -assert len(pay_vals) == len(wd_vals) -# sanity: every non-pin pay mid ≤ matching withdraw mid -for i, (p, w) in enumerate(zip(pay_vals, wd_vals)): - if i == 0 or i >= len(wd_vals) - 2: - continue - if p > w: - pay_vals[i] = w - -wd_path.write_text(" ".join(amt(v) for v in wd_vals) + "\n") -pay_path.write_text(" ".join(amt(v) for v in pay_vals) + "\n") -print(" ".join(amt(v) for v in wd_vals)) -PY -} - -# shellcheck source=metrics.sh -source "$ROOT/metrics.sh" -METRICS_DIR="$REPORT_DIR" -ALT_UNITS_FILE="${REPORT_DIR}/alt_unit_names.json" -export METRICS_DIR CUR WDB CLI_JS ALT_UNITS_FILE -# Ladder can disable host load without killing coin metrics -if [ "${LADDER_LOAD:-1}" = "0" ]; then - METRICS_LOAD=0 - export METRICS_LOAD -fi - -section "ladder · ${CUR} withdraw (0 → random → max · ${LADDER_STEPS} steps)" -info "bank" "$BANK" -info "exchange" "$EX" -info "currency" "$CUR" -# alt_unit_names for human amounts (Kilo-GOA / Mega-GOA / … + GOA in parentheses) -if metrics_load_alt_units "${EX%/}/config"; then - info "alt_unit_names" "from ${EX%/}/config → $ALT_UNITS_FILE" -else - warn "alt_unit_names" "using built-in SI fallback ($ALT_UNITS_FILE)" -fi -info "budget" "${LADDER_TIMEOUT_S}s" -_max_m1=$(python3 -c 'import sys; from decimal import Decimal; m=Decimal(sys.argv[1]); print(m-1 if m>1 else m)' "${LADDER_MAX_AMOUNT}" 2>/dev/null || echo "${LADDER_MAX_AMOUNT}-1") -info "steps" "${LADDER_STEPS} (0 + random≥${LADDER_MIN_AMOUNT} + max-1=${CUR}:${_max_m1} + max=${CUR}:${LADDER_MAX_AMOUNT})" -info "withdraw_scale" "${LADDER_WITHDRAW_SCALE}× pay mids (fund pay ladder)" -info "pay_phase" "$([ "${LADDER_PAY}" = "1" ] && echo enabled || echo disabled)" -info "currency/domain" "${CUR} · ${TALER_DOMAIN:-?} · bank=${BANK}" - -set_group load -section "ladder · load snapshot (before withdraws)" -metrics_report_load "$LOAD_BEFORE" "ladder-start" || true -# Fresh wallet per rung — baseline empty (or last-rung DB if re-used later) -metrics_report_coins "ladder-start" || true - -if ! EXP_PW=$(resolve_explorer_pw); then - err bank "explorer password missing" \ - "set EXP_PW / EXP_PW_FILE or SECRETS_ROOT=…/koopa/host-root (taler-bank/bank-explorer-password.txt)${SECRETS_ROOT:+ · SECRETS_ROOT=${SECRETS_ROOT}}" - secrets_hint 2>/dev/null || true - exit 1 -fi -if [ -n "${SECRETS_ROOT:-}" ]; then - info "explorer secret" "resolved (SECRETS_ROOT=${SECRETS_ROOT} · stage uses stagepaysan secret first when CUR=TESTPAYSAN)" -else - info "explorer secret" "resolved via EXP_PW / EXP_PW_FILE / stage SSH / koopa" -fi -if [ -n "${CLI_JS:-}" ] && [ -f "${CLI_JS}" ]; then - info "wallet-cli" "$CLI_JS" -else - info "wallet-cli" "PATH taler-wallet-cli ($(command -v taler-wallet-cli 2>/dev/null || echo missing))" -fi - -# --- auto-account --- -t0=$(now_ms) -if ! curl -sS -m 30 -o "$SCRATCH/auto-account.json" "${BANK}/intro/auto-account.json"; then - err bank "auto-account.json unreachable" - exit 1 -fi -ms_auto=$(elapsed_ms "$t0") -if ! python3 -c 'import json; d=json.load(open("'"$SCRATCH"'/auto-account.json")); assert d.get("ok") or d.get("username")' 2>/dev/null; then - err bank "auto-account create failed" "$(head -c 120 "$SCRATCH/auto-account.json" | tr '\n' ' ')" - exit 1 -fi -ACCT_USER=$(python3 -c 'import json; print(json.load(open("'"$SCRATCH"'/auto-account.json"))["username"])') -ok "auto-account ${ACCT_USER} (${ms_auto}ms) — personal ${CUR}:0; pool=explorer" -info "auto-account password" "(see $SCRATCH/auto-account.json — not logged)" - -# --- explorer token --- -t0=$(now_ms) -TOK=$(curl -sS -m 20 -u "${EXP_USER}:${EXP_PW}" \ - -H 'Content-Type: application/json' \ - -d '{"scope":"readwrite","duration":{"d_us":3600000000}}' \ - "${BANK}/accounts/${EXP_USER}/token" \ - | python3 -c 'import json,sys; print(json.load(sys.stdin)["access_token"])') -ms_tok=$(elapsed_ms "$t0") -[ -n "$TOK" ] || { err bank "explorer token failed"; exit 1; } -ok "explorer token (${ms_tok}ms)" - -# ONE cumulative wallet for all withdraws + pays (need balance to spend). -# force-select uses *last* reserve_pub from the current accept output. -wallet_prepare() { - local label="${1:-wallet}" - WDB="$SCRATCH/wallet-${label}.sqlite3" - export WDB - if [ ! -f "$WDB" ]; then - wcli exchanges add "$EX" >"$SCRATCH/ex-add-$label.out" 2>&1 || true - wcli exchanges update "$EX" >"$SCRATCH/ex-upd-$label.out" 2>&1 || true - wcli exchanges accept-tos "$EX" >"$SCRATCH/ex-tos-$label.out" 2>&1 || true - fi -} - -t0=$(now_ms) -rm -f "$SCRATCH/wallet-main.sqlite3" -wallet_prepare "main" -ms_tos=$(elapsed_ms "$t0") -ok "wallet exchange + ToS (${ms_tos}ms) — cumulative DB for withdraw+pay (no run-until-done)" - -build_ladder_pair "$SCRATCH/ladder-wd-plan.txt" "$SCRATCH/ladder-pay-plan.txt" -LADDER_LIST=$(tr -d '\n' <"$SCRATCH/ladder-wd-plan.txt") -PAY_LIST=$(tr -d '\n' <"$SCRATCH/ladder-pay-plan.txt") -: "${LADDER_MAX_RUNGS:=99}" -# shellcheck disable=SC2086 -set -- $LADDER_LIST -if [ "$#" -gt "$LADDER_MAX_RUNGS" ]; then - # shellcheck disable=SC2046 - set -- $(printf '%s\n' "$@" | head -n "$LADDER_MAX_RUNGS") -fi -LADDER_N=$# -info "withdraw plan" "$*" -info "withdraw plan (alt)" "$(format_amount_list_alt "$@")" -info "pay plan" "$PAY_LIST" -# shellcheck disable=SC2086 -info "pay plan (alt)" "$(format_amount_list_alt $PAY_LIST)" -printf '%s\n' "$@" >"$SCRATCH/ladder-plan.txt" -printf '%s\n' $PAY_LIST >"$SCRATCH/ladder-pay-plan-lines.txt" 2>/dev/null || true - -OK_N=0 -FAIL_N_L=0 -PAY_OK_N=0 -PAY_FAIL_N=0 -STOP_REASON="" -STOP_AMOUNT="" -declare -a RUNG_JSON=() - -set_group withdraw -section "ladder · phase A · withdraw (${LADDER_N} rungs)" -rung=0 -for AMT in "$@"; do - rung=$((rung + 1)) - if ladder_over; then - STOP_REASON="timeout budget ${LADDER_TIMEOUT_S}s" - warn ladder "time budget exhausted" \ - "problem: LADDER_TIMEOUT_S=${LADDER_TIMEOUT_S}s reached after ${OK_N} ok rungs; remaining amounts not tried" - break - fi - - section "ladder · rung $rung $AMT · $(format_amount_alt "$AMT")" - tag=$(printf '%s' "$AMT" | tr '.:' '__') - # TSV "range" column: fixed pins at ends, random mids (refined after AMT_NUM) - if [ "$rung" -eq 1 ]; then - range_note="pin:0" - elif [ "$rung" -eq "$LADDER_N" ]; then - range_note="pin:max" - elif [ "$rung" -eq $((LADDER_N - 1)) ] && [ "$LADDER_N" -ge 3 ]; then - range_note="pin:max-1" - else - range_note="random" - fi - t_rung=$(now_ms) - ms_mint=0 ms_accept=0 ms_confirm=0 ms_settle=0 - note="" - status="FAIL" - WID="-" - FORCE_SEL_409_LOGGED=0 - - # keep cumulative main wallet - wallet_prepare "main" - - # mint from explorer pool - t0=$(now_ms) - code=$(curl -sS -m 30 -o "$SCRATCH/wd-$tag.json" -w '%{http_code}' \ - -H "Authorization: Bearer ${TOK}" \ - -H 'Content-Type: application/json' \ - -d "{\"amount\":\"${AMT}\"}" \ - "${BANK}/accounts/${EXP_USER}/withdrawals") - ms_mint=$(elapsed_ms "$t0") - WID=$(python3 -c 'import json;d=json.load(open("'"$SCRATCH"'/wd-'"$tag"'.json"));print(d.get("withdrawal_id") or "")' 2>/dev/null || true) - URI=$(python3 -c 'import json;u=json.load(open("'"$SCRATCH"'/wd-'"$tag"'.json")).get("taler_withdraw_uri") or "";print(u.replace(":443/","/"))' 2>/dev/null || true) - # numeric amount (for zero / settle / ceiling special-cases) - AMT_NUM=$(python3 -c 'import sys; print(sys.argv[1].split(":",1)[-1])' "$AMT") - IS_ZERO=0 - python3 -c 'import sys; from decimal import Decimal; sys.exit(0 if Decimal(sys.argv[1])==0 else 1)' "$AMT_NUM" 2>/dev/null && IS_ZERO=1 - IS_MAX_PIN=0 - IS_MAX_M1_PIN=0 - if [ "$AMT_NUM" = "${LADDER_MAX_AMOUNT}" ]; then - IS_MAX_PIN=1 - range_note="pin:max" - elif [ "$AMT_NUM" = "$((LADDER_MAX_AMOUNT - 1))" ] 2>/dev/null || \ - [ "$AMT_NUM" = "$(python3 -c 'print(int("'"$LADDER_MAX_AMOUNT"'")-1)')" ]; then - IS_MAX_M1_PIN=1 - range_note="pin:max-1" - fi - - if [ "$code" != "200" ] && [ "$code" != "201" ] || [ -z "$WID" ] || [ -z "$URI" ]; then - # strip quotes so STOP_REASON never breaks later shell/python argv - note="mint HTTP $code $(head -c 100 "$SCRATCH/wd-$tag.json" 2>/dev/null | tr '\n\"' ' ')" - ms_total=$(elapsed_ms "$t_rung") - if [ "$IS_ZERO" = "1" ]; then - # Probe only: bank may reject GOA:0 — record and continue ladder - status="ZERO_REJECT" - note="zero-withdraw rejected (expected possible): $note" - warn bank "mint $AMT rejected" \ - "problem: bank will not create a GOA:0 withdrawal (zero amount probe). Ladder continues. detail: $note" - echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" - OK_N=$((OK_N + 1)) - continue - fi - # Absolute max is a ceiling probe — bank often returns SQL P0001/5110; do not abort. - if [ "$IS_MAX_PIN" = "1" ]; then - status="CEILING_REJECT" - note="absolute max rejected (ceiling probe; max-1 is the hard pin): $note" - warn bank "mint $AMT rejected (ceiling)" \ - "problem: bank rejects absolute LADDER_MAX_AMOUNT (often SQL P0001/5110). max-1 rung is the last expected success. Ladder continues to report. detail: $note" - echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" - continue - fi - # max-1 can also hit the same libeufin SQL ceiling (5110/P0001) on live GOA — - # treat as soft ceiling when mid rungs already passed; do not block the report. - if [ "$IS_MAX_M1_PIN" = "1" ] && echo "$note" | grep -qE '5110|P0001'; then - status="CEILING_REJECT" - note="max-1 rejected (same ceiling as absolute max): $note" - warn bank "mint $AMT rejected (max-1 ceiling)" \ - "problem: bank rejects max-1 with SQL P0001/5110 (pool ceiling). Mid-rung OK_BANK/OK still count. Ladder continues. detail: $note" - echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" - continue - fi - err bank "mint $AMT failed" "$note" - status="FAIL_MINT" - STOP_REASON="$note" - STOP_AMOUNT="$AMT" - echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" - FAIL_N_L=$((FAIL_N_L + 1)) - break - fi - ok "mint $AMT ($WID) ${ms_mint}ms" - - before=$(wallet_avail) - - # accept - t0=$(now_ms) - if wcli withdraw accept-uri --exchange "$EX" "$URI" >"$SCRATCH/accept-$tag.out" 2>&1; then - ms_accept=$(elapsed_ms "$t0") - ok "accept-uri $AMT ${ms_accept}ms" - else - ms_accept=$(elapsed_ms "$t0") - note="accept-uri failed: $(tail -c 200 "$SCRATCH/accept-$tag.out" | tr '\n' ' ')" - ms_total=$(elapsed_ms "$t_rung") - # Wallet 7006: no denominations for this amount (0, sub-denom dust, etc.) — warn & continue - if [ "$IS_ZERO" = "1" ] || grep -qE 'code: 7006|"code"[[:space:]]*:[[:space:]]*7006|No denominations could be selected' \ - "$SCRATCH/accept-$tag.out" 2>/dev/null; then - if [ "$IS_ZERO" = "1" ]; then - status="ZERO_SKIP" - note="zero-withdraw skip (7006 / no denoms): $note" - warn wallet "accept $AMT skipped" \ - "problem: wallet code 7006 — no coin denominations for GOA:0 (zero amount cannot be withdrawn as coins). Ladder continues. detail: $note" - else - status="SKIP_DENOM" - note="skip amount (wallet 7006 no denoms): $note" - warn wallet "accept $AMT skipped" \ - "problem: wallet code 7006 — no denominations match this amount (below smallest coin or not combinable). Ladder continues with next rung. detail: $note" - fi - echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" - continue - fi - err wallet "accept $AMT" "$note" - status="FAIL_ACCEPT" - STOP_REASON="$note" - STOP_AMOUNT="$AMT" - echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" - FAIL_N_L=$((FAIL_N_L + 1)) - break - fi - - # Confirm ASAP when bank status is selected. No run-until-done (hangs on macOS/wallet). - # Server-side auto-confirm only watches landing withdraw-watch.ids — ladder must confirm itself. - bank_st() { - curl -sS -m 8 "${BANK}/taler-integration/withdrawal-operation/${WID}" 2>/dev/null \ - | python3 -c 'import json,sys; print((json.load(sys.stdin).get("status") or "").strip())' 2>/dev/null || true - } - do_confirm() { - curl -sS -m 15 -o "$SCRATCH/conf-$tag.json" -w '%{http_code}' -X POST \ - -H "Authorization: Bearer ${TOK}" -H 'Content-Type: application/json' -d '{}' \ - "${BANK}/accounts/${EXP_USER}/withdrawals/${WID}/confirm" - } - # Collect reserve_pub candidates for *this* withdrawal (WID + amount). - # Cumulative wallets re-print old reserves in accept/tx dumps — never trust a single "last" blindly. - # Prints unique pubs one per line, preferred order first. - # - # Critical: wallet-cli dumps mix log lines + JSON. Never json.loads(rest_of_file) — - # trailing "Shutdown requested" lines make that always fail. Use JSONDecoder.raw_decode - # and regex on *both* accept and transactions output (reservePub lives in tx, not accept). - extract_rpubs_for_wid() { - python3 - "$WID" "$AMT" "$SCRATCH/accept-$tag.out" "$SCRATCH/tx-$tag.json" "$SCRATCH/used-rpubs.txt" <<'PY' -import json, re, sys -from pathlib import Path - -wid = sys.argv[1] -amt = sys.argv[2] -paths = sys.argv[3:5] -used_path = sys.argv[5] -used = set() -if Path(used_path).is_file(): - used = {ln.strip() for ln in open(used_path) if ln.strip()} - -amt_num = amt.split(":", 1)[-1] if amt else "" -dec = json.JSONDecoder() - -def walk_collect(o, bag, ctx=None): - ctx = dict(ctx or {}) - if isinstance(o, dict): - # First pass: sibling scalars (reservePub + bankConfirmationUrl + amounts - # are peers under withdrawalDetails — order in JSON must not matter). - for k, v in o.items(): - if not isinstance(v, str): - continue - kl = str(k).lower().replace("-", "_") - if kl in ( - "withdrawal_id", "withdraw_id", "wopid", "id", - "transactionid", "transaction_id", - ): - ctx["id"] = v - elif kl in ( - "amount", "rawamount", "instructedamount", - "amounteffective", "amountraw", "transferamount", - ): - ctx["amount"] = v - elif kl in ( - "taler_withdraw_uri", "talerwithdrawuri", "uri", - "bankconfirmationurl", "confirmtransferurl", - ): - ctx["uri"] = v - for k, v in o.items(): - kl = str(k).lower().replace("-", "_") - if kl in ("reserve_pub", "reservepub") and isinstance(v, str) and len(v) >= 40: - bag.append((v, dict(ctx))) - elif not isinstance(v, (str, int, float, bool)) and v is not None: - walk_collect(v, bag, ctx) - elif isinstance(o, list): - for i in o: - walk_collect(i, bag, ctx) - -def score_ctx(pub, ctx, window=""): - score = 0 - cid = str(ctx.get("id") or "") - camt = str(ctx.get("amount") or "") - curi = str(ctx.get("uri") or "") - # bankConfirmationUrl / withdraw URI embeds this op's WID - if wid and wid in curi: - score += 120 - if wid and wid in cid: - score += 100 - if wid and wid in window: - score += 80 - if amt and (camt == amt or camt.endswith(amt_num)): - score += 50 - if amt and amt in window: - score += 30 - if pub in used: - score -= 200 - return score - -def iter_json_objects(text): - """Yield top-level JSON objects from mixed log+JSON wallet dumps.""" - i, n = 0, len(text) - while i < n: - if text[i] == "{": - try: - o, end = dec.raw_decode(text, i) - yield o - i = end - continue - except Exception: - pass - i += 1 - -raw_pubs = [] -scored = [] - -# 1) structured JSON via raw_decode (accept + transactions) -for p in paths: - try: - t = open(p, errors="replace").read() - except Exception: - continue - for o in iter_json_objects(t): - bag = [] - walk_collect(o, bag) - for pub, ctx in bag: - raw_pubs.append(pub) - scored.append((score_ctx(pub, ctx), pub)) - -# 2) regex on *both* accept and tx (reservePub is normally only in transactions) -RPUB_PATS = ( - r"\"reservePub\"\s*:\s*\"([A-Z0-9]{40,})\"", - r"\"reserve_pub\"\s*:\s*\"([A-Z0-9]{40,})\"", - r"reserve[_ ]?pub[\"\s:=]+([A-Z0-9]{40,})", -) -for p in paths: - try: - blob = open(p, errors="replace").read() - except Exception: - continue - for pat in RPUB_PATS: - for m in re.finditer(pat, blob, re.I): - pub = m.group(1) - raw_pubs.append(pub) - window = blob[max(0, m.start() - 600) : m.end() + 600] - scored.append((score_ctx(pub, {}, window) + 10, pub)) - -order = [] -seen = set() -for score, pub in sorted(scored, key=lambda x: x[0], reverse=True): - if pub in seen or pub in used: - continue - seen.add(pub) - order.append(pub) -for pub in reversed(raw_pubs): - if pub in seen or pub in used: - continue - seen.add(pub) - order.append(pub) -for pub in order: - print(pub) -PY - } - - mark_rpub_used() { - local p="$1" - [ -n "$p" ] || return 0 - mkdir -p "$SCRATCH" 2>/dev/null || true - grep -qxF "$p" "$SCRATCH/used-rpubs.txt" 2>/dev/null || echo "$p" >>"$SCRATCH/used-rpubs.txt" - } - - force_select_if_needed() { - local st_now="$1" - [ "$st_now" = "pending" ] || [ -z "$st_now" ] || return 0 - local rpub epayto code_fs any=0 - # refresh tx dump each try (wallet may attach reserve late) - wcli transactions >"$SCRATCH/tx-$tag.json" 2>&1 || true - epayto=$(curl -sS -m 10 "${EX%/}/keys" 2>/dev/null | python3 -c ' -import json,sys -d=json.load(sys.stdin) -acc=d.get("accounts") or [] -for a in acc: - p=a.get("payto_uri") or a.get("payto_address") or "" - if "x-taler-bank" in p or "exchange" in p: - print(p); break -else: - if acc: print(acc[0].get("payto_uri") or "") -' 2>/dev/null || true) - if [ -z "$epayto" ]; then - warn bank "force-select skipped" "problem: exchange payto empty from /keys" - return 0 - fi - # Try candidates until bank leaves pending (200/204) or we exhaust - while IFS= read -r rpub; do - [ -n "$rpub" ] || continue - any=1 - code_fs=$(curl -sS -m 12 -o "$SCRATCH/force-sel-$tag.json" -w '%{http_code}' -X POST \ - -H 'Content-Type: application/json' \ - -d "{\"reserve_pub\":\"${rpub}\",\"selected_exchange\":\"${epayto}\"}" \ - "${BANK}/taler-integration/withdrawal-operation/${WID}" 2>/dev/null || echo "000") - if [ "$code_fs" = "200" ] || [ "$code_fs" = "204" ]; then - info "force-select" "HTTP ${code_fs} rpub=${rpub:0:12}… (ok for WID ${WID:0:8})" - mark_rpub_used "$rpub" - return 0 - fi - if [ "$code_fs" = "409" ]; then - # 5114 = this reserve already bound to another op — not "out of money" - info "force-select" "HTTP 409 rpub=${rpub:0:12}… (stale/used reserve — not balance; trying next)" - mark_rpub_used "$rpub" - continue - fi - info "force-select" "HTTP ${code_fs} rpub=${rpub:0:12}… body=$(tr '\n' ' ' <"$SCRATCH/force-sel-$tag.json" 2>/dev/null | head -c 120)" - # other errors: still try next candidate - done < <(extract_rpubs_for_wid) - if [ "$any" != "1" ]; then - warn bank "force-select skipped" \ - "problem: no reserve_pub for this withdraw (WID=${WID:0:8}…); wallet may not have selected yet" - fi - } - - # No run-until-done (hangs / banned). Read transactions only for reserve_pub candidates. - wcli transactions >"$SCRATCH/tx-$tag.json" 2>&1 || true - - t0=$(now_ms) - conf_ok=0 - st="" - # Poll bank status; force-select while pending; confirm as soon as selected - for i in $(seq 1 "${LADDER_CONFIRM_POLLS}"); do - ladder_over && break - st=$(bank_st) - case "$st" in - selected) - ccode=$(do_confirm) - if [ "$ccode" = "204" ] || [ "$ccode" = "200" ]; then - conf_ok=1 - info "confirm" "HTTP $ccode after selected (poll $i)" - else - note="confirm HTTP $ccode" - fi - break - ;; - confirmed) - conf_ok=1 - break - ;; - aborted) - note="withdrawal aborted by bank" - break - ;; - esac - # force-select while pending — try alternate rpubs on 5114 (not out-of-money) - if [ "$st" = "pending" ] || [ -z "$st" ]; then - if [ "$i" -eq 1 ] || [ "$i" -eq 2 ] || [ $((i % 3)) -eq 0 ]; then - force_select_if_needed "$st" - fi - fi - sleep 0.35 - done - ms_confirm=$(elapsed_ms "$t0") - st=$(printf '%s' "${st:-}" | tr -d '\r\n' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') - if [ "$conf_ok" != "1" ]; then - note="${note:-confirm timeout last=${st:-empty}}" - # Soft: not confirmed — usually stale reserve_pub (5114), NOT empty pool balance - status="SKIP_CONFIRM" - warn bank "confirm $AMT skipped" \ - "problem: bank status='${st:-empty}' after ${LADDER_CONFIRM_POLLS} polls (want selected/confirmed). Usually reserve_pub mismatch (5114), not out-of-money — mint/accept already OK. detail: $note" - ms_total=$(elapsed_ms "$t_rung") - echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" - continue - fi - ok "confirm $AMT ${ms_confirm}ms (client, on selected)" - - # settle: poll wallet balance + bank transfer_done only — never run-until-done - t0=$(now_ms) - settled=0 - xfer="?" - if [ "$IS_ZERO" = "1" ]; then - settled=1 - note="zero-amount: no coin delta expected" - else - for r in $(seq 1 "$LADDER_SETTLE_ROUNDS"); do - ladder_over && break - after=$(wallet_avail) - if python3 -c " -from decimal import Decimal -import sys -sys.exit(0 if Decimal(sys.argv[1]) > Decimal(sys.argv[2]) else 1) -" "$after" "$before" 2>/dev/null; then - settled=1 - break - fi - xfer=$(curl -sS -m 10 "${BANK}/taler-integration/withdrawal-operation/${WID}" \ - | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("transfer_done"), d.get("status"))' 2>/dev/null || echo "?") - # bank done is enough to leave settle without hanging on wallet - if echo "$xfer" | grep -qi True; then - note="bank transfer_done (no run-until-done) avail=${after} $xfer" - break - fi - sleep "$LADDER_SETTLE_SLEEP" - done - fi - ms_settle=$(elapsed_ms "$t0") - after=$(wallet_avail) - ms_total=$(elapsed_ms "$t_rung") - if [ -z "${xfer:-}" ] || [ "$xfer" = "?" ]; then - xfer=$(curl -sS -m 10 "${BANK}/taler-integration/withdrawal-operation/${WID}" \ - | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("transfer_done"), d.get("status"))' 2>/dev/null || echo "?") - fi - - if [ "$settled" = "1" ]; then - status="OK" - note="${note:-avail=${CUR}:${after}}" - ok "settle $AMT → ${CUR}:${after} (settle ${ms_settle}ms, rung ${ms_total}ms)" - OK_N=$((OK_N + 1)) - echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" - metrics_report_coins "ladder-r${rung}-after-${tag}" || true - metrics_record_flow withdrawn "$AMT" || true - elif echo "$xfer" | grep -qi True; then - status="OK_BANK" - note="bank transfer_done avail=${after} $xfer (no run-until-done)" - ok "settle $AMT bank transfer_done (wallet avail=${CUR}:${after}, ${ms_settle}ms)" - OK_N=$((OK_N + 1)) - echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" - metrics_report_coins "ladder-r${rung}-after-${tag}" || true - metrics_record_flow withdrawn "$AMT" || true - else - note="no coins / no transfer_done avail=${after} $xfer" - err wallet "settle $AMT" "$note" - status="FAIL_SETTLE" - STOP_REASON="$note" - STOP_AMOUNT="$AMT" - FAIL_N_L=$((FAIL_N_L + 1)) - echo -e "${rung}\t${range_note}\t${AMT}\t${status}\t${ms_mint}\t${ms_accept}\t${ms_confirm}\t${ms_settle}\t${ms_total}\t${WID}\t${note}" >>"$TSV" - metrics_report_coins "ladder-r${rung}-fail-${tag}" || true - break - fi -done - -# --------------------------------------------------------------------------- -# Phase B — payment ladder (same shape: 0 → low → … → max-1 → max) -# --------------------------------------------------------------------------- -PAY_OK_N=0 -PAY_FAIL_N=0 -if [ "${LADDER_PAY}" = "1" ] && [ -n "${PAY_LIST:-}" ] && [ "$FAIL_N_L" -eq 0 ]; then - set_group pay - section "ladder · phase B · pay" - metrics_report_coins "before-pay-ladder" || true - # Merchant secret (same as e2e) — stage often has only public shop templates - MPW="${E2E_MERCHANT_TOKEN:-${MERCHANT_TOKEN:-}}" - if [ -z "$MPW" ]; then - MPW=$(read_secret "taler-merchant/merchant-${INST}-password.txt" 2>/dev/null || true) - fi - if [ -z "$MPW" ]; then - MPW=$(read_secret "taler-merchant/merchant-goa-demo-cp4zqk-password.txt" 2>/dev/null || true) - fi - if [ -z "$MPW" ] && [ "${CUR}" = "TESTPAYSAN" ]; then - if [ -n "${FRANCPAYSAN_SECRETS:-}" ] && [ -f "${FRANCPAYSAN_SECRETS}/stage/default-instance-token.txt" ]; then - MPW=$(tr -d '\n\r' <"${FRANCPAYSAN_SECRETS}/stage/default-instance-token.txt") - fi - if [ -z "$MPW" ]; then - _remote_mer="${FRANCPAYSAN_REMOTE_MERCHANT_TOKEN:-}" - [ -z "$_remote_mer" ] && [ -n "${FRANCPAYSAN_REMOTE_SECRETS_ROOT:-}" ] && \ - _remote_mer="${FRANCPAYSAN_REMOTE_SECRETS_ROOT}/merchant/secrets/default-instance-token.txt" - for host in ${INSIDE_SSH:+"$INSIDE_SSH"} ${FRANCPAYSAN_SSH:+"$FRANCPAYSAN_SSH"}; do - [ -n "$host" ] && [ -n "$_remote_mer" ] || continue - MPW=$(ssh -o BatchMode=yes -o ConnectTimeout=12 "$host" \ - "tr -d '\\n\\r' <${_remote_mer} 2>/dev/null || true" \ - 2>/dev/null || true) - [ -n "$MPW" ] && break - done - unset _remote_mer - fi - fi - if [ -z "$MPW" ]; then - if [ "${CUR}" = "TESTPAYSAN" ]; then - # Stage: withdraw ladder is the main probe; pays need instance token or - # public templates for *exact* face values (not continuous ladder amounts). - info pay "no merchant token — skip pay ladder on TESTPAYSAN (withdraw maxima still covered)" - info pay "hint: set E2E_MERCHANT_TOKEN or use ./taler-monitoring.sh -d stage.lefrancpaysan.ch e2e for shop templates" - else - warn pay "no merchant token — skip pay ladder (set E2E_MERCHANT_TOKEN or secrets)" - fi - else - ok "merchant token" "instance ${INST}" - case "$MPW" in - secret-token:*) AUTH="Authorization: Bearer ${MPW}" ;; - *) AUTH="Authorization: Bearer secret-token:${MPW}" ;; - esac - wallet_prepare "main" - # shellcheck disable=SC2086 - set -- $PAY_LIST - PAY_N=$# - info "pay rungs" "$PAY_N · $*" - prung=0 - for PAMT in "$@"; do - prung=$((prung + 1)) - if ladder_over; then - warn pay "time budget exhausted" "after ${PAY_OK_N} ok pays" - break - fi - section "ladder · pay rung $prung $PAMT · $(format_amount_alt "$PAMT")" - ptag=$(printf '%s' "$PAMT" | tr '.:' '__') - if [ "$prung" -eq 1 ]; then - prange="pin:0" - elif [ "$prung" -eq "$PAY_N" ]; then - prange="pin:max" - elif [ "$prung" -eq $((PAY_N - 1)) ] && [ "$PAY_N" -ge 3 ]; then - prange="pin:max-1" - else - prange="random" - fi - PNUM=$(python3 -c 'import sys; print(sys.argv[1].split(":",1)[-1])' "$PAMT") - IS_PZERO=0 - python3 -c 'import sys; from decimal import Decimal; sys.exit(0 if Decimal(sys.argv[1])==0 else 1)' "$PNUM" 2>/dev/null && IS_PZERO=1 - IS_PMAX=0 - [ "$PNUM" = "${LADDER_MAX_AMOUNT}" ] && IS_PMAX=1 - t_pay=$(now_ms) - ms_order=0 ms_handle=0 ms_psettle=0 - pnote="" - pstatus="FAIL" - OID="-" - - metrics_report_coins "before-pay-${ptag}" || true - bal=$(wallet_avail) - - if [ "$IS_PZERO" = "1" ]; then - pstatus="ZERO_SKIP" - pnote="zero pay probe skipped" - warn pay "pay $PAMT skipped" "problem: zero amount order not useful. Ladder continues." - ms_total=$(elapsed_ms "$t_pay") - echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t${ms_handle}\t${ms_psettle}\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" - PAY_OK_N=$((PAY_OK_N + 1)) - continue - fi - - # Soft: absolute max pay is a ceiling probe (unlikely affordable / merchant may reject) - if [ "$IS_PMAX" = "1" ]; then - # try once; on fail CEILING_REJECT - : - fi - - # Insufficient balance → soft skip for max pin only; hard fail for mid/max-1 - if ! python3 -c 'import sys; from decimal import Decimal; sys.exit(0 if Decimal(sys.argv[1])>=Decimal(sys.argv[2]) else 1)' "$bal" "$PNUM" 2>/dev/null; then - pnote="insufficient balance avail=${CUR}:${bal} need=${PAMT}" - ms_total=$(elapsed_ms "$t_pay") - if [ "$IS_PMAX" = "1" ]; then - pstatus="CEILING_SKIP" - warn pay "pay $PAMT skipped (ceiling)" "$pnote" - echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t0\t0\t0\t${ms_total}\t-\t${pnote}" >>"$PAY_TSV" - continue - fi - err pay "pay $PAMT" "$pnote" - pstatus="FAIL_BALANCE" - PAY_FAIL_N=$((PAY_FAIL_N + 1)) - FAIL_N_L=$((FAIL_N_L + 1)) - STOP_REASON="$pnote" - STOP_AMOUNT="$PAMT" - echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t0\t0\t0\t${ms_total}\t-\t${pnote}" >>"$PAY_TSV" - break - fi - - t0=$(now_ms) - SUM_JSON=$(python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "ladder pay ${PAMT}" 2>/dev/null || echo '"ladder pay"') - curl -skS -m 20 -o "$SCRATCH/ord-$ptag.json" -X POST \ - -H "$AUTH" -H 'Content-Type: application/json' \ - -d "{\"order\":{\"summary\":${SUM_JSON},\"amount\":\"${PAMT}\",\"fulfillment_message\":\"ok\"},\"create_token\":true}" \ - "${MER}/instances/${INST}/private/orders" 2>"$SCRATCH/ord-$ptag.err" || true - ms_order=$(elapsed_ms "$t0") - OID=$(python3 -c 'import json,re,sys;t=open(sys.argv[1]).read() -try: print(json.loads(t).get("order_id") or "") -except Exception: - m=re.search(r"\"order_id\"\s*:\s*\"([^\"]+)\"",t); print(m.group(1) if m else "") -' "$SCRATCH/ord-$ptag.json" 2>/dev/null || true) - OTOK=$(python3 -c 'import json,re,sys;t=open(sys.argv[1]).read() -try: print(json.loads(t).get("token") or "") -except Exception: - m=re.search(r"\"token\"\s*:\s*\"([^\"]+)\"",t); print(m.group(1) if m else "") -' "$SCRATCH/ord-$ptag.json" 2>/dev/null || true) - if [ -z "$OID" ]; then - pnote="order create failed $(head -c 80 "$SCRATCH/ord-$ptag.json" 2>/dev/null | tr '\n\"' ' ')" - ms_total=$(elapsed_ms "$t_pay") - if [ "$IS_PMAX" = "1" ]; then - pstatus="CEILING_REJECT" - warn pay "pay $PAMT rejected (ceiling)" "$pnote" - echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t0\t0\t${ms_total}\t-\t${pnote}" >>"$PAY_TSV" - continue - fi - err pay "order $PAMT" "$pnote" - pstatus="FAIL_ORDER" - PAY_FAIL_N=$((PAY_FAIL_N + 1)) - FAIL_N_L=$((FAIL_N_L + 1)) - STOP_REASON="$pnote" - STOP_AMOUNT="$PAMT" - echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t0\t0\t${ms_total}\t-\t${pnote}" >>"$PAY_TSV" - break - fi - ok "order $OID ($PAMT) ${ms_order}ms" - - curl -skS -m 12 -o "$SCRATCH/ord-det-$ptag.json" -H "$AUTH" \ - "${MER}/instances/${INST}/private/orders/${OID}" 2>/dev/null || true - PAYURI=$(python3 -c 'import json,sys -try: print(json.load(open(sys.argv[1])).get("taler_pay_uri") or "") -except Exception: print("") -' "$SCRATCH/ord-det-$ptag.json" 2>/dev/null || true) - if [ -z "$PAYURI" ] && [ -n "$OTOK" ]; then - MH=$(python3 -c 'from urllib.parse import urlparse; print(urlparse("'"$MER"'").hostname or "taler.hacktivism.ch")' 2>/dev/null || echo "taler.hacktivism.ch") - PAYURI="taler://pay/${MH}/instances/${INST}/${OID}/?c=${OTOK}" - fi - PAYURI=$(printf '%s' "$PAYURI" | sed 's/:443\//\//g; s/:443?/?/g') - if [ -z "$PAYURI" ]; then - pnote="no pay URI for $OID" - ms_total=$(elapsed_ms "$t_pay") - err pay "uri $PAMT" "$pnote" - pstatus="FAIL_URI" - PAY_FAIL_N=$((PAY_FAIL_N + 1)) - FAIL_N_L=$((FAIL_N_L + 1)) - STOP_REASON="$pnote" - STOP_AMOUNT="$PAMT" - echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t0\t0\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" - break - fi - - t0=$(now_ms) - if ! wcli handle-uri --yes "$PAYURI" >"$SCRATCH/pay-$ptag.out" 2>&1; then - ms_handle=$(elapsed_ms "$t0") - pnote="handle-uri failed $(tail -c 100 "$SCRATCH/pay-$ptag.out" | tr '\n\"' ' ')" - ms_total=$(elapsed_ms "$t_pay") - if [ "$IS_PMAX" = "1" ]; then - pstatus="CEILING_REJECT" - warn pay "pay $PAMT handle failed (ceiling)" "$pnote" - echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t${ms_handle}\t0\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" - continue - fi - err pay "handle $PAMT" "$pnote" - pstatus="FAIL_HANDLE" - PAY_FAIL_N=$((PAY_FAIL_N + 1)) - FAIL_N_L=$((FAIL_N_L + 1)) - STOP_REASON="$pnote" - STOP_AMOUNT="$PAMT" - echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t${ms_handle}\t0\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" - break - fi - ms_handle=$(elapsed_ms "$t0") - ok "handle-uri $PAMT ${ms_handle}ms" - - # Short settle polls: merchant order + wallet tx only — never run-until-done - t0=$(now_ms) - settled=0 - r=0 - while [ "$r" -lt "${LADDER_PAY_SETTLE_ROUNDS}" ]; do - r=$((r + 1)) - wcli transactions >"$SCRATCH/tx-$ptag.out" 2>&1 || true - curl -skS -m 8 -o "$SCRATCH/ord-paid-$ptag.json" -H "$AUTH" \ - "${MER}/instances/${INST}/private/orders/${OID}" 2>/dev/null || true - if grep -qiE 'payment|paid|Payment' "$SCRATCH/tx-$ptag.out" 2>/dev/null \ - || python3 -c 'import json,sys -d=json.load(open(sys.argv[1])) -sys.exit(0 if d.get("paid") is True or str(d.get("order_status","")).lower()=="paid" else 1) -' "$SCRATCH/ord-paid-$ptag.json" 2>/dev/null; then - settled=1 - break - fi - sleep 0.5 - done - ms_psettle=$(elapsed_ms "$t0") - ms_total=$(elapsed_ms "$t_pay") - after=$(wallet_avail) - if [ "$settled" = "1" ]; then - pstatus="OK" - pnote="avail=${CUR}:${after}" - ok "pay settled $PAMT → bal ${CUR}:${after} (total ${ms_total}ms)" - PAY_OK_N=$((PAY_OK_N + 1)) - echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t${ms_handle}\t${ms_psettle}\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" - metrics_report_coins "after-pay-${ptag}" || true - metrics_record_flow spent "$PAMT" || true - else - pnote="not settled order=$OID avail=${CUR}:${after}" - if [ "$IS_PMAX" = "1" ]; then - pstatus="CEILING_REJECT" - warn pay "pay $PAMT not settled (ceiling)" "$pnote" - echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t${ms_handle}\t${ms_psettle}\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" - continue - fi - err pay "settle $PAMT" "$pnote" - pstatus="FAIL_SETTLE" - PAY_FAIL_N=$((PAY_FAIL_N + 1)) - FAIL_N_L=$((FAIL_N_L + 1)) - STOP_REASON="$pnote" - STOP_AMOUNT="$PAMT" - echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t${ms_handle}\t${ms_psettle}\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" - metrics_report_coins "after-pay-fail-${ptag}" || true - break - fi - done - metrics_report_coins "after-pay-ladder" || true - info "pay summary" "ok=$PAY_OK_N fail=$PAY_FAIL_N tsv=$PAY_TSV" - fi -elif [ "${LADDER_PAY}" = "1" ] && [ "$FAIL_N_L" -gt 0 ]; then - warn pay "skipped pay ladder" "withdraw phase already failed" -fi - -ms_phase=$(python3 -c 'import sys,time; print(int((time.time()-float(sys.argv[1]))*1000))' "$SECTION_T0") - -# --- report --- -set_group report -section "ladder · report" -info "auto-account" "$ACCT_USER" -info "ok_rungs" "$OK_N" -info "fail_rungs" "$FAIL_N_L" -info "pay_ok" "$PAY_OK_N" -info "pay_fail" "$PAY_FAIL_N" -info "phase_ms" "$ms_phase" -info "tsv" "$TSV" -info "pay_tsv" "$PAY_TSV" - -# speed summary via python — free-text stop reason via env (JSON " in bank errors -# used to break shell argv quoting → "syntax error near unexpected token '('") -export LADDER_REPORT_STOP_AMOUNT="${STOP_AMOUNT:-}" -export LADDER_REPORT_STOP_REASON="${STOP_REASON:-}" -python3 - "$TSV" "$PAY_TSV" "$JSON" "$OK_N" "$FAIL_N_L" "$PAY_OK_N" "$PAY_FAIL_N" "$ms_phase" "$ACCT_USER" "$CUR" <<'PY' -import csv, json, os, sys, statistics -tsv, pay_tsv, jpath = sys.argv[1:4] -ok_n, fail_n, pay_ok, pay_fail, phase_ms, acct, cur = sys.argv[4:11] -stop_amt = os.environ.get("LADDER_REPORT_STOP_AMOUNT") or None -stop_reason = os.environ.get("LADDER_REPORT_STOP_REASON") or None - -def load_rows(path): - rows = [] - try: - with open(path, newline="") as f: - for row in csv.DictReader(f, delimiter="\t"): - rows.append(row) - except Exception: - pass - return rows - -rows = load_rows(tsv) -prows = load_rows(pay_tsv) - -def nums(rs, key): - out = [] - for row in rs: - try: - out.append(int(row[key])) - except Exception: - pass - return out - -def stats(xs): - if not xs: - return {"n": 0} - return { - "n": len(xs), - "min_ms": min(xs), - "max_ms": max(xs), - "avg_ms": int(sum(xs) / len(xs)), - "p50_ms": int(statistics.median(xs)), - } - -report = { - "currency": cur, - "auto_account": acct, - "ok_rungs": int(ok_n), - "fail_rungs": int(fail_n), - "pay_ok": int(pay_ok), - "pay_fail": int(pay_fail), - "stopped_at_amount": stop_amt or None, - "stop_reason": stop_reason or None, - "phase_ms": int(phase_ms), - "timing": { - "mint": stats(nums(rows, "ms_mint")), - "accept": stats(nums(rows, "ms_accept")), - "confirm": stats(nums(rows, "ms_confirm")), - "settle": stats(nums(rows, "ms_settle")), - "rung_total": stats(nums(rows, "ms_total")), - "pay_order": stats(nums(prows, "ms_order")), - "pay_handle": stats(nums(prows, "ms_handle")), - "pay_settle": stats(nums(prows, "ms_settle")), - "pay_total": stats(nums(prows, "ms_total")), - }, - "rungs": rows, - "pays": prows, -} -json.dump(report, open(jpath, "w"), indent=2) -print("JSON", jpath) -print("--- withdraw speed (ms) ---") -for k in ("mint", "accept", "confirm", "settle", "rung_total"): - v = report["timing"][k] - if v.get("n"): - print(" %s n=%s min=%s p50=%s avg=%s max=%s" % (k.ljust(12), v["n"], v["min_ms"], v["p50_ms"], v["avg_ms"], v["max_ms"])) -print("--- pay speed (ms) ---") -for k in ("pay_order", "pay_handle", "pay_settle", "pay_total"): - v = report["timing"][k] - if v.get("n"): - print(" %s n=%s min=%s p50=%s avg=%s max=%s" % (k.ljust(12), v["n"], v["min_ms"], v["p50_ms"], v["avg_ms"], v["max_ms"])) -print("--- withdraw rungs ---") -for row in rows: - print(" %2s %-16s %-12s total=%sms" % (row.get("rung"), row.get("amount"), row.get("status"), row.get("ms_total"))) -print("--- pay rungs ---") -for row in prows: - print(" %2s %-16s %-12s total=%sms oid=%s" % (row.get("rung"), row.get("amount"), row.get("status"), row.get("ms_total"), row.get("oid"))) -if stop_amt: - print("STOPPED at %s: %s" % (stop_amt, stop_reason)) -else: - print("Completed without hard failure (or budget stop without fail).") -PY - -wcli balance 2>&1 | tee "$REPORT_DIR/balance-final.out" | tail -20 || true - -set_group load -section "ladder · load snapshot (after withdraws)" -metrics_report_load "$LOAD_AFTER" "ladder-end" || true -if [ -f "$LOAD_BEFORE" ] && [ -f "$LOAD_AFTER" ]; then - section "metrics · ladder load delta" - metrics_print_load_delta "$LOAD_BEFORE" "$LOAD_AFTER" || true -fi -# Speed timings → metrics overall (min/p50/avg/max per phase) -if [ -f "$JSON" ]; then - python3 - "$JSON" "${METRICS_DIR}/perf-summary.json" <<'PY' 2>/dev/null || true -import json, sys -rep = json.load(open(sys.argv[1])) -out = {} -for k, v in (rep.get("timing") or {}).items(): - if isinstance(v, dict) and v.get("n"): - out[k] = v -json.dump(out, open(sys.argv[2], "w"), indent=2) -PY -fi -export METRICS_WITHDRAW_TSV="$TSV" -export METRICS_PAY_TSV="$PAY_TSV" -metrics_report_coins "ladder-end" || true -metrics_print_overall "ladder final" || true - -# Keep scratch if LADDER_REPORT_DIR set; else copy key files to /tmp -if [ -z "${LADDER_REPORT_DIR:-}" ]; then - KEEP="/tmp/goa-ladder-report-$(date +%Y%m%d-%H%M%S)" - mkdir -p "$KEEP" - cp -a "$TSV" "$PAY_TSV" "$JSON" "$SCRATCH/auto-account.json" "$SCRATCH/ladder-plan.txt" \ - "$SCRATCH/ladder-wd-plan.txt" "$SCRATCH/ladder-pay-plan.txt" \ - "$REPORT_DIR/balance-final.out" "$LOAD_BEFORE" "$LOAD_AFTER" "$KEEP/" 2>/dev/null || true - info "report_dir" "$KEEP" - echo "$KEEP" >"$SCRATCH/KEEP_PATH" -fi - -if [ "$FAIL_N_L" -gt 0 ]; then - blocker "ladder" "stopped at ${STOP_AMOUNT:-?} — ${STOP_REASON:-error}" - exit 1 -fi -if [ "$OK_N" -eq 0 ]; then - blocker "ladder" "no successful withdraw rungs" - exit 1 -fi -ok "ladder finished withdraw_ok=$OK_N pay_ok=$PAY_OK_N phase=${ms_phase}ms" -exit 0 +exec bash "$ROOT/check_amount_ladder.sh" "$@" diff --git a/ladder_extract_rpubs.py b/ladder_extract_rpubs.py new file mode 100755 index 0000000..1c939e9 --- /dev/null +++ b/ladder_extract_rpubs.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""Extract reserve_pub candidates from wallet accept + transactions dumps. + +Used by check_amount_ladder.sh force-select. Kept as a file (not a bash +heredoc) so shell refactors cannot turn Python ``continue`` into ``return``. + +Usage: + ladder_extract_rpubs.py WID AMT accept.out tx.json used-rpubs.txt +Prints one reserve pub per line, preferred first. +""" +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path + + +def main(argv: list[str]) -> int: + if len(argv) < 6: + print( + "usage: ladder_extract_rpubs.py WID AMT accept.out tx.json used-rpubs.txt", + file=sys.stderr, + ) + return 2 + + wid = argv[1] + amt = argv[2] + paths = argv[3:5] + used_path = argv[5] + used: set[str] = set() + if Path(used_path).is_file(): + used = {ln.strip() for ln in open(used_path) if ln.strip()} + + amt_num = amt.split(":", 1)[-1] if amt else "" + dec = json.JSONDecoder() + + def walk_collect(o, bag, ctx=None): + ctx = dict(ctx or {}) + if isinstance(o, dict): + for k, v in o.items(): + if not isinstance(v, str): + continue + kl = str(k).lower().replace("-", "_") + if kl in ( + "withdrawal_id", + "withdraw_id", + "wopid", + "id", + "transactionid", + "transaction_id", + ): + ctx["id"] = v + elif kl in ( + "amount", + "rawamount", + "instructedamount", + "amounteffective", + "amountraw", + "transferamount", + ): + ctx["amount"] = v + elif kl in ( + "taler_withdraw_uri", + "talerwithdrawuri", + "uri", + "bankconfirmationurl", + "confirmtransferurl", + ): + ctx["uri"] = v + for k, v in o.items(): + kl = str(k).lower().replace("-", "_") + if kl in ("reserve_pub", "reservepub") and isinstance(v, str) and len(v) >= 40: + bag.append((v, dict(ctx))) + elif not isinstance(v, (str, int, float, bool)) and v is not None: + walk_collect(v, bag, ctx) + elif isinstance(o, list): + for i in o: + walk_collect(i, bag, ctx) + + def score_ctx(pub, ctx, window=""): + score = 0 + cid = str(ctx.get("id") or "") + camt = str(ctx.get("amount") or "") + curi = str(ctx.get("uri") or "") + if wid and wid in curi: + score += 120 + if wid and wid in cid: + score += 100 + if wid and wid in window: + score += 80 + if amt and (camt == amt or camt.endswith(amt_num)): + score += 50 + if amt and amt in window: + score += 30 + if pub in used: + score -= 200 + return score + + def iter_json_objects(text: str): + i, n = 0, len(text) + while i < n: + if text[i] == "{": + try: + o, end = dec.raw_decode(text, i) + yield o + i = end + continue + except Exception: + pass + i += 1 + + raw_pubs: list[str] = [] + scored: list[tuple[int, str]] = [] + + for p in paths: + try: + t = open(p, errors="replace").read() + except Exception: + continue + for o in iter_json_objects(t): + bag = [] + walk_collect(o, bag) + for pub, ctx in bag: + raw_pubs.append(pub) + scored.append((score_ctx(pub, ctx), pub)) + + rpub_pats = ( + r"\"reservePub\"\s*:\s*\"([A-Z0-9]{40,})\"", + r"\"reserve_pub\"\s*:\s*\"([A-Z0-9]{40,})\"", + r"reserve[_ ]?pub[\"\s:=]+([A-Z0-9]{40,})", + ) + for p in paths: + try: + blob = open(p, errors="replace").read() + except Exception: + continue + for pat in rpub_pats: + for m in re.finditer(pat, blob, re.I): + pub = m.group(1) + raw_pubs.append(pub) + window = blob[max(0, m.start() - 600) : m.end() + 600] + scored.append((score_ctx(pub, {}, window) + 10, pub)) + + order: list[str] = [] + seen: set[str] = set() + for score, pub in sorted(scored, key=lambda x: x[0], reverse=True): + if pub in seen or pub in used: + continue + seen.add(pub) + order.append(pub) + for pub in reversed(raw_pubs): + if pub in seen or pub in used: + continue + seen.add(pub) + order.append(pub) + for pub in order: + print(pub) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/metrics.sh b/metrics.sh index aac0630..5e6ac42 100644 --- a/metrics.sh +++ b/metrics.sh @@ -70,11 +70,15 @@ path = sys.argv[2] if len(sys.argv) > 2 else "" alt = {} if path: try: - alt = json.load(open(path)) + _loaded = json.load(open(path)) + if isinstance(_loaded, dict): + alt = {str(k): str(v) for k, v in _loaded.items()} except Exception: alt = {} -if not alt: +if not isinstance(alt, dict) or not alt: alt = {"0": "GOA"} +elif "0" not in alt: + alt["0"] = "GOA" def parse(s): if ":" in s: diff --git a/taler-monitoring.sh b/taler-monitoring.sh index 35c2a99..b89fbaa 100755 --- a/taler-monitoring.sh +++ b/taler-monitoring.sh @@ -93,7 +93,8 @@ Phases: sanity public + optional server server server-side only (SSH) e2e withdraw + pay (small amounts; remote aborts on login/KYC) - ladder withdraw/pay amount ladder (GOA ceiling or stage TESTPAYSAN max_wire) + ladder amount ladder (classic: 0→mids→max; any currency / stack ceiling) + max-ladder same script, LADDER_MODE=max — hunt highest withdrawable amount auth401 merchant Basic-auth / case matrix (HTTP 401 paths; may create throwaway instance) aptdeploy koopa podman apt-src smoke: taler-merchant in koopa-taler-deploy-test-apt-src-trixie{,-testing} @@ -265,7 +266,7 @@ while [ $# -gt 0 ]; do CURRENCY_OVERRIDE="$2"; shift 2 ;; --no-probe) NO_PROBE=1; shift ;; - urls|inside|versions|sanity|server|e2e|ladder|goa-ladder|auth401|aptdeploy|apt-deploy|apt_src|surface|ecosystem|mattermost|mail|monpages|pages|devtesting|franken|fake-franken|fake_franken|all|full) PHASES+=("$1"); shift ;; + urls|inside|versions|sanity|server|e2e|ladder|goa-ladder|max-ladder|maxladder|auth401|aptdeploy|apt-deploy|apt_src|surface|ecosystem|mattermost|mail|monpages|pages|devtesting|franken|fake-franken|fake_franken|all|full) PHASES+=("$1"); shift ;; *) # bare domain shorthand: ./taler-monitoring.sh taler.net if [[ "$1" == *.* && "$1" != *://* && "$1" != -* ]]; then @@ -316,7 +317,7 @@ export INSIDE_PROFILE INSIDE_SSH export INSIDE_BANK_CTR INSIDE_EXCHANGE_CTR INSIDE_MERCHANT_CTR export INSIDE_BANK_PORT INSIDE_EXCHANGE_PORT INSIDE_MERCHANT_PORT export INSIDE_DNS_BANK INSIDE_DNS_EXCHANGE INSIDE_DNS_MERCHANT -# Ladder: withdraw then pay — 0 + random mids + max-1 + max (see check_goa_ladder.sh). +# Amount ladder — classic or max-search (see check_amount_ladder.sh; goa-ladder alias). # Defaults so set -u export is safe when vars were never set by caller. : "${LADDER_STEPS:=23}" : "${LADDER_MIN_AMOUNT:=0.000001}" @@ -330,16 +331,23 @@ export INSIDE_DNS_BANK INSIDE_DNS_EXCHANGE INSIDE_DNS_MERCHANT : "${LADDER_INCLUDE_ZERO:=1}" : "${LADDER_INCLUDE_MAX:=1}" : "${LADDER_HIGH_FROM:=1000000}" -: "${LADDER_HIGH_RUNGS:=12}" +: "${LADDER_HIGH_RUNGS:=4}" : "${LADDER_LOAD:=1}" : "${LADDER_PAY:=1}" : "${LADDER_WITHDRAW_SCALE:=1.5}" : "${LADDER_PAY_SETTLE_ROUNDS:=6}" +: "${LADDER_MODE:=classic}" +: "${LADDER_MAX_PROBES:=32}" +: "${LADDER_MAX_TOL_REL:=0.001}" +: "${LADDER_MAX_SEED:=}" +: "${LADDER_PAY_WITH_MAX:=0}" +: "${LADDER_HIGH_RUNGS:=6}" export LADDER_STEPS LADDER_MIN_AMOUNT LADDER_CONFIRM_POLLS LADDER_MAX_RUNGS LADDER_TIMEOUT_S LADDER_REPORT_DIR export LADDER_SETTLE_ROUNDS LADDER_SETTLE_SLEEP EXP_PW_FILE EXP_USER export LADDER_MAX_AMOUNT LADDER_INCLUDE_ZERO LADDER_INCLUDE_MAX export LADDER_HIGH_FROM LADDER_HIGH_RUNGS LADDER_LOAD export LADDER_PAY LADDER_WITHDRAW_SCALE LADDER_PAY_SETTLE_ROUNDS +export LADDER_MODE LADDER_MAX_PROBES LADDER_MAX_TOL_REL LADDER_MAX_SEED LADDER_PAY_WITH_MAX export TALER_DOMAIN_APPLIED=1 i18n_init 2>/dev/null || true # Optional: auth401 and other phases may honor CONTINUE_ON_ERROR @@ -435,7 +443,7 @@ _progress_estimate_phase() { server) n=20 ;; # e2e: bank withdraw ladder emits many INFO lines (coins before/after each note) e2e) n=240 ;; - ladder|goa-ladder) n=120 ;; + ladder|goa-ladder|max-ladder|maxladder) n=120 ;; auth401) n=70 ;; aptdeploy) n=40 ;; surface|ecosystem) n=120 ;; @@ -611,7 +619,15 @@ while [ "$_phase_idx" -lt "$_n_phases" ]; do sanity) run_phase sanity "$ROOT/check_sanity.sh" || ec=1 ;; server) run_phase server "$ROOT/check_server.sh" || ec=1 ;; e2e) run_phase e2e "$ROOT/check_e2e.sh" || ec=1 ;; - ladder|goa-ladder) run_phase ladder "$ROOT/check_goa_ladder.sh" || ec=1 ;; + ladder|goa-ladder) + run_phase ladder "$ROOT/check_amount_ladder.sh" || ec=1 + ;; + max-ladder|maxladder) + # Feature: hunt highest mint+confirm withdrawable amount (any currency) + LADDER_MODE=max + export LADDER_MODE + run_phase ladder "$ROOT/check_amount_ladder.sh" || ec=1 + ;; auth401) run_phase auth401 "$ROOT/check_auth401.sh" || ec=1 ;; aptdeploy|apt-deploy|apt_src) run_phase aptdeploy "$ROOT/check_apt_deploy.sh" || ec=1 ;; surface|ecosystem) run_phase surface "$ROOT/check_surface.sh" || ec=1 ;; From 76a253613a91b637fedbc3948ef6d0550960776c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 14:25:24 +0200 Subject: [PATCH 17/29] release 1.22.0: modular ladder withdraw + pay max-search Split pay into ladder/lib_pay.sh; force-select extract in ladder/extract_rpubs.py. check_goa_ladder.sh is only a compat shim. max-ladder now hunts highest withdrawable and highest payable (never above wallet available), using public free-amount template goa-free on hacktivism. Classic remains default 23-rung wd+pay. Pay phase always runs when LADDER_PAY=1 (visible SKIP_BALANCE when no spendable coins). Report includes best_pay / alt names. --- README.md | 6 +- VERSION | 2 +- VERSIONS.md | 3 +- check_amount_ladder.sh | 320 +++------------- check_goa_ladder.sh | 4 +- ladder/README.md | 25 ++ .../extract_rpubs.py | 0 ladder/lib_pay.sh | 361 ++++++++++++++++++ taler-monitoring.sh | 2 +- 9 files changed, 449 insertions(+), 274 deletions(-) create mode 100644 ladder/README.md rename ladder_extract_rpubs.py => ladder/extract_rpubs.py (100%) create mode 100644 ladder/lib_pay.sh diff --git a/README.md b/README.md index d8cf093..5af4d7f 100644 --- a/README.md +++ b/README.md @@ -277,7 +277,7 @@ Other domains: never SSH. Optional **e2e** aborts cleanly on login/KYC. # Amount ladder (any currency): classic 0 → … → max, then optional pay ./taler-monitoring.sh ladder # classic (current domain) ./taler-monitoring.sh -d stage.lefrancpaysan.ch ladder # TESTPAYSAN maxima -# Max-search: start high, narrow toward highest withdrawable +# Max-search: highest withdrawable + highest payable (≤ available; GOA free template goa-free) ./taler-monitoring.sh max-ladder # Stage auto-reads bank max_wire_transfer_amount (e.g. 2000) and min denom (0.01). # Explorer password: francpaysan-stage-user …/bank-explorer-password.txt (or EXP_PW=). @@ -396,10 +396,12 @@ LADDER_LOAD=0 ./taler-monitoring.sh ladder | `METRICS_LOAD` | `1` | `0` = skip all host/container load probes | | `LADDER_LOAD` | `1` | `0` = skip load in ladder only | | `LADDER_PAY` | `1` | `0` = skip payment ladder after withdraws | -| `LADDER_MODE` | `classic` | default **classic** 23-rung; `max` = hunt highest withdrawable (`max-ladder`) | +| `LADDER_MODE` | `classic` | default **classic** 23-rung wd+pay; `max` = max-search wd+pay (`max-ladder`) | | `LADDER_MAX_PROBES` | `32` | max-search probe budget; `0` = until converge / timeout | | `LADDER_HIGH_RUNGS` | `6` | max-search: first N probes random-high, then bisect | | `LADDER_MAX_TOL_REL` | `0.001` | max-search relative lo/hi gap to stop | +| `LADDER_FREE_TEMPLATE` | `goa-free` | public free-amount pay template (hacktivism) | +| `LADDER_USE_FREE_TEMPLATE` | `1` | create pays via template POST `{"amount":…}` | | `LADDER_WITHDRAW_SCALE` | `1.5` | classic: withdraw mids / pay mids ratio (funding headroom) | | `LADDER_STEPS` | `23` | classic: rungs for withdraw and pay (0 + mids + max−1 + max) | | `METRICS_LOAD_SSH_TIMEOUT` | `90` | seconds for remote load python | diff --git a/VERSION b/VERSION index 3500250..57807d6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.21.0 +1.22.0 diff --git a/VERSIONS.md b/VERSIONS.md index f8d32cd..2f327d9 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,7 +17,8 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| -| **v1.21.0** | 2026-07-19 | **Feature:** generic **amount ladder** (`check_amount_ladder.sh`; compat shim `check_goa_ladder.sh`). Default **classic** 23-rung withdraw (+ pay when enabled); **max-search** (`max-ladder` / `LADDER_MODE=max`) hunts highest mintable amount (probes default 32, `0`=until timeout). **Pay:** never overspend — soft `SKIP_BALANCE` if amount > wallet available. **Alt units:** report/logs show human names (e.g. `8.15 Tera-GOA (GOA:…)`) + raw. **Fixes bundled:** external `ladder_extract_rpubs.py` (no stdin SyntaxError); force-select multi-rpub + raw_decode; 409 tries next rpub; soft `CEILING_REJECT` on mid/high mint 5110/P0001; report python never aborts ladder; helper PATH python≥3.11; wallet-db not `*.json`. CLI-AUTOMATION-NOTES §3/3a/3b/§15–18. | +| **v1.22.0** | 2026-07-19 | **Feature:** modular amount ladder — `ladder/lib_pay.sh` (pay) + withdraw in `check_amount_ladder.sh`; `ladder/extract_rpubs.py`. **max-ladder** hunts highest **withdrawable** and highest **payable** (≤ wallet available). GOA pays use public free-amount template **`goa-free`** (`LADDER_FREE_TEMPLATE`). Classic still default 23-rung wd+pay. `check_goa_ladder.sh` remains a thin compat shim only. | +| **v1.21.0** | 2026-07-19 | **Feature:** generic **amount ladder** (`check_amount_ladder.sh`; compat shim `check_goa_ladder.sh`). Default **classic** 23-rung withdraw (+ pay when enabled); **max-search** (`max-ladder` / `LADDER_MODE=max`) hunts highest mintable amount (probes default 32, `0`=until timeout). **Pay:** never overspend — soft `SKIP_BALANCE` if amount > wallet available. **Alt units:** report/logs show human names (e.g. `8.15 Tera-GOA (GOA:…)`) + raw. **Fixes bundled:** external extract (no stdin SyntaxError); force-select multi-rpub; soft `CEILING_REJECT` on mint 5110/P0001; crash-proof report. CLI-AUTOMATION-NOTES §3/3a/3b/§15–18. | | **v1.20.1** | 2026-07-19 | **Docs:** CLI-AUTOMATION-NOTES — force-select empty scrape + ladder pins (§3/3a); wallet-db must not be `*.json` (§15); `taler-helper-sqlite3` needs Python ≥3.11 (§16); portable wall-clock (§17); GOA withdraw checklist (§18). No code change. *(Superseded notes folded into **v1.21.0** feature release.)* | | **v1.20.0** | 2026-07-19 | **Feature:** domains.conf **`locale`** (l10n) — FP **`fr-CH`**, others **`de-CH`** (aliases ch-FR/ch-DE); UI **`lang=de`** supported (sticky + console tags) but **never** a stock profile default; env `TALER_MON_LOCALE` / `TALER_DOMAIN_LOCALE`. | | **v1.19.1** | 2026-07-19 | **Bugfix / docs:** DEPENDENCIES — e2e requires `taler-helper-sqlite3` on mon PATH + preflight; README domains.conf documents **`lang`** column. | diff --git a/check_amount_ladder.sh b/check_amount_ladder.sh index 9c7e2c1..a4ad861 100755 --- a/check_amount_ladder.sh +++ b/check_amount_ladder.sh @@ -11,24 +11,26 @@ # 4) bank confirm when selected; settle = balance + transfer_done # # Modes (LADDER_MODE): -# classic — random strictly increasing rungs + pins 0, max-1, max (default) -# max — free max-search: start randomly high, narrow toward highest -# amount that mint+accept+confirm (+ bank transfer_done) succeeds +# classic — random strictly increasing rungs + pins 0, max-1, max (default 23) +# max — max-search: highest withdrawable AND highest payable +# (pay never exceeds wallet available; GOA uses public free-amount +# template goa-free by default) # -# Phase A: withdraw. Phase B: pay (classic only by default; off in max mode). +# Phase A: withdraw. Phase B: pay (classic plan list, or max-search pay). # # Stacks auto-tune via bank /config max_wire + exchange min denom when possible. # # Usage: # ./taler-monitoring.sh ladder -# ./taler-monitoring.sh max-ladder # LADDER_MODE=max +# ./taler-monitoring.sh max-ladder # LADDER_MODE=max (wd + pay hunt) # ./taler-monitoring.sh -d stage.lefrancpaysan.ch ladder -# LADDER_MODE=max LADDER_MAX_PROBES=16 ./taler-monitoring.sh ladder +# LADDER_MODE=max LADDER_MAX_PROBES=32 ./taler-monitoring.sh ladder # LADDER_MAX_AMOUNT=100 LADDER_STEPS=7 ./taler-monitoring.sh -d stage.lefrancpaysan.ch ladder # # Env: LADDER_MODE, LADDER_STEPS, LADDER_MAX_AMOUNT, LADDER_MIN_AMOUNT, # LADDER_MAX_PROBES, LADDER_MAX_TOL_REL, LADDER_HIGH_RUNGS, LADDER_STACK_AUTO, -# LADDER_PAY, EXP_PW / EXP_PW_FILE, CLI_JS, MERCHANT_INSTANCE, … +# LADDER_PAY, LADDER_FREE_TEMPLATE, LADDER_FREE_TEMPLATE_INSTANCE, +# EXP_PW / EXP_PW_FILE, CLI_JS, MERCHANT_INSTANCE, … set -euo pipefail ROOT=$(cd "$(dirname "$0")" && pwd) # shellcheck source=lib.sh @@ -71,7 +73,12 @@ elapsed_ms() { # Public variable template for stage pays (optional; fixed templates used when amount maps) : "${LADDER_PAY_TEMPLATE:=}" : "${LADDER_PAY_TEMPLATE_INSTANCE:=fermes-des-collines}" -# classic | max (max = find highest withdrawable; phase alias max-ladder) +# Free-amount public template (hacktivism: goa-free on goa-demo-cp4zqk — any GOA amount) +: "${LADDER_FREE_TEMPLATE:=goa-free}" +: "${LADDER_FREE_TEMPLATE_INSTANCE:=}" +# 1 = create pays via public free template POST (amount in body); 0 = private orders only +: "${LADDER_USE_FREE_TEMPLATE:=1}" +# classic | max (max = highest withdrawable + highest payable; phase alias max-ladder) : "${LADDER_MODE:=classic}" # max-search: probe budget (0 = run until convergence / LADDER_TIMEOUT_S only) : "${LADDER_MAX_PROBES:=32}" @@ -80,6 +87,8 @@ elapsed_ms() { : "${LADDER_HIGH_RUNGS:=6}" # optional seed for reproducible max-search (empty = time-based) : "${LADDER_MAX_SEED:=}" +# After withdraw, wait for spendable available>0 before pay (secs total) +: "${LADDER_PAY_WAIT_AVAILABLE_S:=60}" # Historic libeufin-ish absolute ceiling used as default LADDER_MAX_AMOUNT on GOA LADDER_ABS_CEILING="4503599627370496" @@ -180,13 +189,10 @@ case "${LADDER_MODE}" in LADDER_MODE=classic ;; esac -# max mode: pay ladder off unless caller explicitly set LADDER_PAY=1 *and* LADDER_PAY_WITH_MAX=1 -if [ "$LADDER_MODE" = "max" ]; then - if [ "${LADDER_PAY_WITH_MAX:-0}" != "1" ]; then - LADDER_PAY=0 - fi -fi +# max mode: withdraw + pay hunts (LADDER_PAY=0 still disables pay entirely) +: "${LADDER_FREE_TEMPLATE_INSTANCE:=${MERCHANT_INSTANCE}}" export LADDER_MODE LADDER_MAX_PROBES LADDER_MAX_TOL_REL LADDER_HIGH_RUNGS LADDER_MAX_SEED LADDER_PAY +export LADDER_FREE_TEMPLATE LADDER_FREE_TEMPLATE_INSTANCE LADDER_USE_FREE_TEMPLATE LADDER_PAY_WAIT_AVAILABLE_S SCRATCH=$(mktemp -d) WDB="$SCRATCH/wallet.sqlite3" @@ -435,6 +441,9 @@ PY # shellcheck source=metrics.sh source "$ROOT/metrics.sh" +# Payment module (classic + max-search payable) +# shellcheck source=ladder/lib_pay.sh +source "$ROOT/ladder/lib_pay.sh" METRICS_DIR="$REPORT_DIR" ALT_UNITS_FILE="${REPORT_DIR}/alt_unit_names.json" export METRICS_DIR CUR WDB CLI_JS ALT_UNITS_FILE @@ -540,8 +549,9 @@ ok "wallet exchange + ToS (${ms_tos}ms) — cumulative DB for withdraw+pay (no r : "${LADDER_MAX_RUNGS:=99}" if [ "${LADDER_MODE}" = "max" ]; then - info "ladder mode" "max-search (find highest withdrawable ${CUR})" + info "ladder mode" "max-search (highest withdrawable + highest payable ${CUR})" info "max-search params" "probes≤${LADDER_MAX_PROBES} high_rand=${LADDER_HIGH_RUNGS} tol_rel=${LADDER_MAX_TOL_REL} ceiling=${CUR}:${LADDER_MAX_AMOUNT}" + info "max-search pay" "LADDER_PAY=${LADDER_PAY} free_template=${LADDER_FREE_TEMPLATE}@${LADDER_FREE_TEMPLATE_INSTANCE:-$INST}" LADDER_LIST="" PAY_LIST="" LADDER_N=0 @@ -757,8 +767,8 @@ ladder_withdraw_rung() { # and regex on *both* accept and transactions output (reservePub lives in tx, not accept). extract_rpubs_for_wid() { # External module — never feed extract via bash heredoc (refactor tools - # previously turned Python continue into return → SyntaxError line 91). - local _ext="$ROOT/ladder_extract_rpubs.py" + # lives in ladder/extract_rpubs.py). + local _ext="$ROOT/ladder/extract_rpubs.py" if [ ! -f "$_ext" ]; then warn bank "force-select extract missing" "problem: $_ext not found" return 0 @@ -1122,9 +1132,9 @@ print(min(a,h)) "problem: no OK/OK_BANK within ${rung} probes; hi=$(format_amount_alt "${CUR}:${MAX_HI}")" fi { - echo "# max-search mode" - echo "best=${MAX_BEST_AMT:-}" - echo "best_alt=$(format_amount_alt "${MAX_BEST_AMT:-${CUR}:0}")" + echo "# max-search mode (withdraw)" + echo "best_wd=${MAX_BEST_AMT:-}" + echo "best_wd_alt=$(format_amount_alt "${MAX_BEST_AMT:-${CUR}:0}")" echo "lo=${MAX_LO}" echo "lo_alt=$(format_amount_alt "${CUR}:${MAX_LO}")" echo "hi=${MAX_HI}" @@ -1133,6 +1143,7 @@ print(min(a,h)) } >"$SCRATCH/ladder-plan.txt" printf '%s\n' "${MAX_BEST_AMT:-${CUR}:0}" >"$SCRATCH/ladder-wd-plan.txt" : >"$SCRATCH/ladder-pay-plan.txt" + # pay phase uses max-search (not classic PAY_LIST) PAY_LIST="" else section "ladder · phase A · withdraw (${LADDER_N} rungs)" @@ -1163,255 +1174,15 @@ fi # --------------------------------------------------------------------------- -# Phase B — payment ladder (same shape: 0 → low → … → max-1 → max) +# Phase B — payment (ladder/lib_pay.sh) +# classic: planned pay list · max: highest payable ≤ available (goa-free) # --------------------------------------------------------------------------- PAY_OK_N=0 PAY_FAIL_N=0 -if [ "${LADDER_PAY}" = "1" ] && [ -n "${PAY_LIST:-}" ] && [ "$FAIL_N_L" -eq 0 ]; then - set_group pay - section "ladder · phase B · pay" - metrics_report_coins "before-pay-ladder" || true - # Merchant secret (same as e2e) — stage often has only public shop templates - MPW="${E2E_MERCHANT_TOKEN:-${MERCHANT_TOKEN:-}}" - if [ -z "$MPW" ]; then - MPW=$(read_secret "taler-merchant/merchant-${INST}-password.txt" 2>/dev/null || true) - fi - if [ -z "$MPW" ]; then - MPW=$(read_secret "taler-merchant/merchant-goa-demo-cp4zqk-password.txt" 2>/dev/null || true) - fi - if [ -z "$MPW" ] && [ "${CUR}" = "TESTPAYSAN" ]; then - if [ -n "${FRANCPAYSAN_SECRETS:-}" ] && [ -f "${FRANCPAYSAN_SECRETS}/stage/default-instance-token.txt" ]; then - MPW=$(tr -d '\n\r' <"${FRANCPAYSAN_SECRETS}/stage/default-instance-token.txt") - fi - if [ -z "$MPW" ]; then - _remote_mer="${FRANCPAYSAN_REMOTE_MERCHANT_TOKEN:-}" - [ -z "$_remote_mer" ] && [ -n "${FRANCPAYSAN_REMOTE_SECRETS_ROOT:-}" ] && \ - _remote_mer="${FRANCPAYSAN_REMOTE_SECRETS_ROOT}/merchant/secrets/default-instance-token.txt" - for host in ${INSIDE_SSH:+"$INSIDE_SSH"} ${FRANCPAYSAN_SSH:+"$FRANCPAYSAN_SSH"}; do - [ -n "$host" ] && [ -n "$_remote_mer" ] || continue - MPW=$(ssh -o BatchMode=yes -o ConnectTimeout=12 "$host" \ - "tr -d '\\n\\r' <${_remote_mer} 2>/dev/null || true" \ - 2>/dev/null || true) - [ -n "$MPW" ] && break - done - unset _remote_mer - fi - fi - if [ -z "$MPW" ]; then - if [ "${CUR}" = "TESTPAYSAN" ]; then - # Stage: withdraw ladder is the main probe; pays need instance token or - # public templates for *exact* face values (not continuous ladder amounts). - info pay "no merchant token — skip pay ladder on TESTPAYSAN (withdraw maxima still covered)" - info pay "hint: set E2E_MERCHANT_TOKEN or use ./taler-monitoring.sh -d stage.lefrancpaysan.ch e2e for shop templates" - else - warn pay "no merchant token — skip pay ladder (set E2E_MERCHANT_TOKEN or secrets)" - fi - else - ok "merchant token" "instance ${INST}" - case "$MPW" in - secret-token:*) AUTH="Authorization: Bearer ${MPW}" ;; - *) AUTH="Authorization: Bearer secret-token:${MPW}" ;; - esac - wallet_prepare "main" - # shellcheck disable=SC2086 - set -- $PAY_LIST - PAY_N=$# - _bal0=$(wallet_avail) - info "pay rungs" "$PAY_N · $*" - info "pay budget" "wallet available ${CUR}:${_bal0} · $(format_amount_alt "${CUR}:${_bal0}") — never order more than this" - prung=0 - for PAMT in "$@"; do - prung=$((prung + 1)) - if ladder_over; then - warn pay "time budget exhausted" "after ${PAY_OK_N} ok pays" - break - fi - section "ladder · pay rung $prung $PAMT · $(format_amount_alt "$PAMT")" - ptag=$(printf '%s' "$PAMT" | tr '.:' '__') - if [ "$prung" -eq 1 ]; then - prange="pin:0" - elif [ "$prung" -eq "$PAY_N" ]; then - prange="pin:max" - elif [ "$prung" -eq $((PAY_N - 1)) ] && [ "$PAY_N" -ge 3 ]; then - prange="pin:max-1" - else - prange="random" - fi - PNUM=$(python3 -c 'import sys; print(sys.argv[1].split(":",1)[-1])' "$PAMT") - IS_PZERO=0 - python3 -c 'import sys; from decimal import Decimal; sys.exit(0 if Decimal(sys.argv[1])==0 else 1)' "$PNUM" 2>/dev/null && IS_PZERO=1 - IS_PMAX=0 - [ "$PNUM" = "${LADDER_MAX_AMOUNT}" ] && IS_PMAX=1 - t_pay=$(now_ms) - ms_order=0 ms_handle=0 ms_psettle=0 - pnote="" - pstatus="FAIL" - OID="-" - - metrics_report_coins "before-pay-${ptag}" || true - bal=$(wallet_avail) - - if [ "$IS_PZERO" = "1" ]; then - pstatus="ZERO_SKIP" - pnote="zero pay probe skipped" - warn pay "pay $PAMT skipped" "problem: zero amount order not useful. Ladder continues." - ms_total=$(elapsed_ms "$t_pay") - echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t${ms_handle}\t${ms_psettle}\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" - PAY_OK_N=$((PAY_OK_N + 1)) - continue - fi - - # Never spend more than wallet available (spendable coins only — not pendingIncoming). - # Soft-skip any over-budget rung; do not hard-fail the ladder for insufficient funds. - if ! python3 -c 'import sys; from decimal import Decimal; sys.exit(0 if Decimal(sys.argv[1])>=Decimal(sys.argv[2]) else 1)' "$bal" "$PNUM" 2>/dev/null; then - pnote="skip: would spend more than available avail=${CUR}:${bal} need=${PAMT}" - ms_total=$(elapsed_ms "$t_pay") - pstatus="SKIP_BALANCE" - warn pay "pay $PAMT skipped (balance)" \ - "problem: pay amount exceeds wallet available — never overspend. $(format_amount_alt "${CUR}:${bal}") free vs $(format_amount_alt "$PAMT"). detail: $pnote" - echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t0\t0\t0\t${ms_total}\t-\t${pnote}" >>"$PAY_TSV" - continue - fi - - t0=$(now_ms) - SUM_JSON=$(python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "ladder pay ${PAMT}" 2>/dev/null || echo '"ladder pay"') - curl -skS -m 20 -o "$SCRATCH/ord-$ptag.json" -X POST \ - -H "$AUTH" -H 'Content-Type: application/json' \ - -d "{\"order\":{\"summary\":${SUM_JSON},\"amount\":\"${PAMT}\",\"fulfillment_message\":\"ok\"},\"create_token\":true}" \ - "${MER}/instances/${INST}/private/orders" 2>"$SCRATCH/ord-$ptag.err" || true - ms_order=$(elapsed_ms "$t0") - OID=$(python3 -c 'import json,re,sys;t=open(sys.argv[1]).read() -try: print(json.loads(t).get("order_id") or "") -except Exception: - m=re.search(r"\"order_id\"\s*:\s*\"([^\"]+)\"",t); print(m.group(1) if m else "") -' "$SCRATCH/ord-$ptag.json" 2>/dev/null || true) - OTOK=$(python3 -c 'import json,re,sys;t=open(sys.argv[1]).read() -try: print(json.loads(t).get("token") or "") -except Exception: - m=re.search(r"\"token\"\s*:\s*\"([^\"]+)\"",t); print(m.group(1) if m else "") -' "$SCRATCH/ord-$ptag.json" 2>/dev/null || true) - if [ -z "$OID" ]; then - pnote="order create failed $(head -c 80 "$SCRATCH/ord-$ptag.json" 2>/dev/null | tr '\n\"' ' ')" - ms_total=$(elapsed_ms "$t_pay") - if [ "$IS_PMAX" = "1" ]; then - pstatus="CEILING_REJECT" - warn pay "pay $PAMT rejected (ceiling)" "$pnote" - echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t0\t0\t${ms_total}\t-\t${pnote}" >>"$PAY_TSV" - continue - fi - err pay "order $PAMT" "$pnote" - pstatus="FAIL_ORDER" - PAY_FAIL_N=$((PAY_FAIL_N + 1)) - FAIL_N_L=$((FAIL_N_L + 1)) - STOP_REASON="$pnote" - STOP_AMOUNT="$PAMT" - echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t0\t0\t${ms_total}\t-\t${pnote}" >>"$PAY_TSV" - break - fi - ok "order $OID ($PAMT) ${ms_order}ms" - - curl -skS -m 12 -o "$SCRATCH/ord-det-$ptag.json" -H "$AUTH" \ - "${MER}/instances/${INST}/private/orders/${OID}" 2>/dev/null || true - PAYURI=$(python3 -c 'import json,sys -try: print(json.load(open(sys.argv[1])).get("taler_pay_uri") or "") -except Exception: print("") -' "$SCRATCH/ord-det-$ptag.json" 2>/dev/null || true) - if [ -z "$PAYURI" ] && [ -n "$OTOK" ]; then - MH=$(python3 -c 'from urllib.parse import urlparse; print(urlparse("'"$MER"'").hostname or "taler.hacktivism.ch")' 2>/dev/null || echo "taler.hacktivism.ch") - PAYURI="taler://pay/${MH}/instances/${INST}/${OID}/?c=${OTOK}" - fi - PAYURI=$(printf '%s' "$PAYURI" | sed 's/:443\//\//g; s/:443?/?/g') - if [ -z "$PAYURI" ]; then - pnote="no pay URI for $OID" - ms_total=$(elapsed_ms "$t_pay") - err pay "uri $PAMT" "$pnote" - pstatus="FAIL_URI" - PAY_FAIL_N=$((PAY_FAIL_N + 1)) - FAIL_N_L=$((FAIL_N_L + 1)) - STOP_REASON="$pnote" - STOP_AMOUNT="$PAMT" - echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t0\t0\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" - break - fi - - t0=$(now_ms) - if ! wcli handle-uri --yes "$PAYURI" >"$SCRATCH/pay-$ptag.out" 2>&1; then - ms_handle=$(elapsed_ms "$t0") - pnote="handle-uri failed $(tail -c 100 "$SCRATCH/pay-$ptag.out" | tr '\n\"' ' ')" - ms_total=$(elapsed_ms "$t_pay") - if [ "$IS_PMAX" = "1" ]; then - pstatus="CEILING_REJECT" - warn pay "pay $PAMT handle failed (ceiling)" "$pnote" - echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t${ms_handle}\t0\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" - continue - fi - err pay "handle $PAMT" "$pnote" - pstatus="FAIL_HANDLE" - PAY_FAIL_N=$((PAY_FAIL_N + 1)) - FAIL_N_L=$((FAIL_N_L + 1)) - STOP_REASON="$pnote" - STOP_AMOUNT="$PAMT" - echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t${ms_handle}\t0\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" - break - fi - ms_handle=$(elapsed_ms "$t0") - ok "handle-uri $PAMT ${ms_handle}ms" - - # Short settle polls: merchant order + wallet tx only — never run-until-done - t0=$(now_ms) - settled=0 - r=0 - while [ "$r" -lt "${LADDER_PAY_SETTLE_ROUNDS}" ]; do - r=$((r + 1)) - wcli transactions >"$SCRATCH/tx-$ptag.out" 2>&1 || true - curl -skS -m 8 -o "$SCRATCH/ord-paid-$ptag.json" -H "$AUTH" \ - "${MER}/instances/${INST}/private/orders/${OID}" 2>/dev/null || true - if grep -qiE 'payment|paid|Payment' "$SCRATCH/tx-$ptag.out" 2>/dev/null \ - || python3 -c 'import json,sys -d=json.load(open(sys.argv[1])) -sys.exit(0 if d.get("paid") is True or str(d.get("order_status","")).lower()=="paid" else 1) -' "$SCRATCH/ord-paid-$ptag.json" 2>/dev/null; then - settled=1 - break - fi - sleep 0.5 - done - ms_psettle=$(elapsed_ms "$t0") - ms_total=$(elapsed_ms "$t_pay") - after=$(wallet_avail) - if [ "$settled" = "1" ]; then - pstatus="OK" - pnote="avail=${CUR}:${after}" - ok "pay settled $PAMT → bal ${CUR}:${after} (total ${ms_total}ms)" - PAY_OK_N=$((PAY_OK_N + 1)) - echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t${ms_handle}\t${ms_psettle}\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" - metrics_report_coins "after-pay-${ptag}" || true - metrics_record_flow spent "$PAMT" || true - else - pnote="not settled order=$OID avail=${CUR}:${after}" - if [ "$IS_PMAX" = "1" ]; then - pstatus="CEILING_REJECT" - warn pay "pay $PAMT not settled (ceiling)" "$pnote" - echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t${ms_handle}\t${ms_psettle}\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" - continue - fi - err pay "settle $PAMT" "$pnote" - pstatus="FAIL_SETTLE" - PAY_FAIL_N=$((PAY_FAIL_N + 1)) - FAIL_N_L=$((FAIL_N_L + 1)) - STOP_REASON="$pnote" - STOP_AMOUNT="$PAMT" - echo -e "${prung}\t${prange}\t${PAMT}\t${pstatus}\t${ms_order}\t${ms_handle}\t${ms_psettle}\t${ms_total}\t${OID}\t${pnote}" >>"$PAY_TSV" - metrics_report_coins "after-pay-fail-${ptag}" || true - break - fi - done - metrics_report_coins "after-pay-ladder" || true - info "pay summary" "ok=$PAY_OK_N fail=$PAY_FAIL_N tsv=$PAY_TSV" - fi -elif [ "${LADDER_PAY}" = "1" ] && [ "$FAIL_N_L" -gt 0 ]; then - warn pay "skipped pay ladder" "withdraw phase already failed" -fi +MAX_PAY_BEST_AMT="" +MAX_PAY_LO="0" +MAX_PAY_HI="0" +ladder_run_pay_phase ms_phase=$(python3 -c 'import sys,time; print(int((time.time()-float(sys.argv[1]))*1000))' "$SECTION_T0") @@ -1433,6 +1204,10 @@ export LADDER_REPORT_STOP_AMOUNT="${STOP_AMOUNT:-}" export LADDER_REPORT_STOP_REASON="${STOP_REASON:-}" export LADDER_REPORT_MODE="${LADDER_MODE:-classic}" export LADDER_REPORT_MAX_BEST="${MAX_BEST_AMT:-}" +export LADDER_REPORT_MAX_PAY_BEST="${MAX_PAY_BEST_AMT:-}" +export LADDER_REPORT_MAX_PAY_LO="${MAX_PAY_LO:-}" +export LADDER_REPORT_MAX_PAY_HI="${MAX_PAY_HI:-}" +export LADDER_REPORT_MAX_PAY_BEST_ALT="$(format_amount_alt "${MAX_PAY_BEST_AMT:-${CUR}:0}" 2>/dev/null || true)" export LADDER_REPORT_MAX_LO="${MAX_LO:-}" export LADDER_REPORT_MAX_HI="${MAX_HI:-}" # Human alt_unit_names (e.g. 8.15 Tera-GOA (GOA:…)) for report + final lines @@ -1457,6 +1232,10 @@ try: max_best_alt = os.environ.get("LADDER_REPORT_MAX_BEST_ALT") or None max_lo_alt = os.environ.get("LADDER_REPORT_MAX_LO_ALT") or None max_hi_alt = os.environ.get("LADDER_REPORT_MAX_HI_ALT") or None + max_pay_best = os.environ.get("LADDER_REPORT_MAX_PAY_BEST") or None + max_pay_lo = os.environ.get("LADDER_REPORT_MAX_PAY_LO") or None + max_pay_hi = os.environ.get("LADDER_REPORT_MAX_PAY_HI") or None + max_pay_best_alt = os.environ.get("LADDER_REPORT_MAX_PAY_BEST_ALT") or None alt_path = os.environ.get("LADDER_REPORT_ALT_UNITS") or "" alt = {} if alt_path: @@ -1578,6 +1357,7 @@ try: _best = as_amt(max_best) if max_best else None _lo = as_amt(max_lo) if max_lo is not None and str(max_lo) != "" else None _hi = as_amt(max_hi) if max_hi is not None and str(max_hi) != "" else None + _pb = as_amt(max_pay_best) if max_pay_best else None _max_block = { "best": _best, "best_alt": max_best_alt or (format_alt(_best) if _best else None), @@ -1585,6 +1365,10 @@ try: "lo_alt": max_lo_alt or (format_alt(_lo) if _lo else None), "hi": max_hi, "hi_alt": max_hi_alt or (format_alt(_hi) if _hi else None), + "best_pay": _pb, + "best_pay_alt": max_pay_best_alt or (format_alt(_pb) if _pb else None), + "pay_lo": max_pay_lo, + "pay_hi": max_pay_hi, } report = { @@ -1662,6 +1446,8 @@ try: print(" lo_alt=%s" % la) print(" hi=%s" % (max_hi if max_hi is not None else "-")) print(" hi_alt=%s" % ha) + print(" best_pay=%s" % (max_pay_best or "-")) + print(" best_pay_alt=%s" % (max_pay_best_alt or (format_alt(as_amt(max_pay_best) or "") if max_pay_best else "-"))) except Exception as e: print(" max-search print error: %s" % (e,)) if stop_amt: @@ -1722,7 +1508,7 @@ if [ "$OK_N" -eq 0 ]; then exit 1 fi if [ "${LADDER_MODE}" = "max" ] && [ -n "${MAX_BEST_AMT:-}" ]; then - ok "ladder finished mode=max best=$(format_amount_alt "$MAX_BEST_AMT") · lo=$(format_amount_alt "${CUR}:${MAX_LO}") · hi=$(format_amount_alt "${CUR}:${MAX_HI}") · withdraw_ok=$OK_N phase=${ms_phase}ms" + ok "ladder finished mode=max wd_best=$(format_amount_alt "$MAX_BEST_AMT") · pay_best=$(format_amount_alt "${MAX_PAY_BEST_AMT:-${CUR}:0}") · withdraw_ok=$OK_N pay_ok=$PAY_OK_N phase=${ms_phase}ms" else ok "ladder finished mode=${LADDER_MODE} withdraw_ok=$OK_N pay_ok=$PAY_OK_N phase=${ms_phase}ms" fi diff --git a/check_goa_ladder.sh b/check_goa_ladder.sh index c32f686..1640dc5 100755 --- a/check_goa_ladder.sh +++ b/check_goa_ladder.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Compat shim — use check_amount_ladder.sh (generic currency amount ladder). -# Kept so older host-agent / docs invoking check_goa_ladder.sh still work. +# Compat shim only — prefer: ./taler-monitoring.sh ladder | max-ladder +# Real implementation: check_amount_ladder.sh (+ ladder/ modules). ROOT=$(cd "$(dirname "$0")" && pwd) exec bash "$ROOT/check_amount_ladder.sh" "$@" diff --git a/ladder/README.md b/ladder/README.md new file mode 100644 index 0000000..0ef8e26 --- /dev/null +++ b/ladder/README.md @@ -0,0 +1,25 @@ +# Amount ladder modules + +Glued by `../check_amount_ladder.sh` (phase entry: `ladder` / `max-ladder`). + +| Path | Role | +|------|------| +| `extract_rpubs.py` | Force-select: scrape `reservePub` from wallet dumps (no bash heredoc) | +| `lib_pay.sh` | **Payment** ladder: free-amount template orders, classic pay list, max-search highest **payable** ≤ available | +| *(withdraw stays in `check_amount_ladder.sh`)* | **Withdraw** ladder: classic 23-rung plan + max-search highest **withdrawable** | + +`check_goa_ladder.sh` at repo root is only a **compat shim** (`exec` → `check_amount_ladder.sh`). Not required for new scripts; phase alias `goa-ladder` already uses the amount ladder. + +## Modes + +- **classic** (default): withdraw plan 0→mids→max-1→max, then matching pay plan (never overspend). +- **max** (`max-ladder`): adaptive max withdraw, then adaptive max pay via public free template (hacktivism: `goa-free`). + +## Env (pay) + +| Variable | Default | Meaning | +|----------|---------|---------| +| `LADDER_PAY` | `1` | `0` = skip all pays | +| `LADDER_USE_FREE_TEMPLATE` | `1` | public POST template with `{"amount":…}` | +| `LADDER_FREE_TEMPLATE` | `goa-free` | template id (hacktivism) | +| `LADDER_FREE_TEMPLATE_INSTANCE` | merchant instance | e.g. `goa-demo-cp4zqk` | diff --git a/ladder_extract_rpubs.py b/ladder/extract_rpubs.py similarity index 100% rename from ladder_extract_rpubs.py rename to ladder/extract_rpubs.py diff --git a/ladder/lib_pay.sh b/ladder/lib_pay.sh new file mode 100644 index 0000000..9e8e157 --- /dev/null +++ b/ladder/lib_pay.sh @@ -0,0 +1,361 @@ +# ladder/lib_pay.sh — payment side of amount ladder (sourced by check_amount_ladder.sh) +# Expects: lib.sh + metrics.sh already sourced; CUR BANK MER INST EX SCRATCH WDB +# wcli wallet_avail format_amount_alt ladder_over now_ms elapsed_ms +# LADDER_* env, AUTH optional, PAY_TSV PAY_OK_N PAY_FAIL_N +# +# Public free-amount template (hacktivism): goa-free @ goa-demo-cp4zqk +# Never spends more than wallet available. + +: "${LADDER_FREE_TEMPLATE:=goa-free}" +: "${LADDER_FREE_TEMPLATE_INSTANCE:=}" +: "${LADDER_USE_FREE_TEMPLATE:=1}" +: "${LADDER_PAY_WAIT_AVAILABLE_S:=60}" +: "${LADDER_PAY_SETTLE_ROUNDS:=6}" + +MAX_PAY_BEST_AMT="${MAX_PAY_BEST_AMT:-}" +MAX_PAY_LO="${MAX_PAY_LO:-0}" +MAX_PAY_HI="${MAX_PAY_HI:-0}" + +ladder_pay_resolve_merchant() { + # Sets AUTH (may be empty if free template only) + MPW="${E2E_MERCHANT_TOKEN:-${MERCHANT_TOKEN:-}}" + if [ -z "$MPW" ]; then + MPW=$(read_secret "taler-merchant/merchant-${INST}-password.txt" 2>/dev/null || true) + fi + if [ -z "$MPW" ]; then + MPW=$(read_secret "taler-merchant/merchant-goa-demo-cp4zqk-password.txt" 2>/dev/null || true) + fi + if [ -z "$MPW" ] && [ "${CUR}" = "TESTPAYSAN" ]; then + if [ -n "${FRANCPAYSAN_SECRETS:-}" ] && [ -f "${FRANCPAYSAN_SECRETS}/stage/default-instance-token.txt" ]; then + MPW=$(tr -d '\n\r' <"${FRANCPAYSAN_SECRETS}/stage/default-instance-token.txt") + fi + fi + AUTH="" + if [ -n "$MPW" ]; then + case "$MPW" in + secret-token:*) AUTH="Authorization: Bearer ${MPW}" ;; + *) AUTH="Authorization: Bearer secret-token:${MPW}" ;; + esac + fi + : "${LADDER_FREE_TEMPLATE_INSTANCE:=${INST}}" +} + +# Create order for PAMT → OID OTOK PAYURI. Prefers public free template. +ladder_create_order() { + local pamt="$1" ptag="$2" + local inst mh body tpl_https sum_json + OID=""; OTOK=""; PAYURI="" + inst="${LADDER_FREE_TEMPLATE_INSTANCE:-$INST}" + mh=$(python3 -c 'from urllib.parse import urlparse; print(urlparse("'"${MER}"'").hostname or "taler.hacktivism.ch")' 2>/dev/null || echo "taler.hacktivism.ch") + + if [ "${LADDER_USE_FREE_TEMPLATE}" = "1" ] && [ -n "${LADDER_FREE_TEMPLATE:-}" ]; then + tpl_https="${MER}/instances/${inst}/templates/${LADDER_FREE_TEMPLATE}" + body=$(python3 -c 'import json,sys; print(json.dumps({"amount":sys.argv[1]}))' "$pamt" 2>/dev/null || printf '{"amount":"%s"}' "$pamt") + curl -skS -m 20 -o "$SCRATCH/ord-$ptag.json" -X POST \ + -H 'Content-Type: application/json' \ + -d "$body" \ + "$tpl_https" 2>"$SCRATCH/ord-$ptag.err" || true + OID=$(python3 -c 'import json,re,sys;t=open(sys.argv[1]).read() +try: print(json.loads(t).get("order_id") or "") +except Exception: + m=re.search(r"\"order_id\"\s*:\s*\"([^\"]+)\"",t); print(m.group(1) if m else "") +' "$SCRATCH/ord-$ptag.json" 2>/dev/null || true) + OTOK=$(python3 -c 'import json,re,sys;t=open(sys.argv[1]).read() +try: print(json.loads(t).get("token") or "") +except Exception: + m=re.search(r"\"token\"\s*:\s*\"([^\"]+)\"",t); print(m.group(1) if m else "") +' "$SCRATCH/ord-$ptag.json" 2>/dev/null || true) + if [ -n "$OID" ] && [ -n "$OTOK" ]; then + PAYURI="taler://pay/${mh}/instances/${inst}/${OID}/?c=${OTOK}" + PAYURI=$(printf '%s' "$PAYURI" | sed 's/:443\//\//g; s/:443?/?/g') + return 0 + fi + fi + + [ -n "${AUTH:-}" ] || return 1 + sum_json=$(python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "ladder pay ${pamt}" 2>/dev/null || echo '"ladder pay"') + curl -skS -m 20 -o "$SCRATCH/ord-$ptag.json" -X POST \ + -H "$AUTH" -H 'Content-Type: application/json' \ + -d "{\"order\":{\"summary\":${sum_json},\"amount\":\"${pamt}\",\"fulfillment_message\":\"ok\"},\"create_token\":true}" \ + "${MER}/instances/${INST}/private/orders" 2>"$SCRATCH/ord-$ptag.err" || true + OID=$(python3 -c 'import json,re,sys;t=open(sys.argv[1]).read() +try: print(json.loads(t).get("order_id") or "") +except Exception: + m=re.search(r"\"order_id\"\s*:\s*\"([^\"]+)\"",t); print(m.group(1) if m else "") +' "$SCRATCH/ord-$ptag.json" 2>/dev/null || true) + OTOK=$(python3 -c 'import json,re,sys;t=open(sys.argv[1]).read() +try: print(json.loads(t).get("token") or "") +except Exception: + m=re.search(r"\"token\"\s*:\s*\"([^\"]+)\"",t); print(m.group(1) if m else "") +' "$SCRATCH/ord-$ptag.json" 2>/dev/null || true) + [ -n "$OID" ] || return 1 + curl -skS -m 12 -o "$SCRATCH/ord-det-$ptag.json" -H "$AUTH" \ + "${MER}/instances/${INST}/private/orders/${OID}" 2>/dev/null || true + PAYURI=$(python3 -c 'import json,sys +try: print(json.load(open(sys.argv[1])).get("taler_pay_uri") or "") +except Exception: print("") +' "$SCRATCH/ord-det-$ptag.json" 2>/dev/null || true) + if [ -z "$PAYURI" ] && [ -n "$OTOK" ]; then + PAYURI="taler://pay/${mh}/instances/${INST}/${OID}/?c=${OTOK}" + fi + PAYURI=$(printf '%s' "$PAYURI" | sed 's/:443\//\//g; s/:443?/?/g') + [ -n "$PAYURI" ] +} + +# One pay attempt; never overspends. Always soft (no hard ladder stop). +ladder_pay_attempt() { + local pamt="$1" prange="$2" prung="$3" + local ptag pnum bal t_pay t0 ms_order ms_handle ms_psettle after settled r pstatus pnote + ptag=$(printf '%s' "$pamt" | tr '.:' '__') + pnum=$(python3 -c 'import sys; print(sys.argv[1].split(":",1)[-1])' "$pamt") + t_pay=$(now_ms) + ms_order=0; ms_handle=0; ms_psettle=0 + pnote=""; pstatus="FAIL"; OID="-" + + bal=$(wallet_avail) + if ! python3 -c 'import sys; from decimal import Decimal; sys.exit(0 if Decimal(sys.argv[1])>0 and Decimal(sys.argv[1])>=Decimal(sys.argv[2]) else 1)' "$bal" "$pnum" 2>/dev/null; then + pstatus="SKIP_BALANCE" + pnote="skip: would spend more than available avail=${CUR}:${bal} need=${pamt}" + warn pay "pay $pamt skipped (balance)" \ + "problem: never overspend. $(format_amount_alt "${CUR}:${bal}") free vs $(format_amount_alt "$pamt")" + echo -e "${prung}\t${prange}\t${pamt}\t${pstatus}\t0\t0\t0\t$(elapsed_ms "$t_pay")\t-\t${pnote}" >>"$PAY_TSV" + return 0 + fi + + t0=$(now_ms) + if ! ladder_create_order "$pamt" "$ptag"; then + ms_order=$(elapsed_ms "$t0") + pstatus="FAIL_ORDER" + pnote="order create failed $(head -c 100 "$SCRATCH/ord-$ptag.json" 2>/dev/null | tr '\n\"' ' ')" + warn pay "order $pamt" "$pnote" + echo -e "${prung}\t${prange}\t${pamt}\t${pstatus}\t${ms_order}\t0\t0\t$(elapsed_ms "$t_pay")\t-\t${pnote}" >>"$PAY_TSV" + return 0 + fi + ms_order=$(elapsed_ms "$t0") + ok "order $OID ($pamt) ${ms_order}ms · $(format_amount_alt "$pamt")" + + t0=$(now_ms) + if ! wcli handle-uri --yes "$PAYURI" >"$SCRATCH/pay-$ptag.out" 2>&1; then + ms_handle=$(elapsed_ms "$t0") + pstatus="FAIL_HANDLE" + pnote="handle-uri failed $(tail -c 100 "$SCRATCH/pay-$ptag.out" | tr '\n\"' ' ')" + warn pay "handle $pamt" "$pnote" + echo -e "${prung}\t${prange}\t${pamt}\t${pstatus}\t${ms_order}\t${ms_handle}\t0\t$(elapsed_ms "$t_pay")\t${OID}\t${pnote}" >>"$PAY_TSV" + return 0 + fi + ms_handle=$(elapsed_ms "$t0") + ok "handle-uri $pamt ${ms_handle}ms" + + t0=$(now_ms) + settled=0 + r=0 + while [ "$r" -lt "${LADDER_PAY_SETTLE_ROUNDS}" ]; do + r=$((r + 1)) + wcli transactions >"$SCRATCH/tx-pay-$ptag.out" 2>&1 || true + if [ -n "${AUTH:-}" ]; then + curl -skS -m 8 -o "$SCRATCH/ord-paid-$ptag.json" -H "$AUTH" \ + "${MER}/instances/${INST}/private/orders/${OID}" 2>/dev/null || true + fi + if grep -qiE 'payment|paid|Payment' "$SCRATCH/tx-pay-$ptag.out" 2>/dev/null \ + || python3 -c 'import json,sys +try: + d=json.load(open(sys.argv[1])) + sys.exit(0 if d.get("paid") is True or str(d.get("order_status","")).lower()=="paid" else 1) +except Exception: + sys.exit(1) +' "$SCRATCH/ord-paid-$ptag.json" 2>/dev/null; then + settled=1 + break + fi + sleep 0.5 + done + ms_psettle=$(elapsed_ms "$t0") + after=$(wallet_avail) + if [ "$settled" = "1" ]; then + pstatus="OK" + pnote="avail=${CUR}:${after}" + ok "pay settled $pamt · $(format_amount_alt "$pamt") → bal ${CUR}:${after}" + PAY_OK_N=$((PAY_OK_N + 1)) + echo -e "${prung}\t${prange}\t${pamt}\t${pstatus}\t${ms_order}\t${ms_handle}\t${ms_psettle}\t$(elapsed_ms "$t_pay")\t${OID}\t${pnote}" >>"$PAY_TSV" + metrics_report_coins "after-pay-${ptag}" || true + metrics_record_flow spent "$pamt" || true + else + pstatus="FAIL_SETTLE" + pnote="not settled order=$OID avail=${CUR}:${after}" + warn pay "settle $pamt" "$pnote" + echo -e "${prung}\t${prange}\t${pamt}\t${pstatus}\t${ms_order}\t${ms_handle}\t${ms_psettle}\t$(elapsed_ms "$t_pay")\t${OID}\t${pnote}" >>"$PAY_TSV" + fi + return 0 +} + +ladder_wait_available() { + local wait_s="${1:-$LADDER_PAY_WAIT_AVAILABLE_S}" + local end bal + end=$(( $(date +%s) + wait_s )) + bal=$(wallet_avail) + while python3 -c 'import sys; from decimal import Decimal; sys.exit(0 if Decimal(sys.argv[1])<=0 else 1)' "$bal" 2>/dev/null; do + [ "$(date +%s)" -ge "$end" ] && break + wcli transactions >"$SCRATCH/tx-wait-pay.out" 2>&1 || true + sleep 2 + bal=$(wallet_avail) + done + printf '%s' "$bal" +} + +# Classic planned pays (PAY_LIST) +ladder_run_pay_classic() { + # shellcheck disable=SC2086 + set -- $PAY_LIST + local pay_n=$# prung=0 PAMT prange PNUM + if [ "$pay_n" -eq 0 ]; then + info pay "empty pay plan — skip" + return 0 + fi + info "pay rungs" "$pay_n · $*" + for PAMT in "$@"; do + prung=$((prung + 1)) + ladder_over && break + if [ "$prung" -eq 1 ]; then + prange="pin:0" + elif [ "$prung" -eq "$pay_n" ]; then + prange="pin:max" + elif [ "$prung" -eq $((pay_n - 1)) ] && [ "$pay_n" -ge 3 ]; then + prange="pin:max-1" + else + prange="random" + fi + section "ladder · pay rung $prung $PAMT · $(format_amount_alt "$PAMT")" + PNUM=$(python3 -c 'import sys; print(sys.argv[1].split(":",1)[-1])' "$PAMT") + if python3 -c 'import sys; from decimal import Decimal; sys.exit(0 if Decimal(sys.argv[1])==0 else 1)' "$PNUM" 2>/dev/null; then + echo -e "${prung}\t${prange}\t${PAMT}\tZERO_SKIP\t0\t0\t0\t0\t-\tzero" >>"$PAY_TSV" + PAY_OK_N=$((PAY_OK_N + 1)) + continue + fi + ladder_pay_attempt "$PAMT" "$prange" "$prung" + done +} + +# Max-search: highest payable ≤ available (uses max_search_next from withdraw module) +ladder_run_pay_max_search() { + local bal prung _next _anum prange _act _last _slo _shi + section "ladder · phase B · max-search pay (highest payable ≤ available)" + bal=$(ladder_wait_available "$LADDER_PAY_WAIT_AVAILABLE_S") + MAX_PAY_LO="0" + MAX_PAY_HI="$bal" + MAX_PAY_BEST_AMT="" + if ! python3 -c 'import sys; from decimal import Decimal; sys.exit(0 if Decimal(sys.argv[1])>0 else 1)' "$MAX_PAY_HI" 2>/dev/null; then + warn pay "max-search pay skipped" \ + "problem: wallet available is ${CUR}:0 — need spendable coins (pendingIncoming is not enough)" + return 0 + fi + info "max-search pay" "hi=available $(format_amount_alt "${CUR}:${MAX_PAY_HI}") · template=${LADDER_FREE_TEMPLATE}@${LADDER_FREE_TEMPLATE_INSTANCE}" + info "pay budget" "never order more than $(format_amount_alt "${CUR}:${MAX_PAY_HI}")" + prung=0 + _last="" + while [ "${LADDER_MAX_PROBES}" = "0" ] || [ "$prung" -lt "${LADDER_MAX_PROBES}" ]; do + ladder_over && break + # reuse withdraw amount picker with pay bounds + _slo=${MAX_LO:-0}; _shi=${MAX_HI:-0} + MAX_LO=$MAX_PAY_LO + MAX_HI=$MAX_PAY_HI + if ! type max_search_next >/dev/null 2>&1; then + warn pay "max_search_next missing — cannot run pay max-search" + MAX_LO=$_slo; MAX_HI=$_shi + return 0 + fi + _next=$(max_search_next) + MAX_LO=$_slo; MAX_HI=$_shi + _anum=$(printf '%s' "$_next" | cut -d'|' -f1) + prange=$(printf '%s' "$_next" | cut -d'|' -f2 | sed 's/^max:/pay:/') + _act=$(printf '%s' "$_next" | cut -d'|' -f3) + if [ "$_act" = "done" ]; then + info "max-search pay" "converged best=$(format_amount_alt "${MAX_PAY_BEST_AMT:-${CUR}:0}") · lo=$(format_amount_alt "${CUR}:${MAX_PAY_LO}") · hi=$(format_amount_alt "${CUR}:${MAX_PAY_HI}")" + break + fi + if ! python3 -c 'import sys; from decimal import Decimal; sys.exit(0 if Decimal(sys.argv[1])<=Decimal(sys.argv[2]) else 1)' "$_anum" "$MAX_PAY_HI" 2>/dev/null; then + _anum=$MAX_PAY_HI + fi + if ! python3 -c 'import sys; from decimal import Decimal; sys.exit(0 if Decimal(sys.argv[1])>0 else 1)' "$_anum" 2>/dev/null; then + break + fi + PAMT="${CUR}:${_anum}" + if [ -n "$_last" ] && [ "$PAMT" = "$_last" ]; then + MAX_PAY_HI=$(python3 -c 'from decimal import Decimal; a=Decimal("'"$_anum"'"); print(max(Decimal(0), a-(1 if a>=1 else Decimal("0.000001"))))') + continue + fi + _last=$PAMT + prung=$((prung + 1)) + section "ladder · max pay probe $prung $PAMT · $(format_amount_alt "$PAMT") [lo=$(format_amount_alt "${CUR}:${MAX_PAY_LO}") · hi=$(format_amount_alt "${CUR}:${MAX_PAY_HI}")]" + ladder_pay_attempt "$PAMT" "$prange" "$prung" + _last_st=$(tail -1 "$PAY_TSV" 2>/dev/null | cut -f4) + case "$_last_st" in + OK) + MAX_PAY_LO=$_anum + MAX_PAY_BEST_AMT=$PAMT + info "max-search pay" "success bound lo=$(format_amount_alt "${CUR}:${MAX_PAY_LO}")" + ;; + SKIP_BALANCE|FAIL_ORDER|FAIL_HANDLE|FAIL_SETTLE) + MAX_PAY_HI=$(python3 -c 'from decimal import Decimal; a=Decimal("'"$_anum"'"); h=Decimal("'"$MAX_PAY_HI"'"); print(min(a,h))') + info "max-search pay" "too-high/fail hi:=$(format_amount_alt "${CUR}:${MAX_PAY_HI}") status=${_last_st}" + ;; + esac + if python3 -c 'from decimal import Decimal; import sys; sys.exit(0 if Decimal(sys.argv[1])>0 and Decimal(sys.argv[2])<=Decimal(sys.argv[1]) else 1)' "$MAX_PAY_LO" "$MAX_PAY_HI" 2>/dev/null; then + info "max-search pay" "bounds crossed — stop" + break + fi + done + if [ -n "${MAX_PAY_BEST_AMT:-}" ]; then + ok "max-search best payable $(format_amount_alt "$MAX_PAY_BEST_AMT") · lo=$(format_amount_alt "${CUR}:${MAX_PAY_LO}") · hi=$(format_amount_alt "${CUR}:${MAX_PAY_HI}")" + else + warn pay "max-search no successful pay" "available was ${CUR}:$(wallet_avail)" + fi + { + echo "best_pay=${MAX_PAY_BEST_AMT:-}" + echo "best_pay_alt=$(format_amount_alt "${MAX_PAY_BEST_AMT:-${CUR}:0}" 2>/dev/null || true)" + echo "pay_lo=${MAX_PAY_LO}" + echo "pay_hi=${MAX_PAY_HI}" + } >>"${SCRATCH}/ladder-plan.txt" 2>/dev/null || true + printf '%s\n' "${MAX_PAY_BEST_AMT:-}" >"${SCRATCH}/ladder-pay-plan.txt" 2>/dev/null || true +} + +# Entry: glue from check_amount_ladder.sh after withdraw phase +ladder_run_pay_phase() { + PAY_OK_N=0 + PAY_FAIL_N=0 + if [ "${LADDER_PAY}" != "1" ]; then + info pay "LADDER_PAY=0 — pay phase skipped" + return 0 + fi + if [ "${FAIL_N_L:-0}" -gt 0 ]; then + warn pay "skipped pay ladder" "withdraw phase already failed" + return 0 + fi + + set_group pay + section "ladder · phase B · pay" + metrics_report_coins "before-pay-ladder" || true + ladder_pay_resolve_merchant + + if [ -z "${AUTH:-}" ] && ! { [ "${LADDER_USE_FREE_TEMPLATE}" = "1" ] && [ -n "${LADDER_FREE_TEMPLATE:-}" ]; }; then + warn pay "no merchant token and free template off — skip pay" + return 0 + fi + if [ -n "${AUTH:-}" ]; then + ok "merchant token" "instance ${INST}" + else + info pay "public free template ${LADDER_FREE_TEMPLATE}@${LADDER_FREE_TEMPLATE_INSTANCE} (no merchant token)" + fi + + wallet_prepare "main" + local bal0 + bal0=$(ladder_wait_available "$LADDER_PAY_WAIT_AVAILABLE_S") + info "pay budget" "wallet available ${CUR}:${bal0} · $(format_amount_alt "${CUR}:${bal0}") — never order more than this" + + if [ "${LADDER_MODE}" = "max" ]; then + ladder_run_pay_max_search + else + ladder_run_pay_classic + fi + metrics_report_coins "after-pay-ladder" || true + info "pay summary" "ok=$PAY_OK_N fail=$PAY_FAIL_N tsv=$PAY_TSV" +} diff --git a/taler-monitoring.sh b/taler-monitoring.sh index b89fbaa..73f124a 100755 --- a/taler-monitoring.sh +++ b/taler-monitoring.sh @@ -94,7 +94,7 @@ Phases: server server-side only (SSH) e2e withdraw + pay (small amounts; remote aborts on login/KYC) ladder amount ladder (classic: 0→mids→max; any currency / stack ceiling) - max-ladder same script, LADDER_MODE=max — hunt highest withdrawable amount + max-ladder highest withdrawable + highest payable (free-amount template on GOA) auth401 merchant Basic-auth / case matrix (HTTP 401 paths; may create throwaway instance) aptdeploy koopa podman apt-src smoke: taler-merchant in koopa-taler-deploy-test-apt-src-trixie{,-testing} From 32e7f9668b842fb96450d94b54b3b117331b9a7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 14:26:07 +0200 Subject: [PATCH 18/29] chore: drop check_goa_ladder.sh compat shim No host-agent or suite phase loads the path; taler-monitoring.sh phase alias goa-ladder already runs check_amount_ladder.sh. Update docs and admin-log pointer comments to the amount ladder only. --- CLI-AUTOMATION-NOTES.md | 2 +- TESTS.md | 2 +- VERSIONS.md | 2 +- check_amount_ladder.sh | 2 +- check_goa_ladder.sh | 5 ----- ladder/README.md | 2 +- 6 files changed, 5 insertions(+), 10 deletions(-) delete mode 100755 check_goa_ladder.sh diff --git a/CLI-AUTOMATION-NOTES.md b/CLI-AUTOMATION-NOTES.md index a541d28..a1657f3 100644 --- a/CLI-AUTOMATION-NOTES.md +++ b/CLI-AUTOMATION-NOTES.md @@ -9,7 +9,7 @@ Companions: - Android GUI → [`android-test/GUI-AUTOMATION-NOTES.md`](./android-test/GUI-AUTOMATION-NOTES.md) - Git / suite tree → [`VERSIONS.md`](./VERSIONS.md) -Scripts: `check_e2e.sh`, `check_amount_ladder.sh` (compat `check_goa_ladder.sh`), `taler-monitoring.sh`. +Scripts: `check_e2e.sh`, `check_amount_ladder.sh`, `taler-monitoring.sh`. Legend: diff --git a/TESTS.md b/TESTS.md index 0c89237..192668c 100644 --- a/TESTS.md +++ b/TESTS.md @@ -25,7 +25,7 @@ Every check line has a **global** run number and a **grouped** id: | **sanity** | `check_sanity.sh` | `bank` `exchange` `merchant` | | **server** | `check_server.sh` | (flat `server-NN` or host groups) | | **e2e** | `check_e2e.sh` | `prereq` `load` `bank` `wallet` `bankwd` `settle` `pay` `shop` `paivana` `dig` `report` | -| **ladder** | `check_amount_ladder.sh` (alias `check_goa_ladder.sh`) | `plan` `load` `withdraw` `pay` `report` — modes classic / max (`max-ladder`) | +| **ladder** | `check_amount_ladder.sh` | `plan` `load` `withdraw` `pay` `report` — modes classic / max (`max-ladder`); phase alias `goa-ladder` | **Why groups:** flat `www-001`…`www-080` was hard to talk about. `www.bank-04` / `e2e.bankwd-02` pin the failure to a logical block. diff --git a/VERSIONS.md b/VERSIONS.md index 2f327d9..8ecdba3 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,7 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| -| **v1.22.0** | 2026-07-19 | **Feature:** modular amount ladder — `ladder/lib_pay.sh` (pay) + withdraw in `check_amount_ladder.sh`; `ladder/extract_rpubs.py`. **max-ladder** hunts highest **withdrawable** and highest **payable** (≤ wallet available). GOA pays use public free-amount template **`goa-free`** (`LADDER_FREE_TEMPLATE`). Classic still default 23-rung wd+pay. `check_goa_ladder.sh` remains a thin compat shim only. | +| **v1.22.0** | 2026-07-19 | **Feature:** modular amount ladder — `ladder/lib_pay.sh` (pay) + withdraw in `check_amount_ladder.sh`; `ladder/extract_rpubs.py`. **max-ladder** hunts highest **withdrawable** and highest **payable** (≤ wallet available). GOA pays use public free-amount template **`goa-free`** (`LADDER_FREE_TEMPLATE`). Classic still default 23-rung wd+pay. Phase alias `goa-ladder` → amount ladder (shim removed). | | **v1.21.0** | 2026-07-19 | **Feature:** generic **amount ladder** (`check_amount_ladder.sh`; compat shim `check_goa_ladder.sh`). Default **classic** 23-rung withdraw (+ pay when enabled); **max-search** (`max-ladder` / `LADDER_MODE=max`) hunts highest mintable amount (probes default 32, `0`=until timeout). **Pay:** never overspend — soft `SKIP_BALANCE` if amount > wallet available. **Alt units:** report/logs show human names (e.g. `8.15 Tera-GOA (GOA:…)`) + raw. **Fixes bundled:** external extract (no stdin SyntaxError); force-select multi-rpub; soft `CEILING_REJECT` on mint 5110/P0001; crash-proof report. CLI-AUTOMATION-NOTES §3/3a/3b/§15–18. | | **v1.20.1** | 2026-07-19 | **Docs:** CLI-AUTOMATION-NOTES — force-select empty scrape + ladder pins (§3/3a); wallet-db must not be `*.json` (§15); `taler-helper-sqlite3` needs Python ≥3.11 (§16); portable wall-clock (§17); GOA withdraw checklist (§18). No code change. *(Superseded notes folded into **v1.21.0** feature release.)* | | **v1.20.0** | 2026-07-19 | **Feature:** domains.conf **`locale`** (l10n) — FP **`fr-CH`**, others **`de-CH`** (aliases ch-FR/ch-DE); UI **`lang=de`** supported (sticky + console tags) but **never** a stock profile default; env `TALER_MON_LOCALE` / `TALER_DOMAIN_LOCALE`. | diff --git a/check_amount_ladder.sh b/check_amount_ladder.sh index a4ad861..57f780f 100755 --- a/check_amount_ladder.sh +++ b/check_amount_ladder.sh @@ -2,7 +2,7 @@ # check_amount_ladder.sh — generic withdraw/pay amount ladder (any Taler currency) # # Currency-agnostic bank landing ladder for GOA, TESTPAYSAN, KUDOS, … -# (formerly check_goa_ladder.sh — shim retained for compatibility). +# (formerly check_goa_ladder.sh; use phase ladder / max-ladder / goa-ladder). # # Flow (bank landings): # 1) GET /intro/auto-account.json → personal *account-* (balance 0) diff --git a/check_goa_ladder.sh b/check_goa_ladder.sh deleted file mode 100755 index 1640dc5..0000000 --- a/check_goa_ladder.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -# Compat shim only — prefer: ./taler-monitoring.sh ladder | max-ladder -# Real implementation: check_amount_ladder.sh (+ ladder/ modules). -ROOT=$(cd "$(dirname "$0")" && pwd) -exec bash "$ROOT/check_amount_ladder.sh" "$@" diff --git a/ladder/README.md b/ladder/README.md index 0ef8e26..c08ab3a 100644 --- a/ladder/README.md +++ b/ladder/README.md @@ -8,7 +8,7 @@ Glued by `../check_amount_ladder.sh` (phase entry: `ladder` / `max-ladder`). | `lib_pay.sh` | **Payment** ladder: free-amount template orders, classic pay list, max-search highest **payable** ≤ available | | *(withdraw stays in `check_amount_ladder.sh`)* | **Withdraw** ladder: classic 23-rung plan + max-search highest **withdrawable** | -`check_goa_ladder.sh` at repo root is only a **compat shim** (`exec` → `check_amount_ladder.sh`). Not required for new scripts; phase alias `goa-ladder` already uses the amount ladder. +Phase alias `goa-ladder` maps to `check_amount_ladder.sh` (no separate goa_ladder script). ## Modes From 35cd86bd4398420f0c20eb4091a195ed5803e916 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 14:27:14 +0200 Subject: [PATCH 19/29] docs: point all ladder entrypoints at modular amount ladder Drop remaining check_goa_ladder / old-shim wording. Document ladder/ modules, goa-ladder as phase alias only, and max-ladder as wd+pay max-search. Update admin-log goa-withdraw-ladder comments to prefer taler-monitoring.sh ladder|max-ladder. --- README.md | 2 +- TESTS.md | 4 ++-- check_amount_ladder.sh | 20 ++++++++++---------- ladder/README.md | 18 ++++++++++++------ taler-monitoring.sh | 8 +++++--- 5 files changed, 30 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 5af4d7f..2c08f4e 100644 --- a/README.md +++ b/README.md @@ -306,7 +306,7 @@ E2E_VARIABLE=0 WITHDRAW_AMT=GOA:50 PAY_AMT=GOA:1 ./taler-monitoring.sh e2e # s | **versions** | `deb.taler.net` reachable (InRelease/Packages/pool `.deb`); containers can reach it + have apt source; installed Taler packages vs **trixie** | | **sanity** | bank · exchange · merchant sections (public + server) | | **e2e** | account → credit → withdraw → wallet → confirm → order → pay | -| **ladder** | amount ladder: **classic** (0 + mids + max) or **max-search** (`max-ladder` / `LADDER_MODE=max`) | +| **ladder** | amount ladder modules: classic 23-rung wd+pay, or max-search wd+pay (`max-ladder`) | ```bash # package suite (default trixie on deb.taler.net) diff --git a/TESTS.md b/TESTS.md index 192668c..9320315 100644 --- a/TESTS.md +++ b/TESTS.md @@ -25,7 +25,7 @@ Every check line has a **global** run number and a **grouped** id: | **sanity** | `check_sanity.sh` | `bank` `exchange` `merchant` | | **server** | `check_server.sh` | (flat `server-NN` or host groups) | | **e2e** | `check_e2e.sh` | `prereq` `load` `bank` `wallet` `bankwd` `settle` `pay` `shop` `paivana` `dig` `report` | -| **ladder** | `check_amount_ladder.sh` | `plan` `load` `withdraw` `pay` `report` — modes classic / max (`max-ladder`); phase alias `goa-ladder` | +| **ladder** | `check_amount_ladder.sh` + `ladder/lib_pay.sh` | `plan` `load` `withdraw` `pay` `report` — classic / max-search; phases `ladder` · `max-ladder` · `goa-ladder` (alias) | **Why groups:** flat `www-001`…`www-080` was hard to talk about. `www.bank-04` / `e2e.bankwd-02` pin the failure to a logical block. @@ -141,7 +141,7 @@ Standalone: `host-agent/run-aptdeploy.sh` (on koopa local; else ssh `$DEPLOY_SSH - OSV CVE query when package+version known → **ERROR** on hits - Never SSH / never podman on the *targets* | **sanity.bank-** / **.exchange-** / **.merchant-** | public + optional server-side per component | -| **ladder.plan-** / **.load-** / **.withdraw-** / **.pay-** / **.report-** | amount ladder (GOA ceiling or stage TESTPAYSAN `max_wire`) | +| **ladder.plan-** / **.load-** / **.withdraw-** / **.pay-** / **.report-** | amount ladder classic or max-search (any currency; stage uses `max_wire`) | | **server-** | SSH host ports / processes (flat unless grouped later) | | **auth401.*** | merchant 401 / case matrix (WebUI APIs): **signup** `POST /instances` MIX id, **login** Basic user low vs MIX/UPPER, full **case** matrix, **password** bad/empty, **pwchange** `POST private/auth` + re-login, **admin-create** `POST /management/instances` (if admin token), **bearer**, **durable mon401**, **webui** SPA `toLowerCase` probe (`check_auth401.sh`) | diff --git a/check_amount_ladder.sh b/check_amount_ladder.sh index 57f780f..f62c418 100755 --- a/check_amount_ladder.sh +++ b/check_amount_ladder.sh @@ -1,31 +1,31 @@ #!/usr/bin/env bash # check_amount_ladder.sh — generic withdraw/pay amount ladder (any Taler currency) # -# Currency-agnostic bank landing ladder for GOA, TESTPAYSAN, KUDOS, … -# (formerly check_goa_ladder.sh; use phase ladder / max-ladder / goa-ladder). +# Orchestrator for the modular amount ladder: +# Phase A withdraw — this file (classic plan or max-search) +# Phase B pay — ladder/lib_pay.sh (classic list or max payable) +# Force-select — ladder/extract_rpubs.py +# See ladder/README.md. # # Flow (bank landings): # 1) GET /intro/auto-account.json → personal *account-* (balance 0) # 2) Mint pool withdrawals as explorer + confirm when selected # 3) wallet-cli accept-uri only (no run-until-done) # 4) bank confirm when selected; settle = balance + transfer_done +# 5) pay via free-amount template (GOA: goa-free) or private orders # # Modes (LADDER_MODE): -# classic — random strictly increasing rungs + pins 0, max-1, max (default 23) +# classic — default 23-rung withdraw + matching pay plan (pins 0, max-1, max) # max — max-search: highest withdrawable AND highest payable # (pay never exceeds wallet available; GOA uses public free-amount # template goa-free by default) # -# Phase A: withdraw. Phase B: pay (classic plan list, or max-search pay). -# -# Stacks auto-tune via bank /config max_wire + exchange min denom when possible. -# # Usage: -# ./taler-monitoring.sh ladder -# ./taler-monitoring.sh max-ladder # LADDER_MODE=max (wd + pay hunt) +# ./taler-monitoring.sh ladder # classic +# ./taler-monitoring.sh max-ladder # LADDER_MODE=max +# ./taler-monitoring.sh goa-ladder # alias of ladder (phase name only) # ./taler-monitoring.sh -d stage.lefrancpaysan.ch ladder # LADDER_MODE=max LADDER_MAX_PROBES=32 ./taler-monitoring.sh ladder -# LADDER_MAX_AMOUNT=100 LADDER_STEPS=7 ./taler-monitoring.sh -d stage.lefrancpaysan.ch ladder # # Env: LADDER_MODE, LADDER_STEPS, LADDER_MAX_AMOUNT, LADDER_MIN_AMOUNT, # LADDER_MAX_PROBES, LADDER_MAX_TOL_REL, LADDER_HIGH_RUNGS, LADDER_STACK_AUTO, diff --git a/ladder/README.md b/ladder/README.md index c08ab3a..4737bad 100644 --- a/ladder/README.md +++ b/ladder/README.md @@ -1,18 +1,24 @@ # Amount ladder modules -Glued by `../check_amount_ladder.sh` (phase entry: `ladder` / `max-ladder`). +**Entry:** `../check_amount_ladder.sh` via: + +```bash +./taler-monitoring.sh ladder # classic 23-rung withdraw + pay +./taler-monitoring.sh max-ladder # max-search withdraw + pay +./taler-monitoring.sh goa-ladder # legacy phase name = classic ladder +``` | Path | Role | |------|------| -| `extract_rpubs.py` | Force-select: scrape `reservePub` from wallet dumps (no bash heredoc) | -| `lib_pay.sh` | **Payment** ladder: free-amount template orders, classic pay list, max-search highest **payable** ≤ available | -| *(withdraw stays in `check_amount_ladder.sh`)* | **Withdraw** ladder: classic 23-rung plan + max-search highest **withdrawable** | +| `../check_amount_ladder.sh` | Orchestrator: classic / max **withdraw**, then pay phase | +| `lib_pay.sh` | **Pay** module: free-amount template, classic pay list, max **payable** ≤ available | +| `extract_rpubs.py` | Force-select: scrape `reservePub` from wallet dumps | -Phase alias `goa-ladder` maps to `check_amount_ladder.sh` (no separate goa_ladder script). +There is **no** `check_goa_ladder.sh` — removed; use the phases above. ## Modes -- **classic** (default): withdraw plan 0→mids→max-1→max, then matching pay plan (never overspend). +- **classic** (default): 23-rung withdraw 0→mids→max-1→max, then matching pay plan (never overspend). - **max** (`max-ladder`): adaptive max withdraw, then adaptive max pay via public free template (hacktivism: `goa-free`). ## Env (pay) diff --git a/taler-monitoring.sh b/taler-monitoring.sh index 73f124a..5379f3f 100755 --- a/taler-monitoring.sh +++ b/taler-monitoring.sh @@ -93,8 +93,9 @@ Phases: sanity public + optional server server server-side only (SSH) e2e withdraw + pay (small amounts; remote aborts on login/KYC) - ladder amount ladder (classic: 0→mids→max; any currency / stack ceiling) - max-ladder highest withdrawable + highest payable (free-amount template on GOA) + ladder amount ladder classic (default 23-rung withdraw + pay; any currency) + max-ladder max-search: highest withdrawable + highest payable (≤ available) + goa-ladder alias of ladder (legacy phase name; same check_amount_ladder.sh) auth401 merchant Basic-auth / case matrix (HTTP 401 paths; may create throwaway instance) aptdeploy koopa podman apt-src smoke: taler-merchant in koopa-taler-deploy-test-apt-src-trixie{,-testing} @@ -317,7 +318,8 @@ export INSIDE_PROFILE INSIDE_SSH export INSIDE_BANK_CTR INSIDE_EXCHANGE_CTR INSIDE_MERCHANT_CTR export INSIDE_BANK_PORT INSIDE_EXCHANGE_PORT INSIDE_MERCHANT_PORT export INSIDE_DNS_BANK INSIDE_DNS_EXCHANGE INSIDE_DNS_MERCHANT -# Amount ladder — classic or max-search (see check_amount_ladder.sh; goa-ladder alias). +# Amount ladder — check_amount_ladder.sh + ladder/lib_pay.sh + ladder/extract_rpubs.py +# Phases: ladder|goa-ladder (classic), max-ladder|maxladder (max-search). # Defaults so set -u export is safe when vars were never set by caller. : "${LADDER_STEPS:=23}" : "${LADDER_MIN_AMOUNT:=0.000001}" From fd5ac805664bdd52f2dbb7793a14b1a51d2d32f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 14:34:47 +0200 Subject: [PATCH 20/29] fix 1.22.1: pay-wait withdraw board (start order + pendingIncoming) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit UX fix: after bank OK_BANK, available is often still 0 while pendingIncoming ticks up. The silent wait looked hung. Show OK/OK_BANK withdraws in the order they started, match wallet txs, and update pendingIncoming→done with a timer on the same wallet DB. Short run-pending only; default wait 90s. --- CLI-AUTOMATION-NOTES.md | 1 + VERSION | 2 +- VERSIONS.md | 1 + check_amount_ladder.sh | 29 +++++- ladder/README.md | 1 + ladder/lib_pay.sh | 204 +++++++++++++++++++++++++++++++++++++++- 6 files changed, 232 insertions(+), 6 deletions(-) diff --git a/CLI-AUTOMATION-NOTES.md b/CLI-AUTOMATION-NOTES.md index a1657f3..2f25122 100644 --- a/CLI-AUTOMATION-NOTES.md +++ b/CLI-AUTOMATION-NOTES.md @@ -289,6 +289,7 @@ If only a few changes land, these unlock the most automation: | force-select | multi-rpub try; soft-skip confirm; empty scrape → WARN not balance fail | | higher amounts | mint 5110/P0001 common above live ceiling (§3b); classic soft `CEILING_REJECT`; max-ladder bounds hi; report uses alt_unit_names (Tera-GOA …) | | pay vs balance | never order more than wallet **available** (soft `SKIP_BALANCE`); pendingIncoming does not count as spendable | +| pay wait UX | after withdraw, board lists OK/OK_BANK in **start order** with live wallet `pendingIncoming`/`done` + timer (`LADDER_PAY_WAIT_AVAILABLE_S`, default 90s); same wallet DB as withdraw | | wallet-db path | Prefer `*.sqlite3` (never `*.json` as db path) | | helper / python | mon PATH must include helper + python≥3.11 | diff --git a/VERSION b/VERSION index 57807d6..6245bee 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.22.0 +1.22.1 diff --git a/VERSIONS.md b/VERSIONS.md index 8ecdba3..d4ec0fa 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| +| **v1.22.1** | 2026-07-19 | **Bugfix (UX):** pay wait after withdraw no longer looks hung — shows **withdraw board in start order** (OK/OK_BANK from TSV) with live wallet status (`pendingIncoming` → `done`) + timer + aggregate available/pendingIncoming on the **same** wallet DB. Short `run-pending` nudge only (no run-until-done). Default wait 90s (`LADDER_PAY_WAIT_AVAILABLE_S`). | | **v1.22.0** | 2026-07-19 | **Feature:** modular amount ladder — `ladder/lib_pay.sh` (pay) + withdraw in `check_amount_ladder.sh`; `ladder/extract_rpubs.py`. **max-ladder** hunts highest **withdrawable** and highest **payable** (≤ wallet available). GOA pays use public free-amount template **`goa-free`** (`LADDER_FREE_TEMPLATE`). Classic still default 23-rung wd+pay. Phase alias `goa-ladder` → amount ladder (shim removed). | | **v1.21.0** | 2026-07-19 | **Feature:** generic **amount ladder** (`check_amount_ladder.sh`; compat shim `check_goa_ladder.sh`). Default **classic** 23-rung withdraw (+ pay when enabled); **max-search** (`max-ladder` / `LADDER_MODE=max`) hunts highest mintable amount (probes default 32, `0`=until timeout). **Pay:** never overspend — soft `SKIP_BALANCE` if amount > wallet available. **Alt units:** report/logs show human names (e.g. `8.15 Tera-GOA (GOA:…)`) + raw. **Fixes bundled:** external extract (no stdin SyntaxError); force-select multi-rpub; soft `CEILING_REJECT` on mint 5110/P0001; crash-proof report. CLI-AUTOMATION-NOTES §3/3a/3b/§15–18. | | **v1.20.1** | 2026-07-19 | **Docs:** CLI-AUTOMATION-NOTES — force-select empty scrape + ladder pins (§3/3a); wallet-db must not be `*.json` (§15); `taler-helper-sqlite3` needs Python ≥3.11 (§16); portable wall-clock (§17); GOA withdraw checklist (§18). No code change. *(Superseded notes folded into **v1.21.0** feature release.)* | diff --git a/check_amount_ladder.sh b/check_amount_ladder.sh index f62c418..3bc13ed 100755 --- a/check_amount_ladder.sh +++ b/check_amount_ladder.sh @@ -87,8 +87,8 @@ elapsed_ms() { : "${LADDER_HIGH_RUNGS:=6}" # optional seed for reproducible max-search (empty = time-based) : "${LADDER_MAX_SEED:=}" -# After withdraw, wait for spendable available>0 before pay (secs total) -: "${LADDER_PAY_WAIT_AVAILABLE_S:=60}" +# After withdraw, wait for spendable available>0 before pay (secs total; shows countdown) +: "${LADDER_PAY_WAIT_AVAILABLE_S:=90}" # Historic libeufin-ish absolute ceiling used as default LADDER_MAX_AMOUNT on GOA LADDER_ABS_CEILING="4503599627370496" @@ -308,6 +308,31 @@ print("0") ' "$CUR" 2>/dev/null || echo "0" } +# pendingIncoming for same currency (same wallet DB as withdraw) — "0" if missing +wallet_pending_in() { + wcli balance 2>/dev/null | python3 -c ' +import json,sys +t=sys.stdin.read() +dec=json.JSONDecoder() +i=t.find("{") +if i<0: + print("0"); raise SystemExit +try: + d,_=dec.raw_decode(t,i) +except Exception: + try: + d=json.loads(t[i:t.rfind("}")+1]) + except Exception: + print("0"); raise SystemExit +cur=sys.argv[1] +for b in d.get("balances") or []: + a=b.get("pendingIncoming") or b.get("pending_incoming") or "" + if a.startswith(cur+":"): + print(a.split(":",1)[1]); raise SystemExit +print("0") +' "$CUR" 2>/dev/null || echo "0" +} + # Build paired pay + withdraw ladders (same step count, shared random shape). # Pay: [0] + log-uniform mids + [max-1] + [max] # Wd: [0] + max(mid, mid×scale) mids + [max-1] + [max] (mids higher → enough to spend) diff --git a/ladder/README.md b/ladder/README.md index 4737bad..3c6e826 100644 --- a/ladder/README.md +++ b/ladder/README.md @@ -29,3 +29,4 @@ There is **no** `check_goa_ladder.sh` — removed; use the phases above. | `LADDER_USE_FREE_TEMPLATE` | `1` | public POST template with `{"amount":…}` | | `LADDER_FREE_TEMPLATE` | `goa-free` | template id (hacktivism) | | `LADDER_FREE_TEMPLATE_INSTANCE` | merchant instance | e.g. `goa-demo-cp4zqk` | +| `LADDER_PAY_WAIT_AVAILABLE_S` | `90` | after withdraw, wait for spendable **available**>0; logs **withdraw board in start order** (OK/OK_BANK) with live `pendingIncoming`→`done` + timer (same wallet DB) | diff --git a/ladder/lib_pay.sh b/ladder/lib_pay.sh index 9e8e157..e8ae15a 100644 --- a/ladder/lib_pay.sh +++ b/ladder/lib_pay.sh @@ -9,7 +9,7 @@ : "${LADDER_FREE_TEMPLATE:=goa-free}" : "${LADDER_FREE_TEMPLATE_INSTANCE:=}" : "${LADDER_USE_FREE_TEMPLATE:=1}" -: "${LADDER_PAY_WAIT_AVAILABLE_S:=60}" +: "${LADDER_PAY_WAIT_AVAILABLE_S:=90}" : "${LADDER_PAY_SETTLE_ROUNDS:=6}" MAX_PAY_BEST_AMT="${MAX_PAY_BEST_AMT:-}" @@ -188,17 +188,213 @@ except Exception: return 0 } +# Print withdraw board: OK/OK_BANK rungs in *start order* + wallet tx state. +# Uses TSV (ladder order) + transactions dump. Call after refreshing tx dump. +# Args: left_secs available_num pendingIncoming_num +ladder_print_wd_board() { + local left="${1:-?}" bal="${2:-0}" pend="${3:-0}" + local tsv="${TSV:-}" txf="${SCRATCH:-/tmp}/tx-wait-pay.out" + local board + board=$( + python3 - "$tsv" "$txf" "${CUR:-GOA}" "$left" "$bal" "$pend" <<'PY' 2>/dev/null || true +import json, sys +from pathlib import Path + +tsv, txf, cur, left, bal, pend = sys.argv[1:7] +dec = json.JSONDecoder() + +def num(a: str) -> str: + a = (a or "").strip() + if ":" in a: + return a.split(":", 1)[1] + return a + +def iter_json(text: str): + i, n = 0, len(text) + while i < n: + if text[i] == "{": + try: + o, end = dec.raw_decode(text, i) + yield o + i = end + continue + except Exception: + pass + i += 1 + +# TSV rows in start order (OK / OK_BANK only — successful bank path) +rows = [] +if tsv and Path(tsv).is_file(): + for ln in open(tsv, errors="replace"): + if not ln.strip() or ln.startswith("rung"): + continue + p = ln.rstrip("\n").split("\t") + if len(p) < 10: + continue + status = p[3] + if status not in ("OK", "OK_BANK"): + continue + rows.append({ + "rung": p[0], + "amt": p[2], + "bank": status, + "wid": (p[9] if len(p) > 9 else "")[:12], + }) + +# Wallet withdrawal txs (chronological if timestamp present) +wds = [] +if txf and Path(txf).is_file(): + try: + text = open(txf, errors="replace").read() + except Exception: + text = "" + for o in iter_json(text): + txs = o.get("transactions") if isinstance(o, dict) else None + if not isinstance(txs, list): + continue + for t in txs: + if not isinstance(t, dict): + continue + typ = str(t.get("type") or t.get("Type") or "").lower() + if "withdraw" not in typ: + continue + amt = t.get("amountRaw") or t.get("amountEffective") or t.get("amount") or "" + st = ( + t.get("txState") + or t.get("status") + or t.get("pendingMajor") + or t.get("txMajorState") + or "" + ) + st = str(st).lower() + det = t.get("withdrawalDetails") or t.get("withdrawal_details") or {} + if not isinstance(det, dict): + det = {} + conf = det.get("confirmed") + if conf is True and not st: + st = "done" + elif conf is False and not st: + st = "pending" + # pending* / dialog / kyc → still incoming; done/abort final + if any(x in st for x in ("done", "abort", "fail", "delete")): + phase = "done" if "done" in st else st + elif st in ("", "none", "null"): + phase = "unknown" + else: + phase = "pendingIncoming" + ts = 0 + for k in ("timestamp", "timestamp_ms", "age"): + v = t.get(k) + if isinstance(v, dict): + v = v.get("t_s") or v.get("t_ms") or v.get("t_usec") + try: + ts = int(v or 0) + except Exception: + ts = 0 + if ts: + break + wds.append({"amt": str(amt), "n": num(str(amt)), "phase": phase, "st": st or "?", "ts": ts}) + +wds.sort(key=lambda x: x["ts"]) +used = [False] * len(wds) + +lines = [ + f"withdraw board (start order) · timer {left}s left · available={cur}:{bal} · pendingIncoming={cur}:{pend}" +] +if not rows: + lines.append(" (no OK/OK_BANK rows in TSV yet)") +else: + for i, r in enumerate(rows, 1): + n = num(r["amt"]) + match = None + for j, w in enumerate(wds): + if used[j]: + continue + if w["n"] == n or w["amt"] == r["amt"] or w["amt"].endswith(":" + n): + used[j] = True + match = w + break + if match is None: + wlabel = "wallet=? (no tx match)" + elif match["phase"] == "pendingIncoming": + wlabel = f"wallet=pendingIncoming ({match['st']})" + elif match["phase"] == "done": + wlabel = f"wallet=done ({match['st']})" + else: + wlabel = f"wallet={match['phase']} ({match['st']})" + wid = r["wid"] or "-" + lines.append( + f" #{i:<3} rung={r['rung']:<4} {r['amt']:<22} bank={r['bank']:<8} {wlabel} wid={wid}" + ) + pending_n = sum(1 for w in wds if w["phase"] == "pendingIncoming") + done_n = sum(1 for w in wds if w["phase"] == "done") + lines.append( + f" summary: tsv_ok={len(rows)} · wallet_wd={len(wds)} (pendingIncoming={pending_n} done={done_n})" + ) + +print("\n".join(lines)) +PY + ) + if [ -n "$board" ]; then + while IFS= read -r line || [ -n "$line" ]; do + [ -n "$line" ] && info pay "$line" + done <<<"$board" + else + info pay "timer ${left}s left · available=${CUR}:${bal} · pendingIncoming=${CUR}:${pend} · same wallet (board n/a)" + fi +} + +# Wait for spendable coins on the *same* wallet DB as withdraw (WDB). +# After OK_BANK, available is often still 0 while pendingIncoming > 0 — pay needs available. +# Shows withdraw board in start order + countdown (never looks hung). ladder_wait_available() { local wait_s="${1:-$LADDER_PAY_WAIT_AVAILABLE_S}" - local end bal + local end now left bal pend tick=0 end=$(( $(date +%s) + wait_s )) bal=$(wallet_avail) + pend=$(wallet_pending_in 2>/dev/null || echo "0") + if python3 -c 'import sys; from decimal import Decimal; sys.exit(0 if Decimal(sys.argv[1])>0 else 1)' "$bal" 2>/dev/null; then + printf '%s' "$bal" + return 0 + fi + info pay "same wallet as withdraw (db=$(basename "${WDB:-wallet}")) · available=${CUR}:${bal} · pendingIncoming=${CUR}:${pend}" + info pay "waiting up to ${wait_s}s for spendable available>0 — board lists OK/OK_BANK withdraws in start order" + # initial board before loop + wcli transactions >"$SCRATCH/tx-wait-pay.out" 2>&1 || true + ladder_print_wd_board "$wait_s" "$bal" "$pend" while python3 -c 'import sys; from decimal import Decimal; sys.exit(0 if Decimal(sys.argv[1])<=0 else 1)' "$bal" 2>/dev/null; do - [ "$(date +%s)" -ge "$end" ] && break + now=$(date +%s) + left=$(( end - now )) + if [ "$left" -le 0 ]; then + wcli transactions >"$SCRATCH/tx-wait-pay.out" 2>&1 || true + ladder_print_wd_board "0" "$bal" "$pend" + info pay "timer 0s left — stop waiting (available still ${CUR}:0)" + break + fi + # nudge wallet (same DB); do not run-until-done (can hang) wcli transactions >"$SCRATCH/tx-wait-pay.out" 2>&1 || true + if command -v timeout >/dev/null 2>&1 || command -v gtimeout >/dev/null 2>&1; then + local _to + _to=$(command -v gtimeout 2>/dev/null || command -v timeout) + "$_to" 8 wcli advanced run-pending >"$SCRATCH/run-pending.out" 2>&1 || true + fi + # progress board every ~5s (tick every 2s → every 3rd) + if [ $((tick % 3)) -eq 0 ]; then + ladder_print_wd_board "$left" "$bal" "$pend" + fi sleep 2 + tick=$((tick + 1)) bal=$(wallet_avail) + pend=$(wallet_pending_in 2>/dev/null || echo "0") done + if python3 -c 'import sys; from decimal import Decimal; sys.exit(0 if Decimal(sys.argv[1])>0 else 1)' "$bal" 2>/dev/null; then + wcli transactions >"$SCRATCH/tx-wait-pay.out" 2>&1 || true + ladder_print_wd_board "ready" "$bal" "$pend" + ok pay "spendable coins ready available=${CUR}:${bal} · $(format_amount_alt "${CUR}:${bal}") (same wallet)" + else + warn pay "no spendable coins after ${wait_s}s" \ + "problem: available still ${CUR}:0 (pendingIncoming=${CUR}:${pend}). Board above shows which withdraws still pendingIncoming. Pay will SKIP_BALANCE / skip max-pay until wirewatch+wallet finish. Same wallet as withdraw." + fi printf '%s' "$bal" } @@ -346,7 +542,9 @@ ladder_run_pay_phase() { info pay "public free template ${LADDER_FREE_TEMPLATE}@${LADDER_FREE_TEMPLATE_INSTANCE} (no merchant token)" fi + # Same cumulative wallet as withdraw (WDB=wallet-main.sqlite3) wallet_prepare "main" + info pay "using same wallet as withdraw · WDB=${WDB:-?}" local bal0 bal0=$(ladder_wait_available "$LADDER_PAY_WAIT_AVAILABLE_S") info "pay budget" "wallet available ${CUR}:${bal0} · $(format_amount_alt "${CUR}:${bal0}") — never order more than this" From 5c312833e19d1e50331d8103e1b383080c7bd263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 14:49:08 +0200 Subject: [PATCH 21/29] =?UTF-8?q?docs:=20=C2=A73b=20mint=205110/P0001=20wi?= =?UTF-8?q?th=20dated=20landing=20withdraw=20examples?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Record recurring bank SQL 5110/P0001 as server-side (not mon/wallet), max-search fail band ~824k GOA, and same-day landing recent_withdraws that still show Giga/Mega explorer mints succeeded earlier. --- CLI-AUTOMATION-NOTES.md | 64 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 7 deletions(-) diff --git a/CLI-AUTOMATION-NOTES.md b/CLI-AUTOMATION-NOTES.md index 2f25122..953898f 100644 --- a/CLI-AUTOMATION-NOTES.md +++ b/CLI-AUTOMATION-NOTES.md @@ -67,15 +67,65 @@ Legend: | **Workaround** | Use `LADDER_STEPS≥5` (or higher) so random mids exist; treat zero / absolute-max / max-1-5110 as probes (soft `CEILING_REJECT` in mon ≥1.20.2). | | **Wanted** | Ladder plan docs: pin vs mid semantics; CLI structured error for amount-too-large / zero. | -### 3b. Higher withdraw amounts often fail at mint +### 3b. Mint fails with bank SQL `5110` / `P0001` (server-side; recurring) | | | |--|--| -| **Seen** | Classic GOA ladder high/random mids and ceiling pins — e.g. `GOA:747827072844319` → `FAIL_MINT` / `STOPPED … mint HTTP 500 { code :5110, … P0001 }`; same class of error on max / max-1. | -| **Area** | **bank** / **ops** | -| **Problem** | **Larger amounts** frequently hit bank/libeufin limits (HTTP 500, Taler **5110**, SQL **P0001**). Not specific to pin:max only — mid-high ladder steps fail the same way. Do not treat as wallet/`reserve_pub` when mint never succeeded. | -| **Workaround** | Expect failures above some live ceiling; use max-search / soft ceiling handling; keep mid-range rungs for real withdraw/confirm tests. | -| **Wanted** | Clear amount-too-large from bank (not opaque SQL 500); mon reports ceiling separately from hard fails. | +| **Seen** | **Several times** on GOA (`bank.hacktivism.ch`), classic ladder high/random mids + ceiling pins **and** max-search bisect — not a one-off. Same class of error across 2026-07 runs (magikoopa / mon ladder). | +| **Area** | **bank** (libeufin / taler-corebank + **Postgres**) / **ops** — **not** mon SQL, **not** wallet-cli at mint time | +| **Call site** | mon `POST ${BANK}/accounts/explorer/withdrawals` with `{"amount":"GOA:…"}` only. Wallet/exchange not involved until mint returns 200 + URI. | +| **Response** | HTTP **500**, body e.g. `{ code: 5110, hint: "Unexpected sql error with state P0001" }` | +| **Codes** | **`5110` = `BANK_UNMANAGED_EXCEPTION`** (“no known exception types captured the exception”). **`P0001`** = PostgreSQL SQLSTATE for application `RAISE EXCEPTION` (function/trigger), not a mon bug and not max_wire config alone. | +| **Config check** | Public `/config` still allows huge wires (`max_wire_transfer_amount` ≈ `GOA:4503599627370496.99999999`, `default_debit_threshold` same order). Failing mints are often **far below** that (e.g. ~Kilo-GOA). | +| **API mismatch** | Corebank docs: insufficient funds should be HTTP **409**. Getting **500/5110** means the bank is **leaking** an unmapped SQL error instead of a clean client error. | +| **Problem** | Live mint “ceiling” is **opaque and unstable**. Do **not** treat 5110 as wallet/`reserve_pub` failure when mint never succeeded. Do **not** assume “only absolute pin:max fails” — mid and max-search probes hit the same error. | +| **Workaround (mon ≥1.21)** | Soft `CEILING_REJECT` on 5110/P0001 (classic mids + max pins); **max-ladder** treats mint reject as too-high upper bound and bisects. Keep mid-range rungs for real withdraw/confirm tests. | +| **Wanted (bank)** | Map the SQL condition to a structured error (insufficient funds / amount too large / debit limit) — **not** `BANK_UNMANAGED_EXCEPTION`. Log full `RAISE` text server-side. | + +#### What max-search saw (example run, 2026-07-19) + +Ladder max-search log (magikoopa): + +| Time (local mon log) | Probe | Result | +|----------------------|-------|--------| +| (bisect) | lo ≈ **GOA:822256** (822.256 Kilo-GOA) | still treatable as success path / lower bound | +| probe ~24 | **GOA:824385** (824.385 Kilo-GOA) | mint **HTTP 500 / 5110 / P0001** → soft too-high | +| after | hi := **GOA:824385** | continues bisect | + +So the **live single-mint fail band** was ~**822–825 Kilo-GOA** in that run — **not** “only Tera/Peta pins”. + +#### Landing page shows **much higher** amounts actually gotten (same day) + +Public bank intro stats: `https://bank.hacktivism.ch/intro/stats.json` (and the HTML landing that consumes them). Snapshot **2026-07-19 ~14:47 CEST** (`generated_at_human`). + +**Explorer balance still enormous** (so this is not “pool empty” in the UI sense): + +| Field | Value (snapshot) | +|-------|------------------| +| `balance_explorer` | **4.5036 Peta-GOA** (`GOA:4503599627250215.32499924`) | +| cumulative `withdraws.total_amount` | **4.5036 Peta-GOA** (292 ops) | +| `withdraws.last_24h` | **3.2366 Peta-GOA** (45 ops) | + +**Dated single withdraws that *did* mint** (from `recent_withdraws`, account=`explorer`) — all **before** or around the ~824k fail band above: + +| When (CEST) | Amount (alt) | Raw | +|-------------|--------------|-----| +| **2026-07-19 14:19** | **1.9761 Giga-GOA** | `GOA:1976113563` | +| **2026-07-19 14:19** | **1.4495 Giga-GOA** | `GOA:1449489732` | +| **2026-07-19 14:26** | **201.636 Mega-GOA** | `GOA:201635991` | +| **2026-07-19 14:26** | **144.53 Mega-GOA** | `GOA:144530029` | +| **2026-07-19 14:29** | **3.4099 Mega-GOA** | `GOA:3409891` | +| **2026-07-19 14:29** | **1.8009 Mega-GOA** | `GOA:1800875` | +| **2026-07-19 14:42** | 822.256 Kilo-GOA | `GOA:822256` (matches max-search **lo**) | +| **2026-07-19 14:42** | 424.114 Kilo-GOA | `GOA:424114` | + +Same calendar day, **~1.5–2 hours earlier**, explorer successfully recorded **Giga-GOA** and **Mega-GOA** withdraws — orders of magnitude **above** the ~824k amount that later returned **5110/P0001**. + +**Takeaway:** landing/stats prove higher single withdraws were **possible and recorded**; the 5110 failure is **not** explained by “config max_wire forbids Kilo-GOA” or by the public explorer balance display. Ops should compare **landing `recent_withdraws` + `balance_explorer`** with mon mint 5110 times when filing bank bugs. + +**Earlier mon observations (same error class, larger pins):** classic ladder e.g. `GOA:747827072844319` → mint 500/5110/P0001; absolute max / max-1 ceiling probes — see also classic soft `CEILING_REJECT` and max-ladder hi bound. + +**Exchange intro (context, not the mint path):** `https://exchange.hacktivism.ch/intro/stats.json` — e.g. snapshot same day: `wire_in_amount` ~Peta-GOA scale, `withdraw_amount` ~`GOA:1.79e6` coin-side; bank landing is the right place for **explorer mint** history. --- @@ -287,7 +337,7 @@ If only a few changes land, these unlock the most automation: | ladder explorer | `read_secret` / EXP_PW_FILE | | stage maxima | bank `max_wire` + keys min denom | | force-select | multi-rpub try; soft-skip confirm; empty scrape → WARN not balance fail | -| higher amounts | mint 5110/P0001 common above live ceiling (§3b); classic soft `CEILING_REJECT`; max-ladder bounds hi; report uses alt_unit_names (Tera-GOA …) | +| higher amounts | mint **5110/P0001** recurring server-side (§3b); landing shows much larger **dated** explorer withdraws (Giga/Mega same day); soft `CEILING_REJECT` + max-ladder hi; alt units in report | | pay vs balance | never order more than wallet **available** (soft `SKIP_BALANCE`); pendingIncoming does not count as spendable | | pay wait UX | after withdraw, board lists OK/OK_BANK in **start order** with live wallet `pendingIncoming`/`done` + timer (`LADDER_PAY_WAIT_AVAILABLE_S`, default 90s); same wallet DB as withdraw | | wallet-db path | Prefer `*.sqlite3` (never `*.json` as db path) | From a534ef8d1c6798860cf79275a7459597276456c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 14:50:36 +0200 Subject: [PATCH 22/29] =?UTF-8?q?docs:=20=C2=A73b=20full=20max-search=2051?= =?UTF-8?q?10=20cascade=20(Tera=E2=86=92Kilo)=20with=20dated=20landing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expand CLI-AUTOMATION-NOTES with the 2026-07-19 max-ladder probe table: hi descent while lo=0, OK at 424114/822256, fine bisect fails to 822788, and same-day landing Giga/Mega withdraws for contrast. --- CLI-AUTOMATION-NOTES.md | 114 ++++++++++++++++++++++++++++++---------- 1 file changed, 85 insertions(+), 29 deletions(-) diff --git a/CLI-AUTOMATION-NOTES.md b/CLI-AUTOMATION-NOTES.md index 953898f..69bcbc3 100644 --- a/CLI-AUTOMATION-NOTES.md +++ b/CLI-AUTOMATION-NOTES.md @@ -82,50 +82,106 @@ Legend: | **Workaround (mon ≥1.21)** | Soft `CEILING_REJECT` on 5110/P0001 (classic mids + max pins); **max-ladder** treats mint reject as too-high upper bound and bisects. Keep mid-range rungs for real withdraw/confirm tests. | | **Wanted (bank)** | Map the SQL condition to a structured error (insufficient funds / amount too large / debit limit) — **not** `BANK_UNMANAGED_EXCEPTION`. Log full `RAISE` text server-side. | -#### What max-search saw (example run, 2026-07-19) +#### Full max-search cascade (primary repro log, 2026-07-19, magikoopa / GOA) -Ladder max-search log (magikoopa): +Mon phase **max-ladder** withdraw hunt. Every reject below is the **same** bank body: -| Time (local mon log) | Probe | Result | -|----------------------|-------|--------| -| (bisect) | lo ≈ **GOA:822256** (822.256 Kilo-GOA) | still treatable as success path / lower bound | -| probe ~24 | **GOA:824385** (824.385 Kilo-GOA) | mint **HTTP 500 / 5110 / P0001** → soft too-high | -| after | hi := **GOA:824385** | continues bisect | +```text +mint HTTP 500 { code :5110, hint : Unexpected sql error with state P0001 } +``` -So the **live single-mint fail band** was ~**822–825 Kilo-GOA** in that run — **not** “only Tera/Peta pins”. +Mon treats each as soft too-high (`CEILING_REJECT` / `hi:=amount`). **`lo` stays 0** until the first successful mint. -#### Landing page shows **much higher** amounts actually gotten (same day) +**Phase A — pure descent (no success yet): hi falls Tera → Giga → Mega while lo=0** -Public bank intro stats: `https://bank.hacktivism.ch/intro/stats.json` (and the HTML landing that consumes them). Snapshot **2026-07-19 ~14:47 CEST** (`generated_at_human`). +| Probe | Amount (alt) | Raw `GOA:…` | Result | +|------:|--------------|-------------|--------| +| (earlier) | hi already **467.409 Tera-GOA** | `467409181353380` | 5110 (prior probe) | +| 4 | **154.93 Tera-GOA** | `154929915073879` | 5110 → hi:= | +| 5 | **16.20 Tera-GOA** | `16198385073981` | 5110 → hi:= | +| 6 | **1.417 Tera-GOA** | `1416945306636` | 5110 → hi:= | +| 7 | **282.49 Giga-GOA** | `282485970397` | 5110 → hi:= | +| 8 | **132.18 Giga-GOA** | `132183255296` | 5110 → hi:= | +| 9 | **7.417 Giga-GOA** | `7416937654` | 5110 → hi:= | +| 10 | **711.38 Mega-GOA** | `711383774` | 5110 → hi:= | +| 11 | **395.88 Mega-GOA** | `395879334` | 5110 → hi:= | +| 12 | **119.26 Mega-GOA** | `119263564` | 5110 → hi:= | +| 13 | **5.992 Mega-GOA** | `5992111` | 5110 → hi:= | -**Explorer balance still enormous** (so this is not “pool empty” in the UI sense): +**Phase B — first success raises lo; still 5110 just above** -| Field | Value (snapshot) | -|-------|------------------| +| Probe | Amount (alt) | Raw | Result | +|------:|--------------|-----|--------| +| **14** | **424.114 Kilo-GOA** | **`424114`** | **OK mint** → accept → force-select `PNK5YRKT…` → confirm 204 → settle `transfer_done` (wallet **avail still 0**) · **lo:=424114** | +| 15 | **1.594 Mega-GOA** | `1594158` | 5110 → hi:= | +| **16** | **822.256 Kilo-GOA** | **`822256`** | **OK mint** → force-select `SVKV4E9G…` → confirm → settle · **lo:=822256** (best so far) | + +**Phase C — fine bisect above best: every step 5110 until convergence** + +| Probe | Amount (alt) | Raw | Result | +|------:|--------------|-----|--------| +| 17 | 1.145 Mega-GOA | `1144904` | 5110 | +| 18 | 970.26 Kilo-GOA | `970260` | 5110 | +| 19 | 893.198 Kilo-GOA | `893198` | 5110 | +| 20 | 856.993 Kilo-GOA | `856993` | 5110 | +| 21 | 839.445 Kilo-GOA | `839445` | 5110 | +| 22 | 830.806 Kilo-GOA | `830806` | 5110 | +| 23 | 826.52 Kilo-GOA | `826520` | 5110 | +| 24 | 824.385 Kilo-GOA | `824385` | 5110 | +| 25 | 823.32 Kilo-GOA | `823320` | 5110 | +| 26 | 822.788 Kilo-GOA | `822788` | 5110 → hi:= | + +**Converged (mon):** + +```text +max-search best withdrawable 822.256 Kilo-GOA (GOA:822256) + · lo=822.256 Kilo-GOA · hi=822.788 Kilo-GOA · probes=26 +``` + +**Reading this log** + +1. **Same error from ~467 Tera-GOA down to ~823 Kilo-GOA** — not a single “absurd pin:max” case; the bank returns **5110/P0001** for a **wide continuous range** of amounts. +2. **Successes are real but sparse:** only **`GOA:424114`** and **`GOA:822256`** minted in this hunt (both full bank confirm + `transfer_done`). Everything strictly larger than **822256** in the fine band failed at **mint**. +3. **lo=0 for probes 4–13** means max-search had **no** successful withdraw yet while already rejecting Tera/Giga/Mega — so 5110 is **not** “only after pool drained by this run.” +4. Settle notes **wallet avail=GOA:0** after OK bank path — coins lag / `pendingIncoming` (pay-wait board, § pay wait UX); separate from mint 5110. +5. Landing `recent_withdraws` at **14:42 CEST** lists the same two OK amounts (`424114`, `822256`) with reserves matching force-select prefixes above. + +#### Landing page: much larger withdraws **same day, earlier** (still on explorer) + +Public bank intro: `https://bank.hacktivism.ch/intro/stats.json` (HTML landing uses this). Snapshot **2026-07-19 ~14:47 CEST**. + +**Pool still looks full in the UI:** + +| Field | Snapshot | +|-------|----------| | `balance_explorer` | **4.5036 Peta-GOA** (`GOA:4503599627250215.32499924`) | -| cumulative `withdraws.total_amount` | **4.5036 Peta-GOA** (292 ops) | +| `withdraws.total_amount` | **4.5036 Peta-GOA** (292 ops) | | `withdraws.last_24h` | **3.2366 Peta-GOA** (45 ops) | -**Dated single withdraws that *did* mint** (from `recent_withdraws`, account=`explorer`) — all **before** or around the ~824k fail band above: +**Dated explorer withdraws that succeeded *before* this max-search tight band** (`recent_withdraws`, account=`explorer`): -| When (CEST) | Amount (alt) | Raw | -|-------------|--------------|-----| -| **2026-07-19 14:19** | **1.9761 Giga-GOA** | `GOA:1976113563` | -| **2026-07-19 14:19** | **1.4495 Giga-GOA** | `GOA:1449489732` | -| **2026-07-19 14:26** | **201.636 Mega-GOA** | `GOA:201635991` | -| **2026-07-19 14:26** | **144.53 Mega-GOA** | `GOA:144530029` | -| **2026-07-19 14:29** | **3.4099 Mega-GOA** | `GOA:3409891` | -| **2026-07-19 14:29** | **1.8009 Mega-GOA** | `GOA:1800875` | -| **2026-07-19 14:42** | 822.256 Kilo-GOA | `GOA:822256` (matches max-search **lo**) | -| **2026-07-19 14:42** | 424.114 Kilo-GOA | `GOA:424114` | +| When (CEST) | Amount (alt) | Raw | Notes | +|-------------|--------------|-----|--------| +| **2026-07-19 14:19** | **1.9761 Giga-GOA** | `GOA:1976113563` | ~2h before max-search best | +| **2026-07-19 14:19** | **1.4495 Giga-GOA** | `GOA:1449489732` | same minute | +| **2026-07-19 14:26** | **201.636 Mega-GOA** | `GOA:201635991` | | +| **2026-07-19 14:26** | **144.53 Mega-GOA** | `GOA:144530029` | | +| **2026-07-19 14:29** | **3.4099 Mega-GOA** | `GOA:3409891` | | +| **2026-07-19 14:29** | **1.8009 Mega-GOA** | `GOA:1800875` | | +| **2026-07-19 14:42** | 822.256 Kilo-GOA | `GOA:822256` | = max-search **best** (probe 16) | +| **2026-07-19 14:42** | 424.114 Kilo-GOA | `GOA:424114` | = max-search first OK (probe 14) | -Same calendar day, **~1.5–2 hours earlier**, explorer successfully recorded **Giga-GOA** and **Mega-GOA** withdraws — orders of magnitude **above** the ~824k amount that later returned **5110/P0001**. +**Contrast:** earlier the **same day**, explorer minted **~1.5–2 Giga-GOA** singles; later the same API returns **5110** for **~823 Kilo-GOA** and for all of Tera→Mega probes in the cascade. Landing **balance_explorer** still shows **~Peta-GOA**. So: -**Takeaway:** landing/stats prove higher single withdraws were **possible and recorded**; the 5110 failure is **not** explained by “config max_wire forbids Kilo-GOA” or by the public explorer balance display. Ops should compare **landing `recent_withdraws` + `balance_explorer`** with mon mint 5110 times when filing bank bugs. +- not “config max_wire forbids Kilo-GOA”; +- not “landing says explorer is empty”; +- **live mintability collapsed / became amount-sensitive** relative to history — bank SQL path is wrong or mis-mapped. -**Earlier mon observations (same error class, larger pins):** classic ladder e.g. `GOA:747827072844319` → mint 500/5110/P0001; absolute max / max-1 ceiling probes — see also classic soft `CEILING_REJECT` and max-ladder hi bound. +**Ops checklist when filing bank bug:** attach (1) this max-search probe table, (2) landing `recent_withdraws` + `balance_explorer` at the same timestamps, (3) bank journal / Postgres `P0001` RAISE text for one failing mint. -**Exchange intro (context, not the mint path):** `https://exchange.hacktivism.ch/intro/stats.json` — e.g. snapshot same day: `wire_in_amount` ~Peta-GOA scale, `withdraw_amount` ~`GOA:1.79e6` coin-side; bank landing is the right place for **explorer mint** history. +**Also seen (same error class):** classic ladder e.g. `GOA:747827072844319` → 500/5110/P0001; absolute max / max-1 ceiling pins → soft `CEILING_REJECT`. + +**Exchange intro (not mint path):** `https://exchange.hacktivism.ch/intro/stats.json` — wire-in can still be Peta-scale; use **bank** landing for explorer mint history. --- From 0ae4325d4b4ced031c7bbfafbabf497ce0b9460c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 14:58:06 +0200 Subject: [PATCH 23/29] =?UTF-8?q?docs:=20=C2=A73b=20analysis=20=E2=80=94?= =?UTF-8?q?=20healthy=20load=20but=20we=20do=20not=20manage=20to=20withdra?= =?UTF-8?q?w?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace probe dumps with outcome analysis: stack load is fine, mint 5110 / SKIP_CONFIRM / pendingIncoming with available=0; sparse what-worked table and pay board example (tsv_ok=2, wallet_wd=14, done=0). --- CLI-AUTOMATION-NOTES.md | 141 ++++++++++++---------------------------- 1 file changed, 41 insertions(+), 100 deletions(-) diff --git a/CLI-AUTOMATION-NOTES.md b/CLI-AUTOMATION-NOTES.md index 69bcbc3..7d1cc25 100644 --- a/CLI-AUTOMATION-NOTES.md +++ b/CLI-AUTOMATION-NOTES.md @@ -67,121 +67,62 @@ Legend: | **Workaround** | Use `LADDER_STEPS≥5` (or higher) so random mids exist; treat zero / absolute-max / max-1-5110 as probes (soft `CEILING_REJECT` in mon ≥1.20.2). | | **Wanted** | Ladder plan docs: pin vs mid semantics; CLI structured error for amount-too-large / zero. | -### 3b. Mint fails with bank SQL `5110` / `P0001` (server-side; recurring) +### 3b. We do **not** manage to withdraw (GOA) — bank path broken, not load | | | |--|--| -| **Seen** | **Several times** on GOA (`bank.hacktivism.ch`), classic ladder high/random mids + ceiling pins **and** max-search bisect — not a one-off. Same class of error across 2026-07 runs (magikoopa / mon ladder). | -| **Area** | **bank** (libeufin / taler-corebank + **Postgres**) / **ops** — **not** mon SQL, **not** wallet-cli at mint time | -| **Call site** | mon `POST ${BANK}/accounts/explorer/withdrawals` with `{"amount":"GOA:…"}` only. Wallet/exchange not involved until mint returns 200 + URI. | -| **Response** | HTTP **500**, body e.g. `{ code: 5110, hint: "Unexpected sql error with state P0001" }` | -| **Codes** | **`5110` = `BANK_UNMANAGED_EXCEPTION`** (“no known exception types captured the exception”). **`P0001`** = PostgreSQL SQLSTATE for application `RAISE EXCEPTION` (function/trigger), not a mon bug and not max_wire config alone. | -| **Config check** | Public `/config` still allows huge wires (`max_wire_transfer_amount` ≈ `GOA:4503599627370496.99999999`, `default_debit_threshold` same order). Failing mints are often **far below** that (e.g. ~Kilo-GOA). | -| **API mismatch** | Corebank docs: insufficient funds should be HTTP **409**. Getting **500/5110** means the bank is **leaking** an unmapped SQL error instead of a clean client error. | -| **Problem** | Live mint “ceiling” is **opaque and unstable**. Do **not** treat 5110 as wallet/`reserve_pub` failure when mint never succeeded. Do **not** assume “only absolute pin:max fails” — mid and max-search probes hit the same error. | -| **Workaround (mon ≥1.21)** | Soft `CEILING_REJECT` on 5110/P0001 (classic mids + max pins); **max-ladder** treats mint reject as too-high upper bound and bisects. Keep mid-range rungs for real withdraw/confirm tests. | -| **Wanted (bank)** | Map the SQL condition to a structured error (insufficient funds / amount too large / debit limit) — **not** `BANK_UNMANAGED_EXCEPTION`. Log full `RAISE` text server-side. | +| **Bottom line** | On live **hacktivism GOA**, automation **does not reliably complete a withdraw to spendable coins**. Max-search may report a “best” amount; pay then sees **`available=GOA:0`**. This is **recurring (several runs, 2026-07-19)**. | +| **Area** | **bank** + wire/wallet settle — **not** mon overload, **not** “laptop too slow” | +| **Not the cause: ecosystem load** | Outside-in check **2026-07-19 ~14:56 CEST**: bank/exchange/merchant all **HTTP 200**, config ~**50–150 ms**, exchange `/keys` ~**0.5 s**. Container **loadavg ~1.5–1.7**, RSS bank **~737 MiB** / exchange **~670 MiB**, bank probes **47 ms**, **`wirewatch_running: true`**. Stack is **healthy under moderate load** — failures are **application/SQL/protocol**, not “host on fire.” | +| **Call site (mint)** | `POST ${BANK}/accounts/explorer/withdrawals` → often HTTP **500** `{ code: 5110, hint: "Unexpected sql error with state P0001" }` (**`BANK_UNMANAGED_EXCEPTION`**; Postgres **`P0001`** = `RAISE EXCEPTION`). Should often be clean **409** (insufficient funds); instead opaque **500**. | +| **Config** | `/config` still allows ~**2⁵²** wire max; live mint fails **far below** that. Explorer **balance_explorer** on landing still **~4.5 Peta-GOA**. | -#### Full max-search cascade (primary repro log, 2026-07-19, magikoopa / GOA) +#### Analysis — where withdraw dies -Mon phase **max-ladder** withdraw hunt. Every reject below is the **same** bank body: +| Stage | What we see | Manage to withdraw? | +|-------|-------------|---------------------| +| **Mint (high amounts)** | 5110/P0001 while hi falls Tera→Giga→Mega→Kilo with **lo=0** | **No** — withdrawal never created | +| **Mint (some mid amounts)** | occasional **HTTP 200** + URI | Mint only — not done | +| **Confirm** | force-select 200 then stuck **`selected`**, confirm **HTTP 500** → mon **`SKIP_CONFIRM`** (bounds unchanged) | **No** — bank never wires; **not** out-of-money (§3 / 5114 class) | +| **Bank settle** | rare full path: confirm 204 + `transfer_done` | Bank side “OK_BANK” | +| **Wallet coins** | almost always **`available=0`**, large **`pendingIncoming`**, board **`done=0`** | **No spendable coins** → pay **SKIP_BALANCE** / skip max-pay | + +**Pay-wait board (same cumulative wallet), example:** ```text -mint HTTP 500 { code :5110, hint : Unexpected sql error with state P0001 } +summary: tsv_ok=2 · wallet_wd=14 (pendingIncoming=14 done=0) +timer 0s left — available still GOA:0 +pendingIncoming=GOA:335776 ``` -Mon treats each as soft too-high (`CEILING_REJECT` / `hi:=amount`). **`lo` stays 0** until the first successful mint. +Reading: only **2** TSV rows are full OK/OK_BANK, but the wallet holds **14** withdrawal txs still **pendingIncoming** and **none done**. Max-search leaves many **mint-OK / confirm-failed** (and lagging) withdraws in the DB; they **inflate pendingIncoming** without ever becoming **available**. After **90 s** wait + short `run-pending` nudge: **still no spendable balance** → **we do not manage to withdraw** in the sense that matters (pay). -**Phase A — pure descent (no success yet): hi falls Tera → Giga → Mega while lo=0** +#### What *did* work (sparse) — for contrast -| Probe | Amount (alt) | Raw `GOA:…` | Result | -|------:|--------------|-------------|--------| -| (earlier) | hi already **467.409 Tera-GOA** | `467409181353380` | 5110 (prior probe) | -| 4 | **154.93 Tera-GOA** | `154929915073879` | 5110 → hi:= | -| 5 | **16.20 Tera-GOA** | `16198385073981` | 5110 → hi:= | -| 6 | **1.417 Tera-GOA** | `1416945306636` | 5110 → hi:= | -| 7 | **282.49 Giga-GOA** | `282485970397` | 5110 → hi:= | -| 8 | **132.18 Giga-GOA** | `132183255296` | 5110 → hi:= | -| 9 | **7.417 Giga-GOA** | `7416937654` | 5110 → hi:= | -| 10 | **711.38 Mega-GOA** | `711383774` | 5110 → hi:= | -| 11 | **395.88 Mega-GOA** | `395879334` | 5110 → hi:= | -| 12 | **119.26 Mega-GOA** | `119263564` | 5110 → hi:= | -| 13 | **5.992 Mega-GOA** | `5992111` | 5110 → hi:= | +| When / run | What worked | Amounts | +|------------|-------------|---------| +| Landing **2026-07-19 14:19–14:29 CEST** | Explorer singles **recorded** on bank intro stats | **~1.5–2 Giga-GOA**, **~145–202 Mega-GOA**, **~1.8–3.4 Mega-GOA** | +| Max-search run A (same day) | Full bank path (mint→confirm→`transfer_done`) for **two** amounts only; mon best | **`GOA:424114`**, **`GOA:822256`** (~822 Kilo best) | +| Max-search run B (later) | Even fewer full successes; many mint OK + **SKIP_CONFIRM**; mon best collapsed | e.g. full success around **`GOA:12276`** / **`GOA:17257`** (~**17 Kilo** best) | +| Pay after either | Spendable **available** | **`GOA:0`** (fail) | -**Phase B — first success raises lo; still 5110 just above** +So: **landing/history prove large withdraws were possible earlier the same day**; **automation now does not manage a usable withdraw** (best amount drops run-to-run; coins never spendable). -| Probe | Amount (alt) | Raw | Result | -|------:|--------------|-----|--------| -| **14** | **424.114 Kilo-GOA** | **`424114`** | **OK mint** → accept → force-select `PNK5YRKT…` → confirm 204 → settle `transfer_done` (wallet **avail still 0**) · **lo:=424114** | -| 15 | **1.594 Mega-GOA** | `1594158` | 5110 → hi:= | -| **16** | **822.256 Kilo-GOA** | **`822256`** | **OK mint** → force-select `SVKV4E9G…` → confirm → settle · **lo:=822256** (best so far) | +#### Mon behaviour (workaround only) -**Phase C — fine bisect above best: every step 5110 until convergence** +- 5110 → soft **`CEILING_REJECT`** / max-search **`hi:=`** +- Full OK only raises **`lo`**; **SKIP_CONFIRM** does not +- Pay waits up to **`LADDER_PAY_WAIT_AVAILABLE_S`** (default 90) with withdraw board; then **SKIP_BALANCE** if still 0 +- **Does not fix** bank SQL mapping, stuck confirm, or wirewatch→coins lag -| Probe | Amount (alt) | Raw | Result | -|------:|--------------|-----|--------| -| 17 | 1.145 Mega-GOA | `1144904` | 5110 | -| 18 | 970.26 Kilo-GOA | `970260` | 5110 | -| 19 | 893.198 Kilo-GOA | `893198` | 5110 | -| 20 | 856.993 Kilo-GOA | `856993` | 5110 | -| 21 | 839.445 Kilo-GOA | `839445` | 5110 | -| 22 | 830.806 Kilo-GOA | `830806` | 5110 | -| 23 | 826.52 Kilo-GOA | `826520` | 5110 | -| 24 | 824.385 Kilo-GOA | `824385` | 5110 | -| 25 | 823.32 Kilo-GOA | `823320` | 5110 | -| 26 | 822.788 Kilo-GOA | `822788` | 5110 → hi:= | +#### Wanted (bank / exchange ops) -**Converged (mon):** +1. Map mint **P0001** to structured errors (not **5110**). +2. Confirm after force-select must not stick **selected** + HTTP 500. +3. After `transfer_done`, coins must become wallet **available** (or document delay + API to wait). +4. File bugs with: max-search “what worked” table + landing `recent_withdraws`/`balance_explorer` + load note (healthy) + pay board `tsv_ok` vs `wallet_wd`/`pendingIncoming`/`done`. -```text -max-search best withdrawable 822.256 Kilo-GOA (GOA:822256) - · lo=822.256 Kilo-GOA · hi=822.788 Kilo-GOA · probes=26 -``` - -**Reading this log** - -1. **Same error from ~467 Tera-GOA down to ~823 Kilo-GOA** — not a single “absurd pin:max” case; the bank returns **5110/P0001** for a **wide continuous range** of amounts. -2. **Successes are real but sparse:** only **`GOA:424114`** and **`GOA:822256`** minted in this hunt (both full bank confirm + `transfer_done`). Everything strictly larger than **822256** in the fine band failed at **mint**. -3. **lo=0 for probes 4–13** means max-search had **no** successful withdraw yet while already rejecting Tera/Giga/Mega — so 5110 is **not** “only after pool drained by this run.” -4. Settle notes **wallet avail=GOA:0** after OK bank path — coins lag / `pendingIncoming` (pay-wait board, § pay wait UX); separate from mint 5110. -5. Landing `recent_withdraws` at **14:42 CEST** lists the same two OK amounts (`424114`, `822256`) with reserves matching force-select prefixes above. - -#### Landing page: much larger withdraws **same day, earlier** (still on explorer) - -Public bank intro: `https://bank.hacktivism.ch/intro/stats.json` (HTML landing uses this). Snapshot **2026-07-19 ~14:47 CEST**. - -**Pool still looks full in the UI:** - -| Field | Snapshot | -|-------|----------| -| `balance_explorer` | **4.5036 Peta-GOA** (`GOA:4503599627250215.32499924`) | -| `withdraws.total_amount` | **4.5036 Peta-GOA** (292 ops) | -| `withdraws.last_24h` | **3.2366 Peta-GOA** (45 ops) | - -**Dated explorer withdraws that succeeded *before* this max-search tight band** (`recent_withdraws`, account=`explorer`): - -| When (CEST) | Amount (alt) | Raw | Notes | -|-------------|--------------|-----|--------| -| **2026-07-19 14:19** | **1.9761 Giga-GOA** | `GOA:1976113563` | ~2h before max-search best | -| **2026-07-19 14:19** | **1.4495 Giga-GOA** | `GOA:1449489732` | same minute | -| **2026-07-19 14:26** | **201.636 Mega-GOA** | `GOA:201635991` | | -| **2026-07-19 14:26** | **144.53 Mega-GOA** | `GOA:144530029` | | -| **2026-07-19 14:29** | **3.4099 Mega-GOA** | `GOA:3409891` | | -| **2026-07-19 14:29** | **1.8009 Mega-GOA** | `GOA:1800875` | | -| **2026-07-19 14:42** | 822.256 Kilo-GOA | `GOA:822256` | = max-search **best** (probe 16) | -| **2026-07-19 14:42** | 424.114 Kilo-GOA | `GOA:424114` | = max-search first OK (probe 14) | - -**Contrast:** earlier the **same day**, explorer minted **~1.5–2 Giga-GOA** singles; later the same API returns **5110** for **~823 Kilo-GOA** and for all of Tera→Mega probes in the cascade. Landing **balance_explorer** still shows **~Peta-GOA**. So: - -- not “config max_wire forbids Kilo-GOA”; -- not “landing says explorer is empty”; -- **live mintability collapsed / became amount-sensitive** relative to history — bank SQL path is wrong or mis-mapped. - -**Ops checklist when filing bank bug:** attach (1) this max-search probe table, (2) landing `recent_withdraws` + `balance_explorer` at the same timestamps, (3) bank journal / Postgres `P0001` RAISE text for one failing mint. - -**Also seen (same error class):** classic ladder e.g. `GOA:747827072844319` → 500/5110/P0001; absolute max / max-1 ceiling pins → soft `CEILING_REJECT`. - -**Exchange intro (not mint path):** `https://exchange.hacktivism.ch/intro/stats.json` — wire-in can still be Peta-scale; use **bank** landing for explorer mint history. +Sources: `https://bank.hacktivism.ch/intro/stats.json`, exchange/merchant intro stats, mon max-ladder logs 2026-07-19. --- @@ -393,9 +334,9 @@ If only a few changes land, these unlock the most automation: | ladder explorer | `read_secret` / EXP_PW_FILE | | stage maxima | bank `max_wire` + keys min denom | | force-select | multi-rpub try; soft-skip confirm; empty scrape → WARN not balance fail | -| higher amounts | mint **5110/P0001** recurring server-side (§3b); landing shows much larger **dated** explorer withdraws (Giga/Mega same day); soft `CEILING_REJECT` + max-ladder hi; alt units in report | +| higher amounts / withdraw fail | **§3b: we do not manage to withdraw** (spendable coins) on GOA despite healthy load; mint 5110/P0001 + SKIP_CONFIRM + pendingIncoming pile-up; landing had Giga/Mega earlier same day | | pay vs balance | never order more than wallet **available** (soft `SKIP_BALANCE`); pendingIncoming does not count as spendable | -| pay wait UX | after withdraw, board lists OK/OK_BANK in **start order** with live wallet `pendingIncoming`/`done` + timer (`LADDER_PAY_WAIT_AVAILABLE_S`, default 90s); same wallet DB as withdraw | +| pay wait UX | board: OK/OK_BANK start order + live pendingIncoming/`done`; often `tsv_ok≪wallet_wd` and **done=0** after 90s → skip pay | | wallet-db path | Prefer `*.sqlite3` (never `*.json` as db path) | | helper / python | mon PATH must include helper + python≥3.11 | From dfc7595990369ff5463c3611a730df97ef186e41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 14:58:28 +0200 Subject: [PATCH 24/29] =?UTF-8?q?docs:=20=C2=A73b=20nuance=20=E2=80=94=20w?= =?UTF-8?q?ithdraw=20fails=20at=20least=20not=20in=20time?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spendable coins may arrive later; mon does not get available>0 within the pay wait budget (default 90s). --- CLI-AUTOMATION-NOTES.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/CLI-AUTOMATION-NOTES.md b/CLI-AUTOMATION-NOTES.md index 7d1cc25..b4f7431 100644 --- a/CLI-AUTOMATION-NOTES.md +++ b/CLI-AUTOMATION-NOTES.md @@ -67,11 +67,11 @@ Legend: | **Workaround** | Use `LADDER_STEPS≥5` (or higher) so random mids exist; treat zero / absolute-max / max-1-5110 as probes (soft `CEILING_REJECT` in mon ≥1.20.2). | | **Wanted** | Ladder plan docs: pin vs mid semantics; CLI structured error for amount-too-large / zero. | -### 3b. We do **not** manage to withdraw (GOA) — bank path broken, not load +### 3b. We do **not** manage to withdraw **in time** (GOA) — bank path unreliable, not load | | | |--|--| -| **Bottom line** | On live **hacktivism GOA**, automation **does not reliably complete a withdraw to spendable coins**. Max-search may report a “best” amount; pay then sees **`available=GOA:0`**. This is **recurring (several runs, 2026-07-19)**. | +| **Bottom line** | On live **hacktivism GOA**, automation **does not get spendable coins in time** for the pay phase (default wait **90 s**). Max-search may report a “best” amount and bank may even show `transfer_done`; pay still sees **`available=GOA:0`** with large **`pendingIncoming`**. Coins might arrive **later** (wirewatch / wallet) — **at least not within mon’s budget**. Recurring (several runs, 2026-07-19). | | **Area** | **bank** + wire/wallet settle — **not** mon overload, **not** “laptop too slow” | | **Not the cause: ecosystem load** | Outside-in check **2026-07-19 ~14:56 CEST**: bank/exchange/merchant all **HTTP 200**, config ~**50–150 ms**, exchange `/keys` ~**0.5 s**. Container **loadavg ~1.5–1.7**, RSS bank **~737 MiB** / exchange **~670 MiB**, bank probes **47 ms**, **`wirewatch_running: true`**. Stack is **healthy under moderate load** — failures are **application/SQL/protocol**, not “host on fire.” | | **Call site (mint)** | `POST ${BANK}/accounts/explorer/withdrawals` → often HTTP **500** `{ code: 5110, hint: "Unexpected sql error with state P0001" }` (**`BANK_UNMANAGED_EXCEPTION`**; Postgres **`P0001`** = `RAISE EXCEPTION`). Should often be clean **409** (insufficient funds); instead opaque **500**. | @@ -85,7 +85,7 @@ Legend: | **Mint (some mid amounts)** | occasional **HTTP 200** + URI | Mint only — not done | | **Confirm** | force-select 200 then stuck **`selected`**, confirm **HTTP 500** → mon **`SKIP_CONFIRM`** (bounds unchanged) | **No** — bank never wires; **not** out-of-money (§3 / 5114 class) | | **Bank settle** | rare full path: confirm 204 + `transfer_done` | Bank side “OK_BANK” | -| **Wallet coins** | almost always **`available=0`**, large **`pendingIncoming`**, board **`done=0`** | **No spendable coins** → pay **SKIP_BALANCE** / skip max-pay | +| **Wallet coins** | almost always **`available=0`** within wait, large **`pendingIncoming`**, board **`done=0`** | **Not in time** for pay → **SKIP_BALANCE** / skip max-pay | **Pay-wait board (same cumulative wallet), example:** @@ -95,7 +95,7 @@ timer 0s left — available still GOA:0 pendingIncoming=GOA:335776 ``` -Reading: only **2** TSV rows are full OK/OK_BANK, but the wallet holds **14** withdrawal txs still **pendingIncoming** and **none done**. Max-search leaves many **mint-OK / confirm-failed** (and lagging) withdraws in the DB; they **inflate pendingIncoming** without ever becoming **available**. After **90 s** wait + short `run-pending` nudge: **still no spendable balance** → **we do not manage to withdraw** in the sense that matters (pay). +Reading: only **2** TSV rows are full OK/OK_BANK, but the wallet holds **14** withdrawal txs still **pendingIncoming** and **none done** after the wait. Max-search also leaves many **mint-OK / confirm-failed** withdraws that **inflate pendingIncoming** and may never complete. After **90 s** + short `run-pending`: **still no spendable balance** → **we do not manage to withdraw in time** for pay (whether coins would appear later is unknown to the job). #### What *did* work (sparse) — for contrast @@ -104,9 +104,9 @@ Reading: only **2** TSV rows are full OK/OK_BANK, but the wallet holds **14** wi | Landing **2026-07-19 14:19–14:29 CEST** | Explorer singles **recorded** on bank intro stats | **~1.5–2 Giga-GOA**, **~145–202 Mega-GOA**, **~1.8–3.4 Mega-GOA** | | Max-search run A (same day) | Full bank path (mint→confirm→`transfer_done`) for **two** amounts only; mon best | **`GOA:424114`**, **`GOA:822256`** (~822 Kilo best) | | Max-search run B (later) | Even fewer full successes; many mint OK + **SKIP_CONFIRM**; mon best collapsed | e.g. full success around **`GOA:12276`** / **`GOA:17257`** (~**17 Kilo** best) | -| Pay after either | Spendable **available** | **`GOA:0`** (fail) | +| Pay after either | Spendable **available** within **90 s** | **`GOA:0`** (not in time) | -So: **landing/history prove large withdraws were possible earlier the same day**; **automation now does not manage a usable withdraw** (best amount drops run-to-run; coins never spendable). +So: **landing/history prove large withdraws were possible earlier the same day**; **automation does not manage a usable withdraw in time** (best amount drops run-to-run; coins not available inside mon’s wait). #### Mon behaviour (workaround only) @@ -334,7 +334,7 @@ If only a few changes land, these unlock the most automation: | ladder explorer | `read_secret` / EXP_PW_FILE | | stage maxima | bank `max_wire` + keys min denom | | force-select | multi-rpub try; soft-skip confirm; empty scrape → WARN not balance fail | -| higher amounts / withdraw fail | **§3b: we do not manage to withdraw** (spendable coins) on GOA despite healthy load; mint 5110/P0001 + SKIP_CONFIRM + pendingIncoming pile-up; landing had Giga/Mega earlier same day | +| higher amounts / withdraw fail | **§3b: we do not manage to withdraw in time** on GOA (healthy load); mint 5110/P0001 + SKIP_CONFIRM + pendingIncoming; pay still available=0 after 90s; landing had Giga/Mega earlier same day | | pay vs balance | never order more than wallet **available** (soft `SKIP_BALANCE`); pendingIncoming does not count as spendable | | pay wait UX | board: OK/OK_BANK start order + live pendingIncoming/`done`; often `tsv_ok≪wallet_wd` and **done=0** after 90s → skip pay | | wallet-db path | Prefer `*.sqlite3` (never `*.json` as db path) | From 617d066a701cc1004e59980759f50a8097ea71d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 15:03:05 +0200 Subject: [PATCH 25/29] release 1.23.0: daily GOA /monitoring-ladder and /monitoring-max-ladder Host-agent wrappers + daily timers publish classic and max-search amount ladder reports on bank/exchange/taler.hacktivism.ch. Install enables timers and starts the first round immediately. monpages path allow + goa catalog; Caddy bare redirs for ladder paths. --- VERSION | 2 +- VERSIONS.md | 1 + check_monitoring_pages.sh | 18 ++++++-- host-agent/README.md | 19 ++++++++ host-agent/ROOT-APPLY-MONITORING.md | 4 ++ host-agent/apply-monitoring-live.sh | 6 +++ host-agent/install-host-agent.sh | 22 ++++++++-- host-agent/run-hacktivism-monitoring.sh | 8 ++-- host-agent/run-ladder-monitoring.sh | 43 +++++++++++++++++++ host-agent/run-max-ladder-monitoring.sh | 42 ++++++++++++++++++ host-agent/taler-monitoring-ladder.service | 19 ++++++++ host-agent/taler-monitoring-ladder.timer | 14 ++++++ .../taler-monitoring-max-ladder.service | 19 ++++++++ host-agent/taler-monitoring-max-ladder.timer | 14 ++++++ site-gen/caddy-monitoring-handles.snippet | 9 +++- site-gen/console_to_html.py | 16 +++---- 16 files changed, 237 insertions(+), 19 deletions(-) create mode 100755 host-agent/run-ladder-monitoring.sh create mode 100755 host-agent/run-max-ladder-monitoring.sh create mode 100644 host-agent/taler-monitoring-ladder.service create mode 100644 host-agent/taler-monitoring-ladder.timer create mode 100644 host-agent/taler-monitoring-max-ladder.service create mode 100644 host-agent/taler-monitoring-max-ladder.timer diff --git a/VERSION b/VERSION index 6245bee..a6c2798 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.22.1 +1.23.0 diff --git a/VERSIONS.md b/VERSIONS.md index d4ec0fa..3e7f783 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| +| **v1.23.0** | 2026-07-19 | **Feature:** GOA daily mon pages **`/monitoring-ladder/`** (classic) + **`/monitoring-max-ladder/`** (max-search) on bank/exchange/taler.hacktivism.ch. Host-agent timers once/day + install starts first round immediately. monpages catalog/path allow; Caddy bare redirs. | | **v1.22.1** | 2026-07-19 | **Bugfix (UX):** pay wait after withdraw no longer looks hung — shows **withdraw board in start order** (OK/OK_BANK from TSV) with live wallet status (`pendingIncoming` → `done`) + timer + aggregate available/pendingIncoming on the **same** wallet DB. Short `run-pending` nudge only (no run-until-done). Default wait 90s (`LADDER_PAY_WAIT_AVAILABLE_S`). | | **v1.22.0** | 2026-07-19 | **Feature:** modular amount ladder — `ladder/lib_pay.sh` (pay) + withdraw in `check_amount_ladder.sh`; `ladder/extract_rpubs.py`. **max-ladder** hunts highest **withdrawable** and highest **payable** (≤ wallet available). GOA pays use public free-amount template **`goa-free`** (`LADDER_FREE_TEMPLATE`). Classic still default 23-rung wd+pay. Phase alias `goa-ladder` → amount ladder (shim removed). | | **v1.21.0** | 2026-07-19 | **Feature:** generic **amount ladder** (`check_amount_ladder.sh`; compat shim `check_goa_ladder.sh`). Default **classic** 23-rung withdraw (+ pay when enabled); **max-search** (`max-ladder` / `LADDER_MODE=max`) hunts highest mintable amount (probes default 32, `0`=until timeout). **Pay:** never overspend — soft `SKIP_BALANCE` if amount > wallet available. **Alt units:** report/logs show human names (e.g. `8.15 Tera-GOA (GOA:…)`) + raw. **Fixes bundled:** external extract (no stdin SyntaxError); force-select multi-rpub; soft `CEILING_REJECT` on mint 5110/P0001; crash-proof report. CLI-AUTOMATION-NOTES §3/3a/3b/§15–18. | diff --git a/check_monitoring_pages.sh b/check_monitoring_pages.sh index fea39cd..f4804b7 100755 --- a/check_monitoring_pages.sh +++ b/check_monitoring_pages.sh @@ -100,14 +100,18 @@ _host_allowed() { } # Public mon URL families (suite name "taler-monitoring" ≠ a public path): -# 1) /monitoring(+_err) — landing hosts (bare + slash) -# 2) /taler-monitoring-surface(+_err) — taler.hacktivism.ch only -# 3) /taler-monitoring-aptdeploy(+_err) — taler.hacktivism.ch only +# 1) /monitoring(+_err) — landing hosts (bare + slash) +# 2) /monitoring-ladder(+_err) — GOA classic amount ladder (daily) +# 3) /monitoring-max-ladder(+_err) — GOA max-search ladder (daily) +# 4) /taler-monitoring-surface(+_err) — taler.hacktivism.ch only +# 5) /taler-monitoring-aptdeploy(+_err) — taler.hacktivism.ch only # No /taler-monitoring page. No /taler-monitoring-mail|mattermost pages. _path_allowed() { case "$1" in monitoring|monitoring_err|\ + monitoring-ladder|monitoring-ladder_err|\ + monitoring-max-ladder|monitoring-max-ladder_err|\ taler-monitoring-surface|taler-monitoring-surface_err|\ taler-monitoring-aptdeploy|taler-monitoring-aptdeploy_err) return 0 ;; # explicit deny (never invent a mon page here) @@ -148,6 +152,8 @@ _catalog_urls() { goa) for h in bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch; do printf 'https://%s/monitoring/\n' "$h" + printf 'https://%s/monitoring-ladder/\n' "$h" + printf 'https://%s/monitoring-max-ladder/\n' "$h" done # ecosystem surface (versions / software) + apt-src deploy mon page printf 'https://taler.hacktivism.ch/taler-monitoring-surface/\n' @@ -165,6 +171,12 @@ _catalog_urls() { while IFS= read -r h; do [ -n "$h" ] || continue printf 'https://%s/monitoring/\n' "$h" + case "$h" in + *.hacktivism.ch) + printf 'https://%s/monitoring-ladder/\n' "$h" + printf 'https://%s/monitoring-max-ladder/\n' "$h" + ;; + esac done < <(_landing_allowed) printf 'https://taler.hacktivism.ch/taler-monitoring-surface/\n' printf 'https://taler.hacktivism.ch/taler-monitoring-aptdeploy/\n' diff --git a/host-agent/README.md b/host-agent/README.md index ffbbcef..6c8ea53 100644 --- a/host-agent/README.md +++ b/host-agent/README.md @@ -23,9 +23,28 @@ Thin wrappers only set stack defaults: | `run-aptdeploy.sh` | CLI: ensure + **aptdeploy** (no HTML) | local if hostname=koopa, else **ssh $DEPLOY_SSH** | | `run-aptdeploy-monitoring.sh` | apt-src tests → `/taler-monitoring-aptdeploy*` | `hernani` @ koopa (4h) | | `run-surface-monitoring.sh` | **surface** → only `/taler-monitoring-surface*` (ecosystem + mail + mattermost + versions) | `hernani` @ koopa (hourly) | +| `run-ladder-monitoring.sh` | GOA **classic** amount ladder → `/monitoring-ladder/` on bank/exchange/taler | `hernani` @ koopa (**daily**) | +| `run-max-ladder-monitoring.sh` | GOA **max-search** ladder → `/monitoring-max-ladder/` on bank/exchange/taler | `hernani` @ koopa (**daily**) | Deprecated mail/mattermost wrappers: **`../meta/deprecated/`** (not installed). +### GOA amount ladder pages (daily) + +| URL | Phase | Timer | +|-----|-------|-------| +| `https://{bank,exchange,taler}.hacktivism.ch/monitoring-ladder/` | `ladder` (classic) | `taler-monitoring-ladder.timer` (~05:15 + jitter) | +| `https://{bank,exchange,taler}.hacktivism.ch/monitoring-max-ladder/` | `max-ladder` | `taler-monitoring-max-ladder.timer` (~06:30 + jitter) | + +```bash +systemctl --user enable --now taler-monitoring-ladder.timer +systemctl --user enable --now taler-monitoring-max-ladder.timer +# first round now: +systemctl --user start taler-monitoring-ladder.service +systemctl --user start taler-monitoring-max-ladder.service +``` + +Needs explorer secrets (`SECRETS_ROOT` / bank-explorer password). Long wall: classic ~2h, max ~3h (`RUN_TIMEOUT`). + ### apt-src deploy tests (hacktivism `/monitoring`) | Container | Mode | diff --git a/host-agent/ROOT-APPLY-MONITORING.md b/host-agent/ROOT-APPLY-MONITORING.md index 0a594d1..9687af2 100644 --- a/host-agent/ROOT-APPLY-MONITORING.md +++ b/host-agent/ROOT-APPLY-MONITORING.md @@ -48,9 +48,13 @@ There is **no** public page `/taler-monitoring`. Each family needs **bare and sl | Family | Paths | Host | Source timer | |--------|-------|------|----------------| | **monitoring** | `/monitoring` · `/monitoring/` · `/monitoring_err` · `/monitoring_err/` | bank + exchange + taler (and other landings) | `taler-monitoring-hacktivism.timer` (4h) | +| **monitoring-ladder** | `/monitoring-ladder` · `/` · `_err` | bank + exchange + taler (**GOA**) | `taler-monitoring-ladder.timer` (**daily**) | +| **monitoring-max-ladder** | `/monitoring-max-ladder` · `/` · `_err` | bank + exchange + taler (**GOA**) | `taler-monitoring-max-ladder.timer` (**daily**) | | **taler-monitoring-surface** | bare + slash + `_err` | **taler.hacktivism.ch only** | `taler-monitoring-surface.timer` (1h) | | **taler-monitoring-aptdeploy** | bare + slash + `_err` | **taler.hacktivism.ch only** | `taler-monitoring-aptdeploy.timer` (4h) | +Served via existing Caddy `handle /monitoring*` (file_server under `/var/www/monitoring-sites/{host}/`). + HTML is generated as **hernani** into `$DEPLOY_STAGING (or $HOME/monitoring-sites-staging)/`. diff --git a/host-agent/apply-monitoring-live.sh b/host-agent/apply-monitoring-live.sh index 4916ba8..f14b7e7 100755 --- a/host-agent/apply-monitoring-live.sh +++ b/host-agent/apply-monitoring-live.sh @@ -57,6 +57,12 @@ urls=( https://taler.hacktivism.ch/monitoring/ https://bank.hacktivism.ch/monitoring/ https://exchange.hacktivism.ch/monitoring/ + https://taler.hacktivism.ch/monitoring-ladder/ + https://bank.hacktivism.ch/monitoring-ladder/ + https://exchange.hacktivism.ch/monitoring-ladder/ + https://taler.hacktivism.ch/monitoring-max-ladder/ + https://bank.hacktivism.ch/monitoring-max-ladder/ + https://exchange.hacktivism.ch/monitoring-max-ladder/ https://taler.hacktivism.ch/taler-monitoring-surface https://taler.hacktivism.ch/taler-monitoring-surface/ https://taler.hacktivism.ch/taler-monitoring-surface_err/ diff --git a/host-agent/install-host-agent.sh b/host-agent/install-host-agent.sh index 3468455..d4624c2 100755 --- a/host-agent/install-host-agent.sh +++ b/host-agent/install-host-agent.sh @@ -63,11 +63,19 @@ install_local() { # optional aptdeploy (not a public mon page layout — v1.8.0 simplified overview) install -m 644 "$ROOT/taler-monitoring-aptdeploy.service" "$home_unit/" install -m 644 "$ROOT/taler-monitoring-aptdeploy.timer" "$home_unit/" + # GOA amount ladder pages (daily) + install -m 644 "$ROOT/taler-monitoring-ladder.service" "$home_unit/" + install -m 644 "$ROOT/taler-monitoring-ladder.timer" "$home_unit/" + install -m 644 "$ROOT/taler-monitoring-max-ladder.service" "$home_unit/" + install -m 644 "$ROOT/taler-monitoring-max-ladder.timer" "$home_unit/" + chmod +x "$ROOT/run-ladder-monitoring.sh" "$ROOT/run-max-ladder-monitoring.sh" 2>/dev/null || true systemctl --user daemon-reload systemctl --user enable --now taler-monitoring-hacktivism.path systemctl --user enable --now taler-monitoring-hacktivism.timer systemctl --user enable --now taler-monitoring-surface.timer systemctl --user enable --now taler-monitoring-aptdeploy.timer + systemctl --user enable --now taler-monitoring-ladder.timer + systemctl --user enable --now taler-monitoring-max-ladder.timer # Remove leftover mail/mattermost units if present (moved to meta/deprecated/) systemctl --user disable --now taler-monitoring-mattermost.timer 2>/dev/null || true systemctl --user disable --now taler-monitoring-mail.timer 2>/dev/null || true @@ -81,20 +89,28 @@ install_local() { systemctl --user start taler-monitoring-hacktivism.service || true systemctl --user start taler-monitoring-surface.service || true systemctl --user start taler-monitoring-aptdeploy.service || true + # first ladder rounds immediately (classic then max-search; both daily thereafter) + systemctl --user start taler-monitoring-ladder.service || true + systemctl --user start taler-monitoring-max-ladder.service || true echo "--- status ---" systemctl --user status taler-monitoring-hacktivism.path --no-pager -l | head -15 systemctl --user status taler-monitoring-hacktivism.timer --no-pager -l | head -12 systemctl --user status taler-monitoring-surface.timer --no-pager -l | head -12 systemctl --user status taler-monitoring-aptdeploy.timer --no-pager -l | head -12 + systemctl --user status taler-monitoring-ladder.timer --no-pager -l | head -12 + systemctl --user status taler-monitoring-max-ladder.timer --no-pager -l | head -12 systemctl --user list-timers --all | grep taler-monitoring || true - echo "OK host-agent (v1.8.0 simplified):" + echo "OK host-agent:" echo " · hacktivism 4h → /monitoring on bank/exchange/taler (landing stacks)" + echo " · ladder daily → /monitoring-ladder/ (classic amount ladder, GOA)" + echo " · max-ladder daily → /monitoring-max-ladder/ (max-search, GOA)" echo " · surface 1h → /taler-monitoring-surface* (ecosystem versions/software)" - echo " · aptdeploy 4h (optional tests; not a public mon-page type)" + echo " · aptdeploy 4h (optional tests)" echo " · mail/mattermost: not installed (meta/deprecated/; content in surface)" echo "suite: $mon" - echo "manual surface: systemctl --user start taler-monitoring-surface.service" + echo "manual ladder: systemctl --user start taler-monitoring-ladder.service" + echo "manual max-ladder: systemctl --user start taler-monitoring-max-ladder.service" echo "after upgrades: $ROOT/touch-software-stamp.sh" } diff --git a/host-agent/run-hacktivism-monitoring.sh b/host-agent/run-hacktivism-monitoring.sh index 02132b8..9599cb1 100755 --- a/host-agent/run-hacktivism-monitoring.sh +++ b/host-agent/run-hacktivism-monitoring.sh @@ -16,10 +16,12 @@ export INSIDE_MODE="${INSIDE_MODE:-local-podman}" export LOCAL_STACK="${LOCAL_STACK:-1}" export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" export RUN_TIMEOUT="${RUN_TIMEOUT:-600}" -# stack checks + full monpages inventory (landings + surface + aptdeploy bare/slash). +# stack checks + full monpages inventory (landings + ladder + surface + aptdeploy). # Specialized pages also have their own timers; full outside-in inventory lives here. -# run-aptdeploy-monitoring.sh → monpages job-only for /taler-monitoring-aptdeploy* -# run-surface-monitoring.sh → monpages job-only for /taler-monitoring-surface* +# run-ladder-monitoring.sh → /monitoring-ladder* (daily classic) +# run-max-ladder-monitoring.sh → /monitoring-max-ladder* (daily max-search) +# run-aptdeploy-monitoring.sh → monpages job-only for /taler-monitoring-aptdeploy* +# run-surface-monitoring.sh → monpages job-only for /taler-monitoring-surface* export PHASES="${PHASES:-urls inside versions monpages}" export MON_HOSTS="${MON_HOSTS:-bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch}" export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}" diff --git a/host-agent/run-ladder-monitoring.sh b/host-agent/run-ladder-monitoring.sh new file mode 100755 index 0000000..c51f4f3 --- /dev/null +++ b/host-agent/run-ladder-monitoring.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# run-ladder-monitoring.sh — GOA classic amount ladder → public HTML +# +# Public (landing hosts, same Caddy /monitoring* static root): +# https://{bank,exchange,taler}.hacktivism.ch/monitoring-ladder/ +# https://{bank,exchange,taler}.hacktivism.ch/monitoring-ladder_err/ +# +# Timer: once daily (taler-monitoring-ladder.timer). Long wall clock. +# +set -uo pipefail +AGENT_DIR=$(cd "$(dirname "$0")" && pwd) + +export AGENT_LABEL="${AGENT_LABEL:-ladder-host-agent}" +export STATE_NAME="${STATE_NAME:-taler-ladder-monitoring}" +export TALER_DOMAIN="${TALER_DOMAIN:-hacktivism.ch}" +export TALER_MON_LANG="${TALER_MON_LANG:-en}" +export TALER_MON_LANG_SET=1 +export INSIDE_PODMAN="${INSIDE_PODMAN:-1}" +export INSIDE_MODE="${INSIDE_MODE:-local-podman}" +export LOCAL_STACK="${LOCAL_STACK:-1}" +export SKIP_SSH="${SKIP_SSH:-0}" +export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" +# classic 23-rung + pay can run long +export RUN_TIMEOUT="${RUN_TIMEOUT:-7200}" + +export PHASES="ladder monpages" +export MON_HOSTS="${MON_HOSTS:-bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch}" +export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}" +export DEPLOY_WWW_ROOT="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" + +export HTML_OK_DIR="monitoring-ladder" +export HTML_ERR_DIR="monitoring-ladder_err" +export HTML_URL_OK="/monitoring-ladder/" +export HTML_URL_ERR="/monitoring-ladder_err/" +export PAGE_LABEL="monitoring-ladder" +export MONPAGES_INVENTORY="job" + +# Classic ladder defaults (override in ~/.config/taler-monitoring/env) +export LADDER_MODE="${LADDER_MODE:-classic}" +export LADDER_PAY="${LADDER_PAY:-1}" +export APT_DEPLOY_ENSURE=0 + +exec bash "$AGENT_DIR/run-host-report.sh" "$@" diff --git a/host-agent/run-max-ladder-monitoring.sh b/host-agent/run-max-ladder-monitoring.sh new file mode 100755 index 0000000..c8735a8 --- /dev/null +++ b/host-agent/run-max-ladder-monitoring.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# run-max-ladder-monitoring.sh — GOA max-search amount ladder → public HTML +# +# Public (landing hosts, same Caddy /monitoring* static root): +# https://{bank,exchange,taler}.hacktivism.ch/monitoring-max-ladder/ +# https://{bank,exchange,taler}.hacktivism.ch/monitoring-max-ladder_err/ +# +# Timer: once daily (taler-monitoring-max-ladder.timer). Long wall clock. +# +set -uo pipefail +AGENT_DIR=$(cd "$(dirname "$0")" && pwd) + +export AGENT_LABEL="${AGENT_LABEL:-max-ladder-host-agent}" +export STATE_NAME="${STATE_NAME:-taler-max-ladder-monitoring}" +export TALER_DOMAIN="${TALER_DOMAIN:-hacktivism.ch}" +export TALER_MON_LANG="${TALER_MON_LANG:-en}" +export TALER_MON_LANG_SET=1 +export INSIDE_PODMAN="${INSIDE_PODMAN:-1}" +export INSIDE_MODE="${INSIDE_MODE:-local-podman}" +export LOCAL_STACK="${LOCAL_STACK:-1}" +export SKIP_SSH="${SKIP_SSH:-0}" +export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" +# max-search probes + pay hunt +export RUN_TIMEOUT="${RUN_TIMEOUT:-10800}" + +export PHASES="max-ladder monpages" +export MON_HOSTS="${MON_HOSTS:-bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch}" +export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}" +export DEPLOY_WWW_ROOT="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" + +export HTML_OK_DIR="monitoring-max-ladder" +export HTML_ERR_DIR="monitoring-max-ladder_err" +export HTML_URL_OK="/monitoring-max-ladder/" +export HTML_URL_ERR="/monitoring-max-ladder_err/" +export PAGE_LABEL="monitoring-max-ladder" +export MONPAGES_INVENTORY="job" + +export LADDER_MODE=max +export LADDER_PAY="${LADDER_PAY:-1}" +export APT_DEPLOY_ENSURE=0 + +exec bash "$AGENT_DIR/run-host-report.sh" "$@" diff --git a/host-agent/taler-monitoring-ladder.service b/host-agent/taler-monitoring-ladder.service new file mode 100644 index 0000000..c93bcd8 --- /dev/null +++ b/host-agent/taler-monitoring-ladder.service @@ -0,0 +1,19 @@ +[Unit] +Description=Taler monitoring GOA classic amount ladder → /monitoring-ladder/ +Documentation=file:%h/src/taler-monitoring/host-agent/README.md +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin +Environment=CONTINUE_ON_ERROR=1 +Environment=RUN_TIMEOUT=7200 +EnvironmentFile=-%h/.config/taler-monitoring/env +WorkingDirectory=%h/src/taler-monitoring +ExecStart=%h/src/taler-monitoring/host-agent/run-ladder-monitoring.sh +Nice=10 +TimeoutStartSec=3h + +[Install] +WantedBy=default.target diff --git a/host-agent/taler-monitoring-ladder.timer b/host-agent/taler-monitoring-ladder.timer new file mode 100644 index 0000000..a076315 --- /dev/null +++ b/host-agent/taler-monitoring-ladder.timer @@ -0,0 +1,14 @@ +[Unit] +Description=Daily GOA classic amount ladder mon page +Documentation=file:%h/src/taler-monitoring/host-agent/README.md + +[Timer] +# Once per day (CEST morning-ish on typical hosts); randomize to avoid thundering herd +OnCalendar=*-*-* 05:15:00 +Persistent=true +AccuracySec=15min +RandomizedDelaySec=30min +Unit=taler-monitoring-ladder.service + +[Install] +WantedBy=timers.target diff --git a/host-agent/taler-monitoring-max-ladder.service b/host-agent/taler-monitoring-max-ladder.service new file mode 100644 index 0000000..9943edb --- /dev/null +++ b/host-agent/taler-monitoring-max-ladder.service @@ -0,0 +1,19 @@ +[Unit] +Description=Taler monitoring GOA max-search amount ladder → /monitoring-max-ladder/ +Documentation=file:%h/src/taler-monitoring/host-agent/README.md +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin +Environment=CONTINUE_ON_ERROR=1 +Environment=RUN_TIMEOUT=10800 +EnvironmentFile=-%h/.config/taler-monitoring/env +WorkingDirectory=%h/src/taler-monitoring +ExecStart=%h/src/taler-monitoring/host-agent/run-max-ladder-monitoring.sh +Nice=10 +TimeoutStartSec=4h + +[Install] +WantedBy=default.target diff --git a/host-agent/taler-monitoring-max-ladder.timer b/host-agent/taler-monitoring-max-ladder.timer new file mode 100644 index 0000000..5b16361 --- /dev/null +++ b/host-agent/taler-monitoring-max-ladder.timer @@ -0,0 +1,14 @@ +[Unit] +Description=Daily GOA max-search amount ladder mon page +Documentation=file:%h/src/taler-monitoring/host-agent/README.md + +[Timer] +# Once per day, after classic ladder window +OnCalendar=*-*-* 06:30:00 +Persistent=true +AccuracySec=15min +RandomizedDelaySec=30min +Unit=taler-monitoring-max-ladder.service + +[Install] +WantedBy=timers.target diff --git a/site-gen/caddy-monitoring-handles.snippet b/site-gen/caddy-monitoring-handles.snippet index 5689fff..1fb1e1b 100644 --- a/site-gen/caddy-monitoring-handles.snippet +++ b/site-gen/caddy-monitoring-handles.snippet @@ -5,6 +5,8 @@ # # CORRECT (site-level named matcher — preferred): # @mon_bare path /monitoring /monitoring_err \ +# /monitoring-ladder /monitoring-ladder_err \ +# /monitoring-max-ladder /monitoring-max-ladder_err \ # /taler-monitoring-surface /taler-monitoring-surface_err \ # /taler-monitoring-aptdeploy /taler-monitoring-aptdeploy_err # redir @mon_bare {path}/ 302 @@ -35,6 +37,8 @@ # --- only inside taler.hacktivism.ch { ... } --- @mon_bare path /monitoring /monitoring_err \ + /monitoring-ladder /monitoring-ladder_err \ + /monitoring-max-ladder /monitoring-max-ladder_err \ /taler-monitoring-surface /taler-monitoring-surface_err \ /taler-monitoring-aptdeploy /taler-monitoring-aptdeploy_err \ /taler-monitoring-mail /taler-monitoring-mail_err \ @@ -59,7 +63,10 @@ } # --- bank + exchange + taler (each site block; set root host dir) --- - @mon_bare path /monitoring /monitoring_err +# /monitoring* already covers /monitoring-ladder* and /monitoring-max-ladder* + @mon_bare path /monitoring /monitoring_err \ + /monitoring-ladder /monitoring-ladder_err \ + /monitoring-max-ladder /monitoring-max-ladder_err redir @mon_bare {path}/ 302 handle /monitoring_err* { diff --git a/site-gen/console_to_html.py b/site-gen/console_to_html.py index d49c839..a2c4af1 100755 --- a/site-gen/console_to_html.py +++ b/site-gen/console_to_html.py @@ -281,11 +281,11 @@ def ui(lang: str, key: str, **kwargs) -> str: "mode_redirect": "REDIR", "pages_title": "Monitoring pages (suite HTML)", "pages_summary": "Public sticky-bar reports · phase monpages · part of taler-monitoring", - "pages_i1": "Suite name taler-monitoring ≠ URL. Only three public families (bare + slash each):", - "pages_i2": "1) /monitoring(+_err) on landing hosts · 2) /taler-monitoring-surface(+_err) · 3) /taler-monitoring-aptdeploy(+_err) on taler.hacktivism.ch", - "pages_i3": "No /taler-monitoring page; no /taler-monitoring-mail|mattermost (folded into surface)", + "pages_i1": "Suite name taler-monitoring ≠ URL. Public families (bare + slash each):", + "pages_i2": "1) /monitoring(+_err) · 2) /monitoring-ladder(+_err) daily classic · 3) /monitoring-max-ladder(+_err) daily max-search · 4) /taler-monitoring-surface · 5) /taler-monitoring-aptdeploy (surface/aptdeploy on taler.hacktivism.ch)", + "pages_i3": "No /taler-monitoring page; no /taler-monitoring-mail|mattermost (folded into surface). Ladder pages on GOA bank/exchange/taler landings.", "pages_i4": "monpages: existence + content markers; bare → slash redir (not merchant code 21)", - "pages_i5": "Host-agent: staging → DEPLOY_WWW_ROOT; reverse-proxy handles only those three families", + "pages_i5": "Host-agent: staging → DEPLOY_WWW_ROOT; Caddy /monitoring* also serves ladder and max-ladder paths", "pages_i6": "This page host: {host} · path label: {label}", "stage_title": "mytops stage monitoring · {host}", "stage_summary": "betel only · CHF stage stack · not lifeline/prod · not GOA", @@ -391,11 +391,11 @@ def ui(lang: str, key: str, **kwargs) -> str: "mode_redirect": "REDIR", "pages_title": "Pages de monitoring (HTML de la suite)", "pages_summary": "Rapports publics sticky-bar · phase monpages · partie de taler-monitoring", - "pages_i1": "Nom de suite taler-monitoring ≠ URL. Trois familles publiques seulement (sans et avec /) :", - "pages_i2": "1) /monitoring(+_err) · 2) /taler-monitoring-surface(+_err) · 3) /taler-monitoring-aptdeploy(+_err) sur taler.hacktivism.ch", - "pages_i3": "Pas de page /taler-monitoring ; pas de /taler-monitoring-mail|mattermost (dans surface)", + "pages_i1": "Nom de suite taler-monitoring ≠ URL. Familles publiques (sans et avec /) :", + "pages_i2": "1) /monitoring(+_err) · 2) /monitoring-ladder(+_err) quotidien classic · 3) /monitoring-max-ladder(+_err) max-search · 4) surface · 5) aptdeploy (surface/aptdeploy sur taler.hacktivism.ch)", + "pages_i3": "Pas de page /taler-monitoring ; pas de /taler-monitoring-mail|mattermost (dans surface). Ladder sur landings GOA bank/exchange/taler.", "pages_i4": "monpages : existence + marqueurs ; bare → slash (pas code 21 merchant)", - "pages_i5": "Host-agent : staging → DEPLOY_WWW_ROOT ; reverse-proxy seulement ces trois familles", + "pages_i5": "Host-agent : staging → DEPLOY_WWW_ROOT ; Caddy /monitoring* sert aussi ladder et max-ladder", "stage_title": "Surveillance mytops stage · {host}", "stage_summary": "betel uniquement · pile CHF stage · pas lifeline/prod · pas GOA", "stage_i1": "Hôte : betel (stage). HTML mon : stage.my.taler-ops.ch + stage.taler-ops.ch /monitoring/", From c1a3cba28b57cf63b1abb4300c8e9a3b01edbd21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 15:05:21 +0200 Subject: [PATCH 26/29] fix 1.23.1: only /monitoring-max-ladder for GOA (drop classic ladder page) Hacktivism mon page is max-search only on bank/exchange/taler. Remove classic /monitoring-ladder units, monpages catalog, and Caddy bare redirs. --- VERSION | 2 +- VERSIONS.md | 3 +- check_monitoring_pages.sh | 15 +++----- host-agent/README.md | 10 ++--- host-agent/ROOT-APPLY-MONITORING.md | 5 +-- host-agent/apply-monitoring-live.sh | 3 -- host-agent/install-host-agent.sh | 21 +++++------ host-agent/run-hacktivism-monitoring.sh | 6 +-- host-agent/run-ladder-monitoring.sh | 43 ---------------------- host-agent/taler-monitoring-ladder.service | 19 ---------- host-agent/taler-monitoring-ladder.timer | 14 ------- site-gen/caddy-monitoring-handles.snippet | 5 +-- site-gen/console_to_html.py | 12 +++--- 13 files changed, 34 insertions(+), 124 deletions(-) delete mode 100755 host-agent/run-ladder-monitoring.sh delete mode 100644 host-agent/taler-monitoring-ladder.service delete mode 100644 host-agent/taler-monitoring-ladder.timer diff --git a/VERSION b/VERSION index a6c2798..49e0a31 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.23.0 +1.23.1 diff --git a/VERSIONS.md b/VERSIONS.md index 3e7f783..77bc56f 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,7 +17,8 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| -| **v1.23.0** | 2026-07-19 | **Feature:** GOA daily mon pages **`/monitoring-ladder/`** (classic) + **`/monitoring-max-ladder/`** (max-search) on bank/exchange/taler.hacktivism.ch. Host-agent timers once/day + install starts first round immediately. monpages catalog/path allow; Caddy bare redirs. | +| **v1.23.1** | 2026-07-19 | **Fix / scope:** only **`/monitoring-max-ladder/`** for GOA (drop classic `/monitoring-ladder`). Daily timer + install first-run. monpages catalog/Caddy bare redirs for max-ladder only. | +| **v1.23.0** | 2026-07-19 | **Feature:** GOA daily mon page **`/monitoring-max-ladder/`** (max-search) on bank/exchange/taler.hacktivism.ch *(1.23.0 briefly also had classic ladder; removed in 1.23.1)*. | | **v1.22.1** | 2026-07-19 | **Bugfix (UX):** pay wait after withdraw no longer looks hung — shows **withdraw board in start order** (OK/OK_BANK from TSV) with live wallet status (`pendingIncoming` → `done`) + timer + aggregate available/pendingIncoming on the **same** wallet DB. Short `run-pending` nudge only (no run-until-done). Default wait 90s (`LADDER_PAY_WAIT_AVAILABLE_S`). | | **v1.22.0** | 2026-07-19 | **Feature:** modular amount ladder — `ladder/lib_pay.sh` (pay) + withdraw in `check_amount_ladder.sh`; `ladder/extract_rpubs.py`. **max-ladder** hunts highest **withdrawable** and highest **payable** (≤ wallet available). GOA pays use public free-amount template **`goa-free`** (`LADDER_FREE_TEMPLATE`). Classic still default 23-rung wd+pay. Phase alias `goa-ladder` → amount ladder (shim removed). | | **v1.21.0** | 2026-07-19 | **Feature:** generic **amount ladder** (`check_amount_ladder.sh`; compat shim `check_goa_ladder.sh`). Default **classic** 23-rung withdraw (+ pay when enabled); **max-search** (`max-ladder` / `LADDER_MODE=max`) hunts highest mintable amount (probes default 32, `0`=until timeout). **Pay:** never overspend — soft `SKIP_BALANCE` if amount > wallet available. **Alt units:** report/logs show human names (e.g. `8.15 Tera-GOA (GOA:…)`) + raw. **Fixes bundled:** external extract (no stdin SyntaxError); force-select multi-rpub; soft `CEILING_REJECT` on mint 5110/P0001; crash-proof report. CLI-AUTOMATION-NOTES §3/3a/3b/§15–18. | diff --git a/check_monitoring_pages.sh b/check_monitoring_pages.sh index f4804b7..4ca714a 100755 --- a/check_monitoring_pages.sh +++ b/check_monitoring_pages.sh @@ -100,21 +100,20 @@ _host_allowed() { } # Public mon URL families (suite name "taler-monitoring" ≠ a public path): -# 1) /monitoring(+_err) — landing hosts (bare + slash) -# 2) /monitoring-ladder(+_err) — GOA classic amount ladder (daily) -# 3) /monitoring-max-ladder(+_err) — GOA max-search ladder (daily) -# 4) /taler-monitoring-surface(+_err) — taler.hacktivism.ch only -# 5) /taler-monitoring-aptdeploy(+_err) — taler.hacktivism.ch only -# No /taler-monitoring page. No /taler-monitoring-mail|mattermost pages. +# 1) /monitoring(+_err) — landing hosts (bare + slash) +# 2) /monitoring-max-ladder(+_err) — GOA max-search amount ladder (daily, hacktivism) +# 3) /taler-monitoring-surface(+_err) — taler.hacktivism.ch only +# 4) /taler-monitoring-aptdeploy(+_err) — taler.hacktivism.ch only +# No classic /monitoring-ladder. No /taler-monitoring page. _path_allowed() { case "$1" in monitoring|monitoring_err|\ - monitoring-ladder|monitoring-ladder_err|\ monitoring-max-ladder|monitoring-max-ladder_err|\ taler-monitoring-surface|taler-monitoring-surface_err|\ taler-monitoring-aptdeploy|taler-monitoring-aptdeploy_err) return 0 ;; # explicit deny (never invent a mon page here) + monitoring-ladder|monitoring-ladder_err|\ taler-monitoring|taler-monitoring-mail|taler-monitoring-mattermost) return 1 ;; *) return 1 ;; esac @@ -152,7 +151,6 @@ _catalog_urls() { goa) for h in bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch; do printf 'https://%s/monitoring/\n' "$h" - printf 'https://%s/monitoring-ladder/\n' "$h" printf 'https://%s/monitoring-max-ladder/\n' "$h" done # ecosystem surface (versions / software) + apt-src deploy mon page @@ -173,7 +171,6 @@ _catalog_urls() { printf 'https://%s/monitoring/\n' "$h" case "$h" in *.hacktivism.ch) - printf 'https://%s/monitoring-ladder/\n' "$h" printf 'https://%s/monitoring-max-ladder/\n' "$h" ;; esac diff --git a/host-agent/README.md b/host-agent/README.md index 6c8ea53..cf8969c 100644 --- a/host-agent/README.md +++ b/host-agent/README.md @@ -23,27 +23,23 @@ Thin wrappers only set stack defaults: | `run-aptdeploy.sh` | CLI: ensure + **aptdeploy** (no HTML) | local if hostname=koopa, else **ssh $DEPLOY_SSH** | | `run-aptdeploy-monitoring.sh` | apt-src tests → `/taler-monitoring-aptdeploy*` | `hernani` @ koopa (4h) | | `run-surface-monitoring.sh` | **surface** → only `/taler-monitoring-surface*` (ecosystem + mail + mattermost + versions) | `hernani` @ koopa (hourly) | -| `run-ladder-monitoring.sh` | GOA **classic** amount ladder → `/monitoring-ladder/` on bank/exchange/taler | `hernani` @ koopa (**daily**) | | `run-max-ladder-monitoring.sh` | GOA **max-search** ladder → `/monitoring-max-ladder/` on bank/exchange/taler | `hernani` @ koopa (**daily**) | -Deprecated mail/mattermost wrappers: **`../meta/deprecated/`** (not installed). +Deprecated mail/mattermost wrappers: **`../meta/deprecated/`** (not installed). No classic **`/monitoring-ladder`** page. -### GOA amount ladder pages (daily) +### GOA max-search ladder page (daily) | URL | Phase | Timer | |-----|-------|-------| -| `https://{bank,exchange,taler}.hacktivism.ch/monitoring-ladder/` | `ladder` (classic) | `taler-monitoring-ladder.timer` (~05:15 + jitter) | | `https://{bank,exchange,taler}.hacktivism.ch/monitoring-max-ladder/` | `max-ladder` | `taler-monitoring-max-ladder.timer` (~06:30 + jitter) | ```bash -systemctl --user enable --now taler-monitoring-ladder.timer systemctl --user enable --now taler-monitoring-max-ladder.timer # first round now: -systemctl --user start taler-monitoring-ladder.service systemctl --user start taler-monitoring-max-ladder.service ``` -Needs explorer secrets (`SECRETS_ROOT` / bank-explorer password). Long wall: classic ~2h, max ~3h (`RUN_TIMEOUT`). +Needs explorer secrets (`SECRETS_ROOT` / bank-explorer password). Wall ~3h (`RUN_TIMEOUT=10800`). ### apt-src deploy tests (hacktivism `/monitoring`) diff --git a/host-agent/ROOT-APPLY-MONITORING.md b/host-agent/ROOT-APPLY-MONITORING.md index 9687af2..7d38b35 100644 --- a/host-agent/ROOT-APPLY-MONITORING.md +++ b/host-agent/ROOT-APPLY-MONITORING.md @@ -48,12 +48,11 @@ There is **no** public page `/taler-monitoring`. Each family needs **bare and sl | Family | Paths | Host | Source timer | |--------|-------|------|----------------| | **monitoring** | `/monitoring` · `/monitoring/` · `/monitoring_err` · `/monitoring_err/` | bank + exchange + taler (and other landings) | `taler-monitoring-hacktivism.timer` (4h) | -| **monitoring-ladder** | `/monitoring-ladder` · `/` · `_err` | bank + exchange + taler (**GOA**) | `taler-monitoring-ladder.timer` (**daily**) | -| **monitoring-max-ladder** | `/monitoring-max-ladder` · `/` · `_err` | bank + exchange + taler (**GOA**) | `taler-monitoring-max-ladder.timer` (**daily**) | +| **monitoring-max-ladder** | `/monitoring-max-ladder` · `/` · `_err` | bank + exchange + taler (**GOA** max-search only) | `taler-monitoring-max-ladder.timer` (**daily**) | | **taler-monitoring-surface** | bare + slash + `_err` | **taler.hacktivism.ch only** | `taler-monitoring-surface.timer` (1h) | | **taler-monitoring-aptdeploy** | bare + slash + `_err` | **taler.hacktivism.ch only** | `taler-monitoring-aptdeploy.timer` (4h) | -Served via existing Caddy `handle /monitoring*` (file_server under `/var/www/monitoring-sites/{host}/`). +No classic **`/monitoring-ladder`**. Max-ladder served via Caddy `handle /monitoring*` under `/var/www/monitoring-sites/{host}/`. HTML is generated as **hernani** into `$DEPLOY_STAGING (or $HOME/monitoring-sites-staging)/`. diff --git a/host-agent/apply-monitoring-live.sh b/host-agent/apply-monitoring-live.sh index f14b7e7..dfc4798 100755 --- a/host-agent/apply-monitoring-live.sh +++ b/host-agent/apply-monitoring-live.sh @@ -57,9 +57,6 @@ urls=( https://taler.hacktivism.ch/monitoring/ https://bank.hacktivism.ch/monitoring/ https://exchange.hacktivism.ch/monitoring/ - https://taler.hacktivism.ch/monitoring-ladder/ - https://bank.hacktivism.ch/monitoring-ladder/ - https://exchange.hacktivism.ch/monitoring-ladder/ https://taler.hacktivism.ch/monitoring-max-ladder/ https://bank.hacktivism.ch/monitoring-max-ladder/ https://exchange.hacktivism.ch/monitoring-max-ladder/ diff --git a/host-agent/install-host-agent.sh b/host-agent/install-host-agent.sh index d4624c2..42c87d6 100755 --- a/host-agent/install-host-agent.sh +++ b/host-agent/install-host-agent.sh @@ -63,19 +63,21 @@ install_local() { # optional aptdeploy (not a public mon page layout — v1.8.0 simplified overview) install -m 644 "$ROOT/taler-monitoring-aptdeploy.service" "$home_unit/" install -m 644 "$ROOT/taler-monitoring-aptdeploy.timer" "$home_unit/" - # GOA amount ladder pages (daily) - install -m 644 "$ROOT/taler-monitoring-ladder.service" "$home_unit/" - install -m 644 "$ROOT/taler-monitoring-ladder.timer" "$home_unit/" + # GOA max-search amount ladder page (daily only — no classic /monitoring-ladder) install -m 644 "$ROOT/taler-monitoring-max-ladder.service" "$home_unit/" install -m 644 "$ROOT/taler-monitoring-max-ladder.timer" "$home_unit/" - chmod +x "$ROOT/run-ladder-monitoring.sh" "$ROOT/run-max-ladder-monitoring.sh" 2>/dev/null || true + chmod +x "$ROOT/run-max-ladder-monitoring.sh" 2>/dev/null || true systemctl --user daemon-reload systemctl --user enable --now taler-monitoring-hacktivism.path systemctl --user enable --now taler-monitoring-hacktivism.timer systemctl --user enable --now taler-monitoring-surface.timer systemctl --user enable --now taler-monitoring-aptdeploy.timer - systemctl --user enable --now taler-monitoring-ladder.timer systemctl --user enable --now taler-monitoring-max-ladder.timer + # drop classic ladder mon page if previously installed + systemctl --user disable --now taler-monitoring-ladder.timer 2>/dev/null || true + systemctl --user stop taler-monitoring-ladder.service 2>/dev/null || true + rm -f "$home_unit/taler-monitoring-ladder.service" \ + "$home_unit/taler-monitoring-ladder.timer" 2>/dev/null || true # Remove leftover mail/mattermost units if present (moved to meta/deprecated/) systemctl --user disable --now taler-monitoring-mattermost.timer 2>/dev/null || true systemctl --user disable --now taler-monitoring-mail.timer 2>/dev/null || true @@ -89,8 +91,7 @@ install_local() { systemctl --user start taler-monitoring-hacktivism.service || true systemctl --user start taler-monitoring-surface.service || true systemctl --user start taler-monitoring-aptdeploy.service || true - # first ladder rounds immediately (classic then max-search; both daily thereafter) - systemctl --user start taler-monitoring-ladder.service || true + # first max-ladder round immediately (daily thereafter) systemctl --user start taler-monitoring-max-ladder.service || true echo "--- status ---" @@ -98,18 +99,16 @@ install_local() { systemctl --user status taler-monitoring-hacktivism.timer --no-pager -l | head -12 systemctl --user status taler-monitoring-surface.timer --no-pager -l | head -12 systemctl --user status taler-monitoring-aptdeploy.timer --no-pager -l | head -12 - systemctl --user status taler-monitoring-ladder.timer --no-pager -l | head -12 systemctl --user status taler-monitoring-max-ladder.timer --no-pager -l | head -12 systemctl --user list-timers --all | grep taler-monitoring || true echo "OK host-agent:" echo " · hacktivism 4h → /monitoring on bank/exchange/taler (landing stacks)" - echo " · ladder daily → /monitoring-ladder/ (classic amount ladder, GOA)" - echo " · max-ladder daily → /monitoring-max-ladder/ (max-search, GOA)" + echo " · max-ladder daily → /monitoring-max-ladder/ (GOA max-search, bank/exchange/taler)" echo " · surface 1h → /taler-monitoring-surface* (ecosystem versions/software)" echo " · aptdeploy 4h (optional tests)" + echo " · no classic /monitoring-ladder page" echo " · mail/mattermost: not installed (meta/deprecated/; content in surface)" echo "suite: $mon" - echo "manual ladder: systemctl --user start taler-monitoring-ladder.service" echo "manual max-ladder: systemctl --user start taler-monitoring-max-ladder.service" echo "after upgrades: $ROOT/touch-software-stamp.sh" } diff --git a/host-agent/run-hacktivism-monitoring.sh b/host-agent/run-hacktivism-monitoring.sh index 9599cb1..7ccf7c7 100755 --- a/host-agent/run-hacktivism-monitoring.sh +++ b/host-agent/run-hacktivism-monitoring.sh @@ -16,12 +16,12 @@ export INSIDE_MODE="${INSIDE_MODE:-local-podman}" export LOCAL_STACK="${LOCAL_STACK:-1}" export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" export RUN_TIMEOUT="${RUN_TIMEOUT:-600}" -# stack checks + full monpages inventory (landings + ladder + surface + aptdeploy). +# stack checks + full monpages inventory (landings + max-ladder + surface + aptdeploy). # Specialized pages also have their own timers; full outside-in inventory lives here. -# run-ladder-monitoring.sh → /monitoring-ladder* (daily classic) -# run-max-ladder-monitoring.sh → /monitoring-max-ladder* (daily max-search) +# run-max-ladder-monitoring.sh → /monitoring-max-ladder* (daily max-search, GOA) # run-aptdeploy-monitoring.sh → monpages job-only for /taler-monitoring-aptdeploy* # run-surface-monitoring.sh → monpages job-only for /taler-monitoring-surface* +# No classic /monitoring-ladder page. export PHASES="${PHASES:-urls inside versions monpages}" export MON_HOSTS="${MON_HOSTS:-bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch}" export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}" diff --git a/host-agent/run-ladder-monitoring.sh b/host-agent/run-ladder-monitoring.sh deleted file mode 100755 index c51f4f3..0000000 --- a/host-agent/run-ladder-monitoring.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -# run-ladder-monitoring.sh — GOA classic amount ladder → public HTML -# -# Public (landing hosts, same Caddy /monitoring* static root): -# https://{bank,exchange,taler}.hacktivism.ch/monitoring-ladder/ -# https://{bank,exchange,taler}.hacktivism.ch/monitoring-ladder_err/ -# -# Timer: once daily (taler-monitoring-ladder.timer). Long wall clock. -# -set -uo pipefail -AGENT_DIR=$(cd "$(dirname "$0")" && pwd) - -export AGENT_LABEL="${AGENT_LABEL:-ladder-host-agent}" -export STATE_NAME="${STATE_NAME:-taler-ladder-monitoring}" -export TALER_DOMAIN="${TALER_DOMAIN:-hacktivism.ch}" -export TALER_MON_LANG="${TALER_MON_LANG:-en}" -export TALER_MON_LANG_SET=1 -export INSIDE_PODMAN="${INSIDE_PODMAN:-1}" -export INSIDE_MODE="${INSIDE_MODE:-local-podman}" -export LOCAL_STACK="${LOCAL_STACK:-1}" -export SKIP_SSH="${SKIP_SSH:-0}" -export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" -# classic 23-rung + pay can run long -export RUN_TIMEOUT="${RUN_TIMEOUT:-7200}" - -export PHASES="ladder monpages" -export MON_HOSTS="${MON_HOSTS:-bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch}" -export HTML_OUT="${HTML_OUT:-$HOME/monitoring-sites-staging}" -export DEPLOY_WWW_ROOT="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" - -export HTML_OK_DIR="monitoring-ladder" -export HTML_ERR_DIR="monitoring-ladder_err" -export HTML_URL_OK="/monitoring-ladder/" -export HTML_URL_ERR="/monitoring-ladder_err/" -export PAGE_LABEL="monitoring-ladder" -export MONPAGES_INVENTORY="job" - -# Classic ladder defaults (override in ~/.config/taler-monitoring/env) -export LADDER_MODE="${LADDER_MODE:-classic}" -export LADDER_PAY="${LADDER_PAY:-1}" -export APT_DEPLOY_ENSURE=0 - -exec bash "$AGENT_DIR/run-host-report.sh" "$@" diff --git a/host-agent/taler-monitoring-ladder.service b/host-agent/taler-monitoring-ladder.service deleted file mode 100644 index c93bcd8..0000000 --- a/host-agent/taler-monitoring-ladder.service +++ /dev/null @@ -1,19 +0,0 @@ -[Unit] -Description=Taler monitoring GOA classic amount ladder → /monitoring-ladder/ -Documentation=file:%h/src/taler-monitoring/host-agent/README.md -After=network-online.target -Wants=network-online.target - -[Service] -Type=oneshot -Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin -Environment=CONTINUE_ON_ERROR=1 -Environment=RUN_TIMEOUT=7200 -EnvironmentFile=-%h/.config/taler-monitoring/env -WorkingDirectory=%h/src/taler-monitoring -ExecStart=%h/src/taler-monitoring/host-agent/run-ladder-monitoring.sh -Nice=10 -TimeoutStartSec=3h - -[Install] -WantedBy=default.target diff --git a/host-agent/taler-monitoring-ladder.timer b/host-agent/taler-monitoring-ladder.timer deleted file mode 100644 index a076315..0000000 --- a/host-agent/taler-monitoring-ladder.timer +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Daily GOA classic amount ladder mon page -Documentation=file:%h/src/taler-monitoring/host-agent/README.md - -[Timer] -# Once per day (CEST morning-ish on typical hosts); randomize to avoid thundering herd -OnCalendar=*-*-* 05:15:00 -Persistent=true -AccuracySec=15min -RandomizedDelaySec=30min -Unit=taler-monitoring-ladder.service - -[Install] -WantedBy=timers.target diff --git a/site-gen/caddy-monitoring-handles.snippet b/site-gen/caddy-monitoring-handles.snippet index 1fb1e1b..9ab43a7 100644 --- a/site-gen/caddy-monitoring-handles.snippet +++ b/site-gen/caddy-monitoring-handles.snippet @@ -5,7 +5,6 @@ # # CORRECT (site-level named matcher — preferred): # @mon_bare path /monitoring /monitoring_err \ -# /monitoring-ladder /monitoring-ladder_err \ # /monitoring-max-ladder /monitoring-max-ladder_err \ # /taler-monitoring-surface /taler-monitoring-surface_err \ # /taler-monitoring-aptdeploy /taler-monitoring-aptdeploy_err @@ -37,7 +36,6 @@ # --- only inside taler.hacktivism.ch { ... } --- @mon_bare path /monitoring /monitoring_err \ - /monitoring-ladder /monitoring-ladder_err \ /monitoring-max-ladder /monitoring-max-ladder_err \ /taler-monitoring-surface /taler-monitoring-surface_err \ /taler-monitoring-aptdeploy /taler-monitoring-aptdeploy_err \ @@ -63,9 +61,8 @@ } # --- bank + exchange + taler (each site block; set root host dir) --- -# /monitoring* already covers /monitoring-ladder* and /monitoring-max-ladder* +# /monitoring* already covers /monitoring-max-ladder* @mon_bare path /monitoring /monitoring_err \ - /monitoring-ladder /monitoring-ladder_err \ /monitoring-max-ladder /monitoring-max-ladder_err redir @mon_bare {path}/ 302 diff --git a/site-gen/console_to_html.py b/site-gen/console_to_html.py index a2c4af1..6141c0e 100755 --- a/site-gen/console_to_html.py +++ b/site-gen/console_to_html.py @@ -282,10 +282,10 @@ def ui(lang: str, key: str, **kwargs) -> str: "pages_title": "Monitoring pages (suite HTML)", "pages_summary": "Public sticky-bar reports · phase monpages · part of taler-monitoring", "pages_i1": "Suite name taler-monitoring ≠ URL. Public families (bare + slash each):", - "pages_i2": "1) /monitoring(+_err) · 2) /monitoring-ladder(+_err) daily classic · 3) /monitoring-max-ladder(+_err) daily max-search · 4) /taler-monitoring-surface · 5) /taler-monitoring-aptdeploy (surface/aptdeploy on taler.hacktivism.ch)", - "pages_i3": "No /taler-monitoring page; no /taler-monitoring-mail|mattermost (folded into surface). Ladder pages on GOA bank/exchange/taler landings.", + "pages_i2": "1) /monitoring(+_err) · 2) /monitoring-max-ladder(+_err) daily max-search (GOA) · 3) /taler-monitoring-surface · 4) /taler-monitoring-aptdeploy (surface/aptdeploy on taler.hacktivism.ch)", + "pages_i3": "No classic /monitoring-ladder; no /taler-monitoring page; mail|mattermost folded into surface. Max-ladder on GOA bank/exchange/taler.", "pages_i4": "monpages: existence + content markers; bare → slash redir (not merchant code 21)", - "pages_i5": "Host-agent: staging → DEPLOY_WWW_ROOT; Caddy /monitoring* also serves ladder and max-ladder paths", + "pages_i5": "Host-agent: staging → DEPLOY_WWW_ROOT; Caddy /monitoring* also serves /monitoring-max-ladder*", "pages_i6": "This page host: {host} · path label: {label}", "stage_title": "mytops stage monitoring · {host}", "stage_summary": "betel only · CHF stage stack · not lifeline/prod · not GOA", @@ -392,10 +392,10 @@ def ui(lang: str, key: str, **kwargs) -> str: "pages_title": "Pages de monitoring (HTML de la suite)", "pages_summary": "Rapports publics sticky-bar · phase monpages · partie de taler-monitoring", "pages_i1": "Nom de suite taler-monitoring ≠ URL. Familles publiques (sans et avec /) :", - "pages_i2": "1) /monitoring(+_err) · 2) /monitoring-ladder(+_err) quotidien classic · 3) /monitoring-max-ladder(+_err) max-search · 4) surface · 5) aptdeploy (surface/aptdeploy sur taler.hacktivism.ch)", - "pages_i3": "Pas de page /taler-monitoring ; pas de /taler-monitoring-mail|mattermost (dans surface). Ladder sur landings GOA bank/exchange/taler.", + "pages_i2": "1) /monitoring(+_err) · 2) /monitoring-max-ladder(+_err) quotidien max-search (GOA) · 3) surface · 4) aptdeploy (surface/aptdeploy sur taler.hacktivism.ch)", + "pages_i3": "Pas de /monitoring-ladder classic ; pas de /taler-monitoring ; mail|mattermost dans surface. Max-ladder sur landings GOA.", "pages_i4": "monpages : existence + marqueurs ; bare → slash (pas code 21 merchant)", - "pages_i5": "Host-agent : staging → DEPLOY_WWW_ROOT ; Caddy /monitoring* sert aussi ladder et max-ladder", + "pages_i5": "Host-agent : staging → DEPLOY_WWW_ROOT ; Caddy /monitoring* sert aussi /monitoring-max-ladder*", "stage_title": "Surveillance mytops stage · {host}", "stage_summary": "betel uniquement · pile CHF stage · pas lifeline/prod · pas GOA", "stage_i1": "Hôte : betel (stage). HTML mon : stage.my.taler-ops.ch + stage.taler-ops.ch /monitoring/", From bf04f90538f210ba6459ccd6ca35edcd541364ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 15:15:04 +0200 Subject: [PATCH 27/29] fix 1.23.2: max-ladder RUN_TIMEOUT 3h (not 600s from env) Force MAX_LADDER_RUN_TIMEOUT/10800 in run-max-ladder-monitoring.sh so shared ~/.config/taler-monitoring/env cannot shrink the daily hunt. Install uses systemctl --no-block for long oneshots. --- VERSION | 2 +- VERSIONS.md | 1 + host-agent/install-host-agent.sh | 9 +++++---- host-agent/run-max-ladder-monitoring.sh | 4 ++-- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/VERSION b/VERSION index 49e0a31..14bee92 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.23.1 +1.23.2 diff --git a/VERSIONS.md b/VERSIONS.md index 77bc56f..82ac624 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| +| **v1.23.2** | 2026-07-19 | **Bugfix:** max-ladder host-agent forces **RUN_TIMEOUT=10800** (koopa env had 600 → run aborted mid-ladder); install starts oneshots with **--no-block**. | | **v1.23.1** | 2026-07-19 | **Fix / scope:** only **`/monitoring-max-ladder/`** for GOA (drop classic `/monitoring-ladder`). Daily timer + install first-run. monpages catalog/Caddy bare redirs for max-ladder only. | | **v1.23.0** | 2026-07-19 | **Feature:** GOA daily mon page **`/monitoring-max-ladder/`** (max-search) on bank/exchange/taler.hacktivism.ch *(1.23.0 briefly also had classic ladder; removed in 1.23.1)*. | | **v1.22.1** | 2026-07-19 | **Bugfix (UX):** pay wait after withdraw no longer looks hung — shows **withdraw board in start order** (OK/OK_BANK from TSV) with live wallet status (`pendingIncoming` → `done`) + timer + aggregate available/pendingIncoming on the **same** wallet DB. Short `run-pending` nudge only (no run-until-done). Default wait 90s (`LADDER_PAY_WAIT_AVAILABLE_S`). | diff --git a/host-agent/install-host-agent.sh b/host-agent/install-host-agent.sh index 42c87d6..118ef7f 100755 --- a/host-agent/install-host-agent.sh +++ b/host-agent/install-host-agent.sh @@ -88,11 +88,12 @@ install_local() { systemctl --user daemon-reload 2>/dev/null || true # initial stamp so path exists "$ROOT/touch-software-stamp.sh" - systemctl --user start taler-monitoring-hacktivism.service || true - systemctl --user start taler-monitoring-surface.service || true - systemctl --user start taler-monitoring-aptdeploy.service || true + # oneshot jobs: --no-block so install does not wait hours + systemctl --user start --no-block taler-monitoring-hacktivism.service || true + systemctl --user start --no-block taler-monitoring-surface.service || true + systemctl --user start --no-block taler-monitoring-aptdeploy.service || true # first max-ladder round immediately (daily thereafter) - systemctl --user start taler-monitoring-max-ladder.service || true + systemctl --user start --no-block taler-monitoring-max-ladder.service || true echo "--- status ---" systemctl --user status taler-monitoring-hacktivism.path --no-pager -l | head -15 diff --git a/host-agent/run-max-ladder-monitoring.sh b/host-agent/run-max-ladder-monitoring.sh index c8735a8..ebba236 100755 --- a/host-agent/run-max-ladder-monitoring.sh +++ b/host-agent/run-max-ladder-monitoring.sh @@ -20,8 +20,8 @@ export INSIDE_MODE="${INSIDE_MODE:-local-podman}" export LOCAL_STACK="${LOCAL_STACK:-1}" export SKIP_SSH="${SKIP_SSH:-0}" export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" -# max-search probes + pay hunt -export RUN_TIMEOUT="${RUN_TIMEOUT:-10800}" +# max-search probes + pay hunt — force high wall clock (koopa env often has RUN_TIMEOUT=600) +export RUN_TIMEOUT="${MAX_LADDER_RUN_TIMEOUT:-10800}" export PHASES="max-ladder monpages" export MON_HOSTS="${MON_HOSTS:-bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch}" From a103eeab9d66a23554f5c2e20c66c2f0e69ec356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 15:19:25 +0200 Subject: [PATCH 28/29] fix 1.23.3: ladder requires wallet-cli; settle for coins so pay can run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of stale intro payments: koopa max-ladder minted but accept-uri failed with taler-wallet-cli not found (SKIP_ACCEPT ×16). Fail early without CLI/helper; after bank transfer_done keep polling + run-pending for available coins; pay wait 180s; host-agent max-ladder sets WALLET_CLI PATH. --- VERSION | 2 +- VERSIONS.md | 1 + check_amount_ladder.sh | 50 ++++++++++++++++++++----- host-agent/run-host-report.sh | 1 + host-agent/run-max-ladder-monitoring.sh | 6 +++ ladder/lib_pay.sh | 2 +- 6 files changed, 51 insertions(+), 11 deletions(-) diff --git a/VERSION b/VERSION index 14bee92..ac1df3f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.23.2 +1.23.3 diff --git a/VERSIONS.md b/VERSIONS.md index 82ac624..32c243b 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| +| **v1.23.3** | 2026-07-19 | **Bugfix:** payments stale because max-ladder never got coins — root cause on koopa was **taler-wallet-cli missing** (all mint-OK → SKIP_ACCEPT). Hard-fail ladder prereq if CLI/helper missing; settle keeps polling after transfer_done + short run-pending; pay wait default 180s; max-ladder wrapper sets PATH/WALLET_CLI. | | **v1.23.2** | 2026-07-19 | **Bugfix:** max-ladder host-agent forces **RUN_TIMEOUT=10800** (koopa env had 600 → run aborted mid-ladder); install starts oneshots with **--no-block**. | | **v1.23.1** | 2026-07-19 | **Fix / scope:** only **`/monitoring-max-ladder/`** for GOA (drop classic `/monitoring-ladder`). Daily timer + install first-run. monpages catalog/Caddy bare redirs for max-ladder only. | | **v1.23.0** | 2026-07-19 | **Feature:** GOA daily mon page **`/monitoring-max-ladder/`** (max-search) on bank/exchange/taler.hacktivism.ch *(1.23.0 briefly also had classic ladder; removed in 1.23.1)*. | diff --git a/check_amount_ladder.sh b/check_amount_ladder.sh index 3bc13ed..67b6001 100755 --- a/check_amount_ladder.sh +++ b/check_amount_ladder.sh @@ -88,7 +88,7 @@ elapsed_ms() { # optional seed for reproducible max-search (empty = time-based) : "${LADDER_MAX_SEED:=}" # After withdraw, wait for spendable available>0 before pay (secs total; shows countdown) -: "${LADDER_PAY_WAIT_AVAILABLE_S:=90}" +: "${LADDER_PAY_WAIT_AVAILABLE_S:=180}" # Historic libeufin-ish absolute ceiling used as default LADDER_MAX_AMOUNT on GOA LADDER_ABS_CEILING="4503599627370496" @@ -99,8 +99,20 @@ if [ -z "${CLI_JS}" ] || [ ! -f "${CLI_JS}" ]; then CLI_JS=$(find_wallet_cli 2>/dev/null || true) fi if [ -z "${CLI_JS}" ] || [ ! -f "${CLI_JS}" ]; then - # allow PATH wrapper as last resort (wcli falls back to taler-wallet-cli) - CLI_JS="" + # PATH wrapper last resort + if command -v taler-wallet-cli >/dev/null 2>&1; then + CLI_JS="" + else + err prereq "taler-wallet-cli not found" \ + "problem: no CLI_JS / WALLET_CLI .mjs and taler-wallet-cli not on PATH — accept-uri/pay cannot run (payments stay stale). Install wallet-cli (e.g. ~/.local/bin/taler-wallet-cli.mjs) + taler-helper-sqlite3; set WALLET_CLI= in ~/.config/taler-monitoring/env" + exit 1 + fi +fi +# helper is required for real wallet DB (otherwise accept "works" empty / no coins) +if ! command -v taler-helper-sqlite3 >/dev/null 2>&1; then + err prereq "taler-helper-sqlite3 not on PATH" \ + "problem: wallet-cli needs taler-helper-sqlite3 for sqlite backend — put it on PATH next to wallet-cli" + exit 1 fi CUR="${EXPECT_CURRENCY:-GOA}" @@ -522,9 +534,16 @@ else info "explorer secret" "resolved via EXP_PW / EXP_PW_FILE / stage SSH / koopa" fi if [ -n "${CLI_JS:-}" ] && [ -f "${CLI_JS}" ]; then - info "wallet-cli" "$CLI_JS" + ok "wallet-cli" "$CLI_JS" else - info "wallet-cli" "PATH taler-wallet-cli ($(command -v taler-wallet-cli 2>/dev/null || echo missing))" + ok "wallet-cli" "PATH $(command -v taler-wallet-cli)" +fi +ok "wallet-helper" "$(command -v taler-helper-sqlite3)" +# prove wcli can start (avoids minting 32 probes then SKIP_ACCEPT command-not-found) +if ! wcli balance >"$SCRATCH/wcli-smoke.out" 2>&1; then + err prereq "wallet-cli smoke failed" \ + "problem: wcli balance failed — fix CLI_JS/PATH/helper. detail: $(tr '\n' ' ' <"$SCRATCH/wcli-smoke.out" | head -c 240)" + exit 1 fi # --- auto-account --- @@ -910,10 +929,13 @@ else: fi ok "confirm $AMT ${ms_confirm}ms (client, on selected)" - # settle: poll wallet balance + bank transfer_done only — never run-until-done + # settle: poll wallet balance + bank transfer_done — never run-until-done. + # Prefer real coin delta (status OK). If bank transfer_done first, keep polling + # briefly for available coins (OK) before accepting OK_BANK (pay needs available). t0=$(now_ms) settled=0 xfer="?" + bank_done=0 if [ "$IS_ZERO" = "1" ]; then settled=1 note="zero-amount: no coin delta expected" @@ -931,13 +953,23 @@ sys.exit(0 if Decimal(sys.argv[1]) > Decimal(sys.argv[2]) else 1) fi xfer=$(curl -sS -m 10 "${BANK}/taler-integration/withdrawal-operation/${WID}" \ | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("transfer_done"), d.get("status"))' 2>/dev/null || echo "?") - # bank done is enough to leave settle without hanging on wallet if echo "$xfer" | grep -qi True; then - note="bank transfer_done (no run-until-done) avail=${after} $xfer" - break + bank_done=1 + # nudge wallet to pick up wire (bounded; not run-until-done) + wcli transactions >"$SCRATCH/tx-settle-$tag.out" 2>&1 || true + if command -v timeout >/dev/null 2>&1 || command -v gtimeout >/dev/null 2>&1; then + _to=$(command -v gtimeout 2>/dev/null || command -v timeout) + "$_to" 8 wcli advanced run-pending >"$SCRATCH/run-pending-settle.out" 2>&1 || true + fi + # after bank done, still prefer coins; allow remaining rounds for available + note="bank transfer_done waiting coins avail=${after} $xfer" fi sleep "$LADDER_SETTLE_SLEEP" done + # If bank done but still no coin delta after all rounds, OK_BANK (pay-wait may finish) + if [ "$settled" != "1" ] && [ "$bank_done" = "1" ]; then + note="bank transfer_done avail=$(wallet_avail) $xfer (no run-until-done)" + fi fi ms_settle=$(elapsed_ms "$t0") after=$(wallet_avail) diff --git a/host-agent/run-host-report.sh b/host-agent/run-host-report.sh index c34ba5b..1f75caf 100755 --- a/host-agent/run-host-report.sh +++ b/host-agent/run-host-report.sh @@ -40,6 +40,7 @@ _wrap_keys=( APT_DEPLOY_ENSURE SURFACE_SCOPE TALER_DOMAIN_FROM_CLI STRICT_EXIT SOURCE_SUITE_PATH MONPAGES_INVENTORY MONPAGES_CHECK_BARE MONPAGES_BARE_STRICT MONPAGES_REQUIRE_PUBLIC CHECK_BANK MERCHANT_REQUIRED CHECK_LANDING + WALLET_CLI CLI_JS PATH LADDER_MODE LADDER_PAY LADDER_PAY_WAIT_AVAILABLE_S ) for _k in "${_wrap_keys[@]}"; do if [ -n "${!_k+x}" ]; then diff --git a/host-agent/run-max-ladder-monitoring.sh b/host-agent/run-max-ladder-monitoring.sh index ebba236..813fedd 100755 --- a/host-agent/run-max-ladder-monitoring.sh +++ b/host-agent/run-max-ladder-monitoring.sh @@ -37,6 +37,12 @@ export MONPAGES_INVENTORY="job" export LADDER_MODE=max export LADDER_PAY="${LADDER_PAY:-1}" +export LADDER_PAY_WAIT_AVAILABLE_S="${LADDER_PAY_WAIT_AVAILABLE_S:-180}" export APT_DEPLOY_ENSURE=0 +# Prefer host wallet-cli install used by mon (find_wallet_cli / WALLET_CLI) +export PATH="${HOME}/.local/bin:${PATH:-}" +if [ -z "${WALLET_CLI:-}" ] && [ -f "${HOME}/.local/bin/taler-wallet-cli.mjs" ]; then + export WALLET_CLI="${HOME}/.local/bin/taler-wallet-cli.mjs" +fi exec bash "$AGENT_DIR/run-host-report.sh" "$@" diff --git a/ladder/lib_pay.sh b/ladder/lib_pay.sh index e8ae15a..6565033 100644 --- a/ladder/lib_pay.sh +++ b/ladder/lib_pay.sh @@ -9,7 +9,7 @@ : "${LADDER_FREE_TEMPLATE:=goa-free}" : "${LADDER_FREE_TEMPLATE_INSTANCE:=}" : "${LADDER_USE_FREE_TEMPLATE:=1}" -: "${LADDER_PAY_WAIT_AVAILABLE_S:=90}" +: "${LADDER_PAY_WAIT_AVAILABLE_S:=180}" : "${LADDER_PAY_SETTLE_ROUNDS:=6}" MAX_PAY_BEST_AMT="${MAX_PAY_BEST_AMT:-}" From d42004386370ed7f02665fa61e1f1637d4e2daf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Sun, 19 Jul 2026 15:38:25 +0200 Subject: [PATCH 29/29] fix 1.23.4: normalize taler:// :443/:80 + mon scan all landings Shared lib.sh normalize_taler_uri for ladder/e2e/pay. urls phase checks public bank/exchange/merchant intros and bank mint JSON for residual default ports (ERROR on GOA/local). Unit test included. --- CLI-AUTOMATION-NOTES.md | 4 +- TESTS.md | 3 ++ VERSION | 2 +- VERSIONS.md | 1 + check_amount_ladder.sh | 7 ++- check_e2e.sh | 12 ++--- check_urls.sh | 84 +++++++++++++++++++++++++++++++ ladder/lib_pay.sh | 4 +- lib.sh | 47 +++++++++++++++++ tests/test_normalize_taler_uri.sh | 73 +++++++++++++++++++++++++++ 10 files changed, 225 insertions(+), 12 deletions(-) create mode 100755 tests/test_normalize_taler_uri.sh diff --git a/CLI-AUTOMATION-NOTES.md b/CLI-AUTOMATION-NOTES.md index b4f7431..22f75e5 100644 --- a/CLI-AUTOMATION-NOTES.md +++ b/CLI-AUTOMATION-NOTES.md @@ -145,8 +145,8 @@ Sources: `https://bank.hacktivism.ch/intro/stats.json`, exchange/merchant intro | **Seen** | ladder/e2e strip ports; QR checks; mint URIs with `:443` | | **Area** | **cli** / **bank** / **doc** | | **Problem** | Some bank/wallet outputs include `taler://…host:443/…`. QR validators and some clients reject or double-normalize inconsistently. | -| **Workaround** | `sed 's/:443\//\//g'` everywhere in automation. | -| **Wanted** | Canonical form without default ports in **all** wallet-cli and bank integration outputs; document as invariant. | +| **Workaround** | mon ≥1.23.4: `normalize_taler_uri` (lib.sh) on ladder/e2e/pay; bank landing `demo-withdraw-api.py` normalizes mint JSON; **urls** phase fails GOA/local if public landings still emit `:443`/`:80` in `taler://` (`www.taler-uri-ports-*`, unit `tests/test_normalize_taler_uri.sh`). | +| **Wanted** | libeufin never emits default ports (source fix). | --- diff --git a/TESTS.md b/TESTS.md index 9320315..7364968 100644 --- a/TESTS.md +++ b/TESTS.md @@ -59,10 +59,13 @@ Numbering follows **executed** checks (early skip may shift later NN inside the | **www.webui-** | SPA fingerprints: **`/webui/version.txt`**, **`/webui/version-overlay.txt`**, `index.html` / `index.js`; optional pin via `EXPECT_WEBUI_VERSION` / `EXPECT_WEBUI_OVERLAY` / `WEBUI_OVERLAY_DENY` | | **www.paivana-** | local GOA paywall front (redirect to template) | | **www.landing-** | own-stack intro links; static assets (`qrcode.min.js` hard; `og-goa-shop.png` hard only GOA/local); **demo-withdraw.json** GOA-only; shop-pay soft; stage merchant shop: `shop-ui.js` + `shops.css` + `qrcode.min.js`; cross-links local | +| **www.taler-uri-ports-** | **no default `:443`/`:80`** in public `taler://` URIs — bank/exchange/merchant `/intro/` HTML, `demo-withdraw.json`, `auto-account.json`, bank `stats.json` (ERROR on GOA/local). Unit: `tests/test_normalize_taler_uri.sh` | | **www.qr-** | QR payloads: harvest `taler://` / `payto://` / app `data-qr-url` from landings + mint JSON; **form** check; **qrencode → zbarimg** exact roundtrip; optional static QR images. Needs `qrencode` + `zbar-tools`. Skip: `QR_CHECK=0` | **QR rule:** `taler://withdraw/HOST/taler-integration/UUID` (no default `:443`/`:80`); `taler://pay/` / `pay-template/`; `payto://…` shape OK; decoded PNG must equal payload. Form errors on withdraw/pay/payto are **ERROR** on local/GOA. +**Default port rule:** public and automation URIs must never keep libeufin-style `host:443` / `host:80`. Consumers use `normalize_taler_uri` (lib.sh); bank landing API normalizes in `demo-withdraw-api.py`. + **Legal docs rule:** HTTP 200, non-empty body, not plain `not configured`, not merchant API JSON `code:21`. Local stack may require content needle (terms/privacy/FADP/GOA…). **Performance rule:** Measured from the **monitoring runner** (public via Caddy), not container loopback. Latency (ms) on every perf line + **perf summary** (n / min / p50 / avg / max). diff --git a/VERSION b/VERSION index ac1df3f..27ddcc1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.23.3 +1.23.4 diff --git a/VERSIONS.md b/VERSIONS.md index 32c243b..1da8044 100644 --- a/VERSIONS.md +++ b/VERSIONS.md @@ -17,6 +17,7 @@ Git tags: `vMAJOR.FEATURE.FIX` (e.g. `v1.8.0`). File `VERSION` omits the `v` pre | Tag | Date (UTC) | Notes | |-----|------------|--------| +| **v1.23.4** | 2026-07-19 | **Bugfix / mon:** shared **normalize_taler_uri** (strip taler:// host :443/:80); ladder/e2e/pay use it. **urls** scans bank+exchange+merchant intros + demo-withdraw/auto-account/stats for default ports (**ERROR** on GOA). Unit test tests/test_normalize_taler_uri.sh. | | **v1.23.3** | 2026-07-19 | **Bugfix:** payments stale because max-ladder never got coins — root cause on koopa was **taler-wallet-cli missing** (all mint-OK → SKIP_ACCEPT). Hard-fail ladder prereq if CLI/helper missing; settle keeps polling after transfer_done + short run-pending; pay wait default 180s; max-ladder wrapper sets PATH/WALLET_CLI. | | **v1.23.2** | 2026-07-19 | **Bugfix:** max-ladder host-agent forces **RUN_TIMEOUT=10800** (koopa env had 600 → run aborted mid-ladder); install starts oneshots with **--no-block**. | | **v1.23.1** | 2026-07-19 | **Fix / scope:** only **`/monitoring-max-ladder/`** for GOA (drop classic `/monitoring-ladder`). Daily timer + install first-run. monpages catalog/Caddy bare redirs for max-ladder only. | diff --git a/check_amount_ladder.sh b/check_amount_ladder.sh index 67b6001..2f5da98 100755 --- a/check_amount_ladder.sh +++ b/check_amount_ladder.sh @@ -671,7 +671,12 @@ ladder_withdraw_rung() { "${BANK}/accounts/${EXP_USER}/withdrawals") ms_mint=$(elapsed_ms "$t0") WID=$(python3 -c 'import json;d=json.load(open("'"$SCRATCH"'/wd-'"$tag"'.json"));print(d.get("withdrawal_id") or "")' 2>/dev/null || true) - URI=$(python3 -c 'import json;u=json.load(open("'"$SCRATCH"'/wd-'"$tag"'.json")).get("taler_withdraw_uri") or "";print(u.replace(":443/","/"))' 2>/dev/null || true) + # libeufin often emits taler://withdraw/host:443/… — strip default ports (lib.sh) + URI=$(python3 -c 'import json;print(json.load(open("'"$SCRATCH"'/wd-'"$tag"'.json")).get("taler_withdraw_uri") or "")' 2>/dev/null || true) + URI=$(normalize_taler_uri "$URI") + if taler_uri_has_default_port "$URI"; then + warn bank "withdraw URI still has default port" "problem: $URI" + fi # numeric amount (for zero / settle / ceiling special-cases) AMT_NUM=$(python3 -c 'import sys; print(sys.argv[1].split(":",1)[-1])' "$AMT") IS_ZERO=0 diff --git a/check_e2e.sh b/check_e2e.sh index d219c73..9e20c62 100755 --- a/check_e2e.sh +++ b/check_e2e.sh @@ -944,7 +944,7 @@ e2e_one_withdraw() { "$BANK/accounts/${USER}/withdrawals" WID=$(python3 -c 'import json;d=json.load(open("'"$SCRATCH"'/wd-'"$tag"'.json"));print(d.get("withdrawal_id") or d.get("id") or "")' 2>/dev/null || true) URI=$(python3 -c 'import json;d=json.load(open("'"$SCRATCH"'/wd-'"$tag"'.json"));print(d.get("taler_withdraw_uri") or "")' 2>/dev/null || true) - URI_CLEAN=$(python3 -c 'import json;u=json.load(open("'"$SCRATCH"'/wd-'"$tag"'.json")).get("taler_withdraw_uri") or "";print(u.replace(":443/","/"))' 2>/dev/null || true) + URI_CLEAN=$(normalize_taler_uri "$URI") if [ -z "$WID" ] || [ -z "$URI_CLEAN" ]; then warn "Bank withdraw $WITHDRAW_AMT" "create failed — $(head -c 80 "$SCRATCH/wd-$tag.json" | tr '\n' ' ')" return 1 @@ -952,13 +952,13 @@ e2e_one_withdraw() { ok "Bank withdrawal created $WITHDRAW_AMT ($WID)" cp "$SCRATCH/wd-$tag.json" "$SCRATCH/wd.json" - USE_URI="$URI" - [ -z "$USE_URI" ] && USE_URI="$URI_CLEAN" + # Always prefer stripped URI (libeufin often emits host:443) + USE_URI="${URI_CLEAN:-$URI}" if wcli withdraw accept-uri --exchange "${EXCHANGE_PUBLIC}/" "$USE_URI" >"$SCRATCH/accept-$tag.out" 2>&1; then ok "wallet accept $WITHDRAW_AMT" - elif [ "$USE_URI" != "$URI_CLEAN" ] && [ -n "$URI_CLEAN" ] \ - && wcli withdraw accept-uri --exchange "${EXCHANGE_PUBLIC}/" "$URI_CLEAN" >"$SCRATCH/accept-$tag.out" 2>&1; then - ok "wallet accept $WITHDRAW_AMT (no :443)" + elif [ -n "$URI" ] && [ "$USE_URI" != "$URI" ] \ + && wcli withdraw accept-uri --exchange "${EXCHANGE_PUBLIC}/" "$URI" >"$SCRATCH/accept-$tag.out" 2>&1; then + ok "wallet accept $WITHDRAW_AMT (raw URI fallback)" else # Prefer the real Error/ENOENT line over noise (perl -e source, stack frames) _acc_err=$( diff --git a/check_urls.sh b/check_urls.sh index 472aba5..f2557ff 100755 --- a/check_urls.sh +++ b/check_urls.sh @@ -1339,6 +1339,90 @@ if [ "${LOCAL_STACK:-1}" = "1" ] || [ -n "${BANK_PUBLIC:-}" ]; then fi fi +# --------------------------------------------------------------------------- +# taler:// URIs must not carry default ports :443 / :80 (wallet / QR rule) +# Scans public landings on this stack (bank + exchange + merchant intros, +# bank mint JSON). Fail hard on GOA/local; soft warn elsewhere. +# --------------------------------------------------------------------------- +set_group landings +section "www · taler:// default ports (:443/:80 must be absent)" +_port_scan_fail=0 +_port_scan_ok=0 +_port_scan_one() { + local label="$1" url="$2" + local f code n detail + f="$tmp/portscan-$$.body" + code=$(http_body "$url" "$f") + if [ "$code" != "200" ]; then + if [ "${LOCAL_STACK:-1}" = "1" ] || [ "${EXPECT_CURRENCY:-}" = "GOA" ]; then + # mint JSON is hard; HTML intros soft if missing + case "$url" in + *demo-withdraw.json|*auto-account.json) + fail "taler-uri ports · $label" "HTTP $code · $url" + _port_scan_fail=$((_port_scan_fail + 1)) + ;; + *) + warn "taler-uri ports · $label" "HTTP $code · skip · $url" + ;; + esac + else + info "taler-uri ports · $label" "HTTP $code · skip" + fi + rm -f "$f" + return 0 + fi + detail=$(python3 - "$f" <<'PY' 2>/dev/null || true +import re, sys, json +from pathlib import Path +raw = Path(sys.argv[1]).read_text(errors="replace") +# collect taler:// from JSON fields and raw HTML/JS +uris = set(re.findall(r"taler://[A-Za-z0-9._~:/?#\[\]@!$&'()*+,;=%-]+", raw)) +# JSON-escaped sequences +uris.update(re.findall(r"taler:\\/\\/[A-Za-z0-9._~:/?#\[\]@!$&'()*+,;=%\\-]+", raw)) +uris = {u.replace("\\/", "/") for u in uris} +bad = [] +pat = re.compile(r"taler://\S+:(443|80)(/|$|\?|#)", re.I) +for u in sorted(uris): + if pat.search(u): + bad.append(u[:120]) +if bad: + print("BAD\t%d\t%s" % (len(bad), bad[0])) +else: + print("OK\t%d\t" % len(uris)) +PY +) + n_kind=${detail%%$'\t'*} + rest=${detail#*$'\t'} + n_count=${rest%%$'\t'*} + sample=${rest#*$'\t'} + if [ "$n_kind" = "BAD" ]; then + if [ "${LOCAL_STACK:-1}" = "1" ] || [ "${EXPECT_CURRENCY:-}" = "GOA" ]; then + fail "taler-uri ports · $label" "default :443/:80 still present · n=${n_count} e.g. ${sample}" + _port_scan_fail=$((_port_scan_fail + 1)) + else + warn "taler-uri ports · $label" "default port in URI · ${sample}" + fi + else + ok "taler-uri ports · $label" "no default ports · taler_uris=${n_count:-0} · $url" + _port_scan_ok=$((_port_scan_ok + 1)) + fi + rm -f "$f" +} +if [ "${CHECK_LANDING:-1}" = "1" ]; then + _port_scan_one "bank intro HTML" "${BANK_PUBLIC}/intro/" + _port_scan_one "bank demo-withdraw.json" "${BANK_PUBLIC}/intro/demo-withdraw.json" + _port_scan_one "bank auto-account.json" "${BANK_PUBLIC}/intro/auto-account.json" + _port_scan_one "exchange intro HTML" "${EXCHANGE_PUBLIC}/intro/" + _port_scan_one "merchant intro HTML" "${MERCHANT_PUBLIC}/intro/" + # stats may embed demo withdraw URI + _port_scan_one "bank stats.json" "${BANK_PUBLIC}/intro/stats.json" + if [ "$_port_scan_fail" -eq 0 ]; then + ok "taler-uri ports summary" "clean · ${_port_scan_ok} resource(s) checked (bank+exchange+merchant)" + fi +else + info "taler-uri ports" "skipped (CHECK_LANDING=0)" +fi + # Merchant shop assets — GOA: shop-pay.*; stage TESTPAYSAN: /assets/shop-ui.js + # shops.css + QR encoder (shop-ui modal needs /intro/qrcode.min.js or /qrcode.min.js). _ma=0 diff --git a/ladder/lib_pay.sh b/ladder/lib_pay.sh index 6565033..01ba819 100644 --- a/ladder/lib_pay.sh +++ b/ladder/lib_pay.sh @@ -67,7 +67,7 @@ except Exception: ' "$SCRATCH/ord-$ptag.json" 2>/dev/null || true) if [ -n "$OID" ] && [ -n "$OTOK" ]; then PAYURI="taler://pay/${mh}/instances/${inst}/${OID}/?c=${OTOK}" - PAYURI=$(printf '%s' "$PAYURI" | sed 's/:443\//\//g; s/:443?/?/g') + PAYURI=$(normalize_taler_uri "$PAYURI") return 0 fi fi @@ -98,7 +98,7 @@ except Exception: print("") if [ -z "$PAYURI" ] && [ -n "$OTOK" ]; then PAYURI="taler://pay/${mh}/instances/${INST}/${OID}/?c=${OTOK}" fi - PAYURI=$(printf '%s' "$PAYURI" | sed 's/:443\//\//g; s/:443?/?/g') + PAYURI=$(normalize_taler_uri "$PAYURI") [ -n "$PAYURI" ] } diff --git a/lib.sh b/lib.sh index 9ced76b..9187afd 100755 --- a/lib.sh +++ b/lib.sh @@ -1504,6 +1504,53 @@ load_monitoring_secrets_env() { export E2E_BANK_ADMIN_PASS E2E_MERCHANT_TOKEN SECRETS_ROOT KOOPA_ADMIN_SECRETS } +# Strip default HTTPS/HTTP ports from taler:// URIs (libeufin often emits host:443). +# Wallets and mon QR rules reject default :443/:80 in the authority. +# Usage: normalize_taler_uri "taler://withdraw/host:443/…" → prints cleaned URI +normalize_taler_uri() { + local u="${1-}" + if [ -z "$u" ]; then + return 0 + fi + printf '%s' "$u" | python3 -c ' +import re, sys +u = sys.stdin.read().strip() +# any taler://SCHEME/HOST:443|80/... +u = re.sub( + r"(taler://[A-Za-z0-9._-]+/)([^/?#:]+):443(?=/|$|\?|#)", + r"\1\2", + u, + flags=re.I, +) +u = re.sub( + r"(taler://[A-Za-z0-9._-]+/)([^/?#:]+):80(?=/|$|\?|#)", + r"\1\2", + u, + flags=re.I, +) +# leftover path/query forms +for a, b in ( + (":443/", "/"), + (":443?", "?"), + (":443#", "#"), + (":80/", "/"), + (":80?", "?"), + (":80#", "#"), +): + u = u.replace(a, b) +# bare trailing :443 / :80 on host (no slash) +u = re.sub(r"(taler://[A-Za-z0-9._-]+/[^/?#:]+):443$", r"\1", u, flags=re.I) +u = re.sub(r"(taler://[A-Za-z0-9._-]+/[^/?#:]+):80$", r"\1", u, flags=re.I) +sys.stdout.write(u) +' 2>/dev/null || printf '%s' "$u" +} + +# True (exit 0) if URI still has default port in taler:// authority. +taler_uri_has_default_port() { + # host:443 or host:80 before next path/query/end + python3 -c "import re,sys; u=sys.argv[1] if len(sys.argv)>1 else ''; sys.exit(0 if re.search(r'taler://\\S+:(443|80)(/|$|\\?|#)', u, re.I) else 1)" "${1-}" 2>/dev/null +} + find_wallet_cli() { # Must return a path suitable for: node "$WALLET_CLI" … # Prefer *.mjs / bundled entry; plain /usr/bin/taler-wallet-cli is often a diff --git a/tests/test_normalize_taler_uri.sh b/tests/test_normalize_taler_uri.sh new file mode 100755 index 0000000..1db180b --- /dev/null +++ b/tests/test_normalize_taler_uri.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Unit tests for normalize_taler_uri / taler_uri_has_default_port (lib.sh). +# Run: ./tests/test_normalize_taler_uri.sh +set -euo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +# shellcheck source=../lib.sh +source "$ROOT/lib.sh" + +pass=0 +fail=0 +assert_eq() { + local name="$1" got="$2" want="$3" + if [ "$got" = "$want" ]; then + pass=$((pass + 1)) + else + echo "FAIL $name" >&2 + echo " got: $got" >&2 + echo " want: $want" >&2 + fail=$((fail + 1)) + fi +} +assert_true() { + local name="$1" + if "$@"; then + # shift name + : + fi +} + +# --- strip :443 / :80 --- +assert_eq "withdraw :443/" \ + "$(normalize_taler_uri 'taler://withdraw/bank.hacktivism.ch:443/taler-integration/abc-123')" \ + "taler://withdraw/bank.hacktivism.ch/taler-integration/abc-123" + +assert_eq "withdraw :443 end" \ + "$(normalize_taler_uri 'taler://withdraw/bank.example:443')" \ + "taler://withdraw/bank.example" + +assert_eq "pay :443" \ + "$(normalize_taler_uri 'taler://pay/taler.hacktivism.ch:443/instances/x/orders/y?c=z')" \ + "taler://pay/taler.hacktivism.ch/instances/x/orders/y?c=z" + +assert_eq "pay-template :80" \ + "$(normalize_taler_uri 'taler://pay-template/shop.example:80/instances/i/t')" \ + "taler://pay-template/shop.example/instances/i/t" + +assert_eq "already clean" \ + "$(normalize_taler_uri 'taler://withdraw/bank.hacktivism.ch/taler-integration/uuid')" \ + "taler://withdraw/bank.hacktivism.ch/taler-integration/uuid" + +assert_eq "empty" "$(normalize_taler_uri '')" "" + +# non-default port kept +assert_eq "keep :8443" \ + "$(normalize_taler_uri 'taler://withdraw/bank.example:8443/taler-integration/x')" \ + "taler://withdraw/bank.example:8443/taler-integration/x" + +# has_default_port detector +if taler_uri_has_default_port 'taler://withdraw/h:443/p'; then + pass=$((pass + 1)) +else + echo "FAIL detector should match :443" >&2 + fail=$((fail + 1)) +fi +if taler_uri_has_default_port 'taler://withdraw/h/p'; then + echo "FAIL detector should not match clean URI" >&2 + fail=$((fail + 1)) +else + pass=$((pass + 1)) +fi + +echo "normalize_taler_uri tests: pass=$pass fail=$fail" +[ "$fail" -eq 0 ]