monitoring/merge: origin ladder+metrics + stage domains/profiles
This commit is contained in:
Hernâni Marques
commit
d6f4f649c1
86 changed files with 8199 additions and 256 deletions
|
|
@ -5,15 +5,32 @@
|
|||
| `taler-merchant/` | podman `taler-hacktivism`: `/root`, `/usr/local/bin` |
|
||||
| `taler-exchange/` | podman `taler-hacktivism-exchange-ansible`: `/root`, `/usr/local/bin` |
|
||||
| `taler-hacktivism-bank/` | podman `taler-hacktivism-bank`: `/root`, `/usr/local/bin` |
|
||||
| `taler-bank/` | bank host helpers + **`maintenance/`** (debit limits, dry-run) |
|
||||
| `taler-shared/` | host **ensure-taler-apps** (post-container app start + auto-confirm) |
|
||||
| `taler-sanity/` | host root checks (stack, settlement, helpers) |
|
||||
| `taler-monitoring/` | **outside-in** public URL walk (`/config` → keys/terms/integration/webui) |
|
||||
| `taler-shared/` | shared helpers (`upgrade-goa-debs.sh` = apt upgrade **inside** bank/exchange/merchant containers) |
|
||||
| `taler-landing/` | **hernani user systemd**: central landing `stats.json` (full bank scan + alt units) |
|
||||
| `monitoring/` | host `/home/hernani/scripts` (tor relay stats) |
|
||||
| `nym/` | **koopa-nym** build/up/status (nym.com nym-node) |
|
||||
| `taler-wallet-cli/` | thin wrappers; **benchmarks live in** `../benchmarks/` |
|
||||
| `castopod/` | host `hernani` podman-compose `~/koopa-castopod` — see `castopod/README.md` |
|
||||
|
||||
**Secrets:** never in this tree — sibling **`../koopa-admin-secrets`** (`koopa/host-root/<service>/` ↔ `/root/` on host; `containers/…/secrets/` for in-container).
|
||||
|
||||
## Boot automation (host user systemd)
|
||||
|
||||
Merchant/bank containers stay **`sleep infinity`**. After the container unit
|
||||
starts, **`taler-merchant-apps.service`** / **`taler-bank-apps.service`** run
|
||||
`~/.local/bin/ensure-taler-apps.sh` (install via
|
||||
`taler-shared/install-ensure-taler-apps.sh`). That runs the in-container
|
||||
`start_base` + `start_*.sh` (and bank auto-confirm **2 s**).
|
||||
|
||||
Landing **stats.json** (bank / exchange / merchant intros) is refreshed by
|
||||
**`taler-landing-stats.timer`** as user **hernani** (install:
|
||||
`taler-landing/install-landing-stats-host.sh`). Full bank ledger scan + GOA
|
||||
alt-unit fields; see `taler-landing/README.md`.
|
||||
|
||||
## Manual start model (all three)
|
||||
|
||||
1. **root** runs `/root/start_base_services_for_taler_*.sh`
|
||||
|
|
|
|||
9
scripts/nym/README.md
Normal file
9
scripts/nym/README.md
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
# scripts/nym — helpers for `koopa-nym`
|
||||
|
||||
| Script | Role |
|
||||
|--------|------|
|
||||
| `build.sh` | `podman build` image `localhost/koopa-nym:latest` |
|
||||
| `up.sh` | compose up in `~/koopa-nym` or `configs/nym` |
|
||||
| `status.sh` | container + HTTP API probe |
|
||||
|
||||
Config mirror: `configs/nym/`. Live host tree: `~/koopa-nym/`.
|
||||
18
scripts/nym/build.sh
Executable file
18
scripts/nym/build.sh
Executable file
|
|
@ -0,0 +1,18 @@
|
|||
#!/bin/bash
|
||||
# Download nym-node + build localhost/koopa-nym:latest
|
||||
set -euo pipefail
|
||||
ROOT=$(cd "$(dirname "$0")/../.." && pwd)
|
||||
SRC="${NYM_SRC:-$ROOT/configs/nym}"
|
||||
TAG="${NYM_IMAGE:-localhost/koopa-nym:latest}"
|
||||
REL="${NYM_RELEASE_TAG:-nym-binaries-v2026.13-ziller}"
|
||||
URL="${NYM_NODE_URL:-https://github.com/nymtech/nym/releases/download/${REL}/nym-node}"
|
||||
cd "$SRC"
|
||||
if [ ! -x nym-node ] || [ "${NYM_FORCE_DOWNLOAD:-0}" = "1" ]; then
|
||||
echo "download $URL"
|
||||
curl -fsSL -L -o nym-node.tmp "$URL"
|
||||
chmod +x nym-node.tmp
|
||||
mv nym-node.tmp nym-node
|
||||
fi
|
||||
./nym-node --version || true
|
||||
podman build -t "$TAG" -f Containerfile .
|
||||
echo "built $TAG"
|
||||
13
scripts/nym/status.sh
Executable file
13
scripts/nym/status.sh
Executable file
|
|
@ -0,0 +1,13 @@
|
|||
#!/bin/bash
|
||||
# Probe koopa-nym container + local HTTP API
|
||||
set -euo pipefail
|
||||
HTTP="${NYM_HTTP:-http://127.0.0.1:9080}"
|
||||
echo "=== podman ==="
|
||||
podman ps -a --filter name=koopa-nym --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' || true
|
||||
echo "=== listen ==="
|
||||
ss -lntp 2>/dev/null | grep -E '9080|1789|1790|19000|51822' || true
|
||||
echo "=== api /roles ==="
|
||||
curl -sS -m 5 "${HTTP}/api/v1/roles" 2>/dev/null | head -c 500 || echo "(unreachable)"
|
||||
echo
|
||||
echo "=== api /health (if present) ==="
|
||||
curl -sS -m 3 -o /dev/null -w "http=%{http_code}\n" "${HTTP}/api/v1/health" 2>/dev/null || true
|
||||
22
scripts/nym/up.sh
Executable file
22
scripts/nym/up.sh
Executable file
|
|
@ -0,0 +1,22 @@
|
|||
#!/bin/bash
|
||||
# Start koopa-nym via compose (prefers ~/koopa-nym on live host)
|
||||
set -euo pipefail
|
||||
ROOT=$(cd "$(dirname "$0")/../.." && pwd)
|
||||
if [ -d "${HOME}/koopa-nym" ] && [ -f "${HOME}/koopa-nym/compose.yml" ]; then
|
||||
DIR="${HOME}/koopa-nym"
|
||||
else
|
||||
DIR="${NYM_SRC:-$ROOT/configs/nym}"
|
||||
fi
|
||||
cd "$DIR"
|
||||
if [ ! -f .env ] && [ -f .env.example ]; then
|
||||
echo "note: no .env — copy .env.example and set NYMNODE_PUBLIC_IPS" >&2
|
||||
fi
|
||||
if command -v podman-compose >/dev/null 2>&1; then
|
||||
podman-compose up -d --build
|
||||
elif podman compose version >/dev/null 2>&1; then
|
||||
podman compose up -d --build
|
||||
else
|
||||
echo "need podman compose or podman-compose" >&2
|
||||
exit 1
|
||||
fi
|
||||
podman ps --filter name=koopa-nym
|
||||
|
|
@ -11,6 +11,7 @@ Container: **`taler-hacktivism-bank`** (libeufin-bank, GOA, **no IBAN**).
|
|||
| `landing-stats-install.sh` | host only | root/podman — copies + runs + optional cron |
|
||||
| `demo-withdraw-api.py` | `/usr/local/bin/` | root — loopback **:19096** |
|
||||
| `install-demo-withdraw-api.sh` | host only | installs API + nginx + auto-confirm |
|
||||
| `maintenance/raise-debit-limits.sh` | host or bank ctr | raise all accounts’ `debit_threshold` (dry-run; `--no-dry`) |
|
||||
| `auto-confirm-withdrawals.sh` | `/usr/local/bin/` | root — **explorer-only** confirm loop |
|
||||
| `refresh-demo-withdraw.sh` | `/usr/local/bin/` | refresh static `withdraw.uri` |
|
||||
| `credit-account.sh` | host/ops | admin → user credit |
|
||||
|
|
@ -60,12 +61,20 @@ Requires **python3** in the bank container. Env: `BANK_URL`, `BANK_USER`/`BANK_P
|
|||
|
||||
```bash
|
||||
# loop inside container — refuses non-explorer unless ALLOW_NON_EXPLORER=1
|
||||
auto-confirm-withdrawals.sh --loop 4
|
||||
auto-confirm-withdrawals.sh --loop 2
|
||||
```
|
||||
|
||||
Only confirms withdrawals owned by **`explorer`** when status is `selected`
|
||||
(community demo path). Does not confirm arbitrary customer withdraws.
|
||||
|
||||
**Stable ops (via `koopa-external` if LAN `koopa` is down):**
|
||||
|
||||
- One process only (`flock` on `/var/run/auto-confirm-withdrawals.lock`)
|
||||
- `QUIET=1` + summary `tick checked=… selected=…` each loop
|
||||
- Watch list capped (`WATCH_MAX=80`, prune of bloated `withdraw-watch.ids`)
|
||||
- Status via `taler-integration/withdrawal-operation/{id}`
|
||||
- Reinstall: `./install-demo-withdraw-api.sh` on host with podman access to `taler-hacktivism-bank`
|
||||
|
||||
## Config
|
||||
|
||||
See `configs/taler-hacktivism-bank/` and `configs/bank-landing/`.
|
||||
|
|
|
|||
|
|
@ -1,33 +1,39 @@
|
|||
#!/bin/bash
|
||||
# Auto-confirm bank withdrawals for the community demo pool ONLY.
|
||||
#
|
||||
# Only account: explorer (override only if you really mean another pool user
|
||||
# via BANK_USER, but still confirms with that user's token only — never
|
||||
# confirms other customers' withdrawals).
|
||||
# Auto-confirm bank withdrawals for the community demo pool ONLY (explorer).
|
||||
#
|
||||
# Run once: auto-confirm-withdrawals.sh
|
||||
# Loop: auto-confirm-withdrawals.sh --loop [SECS]
|
||||
#
|
||||
# Env:
|
||||
# BANK_URL (default http://127.0.0.1:9012)
|
||||
# BANK_USER (default explorer) — must be the pool account
|
||||
# BANK_USER (default explorer)
|
||||
# BANK_PASS or /root/bank-explorer-password.txt
|
||||
# LANDING_DIR (default /var/www/bank-landing)
|
||||
# ALLOW_NON_EXPLORER=1 — allow BANK_USER other than explorer (off by default)
|
||||
# ALLOW_NON_EXPLORER=1
|
||||
# WATCH_MAX max IDs kept in withdraw-watch.ids after prune (default 80)
|
||||
# QUIET=1 less skip noise (default 1 in --loop)
|
||||
# LOCK_FILE default /var/run/auto-confirm-withdrawals.lock
|
||||
set -euo pipefail
|
||||
|
||||
BANK="${BANK_URL:-http://127.0.0.1:9012}"
|
||||
BANK="${BANK%/}"
|
||||
USER="${BANK_USER:-explorer}"
|
||||
LANDING_DIR="${LANDING_DIR:-/var/www/bank-landing}"
|
||||
WATCH_FILE="${LANDING_DIR}/withdraw-watch.ids"
|
||||
URI_FILE="${LANDING_DIR}/withdraw.uri"
|
||||
WATCH_MAX="${WATCH_MAX:-80}"
|
||||
LOCK_FILE="${LOCK_FILE:-/var/run/auto-confirm-withdrawals.lock}"
|
||||
LOOP=0
|
||||
SLEEP=5
|
||||
SLEEP=2
|
||||
if [ "${1:-}" = "--loop" ]; then
|
||||
LOOP=1
|
||||
SLEEP="${2:-5}"
|
||||
SLEEP="${2:-2}"
|
||||
fi
|
||||
# Quiet by default in loop mode
|
||||
if [ -z "${QUIET+x}" ]; then
|
||||
if [ "$LOOP" -eq 1 ]; then QUIET=1; else QUIET=0; fi
|
||||
fi
|
||||
|
||||
# Safety: only the shared community account unless explicitly overridden
|
||||
if [ "$USER" != "explorer" ] && [ "${ALLOW_NON_EXPLORER:-0}" != "1" ]; then
|
||||
echo "refusing BANK_USER=$USER — auto-confirm is for explorer only (set ALLOW_NON_EXPLORER=1 to override)" >&2
|
||||
exit 1
|
||||
|
|
@ -41,14 +47,20 @@ if [ -z "$PASS" ]; then
|
|||
fi
|
||||
[ -n "$PASS" ] || { echo "no password for $USER" >&2; exit 1; }
|
||||
|
||||
# JSON field extract without python (bank container may lack python3)
|
||||
json_str() {
|
||||
# json_str FIELD < json-text-or-file
|
||||
local field="$1"
|
||||
local data
|
||||
if [ -f "${2:-}" ]; then data=$(cat "$2"); else data=$(cat); fi
|
||||
printf '%s' "$data" | sed -n "s/.*\"${field}\"[[:space:]]*:[[:space:]]*\"\\([^\"]*\\)\".*/\\1/p" | head -1
|
||||
}
|
||||
# Single instance (loop mode)
|
||||
if [ "$LOOP" -eq 1 ]; then
|
||||
mkdir -p "$(dirname "$LOCK_FILE")" 2>/dev/null || true
|
||||
exec 9>"$LOCK_FILE"
|
||||
if command -v flock >/dev/null 2>&1; then
|
||||
if ! flock -n 9; then
|
||||
echo "auto-confirm already running (lock $LOCK_FILE) — exit"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
log() { printf '%s\n' "$*"; }
|
||||
qlog() { [ "${QUIET}" = "1" ] || log "$@"; }
|
||||
|
||||
token() {
|
||||
curl -sS -m 12 -u "${USER}:${PASS}" \
|
||||
|
|
@ -57,83 +69,169 @@ token() {
|
|||
"${BANK}/accounts/${USER}/token"
|
||||
}
|
||||
|
||||
# IDs created for the community pool (landing + watch list only)
|
||||
# Prefer taler-integration status (has selection_done / transfer_done)
|
||||
status_json() {
|
||||
local wid="$1"
|
||||
local j
|
||||
j=$(curl -sS -m 8 "${BANK}/taler-integration/withdrawal-operation/${wid}" 2>/dev/null || true)
|
||||
if [ -z "$j" ] || ! printf '%s' "$j" | grep -q '"status"'; then
|
||||
j=$(curl -sS -m 8 "${BANK}/withdrawals/${wid}" 2>/dev/null || true)
|
||||
fi
|
||||
printf '%s' "$j"
|
||||
}
|
||||
|
||||
field() {
|
||||
# field name from json on stdin/arg
|
||||
local name="$1" data="${2:-}"
|
||||
if [ -z "$data" ]; then data=$(cat); fi
|
||||
printf '%s' "$data" | sed -n "s/.*\"${name}\"[[:space:]]*:[[:space:]]*\"\\([^\"]*\\)\".*/\\1/p" | head -1
|
||||
}
|
||||
|
||||
known_ids() {
|
||||
if [ -f "${LANDING_DIR}/withdraw.uri" ]; then
|
||||
basename "$(tr -d '\n' <"${LANDING_DIR}/withdraw.uri")"
|
||||
if [ -f "$URI_FILE" ]; then
|
||||
basename "$(tr -d '\n' <"$URI_FILE")"
|
||||
fi
|
||||
if [ -f "${LANDING_DIR}/withdraw-watch.ids" ]; then
|
||||
# strip empty / comments
|
||||
grep -E '^[0-9a-fA-F-]{36}$' "${LANDING_DIR}/withdraw-watch.ids" || true
|
||||
if [ -f "$WATCH_FILE" ]; then
|
||||
grep -E '^[0-9a-fA-F-]{36}$' "$WATCH_FILE" || true
|
||||
fi
|
||||
}
|
||||
|
||||
# Keep file small: drop confirmed/aborted; keep pending/selected + tail
|
||||
prune_watch() {
|
||||
[ -f "$WATCH_FILE" ] || return 0
|
||||
local tmp keep=0
|
||||
tmp=$(mktemp)
|
||||
# Prefer newest IDs; re-check status for tail of file only would be slow —
|
||||
# keep last WATCH_MAX unique lines and re-append selected ones found this round.
|
||||
if [ -f "${LANDING_DIR}/.auto-confirm-selected" ]; then
|
||||
cat "${LANDING_DIR}/.auto-confirm-selected" >>"$tmp" 2>/dev/null || true
|
||||
fi
|
||||
tail -n "$((WATCH_MAX * 3))" "$WATCH_FILE" 2>/dev/null \
|
||||
| grep -E '^[0-9a-fA-F-]{36}$' \
|
||||
| awk 'NF && !seen[$0]++' \
|
||||
| tail -n "$WATCH_MAX" >>"$tmp" || true
|
||||
# unique preserve order
|
||||
awk 'NF && !seen[$0]++' "$tmp" >"${tmp}.2"
|
||||
mv "${tmp}.2" "$WATCH_FILE"
|
||||
rm -f "$tmp"
|
||||
keep=$(wc -l <"$WATCH_FILE" | tr -d ' ')
|
||||
qlog "prune watch list → ${keep} ids"
|
||||
}
|
||||
|
||||
confirm_one() {
|
||||
local wid="$1"
|
||||
local tok="$2"
|
||||
local info st uname conf
|
||||
|
||||
# Public status — must belong to explorer (pool), not another customer
|
||||
info=$(curl -sS -m 10 "${BANK}/withdrawals/${wid}" 2>/dev/null || true)
|
||||
info=$(status_json "$wid")
|
||||
[ -n "$info" ] || return 0
|
||||
|
||||
st=$(printf '%s' "$info" | sed -n 's/.*"status"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -1)
|
||||
uname=$(printf '%s' "$info" | sed -n 's/.*"username"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -1)
|
||||
st=$(field status "$info")
|
||||
uname=$(field username "$info")
|
||||
|
||||
# Only confirm withdrawals owned by the pool account
|
||||
if [ -n "$uname" ] && [ "$uname" != "$USER" ]; then
|
||||
echo "skip $wid owner=$uname (only confirm $USER)"
|
||||
return 0
|
||||
fi
|
||||
# If API omits username, still only confirm via explorer token (cannot confirm others)
|
||||
if [ -z "$uname" ]; then
|
||||
# require sender_wire / payto to mention explorer when present
|
||||
case "$info" in
|
||||
*explorer*) ;;
|
||||
*)
|
||||
# still try only if status selected — confirm endpoint is under explorer account
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
if [ "$st" != "selected" ]; then
|
||||
echo "skip $wid status=${st:-?} owner=${uname:-?}"
|
||||
qlog "skip $wid owner=$uname (only confirm $USER)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "confirming $wid as $USER (community pool) ..."
|
||||
conf=$(curl -sS -m 15 -o /tmp/acw-conf.out -w '%{http_code}' \
|
||||
-X POST \
|
||||
-H "Authorization: Bearer ${tok}" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{}' \
|
||||
"${BANK}/accounts/${USER}/withdrawals/${wid}/confirm")
|
||||
echo " HTTP $conf $(head -c 200 /tmp/acw-conf.out 2>/dev/null || true)"
|
||||
curl -sS -m 8 "${BANK}/withdrawals/${wid}" 2>/dev/null \
|
||||
| sed -n 's/.*"status"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/ now status=\1/p' | head -1 || true
|
||||
case "$st" in
|
||||
selected)
|
||||
log "confirming $wid as $USER (status=selected) ..."
|
||||
conf=$(curl -sS -m 15 -o /tmp/acw-conf.out -w '%{http_code}' \
|
||||
-X POST \
|
||||
-H "Authorization: Bearer ${tok}" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{}' \
|
||||
"${BANK}/accounts/${USER}/withdrawals/${wid}/confirm")
|
||||
log " HTTP $conf $(head -c 160 /tmp/acw-conf.out 2>/dev/null || true)"
|
||||
# remember for prune keep
|
||||
echo "$wid" >>"${LANDING_DIR}/.auto-confirm-selected"
|
||||
# verify
|
||||
info=$(status_json "$wid")
|
||||
st=$(field status "$info")
|
||||
log " now status=${st:-?}"
|
||||
;;
|
||||
confirmed|aborted)
|
||||
qlog "skip $wid status=$st"
|
||||
;;
|
||||
pending|"")
|
||||
qlog "skip $wid status=${st:-?} (waiting wallet select)"
|
||||
;;
|
||||
*)
|
||||
qlog "skip $wid status=${st:-?}"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
once() {
|
||||
local tjson tok ids
|
||||
local tjson tok ids n_sel=0 n_conf=0 n_pend=0 n_other=0 n=0
|
||||
tjson=$(token)
|
||||
tok=$(printf '%s' "$tjson" | sed -n 's/.*"access_token"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -1)
|
||||
if [ -z "$tok" ]; then
|
||||
echo "token fail for $USER: $tjson" >&2
|
||||
return 1
|
||||
fi
|
||||
ids=$(known_ids | sort -u)
|
||||
|
||||
# Cap work per loop: last WATCH_MAX ids only (newest at end of file)
|
||||
ids=$( {
|
||||
[ -f "$URI_FILE" ] && basename "$(tr -d '\n' <"$URI_FILE")"
|
||||
if [ -f "$WATCH_FILE" ]; then
|
||||
grep -E '^[0-9a-fA-F-]{36}$' "$WATCH_FILE" | awk 'NF && !seen[$0]++' | tail -n "$WATCH_MAX"
|
||||
fi
|
||||
} | awk 'NF && !seen[$0]++' )
|
||||
|
||||
if [ -z "$ids" ]; then
|
||||
echo "no withdrawal ids to watch (landing withdraw.uri / withdraw-watch.ids)"
|
||||
qlog "no withdrawal ids to watch"
|
||||
return 0
|
||||
fi
|
||||
|
||||
: >"${LANDING_DIR}/.auto-confirm-selected.tmp"
|
||||
while read -r wid; do
|
||||
[ -n "$wid" ] || continue
|
||||
confirm_one "$wid" "$tok"
|
||||
n=$((n + 1))
|
||||
info=$(status_json "$wid")
|
||||
st=$(field status "$info")
|
||||
case "$st" in
|
||||
selected)
|
||||
n_sel=$((n_sel + 1))
|
||||
echo "$wid" >>"${LANDING_DIR}/.auto-confirm-selected.tmp"
|
||||
confirm_one "$wid" "$tok"
|
||||
;;
|
||||
confirmed) n_conf=$((n_conf + 1)); qlog "skip $wid status=confirmed" ;;
|
||||
pending) n_pend=$((n_pend + 1)); qlog "skip $wid status=pending" ;;
|
||||
aborted) n_other=$((n_other + 1)); qlog "skip $wid status=aborted" ;;
|
||||
*) n_other=$((n_other + 1)); qlog "skip $wid status=${st:-?}" ;;
|
||||
esac
|
||||
done <<<"$ids"
|
||||
|
||||
if [ -s "${LANDING_DIR}/.auto-confirm-selected.tmp" ]; then
|
||||
mv "${LANDING_DIR}/.auto-confirm-selected.tmp" "${LANDING_DIR}/.auto-confirm-selected"
|
||||
else
|
||||
rm -f "${LANDING_DIR}/.auto-confirm-selected.tmp"
|
||||
fi
|
||||
|
||||
# Always one summary line per loop (monitoring-friendly)
|
||||
log "tick checked=$n selected=$n_sel confirmed=$n_conf pending=$n_pend other=$n_other"
|
||||
|
||||
# Periodic prune (every loop is ok now that we only scan tail)
|
||||
if [ "$n" -gt "$((WATCH_MAX / 2))" ] || [ -f "$WATCH_FILE" ]; then
|
||||
wc=$(wc -l <"$WATCH_FILE" 2>/dev/null | tr -d ' ' || echo 0)
|
||||
if [ "${wc:-0}" -gt "$((WATCH_MAX * 2))" ]; then
|
||||
prune_watch
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
if [ "$LOOP" -eq 1 ]; then
|
||||
echo "auto-confirm loop every ${SLEEP}s for pool user=$USER only"
|
||||
log "auto-confirm loop every ${SLEEP}s user=$USER watch_max=$WATCH_MAX quiet=$QUIET"
|
||||
# initial prune if bloated
|
||||
if [ -f "$WATCH_FILE" ]; then
|
||||
wc=$(wc -l <"$WATCH_FILE" | tr -d ' ')
|
||||
if [ "$wc" -gt "$((WATCH_MAX * 2))" ]; then
|
||||
log "watch list bloated ($wc) — pruning"
|
||||
prune_watch
|
||||
fi
|
||||
fi
|
||||
while true; do
|
||||
once || true
|
||||
sleep "$SLEEP"
|
||||
|
|
|
|||
|
|
@ -42,6 +42,27 @@ def public_webui_url() -> str:
|
|||
return f"{BANK_PUBLIC}/webui/"
|
||||
|
||||
|
||||
def normalize_taler_withdraw_uri(uri: str) -> str:
|
||||
"""Strip default :443/:80 from taler://withdraw host (wallet base-URL fix).
|
||||
|
||||
libeufin builds authority from https BASE_URL and includes port 443; Android /
|
||||
iOS wallets then fail host parsing or TLS. Path and id stay unchanged.
|
||||
"""
|
||||
if not uri:
|
||||
return uri
|
||||
# taler://withdraw/bank.example:443/taler-integration/UUID
|
||||
uri = re.sub(
|
||||
r"(taler://withdraw/)([^/?#]+):443(?=/|$)",
|
||||
r"\1\2",
|
||||
uri,
|
||||
)
|
||||
uri = re.sub(
|
||||
r"(taler://withdraw/)([^/?#]+):80(?=/|$)",
|
||||
r"\1\2",
|
||||
uri,
|
||||
)
|
||||
return uri
|
||||
|
||||
def load_pass() -> str:
|
||||
p = os.environ.get("BANK_PASS", "").strip()
|
||||
if p:
|
||||
|
|
@ -104,9 +125,9 @@ def mint_withdraw() -> dict:
|
|||
raise RuntimeError(f"no taler_withdraw_uri: {wd}")
|
||||
if not wid:
|
||||
wid = uri.rstrip("/").split("/")[-1]
|
||||
# Keep host:port from libeufin (e.g. bank.hacktivism.ch:443). Stripping :443
|
||||
# breaks taler-integration withdraw links / main landing QR on HTTPS banks.
|
||||
uri = str(uri).strip()
|
||||
# libeufin emits taler://withdraw/host:443/... from https BASE_URL; mobile
|
||||
# wallets and many desktop builds reject/mis-parse default port :443.
|
||||
uri = normalize_taler_withdraw_uri(str(uri).strip())
|
||||
LANDING.mkdir(parents=True, exist_ok=True)
|
||||
(LANDING / "withdraw.uri").write_text(uri + "\n")
|
||||
(LANDING / "withdraw.amount").write_text(AMOUNT + "\n")
|
||||
|
|
|
|||
305
scripts/taler-bank/goa-withdraw-ladder.sh
Executable file
305
scripts/taler-bank/goa-withdraw-ladder.sh
Executable file
|
|
@ -0,0 +1,305 @@
|
|||
#!/usr/bin/env bash
|
||||
# Systematic GOA withdraw ladder on bank.hacktivism.ch
|
||||
#
|
||||
# Per landing / intro docs:
|
||||
# 1) GET /intro/auto-account.json → personal goa-account-* (+ first pool URI)
|
||||
# 2) Shared pool withdrawals (explorer) with server auto-confirm
|
||||
# 3) wallet-cli: ToS + accept-uri + run-until-done each rung
|
||||
# 4) amounts from atomic-GOA up until bank/wallet fails
|
||||
#
|
||||
# Prefer the monitoring phase (random increasing ranges + timings + report):
|
||||
# ../taler-monitoring/taler-monitoring.sh ladder
|
||||
# LADDER_MAX_RUNGS=10 ../taler-monitoring/check_goa_ladder.sh
|
||||
#
|
||||
# Usage (standalone, fixed ladder):
|
||||
# ./goa-withdraw-ladder.sh
|
||||
# MAX_RUNGS=12 ./goa-withdraw-ladder.sh
|
||||
# AMOUNTS='GOA:0.000001 GOA:0.01 GOA:1 GOA:10 GOA:100' ./goa-withdraw-ladder.sh
|
||||
#
|
||||
# Env:
|
||||
# BANK default https://bank.hacktivism.ch
|
||||
# EXCHANGE default https://exchange.hacktivism.ch/
|
||||
# EXP_USER default explorer
|
||||
# EXP_PW_FILE explorer password file (for mint + confirm)
|
||||
# WALLET_CLI path to taler-wallet-cli.mjs or binary
|
||||
# WDB wallet sqlite path
|
||||
# SHOTDIR result directory
|
||||
set -euo pipefail
|
||||
export PATH="/tmp/py313bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:${PATH}"
|
||||
|
||||
BANK="${BANK:-https://bank.hacktivism.ch}"
|
||||
BANK="${BANK%/}"
|
||||
EXCHANGE="${EXCHANGE:-https://exchange.hacktivism.ch/}"
|
||||
EXP_USER="${EXP_USER:-explorer}"
|
||||
EXP_PW_FILE="${EXP_PW_FILE:-/Users/newkamek/src/koopa/koopa-admin-secrets/koopa/host-root/taler-bank/bank-explorer-password.txt}"
|
||||
SHOTDIR="${SHOTDIR:-/tmp/goa-ladder-$(date +%Y%m%d-%H%M%S)}"
|
||||
WDB="${WDB:-$SHOTDIR/wallet.db}"
|
||||
MAX_RUNGS="${MAX_RUNGS:-24}"
|
||||
CLI_JS="${WALLET_CLI_JS:-/Users/newkamek/src/taler-typescript-core/packages/taler-wallet-cli/bin/taler-wallet-cli.mjs}"
|
||||
mkdir -p "$SHOTDIR"
|
||||
LOG="$SHOTDIR/ladder.log"
|
||||
RESULTS="$SHOTDIR/results.tsv"
|
||||
echo -e "rung\tamount\tstatus\twid\tnote" >"$RESULTS"
|
||||
|
||||
log() { printf '%s\n' "$*" | tee -a "$LOG"; }
|
||||
die() { log "ERROR: $*"; exit 1; }
|
||||
|
||||
wcli() {
|
||||
if [ -f "$CLI_JS" ]; then
|
||||
node "$CLI_JS" --wallet-db="$WDB" --no-throttle "$@"
|
||||
else
|
||||
taler-wallet-cli --wallet-db="$WDB" --no-throttle "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
# Default ladder via Python: fixed 0 + random low picks + random high ranges + fixed max.
|
||||
# Prefer taler-monitoring check_goa_ladder.sh for full control (LADDER_* env).
|
||||
default_amounts() {
|
||||
python3 - <<'PY'
|
||||
import math, random
|
||||
from decimal import Decimal, ROUND_HALF_UP
|
||||
|
||||
MAX = Decimal("4503599627370496")
|
||||
HIGH_FROM = Decimal("1000000")
|
||||
HIGH_RUNGS = 12
|
||||
CUR = "GOA"
|
||||
|
||||
def fmt(v: Decimal) -> str:
|
||||
q = v.quantize(Decimal("0.00000001"), rounding=ROUND_HALF_UP)
|
||||
if q == q.to_integral():
|
||||
return "%s:%s" % (CUR, format(int(q), "d"))
|
||||
return "%s:%s" % (CUR, format(q, "f").rstrip("0").rstrip("."))
|
||||
|
||||
def logu(lo, hi):
|
||||
lo = max(lo, 1e-12)
|
||||
if hi <= lo:
|
||||
return hi
|
||||
return math.exp(random.uniform(math.log(lo), math.log(hi)))
|
||||
|
||||
out = [fmt(Decimal(0))]
|
||||
prev = Decimal(0)
|
||||
# low bands (same idea as monitoring LADDER_RANGES without 0:0)
|
||||
bands = [
|
||||
(1e-6, 9e-6), (1e-5, 9e-5), (1e-4, 9e-4), (1e-3, 9e-3), (0.01, 0.09),
|
||||
(0.1, 0.9), (1, 9), (10, 49), (50, 99), (100, 499), (500, 999),
|
||||
(1000, 4999), (5000, 9999), (1e4, 5e4), (5e4, 1e5), (1e5, 5e5), (5e5, 2e6),
|
||||
]
|
||||
for lo, hi in bands:
|
||||
lo2 = max(lo, float(prev) * 1.0000001 if prev > 0 else lo)
|
||||
if lo2 > hi:
|
||||
continue
|
||||
v = Decimal(str(logu(lo2, hi))).quantize(Decimal("0.00000001"))
|
||||
if v <= prev:
|
||||
continue
|
||||
if v >= MAX:
|
||||
continue
|
||||
out.append(fmt(v))
|
||||
prev = v
|
||||
|
||||
band_lo = max(float(HIGH_FROM), float(prev) * 1.0000001)
|
||||
band_hi = float(MAX - 1)
|
||||
if band_lo < band_hi and HIGH_RUNGS > 0:
|
||||
cuts = [band_lo, band_hi]
|
||||
for _ in range(HIGH_RUNGS - 1):
|
||||
cuts.append(logu(band_lo, band_hi))
|
||||
cuts = sorted(set(cuts))
|
||||
while len(cuts) < HIGH_RUNGS + 1:
|
||||
cuts.append(logu(band_lo, band_hi))
|
||||
cuts = sorted(set(cuts))
|
||||
segs = [(cuts[i], cuts[i + 1]) for i in range(len(cuts) - 1) if cuts[i + 1] > cuts[i] * 1.001]
|
||||
segs.sort(key=lambda t: t[0])
|
||||
for r_lo, r_hi in segs[:HIGH_RUNGS]:
|
||||
floor = max(r_lo, float(prev) * 1.0000001)
|
||||
if floor >= r_hi:
|
||||
continue
|
||||
v = Decimal(str(logu(floor, r_hi))).quantize(Decimal(1))
|
||||
if v <= prev or v >= MAX:
|
||||
continue
|
||||
out.append(fmt(v))
|
||||
prev = v
|
||||
|
||||
out.append(fmt(MAX))
|
||||
print("\n".join(out))
|
||||
PY
|
||||
}
|
||||
|
||||
if [ -n "${AMOUNTS:-}" ]; then
|
||||
# shellcheck disable=SC2206
|
||||
LADDER=($AMOUNTS)
|
||||
else
|
||||
LADDER=()
|
||||
while IFS= read -r line; do
|
||||
[ -n "$line" ] || continue
|
||||
LADDER+=("$line")
|
||||
[ "${#LADDER[@]}" -ge "$MAX_RUNGS" ] && break
|
||||
done < <(default_amounts)
|
||||
fi
|
||||
|
||||
[ -f "$EXP_PW_FILE" ] || die "missing explorer password: $EXP_PW_FILE"
|
||||
EXP_PW="$(tr -d '\n' <"$EXP_PW_FILE")"
|
||||
|
||||
explorer_token() {
|
||||
curl -sS -m 20 -u "${EXP_USER}:${EXP_PW}" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"scope":"readwrite","duration":{"d_us":3600000000}}' \
|
||||
"${BANK}/accounts/${EXP_USER}/token" \
|
||||
| python3 -c 'import json,sys; print(json.load(sys.stdin)["access_token"])'
|
||||
}
|
||||
|
||||
confirm_when_selected() {
|
||||
local wid="$1" tok="$2"
|
||||
local i st
|
||||
for i in $(seq 1 30); do
|
||||
st=$(curl -sS -m 12 "${BANK}/taler-integration/withdrawal-operation/${wid}" \
|
||||
| python3 -c 'import json,sys; print(json.load(sys.stdin).get("status",""))' 2>/dev/null || true)
|
||||
case "$st" in
|
||||
selected)
|
||||
code=$(curl -sS -m 20 -o "$SHOTDIR/conf-${wid}.json" -w '%{http_code}' -X POST \
|
||||
-H "Authorization: Bearer ${tok}" \
|
||||
-H 'Content-Type: application/json' -d '{}' \
|
||||
"${BANK}/accounts/${EXP_USER}/withdrawals/${wid}/confirm")
|
||||
log " auto-confirm HTTP $code (status was selected)"
|
||||
return 0
|
||||
;;
|
||||
confirmed)
|
||||
log " already confirmed"
|
||||
return 0
|
||||
;;
|
||||
aborted)
|
||||
log " aborted"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
sleep 1
|
||||
done
|
||||
log " timeout waiting for selected (last=$st)"
|
||||
return 1
|
||||
}
|
||||
|
||||
mint_withdraw() {
|
||||
local amount="$1"
|
||||
curl -sS -m 30 -H "Authorization: Bearer ${TOK}" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"amount\":\"${amount}\"}" \
|
||||
"${BANK}/accounts/${EXP_USER}/withdrawals"
|
||||
}
|
||||
|
||||
wallet_avail() {
|
||||
wcli balance 2>/dev/null | python3 -c '
|
||||
import json,sys,re
|
||||
t=sys.stdin.read()
|
||||
i=t.find("{")
|
||||
if i<0: print("0"); raise SystemExit
|
||||
d=json.loads(t[i:t.rfind("}")+1])
|
||||
for b in d.get("balances") or []:
|
||||
a=b.get("available") or ""
|
||||
if a.startswith("GOA:"):
|
||||
print(a.split(":",1)[1]); raise SystemExit
|
||||
print("0")
|
||||
' 2>/dev/null || echo "0"
|
||||
}
|
||||
|
||||
# --- main ---
|
||||
log "SHOTDIR=$SHOTDIR"
|
||||
log "BANK=$BANK EXCHANGE=$EXCHANGE"
|
||||
log "ladder (${#LADDER[@]} rungs): ${LADDER[*]}"
|
||||
|
||||
log "=== 1) auto-account (bank.hacktivism.ch/intro) ==="
|
||||
curl -sS -m 30 "${BANK}/intro/auto-account.json" | tee "$SHOTDIR/auto-account.json" | python3 -m json.tool | tee -a "$LOG" | head -40
|
||||
ACCT_USER=$(python3 -c 'import json; print(json.load(open("'"$SHOTDIR"'/auto-account.json"))["username"])')
|
||||
ACCT_PASS=$(python3 -c 'import json; print(json.load(open("'"$SHOTDIR"'/auto-account.json"))["password"])')
|
||||
log "personal account: $ACCT_USER (balance GOA:0 — pool withdraws use explorer + auto-confirm)"
|
||||
echo "$ACCT_USER" >"$SHOTDIR/account-user.txt"
|
||||
echo "$ACCT_PASS" >"$SHOTDIR/account-pass.txt"
|
||||
|
||||
log "=== 2) explorer token + wallet init ==="
|
||||
TOK=$(explorer_token)
|
||||
log "explorer token ok"
|
||||
|
||||
wcli exchanges add "$EXCHANGE" 2>&1 | tee "$SHOTDIR/ex-add.out" | tail -5 || true
|
||||
wcli exchanges update "$EXCHANGE" 2>&1 | tee "$SHOTDIR/ex-upd.out" | tail -5 || true
|
||||
wcli exchanges accept-tos "$EXCHANGE" 2>&1 | tee "$SHOTDIR/ex-tos.out" | tail -5 || true
|
||||
|
||||
n=0
|
||||
fail_rung=""
|
||||
for amt in "${LADDER[@]}"; do
|
||||
n=$((n + 1))
|
||||
log ""
|
||||
log "======== rung $n / ${#LADDER[@]} amount=$amt ========"
|
||||
WD=$(mint_withdraw "$amt" 2>&1) || true
|
||||
echo "$WD" | tee "$SHOTDIR/wd-${n}.json" >/dev/null
|
||||
if ! echo "$WD" | python3 -c 'import json,sys; json.load(sys.stdin)' 2>/dev/null; then
|
||||
log "FAIL mint: $WD"
|
||||
echo -e "${n}\t${amt}\tFAIL_MINT\t-\t$(echo "$WD" | tr '\n' ' ' | head -c 200)" >>"$RESULTS"
|
||||
fail_rung="$amt"
|
||||
break
|
||||
fi
|
||||
WID=$(echo "$WD" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("withdrawal_id",""))')
|
||||
URI=$(echo "$WD" | python3 -c 'import json,sys; u=json.load(sys.stdin).get("taler_withdraw_uri",""); print(u.replace(":443/","/"))')
|
||||
[ -n "$WID" ] && [ -n "$URI" ] || {
|
||||
log "FAIL parse mint: $WD"
|
||||
echo -e "${n}\t${amt}\tFAIL_PARSE\t-\t" >>"$RESULTS"
|
||||
fail_rung="$amt"
|
||||
break
|
||||
}
|
||||
log " WID=$WID"
|
||||
log " URI=$URI"
|
||||
|
||||
before=$(wallet_avail)
|
||||
if ! wcli withdraw accept-uri --exchange "$EXCHANGE" "$URI" 2>&1 | tee "$SHOTDIR/accept-${n}.out" | tail -20; then
|
||||
# still try confirm if selected
|
||||
true
|
||||
fi
|
||||
if ! confirm_when_selected "$WID" "$TOK"; then
|
||||
log "FAIL confirm $amt"
|
||||
echo -e "${n}\t${amt}\tFAIL_CONFIRM\t${WID}\t" >>"$RESULTS"
|
||||
fail_rung="$amt"
|
||||
# keep going? user asked until it no longer works — stop
|
||||
break
|
||||
fi
|
||||
|
||||
settled=0
|
||||
for r in $(seq 1 20); do
|
||||
wcli run-until-done 2>&1 | tee -a "$SHOTDIR/rud-${n}.out" >/dev/null || true
|
||||
after=$(wallet_avail)
|
||||
# progress if available increased (float-safe string compare via python)
|
||||
if python3 -c "import sys; sys.exit(0 if float(sys.argv[1])>float(sys.argv[2]) else 1)" "$after" "$before" 2>/dev/null; then
|
||||
settled=1
|
||||
log " settled avail GOA:$after (was $before)"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
after=$(wallet_avail)
|
||||
if [ "$settled" = "1" ]; then
|
||||
log "OK $amt → wallet GOA:$after"
|
||||
echo -e "${n}\t${amt}\tOK\t${WID}\tavail=${after}" >>"$RESULTS"
|
||||
else
|
||||
# bank may be confirmed but wire lag — check transfer_done
|
||||
xfer=$(curl -sS -m 10 "${BANK}/taler-integration/withdrawal-operation/${WID}" \
|
||||
| python3 -c 'import json,sys; d=json.load(sys.stdin); print(d.get("transfer_done"), d.get("status"))')
|
||||
log " not settled yet transfer_done/status=$xfer avail=$after"
|
||||
if echo "$xfer" | grep -q True; then
|
||||
echo -e "${n}\t${amt}\tOK_BANK_LAG\t${WID}\tavail=${after}" >>"$RESULTS"
|
||||
log "OK bank confirmed (wallet lag) $amt"
|
||||
else
|
||||
echo -e "${n}\t${amt}\tFAIL_SETTLE\t${WID}\tavail=${after} $xfer" >>"$RESULTS"
|
||||
fail_rung="$amt"
|
||||
break
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
log ""
|
||||
log "=== final balance ==="
|
||||
wcli balance 2>&1 | tee "$SHOTDIR/balance-final.out"
|
||||
log ""
|
||||
log "=== results ==="
|
||||
column -t -s $'\t' "$RESULTS" 2>/dev/null || cat "$RESULTS"
|
||||
log "SHOTDIR=$SHOTDIR"
|
||||
if [ -n "$fail_rung" ]; then
|
||||
log "STOPPED at first failure: $fail_rung"
|
||||
exit 2
|
||||
fi
|
||||
log "All rungs OK"
|
||||
exit 0
|
||||
|
|
@ -61,18 +61,31 @@ else
|
|||
fi
|
||||
'
|
||||
|
||||
# start/restart API
|
||||
# start/restart API + single auto-confirm loop (flock inside script)
|
||||
podman exec -u root "$CTR" bash -lc '
|
||||
pkill -f "demo-withdraw-api.py" 2>/dev/null || true
|
||||
# stop demo-withdraw by pid (avoid pkill -f self-match)
|
||||
ps -eo pid=,args= | awk "/demo-withdraw-api\\.py/ && !/awk/ {print \$1}" | while read p; do kill \$p 2>/dev/null || true; done
|
||||
sleep 0.3
|
||||
nohup python3 /usr/local/bin/demo-withdraw-api.py \
|
||||
>>/var/log/demo-withdraw-api.log 2>&1 </dev/null &
|
||||
echo "api pid $!"
|
||||
# auto-confirm loop
|
||||
pkill -f "auto-confirm-withdrawals.sh --loop" 2>/dev/null || true
|
||||
nohup /usr/local/bin/auto-confirm-withdrawals.sh --loop 4 \
|
||||
# stop auto-confirm by pid
|
||||
ps -eo pid=,args= | awk "/auto-confirm-withdrawals\\.sh --loop/ && !/awk/ {print \$1}" | while read p; do kill \$p 2>/dev/null || true; done
|
||||
sleep 0.5
|
||||
if [ -f /var/log/auto-confirm-withdrawals.log ]; then
|
||||
sz=$(wc -c </var/log/auto-confirm-withdrawals.log)
|
||||
if [ "${sz:-0}" -gt 5000000 ]; then
|
||||
tail -100 /var/log/auto-confirm-withdrawals.log > /tmp/ac-keep.log
|
||||
: > /var/log/auto-confirm-withdrawals.log
|
||||
cat /tmp/ac-keep.log >> /var/log/auto-confirm-withdrawals.log
|
||||
fi
|
||||
fi
|
||||
nohup env QUIET=1 WATCH_MAX=80 \
|
||||
/usr/local/bin/auto-confirm-withdrawals.sh --loop 2 \
|
||||
>>/var/log/auto-confirm-withdrawals.log 2>&1 </dev/null &
|
||||
echo "auto-confirm pid $!"
|
||||
sleep 1
|
||||
ps -eo pid=,args= | awk "/auto-confirm-withdrawals\\.sh --loop/ && !/awk/"
|
||||
curl -sS -m 8 http://127.0.0.1:19096/demo-withdraw.json | head -c 300; echo
|
||||
'
|
||||
|
||||
|
|
|
|||
|
|
@ -16,13 +16,16 @@ BANK_USER="${BANK_USER:-explorer}"
|
|||
ADMIN_USER="${ADMIN_USER:-admin}"
|
||||
# Per-account transaction window (libeufin delta). Was -100 → systematically
|
||||
# undercounted credits/withdraws on active accounts (broken public stats).
|
||||
TX_DELTA="${TX_DELTA:--50000}"
|
||||
# Prefer the host collector (hernani systemd taler-landing-stats) for full
|
||||
# pagination; these defaults keep the in-container fallback deep enough.
|
||||
TX_DELTA="${TX_DELTA:--200000}"
|
||||
# Max accounts to list + scan (was 80 → missed later accounts; bank has 100+).
|
||||
MAX_SCAN_ACCOUNTS="${MAX_SCAN_ACCOUNTS:-500}"
|
||||
# 0 or empty → no head limit (scan every listed account).
|
||||
MAX_SCAN_ACCOUNTS="${MAX_SCAN_ACCOUNTS:-0}"
|
||||
# Account-list page size for GET /accounts?delta=… (must cover all users)
|
||||
ACCOUNTS_DELTA="${ACCOUNTS_DELTA:--500}"
|
||||
ACCOUNTS_DELTA="${ACCOUNTS_DELTA:--10000}"
|
||||
# curl timeout per account (deeper history needs more headroom)
|
||||
TX_CURL_TIMEOUT="${TX_CURL_TIMEOUT:-25}"
|
||||
TX_CURL_TIMEOUT="${TX_CURL_TIMEOUT:-45}"
|
||||
export TZ="${TZ:-Europe/Zurich}"
|
||||
|
||||
PASS="${BANK_PASS:-}"
|
||||
|
|
@ -218,15 +221,23 @@ fi
|
|||
SCAN_OK=0
|
||||
SCAN_EMPTY=0
|
||||
SCAN_FAIL=0
|
||||
# Scan customer accounts only (not exchange/admin — exchange credits would double-count)
|
||||
# Scan ALL accounts except exchange (withdraw debits live on customers + admin
|
||||
# top-ups; exchange credits would double-count the same GOA).
|
||||
# Prefer host collect_bank_stats.py (hernani timer) for full pagination.
|
||||
{
|
||||
echo "$BANK_USER"
|
||||
grep -Eve "^(admin|exchange|${BANK_USER})$" "$WORKDIR/usernames.txt" 2>/dev/null || true
|
||||
} | awk 'NF && !seen[$0]++' | head -n "$MAX_SCAN_ACCOUNTS" >"$WORKDIR/scan-users.txt"
|
||||
# include admin + every auto-account; drop only exchange
|
||||
grep -Eve "^(exchange|${BANK_USER})$" "$WORKDIR/usernames.txt" 2>/dev/null || true
|
||||
} | awk 'NF && !seen[$0]++' >"$WORKDIR/scan-users.all"
|
||||
if [ -n "${MAX_SCAN_ACCOUNTS}" ] && [ "${MAX_SCAN_ACCOUNTS}" -gt 0 ] 2>/dev/null; then
|
||||
head -n "$MAX_SCAN_ACCOUNTS" "$WORKDIR/scan-users.all" >"$WORKDIR/scan-users.txt"
|
||||
else
|
||||
cp "$WORKDIR/scan-users.all" "$WORKDIR/scan-users.txt"
|
||||
fi
|
||||
|
||||
while IFS= read -r uname; do
|
||||
[ -n "$uname" ] || continue
|
||||
case "$uname" in admin|exchange) continue ;; esac
|
||||
case "$uname" in exchange) continue ;; esac
|
||||
# Safe filename (usernames are mostly [A-Za-z0-9_-])
|
||||
safe=$(printf '%s' "$uname" | tr -c 'A-Za-z0-9._-' '_')
|
||||
code=$(curl -sS -m "${TX_CURL_TIMEOUT}" -o "$WORKDIR/tx-${safe}.json" -w '%{http_code}' \
|
||||
|
|
@ -568,7 +579,7 @@ cat >"$TMP" <<EOF
|
|||
"total_in_value": ${TOTAL_IN_N:-0},
|
||||
"total_out": $(json_str "$TOTAL_OUT_AMT"),
|
||||
"total_out_value": ${TOTAL_OUT_N:-0},
|
||||
"note": "incoming=credits; withdraw=Taler withdrawal debits; excl. admin+exchange accounts"
|
||||
"note": "incoming=credits; withdraw=Taler withdrawal debits; excl. exchange only (admin+explorer+all users scanned)"
|
||||
},
|
||||
"withdraws": {
|
||||
"count": ${N_WD:-0},
|
||||
|
|
|
|||
47
scripts/taler-bank/maintenance/README.md
Normal file
47
scripts/taler-bank/maintenance/README.md
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
# taler-bank maintenance
|
||||
|
||||
One-shot ops tools for **libeufin-bank** on koopa (`taler-hacktivism-bank`).
|
||||
Pattern matches **mytops-admin-log** `scripts/taler-merchant/maintenance/`: **dry-run by default**, `--no-dry` to apply.
|
||||
|
||||
## `raise-debit-limits.sh`
|
||||
|
||||
Raise **`debit_threshold`** on every bank account (overdraft / withdraw headroom).
|
||||
Optionally set **`DEFAULT_DEBT_LIMIT`** for new accounts.
|
||||
|
||||
| | |
|
||||
|--|--|
|
||||
| **Default amount** | `GOA:4503599627370496` (libeufin amount ceiling ≈ 2⁵²) |
|
||||
| **Auth** | admin password: `BANK_ADMIN_PASS` or `bank-admin-password.txt` |
|
||||
| **Bank** | `http://127.0.0.1:9012` on koopa, or `podman exec` into `taler-hacktivism-bank` |
|
||||
|
||||
### Usage
|
||||
|
||||
```bash
|
||||
# on koopa (hernani), after bank is up:
|
||||
cd ~/path/to/koopa-admin-log # or copy script
|
||||
|
||||
# preview
|
||||
./scripts/taler-bank/maintenance/raise-debit-limits.sh
|
||||
|
||||
# apply to all accounts
|
||||
./scripts/taler-bank/maintenance/raise-debit-limits.sh --no-dry
|
||||
|
||||
# apply + DEFAULT_DEBT_LIMIT in container overrides + bank restart
|
||||
./scripts/taler-bank/maintenance/raise-debit-limits.sh --no-dry --set-default
|
||||
|
||||
# custom amount / single user
|
||||
./scripts/taler-bank/maintenance/raise-debit-limits.sh --no-dry --amount GOA:1000000 --only explorer
|
||||
```
|
||||
|
||||
Via container if host cannot reach :9012:
|
||||
|
||||
```bash
|
||||
podman exec -u root -i taler-hacktivism-bank bash -s -- --no-dry \
|
||||
< scripts/taler-bank/maintenance/raise-debit-limits.sh
|
||||
```
|
||||
|
||||
### Notes
|
||||
|
||||
- Does **not** invent balance; only raises how far accounts may go into **debit**.
|
||||
- Exchange coin denominations (largest `GOA:1000`) are separate — large withdraws still work with many coins.
|
||||
- Secrets never live in this repo — see `SECRETS.md` / `koopa-admin-secrets`.
|
||||
280
scripts/taler-bank/maintenance/raise-debit-limits.sh
Executable file
280
scripts/taler-bank/maintenance/raise-debit-limits.sh
Executable file
|
|
@ -0,0 +1,280 @@
|
|||
#!/usr/bin/env bash
|
||||
# Raise libeufin-bank debit thresholds for all accounts (and optionally the default).
|
||||
#
|
||||
# Style: like mytops-admin-log scripts/taler-merchant/maintenance/*.sh
|
||||
# dry-run by default; pass --no-dry to apply.
|
||||
#
|
||||
# Where to run:
|
||||
# - on koopa host (preferred): talks to BANK_URL (default http://127.0.0.1:9012)
|
||||
# - or: podman exec -u root -i taler-hacktivism-bank bash -s < this-script -- --no-dry
|
||||
#
|
||||
# Auth (first match wins):
|
||||
# BANK_ADMIN_PASS
|
||||
# /root/bank-admin-password.txt (inside bank container or host root secrets mirror)
|
||||
# $KOOPA_SECRETS/.../bank-admin-password.txt
|
||||
# ../koopa-admin-secrets/koopa/host-root/taler-bank/bank-admin-password.txt (relative to repo)
|
||||
#
|
||||
# Env / flags:
|
||||
# --no-dry apply changes
|
||||
# --amount GOA:N threshold (default: libeufin ceiling GOA:4503599627370496)
|
||||
# --set-default also set DEFAULT_DEBT_LIMIT in bank-overrides.conf (+ bank restart)
|
||||
# --only USER[,USER] only these usernames
|
||||
# BANK_URL default http://127.0.0.1:9012
|
||||
# BANK_CTR if set and BANK not reachable on host, use podman exec into container
|
||||
# (default: taler-hacktivism-bank when host curl fails)
|
||||
set -euo pipefail
|
||||
|
||||
echo "[INFO] raise-debit-limits — libeufin-bank account debit_threshold"
|
||||
echo "[INFO] Dry-run unless --no-dry. Tested against bank.hacktivism.ch (GOA)."
|
||||
|
||||
DRY_RUN=true
|
||||
SET_DEFAULT=false
|
||||
AMOUNT="${AMOUNT:-GOA:4503599627370496}"
|
||||
ONLY_USERS=""
|
||||
BANK_URL="${BANK_URL:-http://127.0.0.1:9012}"
|
||||
BANK_URL="${BANK_URL%/}"
|
||||
BANK_CTR="${BANK_CTR:-taler-hacktivism-bank}"
|
||||
OVERRIDE_CONF="${OVERRIDE_CONF:-/etc/libeufin/bank-overrides.conf}"
|
||||
|
||||
usage() {
|
||||
sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//'
|
||||
exit 0
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--no-dry) DRY_RUN=false; echo "[WARN] Dry-run disabled — will PATCH accounts."; shift ;;
|
||||
--set-default) SET_DEFAULT=true; shift ;;
|
||||
--amount) AMOUNT="${2:?}"; shift 2 ;;
|
||||
--only) ONLY_USERS="${2:?}"; shift 2 ;;
|
||||
-h|--help) usage ;;
|
||||
*)
|
||||
echo "[ERROR] unknown arg: $1" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if $DRY_RUN; then
|
||||
echo "[INFO] Dry-run active (no HTTP PATCH, no conf write)."
|
||||
else
|
||||
echo "[WARN] APPLY mode amount=${AMOUNT}"
|
||||
fi
|
||||
|
||||
# --- admin password ---
|
||||
find_admin_pass() {
|
||||
if [[ -n "${BANK_ADMIN_PASS:-}" ]]; then
|
||||
printf '%s' "$BANK_ADMIN_PASS"
|
||||
return 0
|
||||
fi
|
||||
local f
|
||||
for f in \
|
||||
/root/bank-admin-password.txt \
|
||||
"${KOOPA_SECRETS:-}/koopa/host-root/taler-bank/bank-admin-password.txt" \
|
||||
"$(cd "$(dirname "$0")/../../../.." 2>/dev/null && pwd)/koopa-admin-secrets/koopa/host-root/taler-bank/bank-admin-password.txt" \
|
||||
"$HOME/src/koopa/koopa-admin-secrets/koopa/host-root/taler-bank/bank-admin-password.txt"
|
||||
do
|
||||
[[ -n "$f" && -f "$f" && -r "$f" ]] || continue
|
||||
tr -d '\n' <"$f"
|
||||
return 0
|
||||
done
|
||||
# hernani@koopa: secret lives in the bank container
|
||||
if command -v podman >/dev/null 2>&1 \
|
||||
&& podman inspect -f '{{.State.Running}}' "${BANK_CTR:-taler-hacktivism-bank}" 2>/dev/null | grep -qx true; then
|
||||
podman exec "${BANK_CTR:-taler-hacktivism-bank}" cat /root/bank-admin-password.txt 2>/dev/null | tr -d '\n'
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
ADMIN_PASS="$(find_admin_pass)" || {
|
||||
echo "[ERROR] no admin password (BANK_ADMIN_PASS or bank-admin-password.txt)" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Prefer host URL; fall back to podman exec curl inside bank container
|
||||
USE_PODMAN=false
|
||||
if ! curl -sf -m 3 "${BANK_URL}/config" >/dev/null 2>&1; then
|
||||
if command -v podman >/dev/null 2>&1 && podman inspect -f '{{.State.Running}}' "$BANK_CTR" 2>/dev/null | grep -qx true; then
|
||||
echo "[INFO] ${BANK_URL} not reachable — using podman exec ${BANK_CTR}"
|
||||
USE_PODMAN=true
|
||||
BANK_URL="http://127.0.0.1:9012"
|
||||
else
|
||||
echo "[ERROR] bank not reachable at ${BANK_URL} and container ${BANK_CTR} not running" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
bank_curl() {
|
||||
# bank_curl [curl-args...] URL_PATH_or_absolute
|
||||
# Last arg is URL path starting with / or full URL
|
||||
local args=("$@")
|
||||
local n=$((${#args[@]} - 1))
|
||||
local url="${args[$n]}"
|
||||
unset "args[$n]"
|
||||
if [[ "$url" != http* ]]; then
|
||||
url="${BANK_URL}${url}"
|
||||
fi
|
||||
if $USE_PODMAN; then
|
||||
podman exec -u root -i "$BANK_CTR" curl -sS -m 30 "${args[@]}" "$url"
|
||||
else
|
||||
curl -sS -m 30 "${args[@]}" "$url"
|
||||
fi
|
||||
}
|
||||
|
||||
bank_curl_code() {
|
||||
# like bank_curl but print HTTP code on stdout after body to fd3... simpler: write body to file
|
||||
local out="$1"; shift
|
||||
local args=("$@")
|
||||
local n=$((${#args[@]} - 1))
|
||||
local url="${args[$n]}"
|
||||
unset "args[$n]"
|
||||
if [[ "$url" != http* ]]; then
|
||||
url="${BANK_URL}${url}"
|
||||
fi
|
||||
if $USE_PODMAN; then
|
||||
podman exec -u root -i "$BANK_CTR" curl -sS -m 30 -o /tmp/raise-debt-body -w '%{http_code}' "${args[@]}" "$url"
|
||||
else
|
||||
curl -sS -m 30 -o "$out" -w '%{http_code}' "${args[@]}" "$url"
|
||||
fi
|
||||
}
|
||||
|
||||
echo "[INFO] bank=${BANK_URL} podman=${USE_PODMAN} amount=${AMOUNT}"
|
||||
|
||||
TOK_JSON=$(bank_curl -u "admin:${ADMIN_PASS}" -H 'Content-Type: application/json' \
|
||||
-d '{"scope":"readwrite"}' /accounts/admin/token)
|
||||
TOKEN=$(printf '%s' "$TOK_JSON" | python3 -c 'import sys,json; print(json.load(sys.stdin)["access_token"])')
|
||||
[[ -n "$TOKEN" ]] || { echo "[ERROR] admin token failed: $TOK_JSON" >&2; exit 1; }
|
||||
echo "[INFO] admin token OK"
|
||||
|
||||
ACCS_JSON=$(bank_curl -H "Authorization: Bearer ${TOKEN}" "/accounts?limit=500")
|
||||
export ACCS_JSON AMOUNT ONLY_USERS DRY_RUN TOKEN
|
||||
export BANK_URL USE_PODMAN BANK_CTR
|
||||
|
||||
python3 <<'PY'
|
||||
import json, os, sys, subprocess, urllib.request
|
||||
|
||||
amount = os.environ["AMOUNT"]
|
||||
only = {u.strip() for u in os.environ.get("ONLY_USERS", "").split(",") if u.strip()}
|
||||
dry = os.environ.get("DRY_RUN", "true").lower() in ("1", "true", "yes")
|
||||
token = os.environ["TOKEN"]
|
||||
bank = os.environ["BANK_URL"].rstrip("/")
|
||||
use_podman = os.environ.get("USE_PODMAN", "false").lower() in ("1", "true", "yes")
|
||||
ctr = os.environ.get("BANK_CTR", "taler-hacktivism-bank")
|
||||
|
||||
accs = json.loads(os.environ["ACCS_JSON"]).get("accounts") or []
|
||||
if only:
|
||||
accs = [a for a in accs if a.get("username") in only]
|
||||
print(f"[INFO] accounts to process: {len(accs)}")
|
||||
|
||||
def http_patch(user: str, body: dict) -> int:
|
||||
data = json.dumps(body).encode()
|
||||
url = f"{bank}/accounts/{user}"
|
||||
if use_podman:
|
||||
# avoid putting token in process list longer than needed — still visible
|
||||
cmd = [
|
||||
"podman", "exec", "-u", "root", "-i", ctr,
|
||||
"curl", "-sS", "-m", "30", "-o", "/dev/null", "-w", "%{http_code}",
|
||||
"-X", "PATCH",
|
||||
"-H", f"Authorization: Bearer {token}",
|
||||
"-H", "Content-Type: application/json",
|
||||
"-d", json.dumps(body),
|
||||
url.replace(bank, "http://127.0.0.1:9012") if bank.startswith("http") else f"http://127.0.0.1:9012/accounts/{user}",
|
||||
]
|
||||
# always use in-container localhost for podman path
|
||||
cmd[-1] = f"http://127.0.0.1:9012/accounts/{user}"
|
||||
out = subprocess.check_output(cmd, text=True).strip()
|
||||
return int(out)
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
data=data,
|
||||
method="PATCH",
|
||||
headers={
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return r.status
|
||||
except Exception as e:
|
||||
code = getattr(e, "code", None)
|
||||
if code is not None:
|
||||
return int(code)
|
||||
print(f"[ERROR] {user}: {e}", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
ok = fail = skip = 0
|
||||
for a in accs:
|
||||
u = a["username"]
|
||||
old = a.get("debit_threshold") or "?"
|
||||
bal = a.get("balance") or {}
|
||||
if old == amount:
|
||||
print(f"[SKIP] {u}: already {amount}")
|
||||
skip += 1
|
||||
continue
|
||||
if dry:
|
||||
print(f"[DRY] {u}: {old} -> {amount} (balance={bal})")
|
||||
ok += 1
|
||||
continue
|
||||
code = http_patch(u, {"debit_threshold": amount})
|
||||
if code in (200, 204):
|
||||
print(f"[EXECUTE] {u}: {old} -> {amount} HTTP {code}")
|
||||
ok += 1
|
||||
else:
|
||||
print(f"[FAIL] {u}: {old} -> {amount} HTTP {code}")
|
||||
fail += 1
|
||||
|
||||
print(f"[INFO] done ok={ok} skip={skip} fail={fail} dry={dry}")
|
||||
sys.exit(1 if fail else 0)
|
||||
PY
|
||||
|
||||
# Optional: DEFAULT_DEBT_LIMIT in overrides (new accounts)
|
||||
if $SET_DEFAULT; then
|
||||
echo "[INFO] --set-default: DEFAULT_DEBT_LIMIT in ${OVERRIDE_CONF}"
|
||||
if $USE_PODMAN; then
|
||||
conf_path="$OVERRIDE_CONF"
|
||||
if $DRY_RUN; then
|
||||
echo "[DRY] sed DEFAULT_DEBT_LIMIT = ${AMOUNT} in container:${conf_path}"
|
||||
echo "[DRY] restart libeufin-bank serve"
|
||||
else
|
||||
podman exec -u root "$BANK_CTR" bash -lc "
|
||||
set -e
|
||||
conf='$OVERRIDE_CONF'
|
||||
cp -a \"\$conf\" \"\${conf}.bak-\$(date +%Y%m%d%H%M%S)\"
|
||||
if grep -q '^DEFAULT_DEBT_LIMIT' \"\$conf\"; then
|
||||
sed -i 's/^DEFAULT_DEBT_LIMIT = .*/DEFAULT_DEBT_LIMIT = ${AMOUNT}/' \"\$conf\"
|
||||
else
|
||||
echo 'DEFAULT_DEBT_LIMIT = ${AMOUNT}' >>\"\$conf\"
|
||||
fi
|
||||
grep '^DEFAULT_DEBT_LIMIT' \"\$conf\"
|
||||
# restart bank process (manual stack)
|
||||
pids=\$(ps -eo pid=,args= | awk '/libeufin-bank serve/ && !/awk/ {print \$1}')
|
||||
for p in \$pids; do kill \$p 2>/dev/null || true; done
|
||||
sleep 1
|
||||
runuser -u libeufin-bank -- nohup /usr/bin/libeufin-bank serve -c /etc/libeufin/libeufin-bank.conf \
|
||||
>>/var/log/libeufin-bank/serve.log 2>&1 &
|
||||
for i in \$(seq 1 30); do
|
||||
curl -sf -m 2 http://127.0.0.1:9012/config >/dev/null && break
|
||||
sleep 0.5
|
||||
done
|
||||
curl -sS http://127.0.0.1:9012/config | python3 -c 'import sys,json; d=json.load(sys.stdin); print(\"[INFO] default_debit_threshold\", d.get(\"default_debit_threshold\"))'
|
||||
"
|
||||
echo "[EXECUTE] DEFAULT_DEBT_LIMIT + bank restart"
|
||||
fi
|
||||
else
|
||||
if [[ ! -f "$OVERRIDE_CONF" ]]; then
|
||||
echo "[WARN] ${OVERRIDE_CONF} not on host — skip --set-default (use from inside container or with podman path)"
|
||||
elif $DRY_RUN; then
|
||||
echo "[DRY] sed DEFAULT_DEBT_LIMIT = ${AMOUNT} in ${OVERRIDE_CONF}"
|
||||
else
|
||||
cp -a "$OVERRIDE_CONF" "${OVERRIDE_CONF}.bak-$(date +%Y%m%d%H%M%S)"
|
||||
sed -i "s/^DEFAULT_DEBT_LIMIT = .*/DEFAULT_DEBT_LIMIT = ${AMOUNT}/" "$OVERRIDE_CONF"
|
||||
echo "[EXECUTE] wrote ${OVERRIDE_CONF}"
|
||||
grep '^DEFAULT_DEBT_LIMIT' "$OVERRIDE_CONF" || true
|
||||
echo "[WARN] restart libeufin-bank yourself if conf is mounted from host"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "[INFO] Done."
|
||||
|
|
@ -50,13 +50,16 @@ curl -sS -m 15 \
|
|||
"${BANK}/accounts/${USER}/withdrawals" >"$WORKDIR/wd.out"
|
||||
|
||||
URI=$(python3 - "$WORKDIR/wd.out" <<'PY'
|
||||
import json,sys
|
||||
d=json.load(open(sys.argv[1]))
|
||||
print(d.get("taler_withdraw_uri") or "")
|
||||
if not d.get("taler_withdraw_uri"):
|
||||
sys.stderr.write(open(sys.argv[1]).read()+"\n")
|
||||
import json, re, sys
|
||||
d = json.load(open(sys.argv[1]))
|
||||
uri = d.get("taler_withdraw_uri") or ""
|
||||
if not uri:
|
||||
sys.stderr.write(open(sys.argv[1]).read() + "\n")
|
||||
sys.exit(1)
|
||||
print(d.get("withdrawal_id",""), file=sys.stderr)
|
||||
# strip default HTTPS port for wallet apps
|
||||
uri = re.sub(r"(taler://withdraw/[^/:]+):443(?=/|$)", r"\1", uri)
|
||||
print(uri)
|
||||
print(d.get("withdrawal_id", ""), file=sys.stderr)
|
||||
PY
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -38,6 +38,8 @@ URI=$(printf '%s' "$WD" | sed -n 's/.*"taler_withdraw_uri"[[:space:]]*:[[:space:
|
|||
WID=$(printf '%s' "$WD" | sed -n 's/.*"withdrawal_id"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p')
|
||||
[ -n "$URI" ] || { echo "no URI from: $WD" >&2; exit 1; }
|
||||
[ -n "$WID" ] || WID=$(basename "$URI")
|
||||
# libeufin emits host:443 — strip default HTTPS port for wallet apps
|
||||
URI=$(printf '%s' "$URI" | sed -E 's|(taler://withdraw/[^/:]+):443(/)|\1\2|; s|(taler://withdraw/[^/:]+):443$|\1|')
|
||||
|
||||
mkdir -p "$LANDING_DIR"
|
||||
printf '%s\n' "$URI" >"$LANDING_DIR/withdraw.uri"
|
||||
|
|
|
|||
112
scripts/taler-landing/README.md
Normal file
112
scripts/taler-landing/README.md
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
# taler-landing — central stats (hernani user systemd)
|
||||
|
||||
Public landing numbers on
|
||||
|
||||
- https://bank.hacktivism.ch/intro/ → `stats.json`
|
||||
- https://exchange.hacktivism.ch/intro/ → `stats.json`
|
||||
- https://taler.hacktivism.ch/intro/ → `stats.json`
|
||||
|
||||
are produced by **one host process** as user **`hernani`**, not by three unrelated in-container crons.
|
||||
|
||||
## Why host / user systemd
|
||||
|
||||
| Concern | Approach |
|
||||
|--------|----------|
|
||||
| **All accounts / all money** | Python full scan + tx pagination (not capped at 80 accounts) |
|
||||
| **High ladder GOA** | `amount_alt` / UI alt units (Kilo-/Mega-/Peta-GOA) — tiles stay short |
|
||||
| **No root cron** | `systemctl --user` as hernani + `loginctl enable-linger` |
|
||||
| **Failure safety** | failed run only updates `stats-run.json`; last good `stats.json` stays |
|
||||
|
||||
## Install (on koopa as hernani)
|
||||
|
||||
```bash
|
||||
cd ~/src/koopa/koopa-admin-log
|
||||
./scripts/taler-landing/install-landing-stats-host.sh
|
||||
|
||||
# timer without login session:
|
||||
sudo loginctl enable-linger hernani
|
||||
|
||||
# run once:
|
||||
systemctl --user start taler-landing-stats.service
|
||||
journalctl --user -u taler-landing-stats.service -n 50 --no-pager
|
||||
systemctl --user list-timers 'taler-landing-stats*'
|
||||
```
|
||||
|
||||
Units:
|
||||
|
||||
- `configs/systemd/user/taler-landing-stats.service` — oneshot collector
|
||||
- `configs/systemd/user/taler-landing-stats.timer` — every **2 minutes** after boot
|
||||
|
||||
Installed paths:
|
||||
|
||||
| Path | Role |
|
||||
|------|------|
|
||||
| `~/.local/bin/collect-landing-stats.sh` | orchestrator |
|
||||
| `~/.local/lib/taler-landing/*.py` | bank scan + alt enrich |
|
||||
| `~/.local/state/taler-landing-stats/` | logs |
|
||||
| `~/.config/systemd/user/taler-landing-stats.*` | user units |
|
||||
|
||||
## What the collector does
|
||||
|
||||
1. **Bank** — `collect_bank_stats.py`
|
||||
- admin token → list **all** accounts
|
||||
- for each account (except `SCAN_SKIP`, default **`exchange`**) page through **all** transactions
|
||||
- sum credits / Taler withdraws / other debits
|
||||
- emit `amount` + `amount_alt` / `amount_full`
|
||||
- `podman cp` → `taler-hacktivism-bank:/var/www/bank-landing/stats.json`
|
||||
|
||||
2. **Exchange** — run `landing-stats-exchange.sh` inside exchange container, then **enrich** alt fields on the host and write back.
|
||||
|
||||
3. **Merchant** — same for merchant container.
|
||||
|
||||
4. **Resources (all three)** — `collect_container_resources.sh` runs
|
||||
`mem-snapshot` **inside** each podman container (not host `/proc`):
|
||||
container RSS (+ cgroup limit label), postgres/java/taler/nginx groups,
|
||||
top-10 processes, loadavg. Merged into `performance.memory` /
|
||||
`performance.loadavg` via `merge_resources.py`.
|
||||
|
||||
`exchange` is skipped in the bank **flow** scan so the same GOA is not counted once as customer withdraw and again as exchange credit. **Admin**, **explorer**, and every auto-account are included.
|
||||
|
||||
## Secrets
|
||||
|
||||
Preferred order:
|
||||
|
||||
1. `~/src/koopa/koopa-admin-secrets/koopa/host-root/taler-bank/bank-admin-password.txt`
|
||||
2. `~/.config/taler-landing/bank-*-password.txt`
|
||||
3. `podman exec taler-hacktivism-bank cat /root/bank-admin-password.txt`
|
||||
|
||||
Explorer password optional (shared-pool balance).
|
||||
|
||||
## Env overrides
|
||||
|
||||
| Env | Default | Meaning |
|
||||
|-----|---------|---------|
|
||||
| `BANK_URL` | `http://127.0.0.1:9012` | libeufin loopback |
|
||||
| `SCAN_SKIP` | `exchange` | usernames excluded from flow |
|
||||
| `TX_PAGE` | `500` | transactions page size |
|
||||
| `MAX_TX_PAGES` | `0` | `0` = unlimited pages / account |
|
||||
| `ACCOUNTS_DELTA` | `-10000` | account list window |
|
||||
| `ADMIN_LOG` | `%h/src/koopa/koopa-admin-log` | refresh in-container scripts |
|
||||
|
||||
## UI alt names
|
||||
|
||||
`configs/shared/goa-amount.js` formats large amounts as e.g. `1.23 Mega-GOA` (tooltip = full `GOA:…`). Landings load it as `/intro/goa-amount.js` (deploy via `deploy-landings.sh`).
|
||||
|
||||
## Legacy in-container bank cron
|
||||
|
||||
`scripts/taler-bank/landing-stats.sh` remains a **fallback** inside the bank container. Once the hernani timer is healthy, disable the old in-container minutely cron to avoid races:
|
||||
|
||||
```bash
|
||||
podman exec taler-hacktivism-bank crontab -l # inspect
|
||||
# remove landing-stats.sh line if present
|
||||
```
|
||||
|
||||
## Manual run
|
||||
|
||||
```bash
|
||||
~/.local/bin/collect-landing-stats.sh
|
||||
# or from checkout:
|
||||
./scripts/taler-landing/collect-landing-stats.sh
|
||||
|
||||
curl -sS https://bank.hacktivism.ch/intro/stats.json | python3 -m json.tool | head
|
||||
```
|
||||
287
scripts/taler-landing/collect-landing-stats.sh
Executable file
287
scripts/taler-landing/collect-landing-stats.sh
Executable file
|
|
@ -0,0 +1,287 @@
|
|||
#!/usr/bin/env bash
|
||||
# Central landing-stats collector for hernani@koopa (user systemd timer).
|
||||
#
|
||||
# - Bank: full account+ledger scan via collect_bank_stats.py → bank container
|
||||
# - Exchange / merchant: existing in-container scripts, then amount_alt enrich
|
||||
# - All three: container RSS / loadavg / top procs via mem-snapshot (podman exec)
|
||||
# - Never wipes a good stats.json on failure (writes stats-run.json only)
|
||||
#
|
||||
# Install: scripts/taler-landing/install-landing-stats-host.sh
|
||||
set -euo pipefail
|
||||
|
||||
export TZ="${TZ:-Europe/Zurich}"
|
||||
export PATH="${HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin${PATH:+:$PATH}"
|
||||
|
||||
log() { printf '%s %s\n' "$(date -Iseconds)" "$*"; }
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")" && pwd)"
|
||||
# When installed to ~/.local/bin, libs live in ~/.local/lib/taler-landing
|
||||
if [ -f "$ROOT/collect_bank_stats.py" ]; then
|
||||
LIB="$ROOT"
|
||||
elif [ -f "${HOME}/.local/lib/taler-landing/collect_bank_stats.py" ]; then
|
||||
LIB="${HOME}/.local/lib/taler-landing"
|
||||
elif [ -f "${HOME}/src/koopa/koopa-admin-log/scripts/taler-landing/collect_bank_stats.py" ]; then
|
||||
LIB="${HOME}/src/koopa/koopa-admin-log/scripts/taler-landing"
|
||||
else
|
||||
LIB="$ROOT"
|
||||
fi
|
||||
|
||||
ADMIN_LOG="${ADMIN_LOG:-${HOME}/src/koopa/koopa-admin-log}"
|
||||
SECRETS_BANK="${SECRETS_BANK:-${HOME}/src/koopa/koopa-admin-secrets/koopa/host-root/taler-bank}"
|
||||
|
||||
BANK_CTR="${BANK_CTR:-taler-hacktivism-bank}"
|
||||
EX_CTR="${EX_CTR:-taler-hacktivism-exchange-ansible}"
|
||||
MER_CTR="${MER_CTR:-taler-hacktivism}"
|
||||
|
||||
BANK_URL="${BANK_URL:-http://127.0.0.1:9012}"
|
||||
BANK_PUBLIC_URL="${BANK_PUBLIC_URL:-https://bank.hacktivism.ch}"
|
||||
EXCHANGE_CONFIG_URL="${EXCHANGE_CONFIG_URL:-https://exchange.hacktivism.ch/config}"
|
||||
|
||||
BANK_LANDING_IN="${BANK_LANDING_IN:-/var/www/bank-landing}"
|
||||
EX_LANDING_IN="${EX_LANDING_IN:-/var/www/exchange-landing}"
|
||||
MER_LANDING_IN="${MER_LANDING_IN:-/var/www/merchant-landing}"
|
||||
|
||||
WORKDIR="${LANDING_STATS_WORKDIR:-${XDG_RUNTIME_DIR:-/tmp}/taler-landing-stats}"
|
||||
mkdir -p "$WORKDIR"
|
||||
LOG_DIR="${LANDING_STATS_LOGDIR:-${HOME}/.local/state/taler-landing-stats}"
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
PY="${PYTHON:-python3}"
|
||||
ec_bank=0
|
||||
ec_ex=0
|
||||
ec_mer=0
|
||||
|
||||
# Resolve helper scripts (checkout or ~/.local/lib)
|
||||
COLLECT_RES="$LIB/collect_container_resources.sh"
|
||||
MERGE_RES="$LIB/merge_resources.py"
|
||||
if [ ! -f "$COLLECT_RES" ] && [ -f "$ROOT/collect_container_resources.sh" ]; then
|
||||
COLLECT_RES="$ROOT/collect_container_resources.sh"
|
||||
fi
|
||||
if [ ! -f "$MERGE_RES" ] && [ -f "$ROOT/merge_resources.py" ]; then
|
||||
MERGE_RES="$ROOT/merge_resources.py"
|
||||
fi
|
||||
MEM_SRC="${MEM_SNAPSHOT_SRC:-$ADMIN_LOG/scripts/taler-shared/mem-snapshot.sh}"
|
||||
|
||||
merge_container_resources() {
|
||||
local label="$1" ctr="$2" stats_file="$3"
|
||||
[ -f "$stats_file" ] || return 0
|
||||
if ! ctr_running "$ctr"; then
|
||||
log "WARN: $label resources: $ctr not running"
|
||||
return 1
|
||||
fi
|
||||
if [ ! -x "$COLLECT_RES" ] && [ -f "$COLLECT_RES" ]; then
|
||||
chmod +x "$COLLECT_RES" 2>/dev/null || true
|
||||
fi
|
||||
if [ ! -f "$COLLECT_RES" ] || [ ! -f "$MERGE_RES" ]; then
|
||||
log "WARN: $label resources: helpers missing ($COLLECT_RES / $MERGE_RES)"
|
||||
return 1
|
||||
fi
|
||||
local resf="$WORKDIR/${label}-resources.json"
|
||||
set +e
|
||||
ADMIN_LOG="$ADMIN_LOG" MEM_SNAPSHOT_SRC="$MEM_SRC" \
|
||||
bash "$COLLECT_RES" "$ctr" "$resf" >>"$LOG_DIR/${label}-resources.log" 2>&1
|
||||
local ec=$?
|
||||
set -e
|
||||
if [ "$ec" -ne 0 ] || [ ! -s "$resf" ]; then
|
||||
log "WARN: $label resources: collect failed (ec=$ec)"
|
||||
return 1
|
||||
fi
|
||||
if ! "$PY" -c 'import json,sys; d=json.load(open(sys.argv[1])); sys.exit(0 if d.get("ok") and d.get("memory") else 1)' "$resf" 2>/dev/null; then
|
||||
log "WARN: $label resources: bad payload"
|
||||
return 1
|
||||
fi
|
||||
"$PY" "$MERGE_RES" "$stats_file" "$resf" >>"$LOG_DIR/${label}-resources.log" 2>&1
|
||||
log "$label: resources merged (RSS + loadavg from $ctr)"
|
||||
return 0
|
||||
}
|
||||
|
||||
read_pass() {
|
||||
local name="$1" f
|
||||
for f in \
|
||||
"${SECRETS_BANK}/${name}" \
|
||||
"${HOME}/.config/taler-landing/${name}" \
|
||||
"/run/user/$(id -u)/taler-landing/${name}"
|
||||
do
|
||||
if [ -r "$f" ]; then
|
||||
tr -d '\n' <"$f"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
# container (hernani can podman exec root files inside owned containers)
|
||||
if podman inspect -f '{{.State.Running}}' "$BANK_CTR" 2>/dev/null | grep -qx true; then
|
||||
if podman exec "$BANK_CTR" test -r "/root/${name}" 2>/dev/null; then
|
||||
podman exec "$BANK_CTR" cat "/root/${name}" 2>/dev/null | tr -d '\n'
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
ctr_running() {
|
||||
podman inspect -f '{{.State.Running}}' "$1" 2>/dev/null | grep -qx true
|
||||
}
|
||||
|
||||
publish_json() {
|
||||
local ctr="$1" dest_dir="$2" src_stats="$3" src_run="$4"
|
||||
if ! ctr_running "$ctr"; then
|
||||
log "WARN: $ctr not running — skip publish $dest_dir"
|
||||
return 1
|
||||
fi
|
||||
podman exec "$ctr" mkdir -p "$dest_dir" 2>/dev/null || true
|
||||
if [ -f "$src_stats" ]; then
|
||||
podman cp "$src_stats" "${ctr}:${dest_dir}/stats.json"
|
||||
fi
|
||||
if [ -f "$src_run" ]; then
|
||||
podman cp "$src_run" "${ctr}:${dest_dir}/stats-run.json"
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bank — full scan (all accounts / all money except SCAN_SKIP)
|
||||
# ---------------------------------------------------------------------------
|
||||
collect_bank() {
|
||||
log "bank: collect full stats via $LIB/collect_bank_stats.py"
|
||||
local admin_pass explorer_pass
|
||||
admin_pass="$(read_pass bank-admin-password.txt || true)"
|
||||
explorer_pass="$(read_pass bank-explorer-password.txt || true)"
|
||||
if [ -z "$admin_pass" ]; then
|
||||
log "ERROR: bank admin password not found (secrets or container)"
|
||||
printf '%s\n' '{"ok":false,"error":"no admin password","at_human":"'"$(date +"%Y-%m-%d %H:%M %Z")"'"}' \
|
||||
>"$WORKDIR/bank-stats-run.json"
|
||||
publish_json "$BANK_CTR" "$BANK_LANDING_IN" "" "$WORKDIR/bank-stats-run.json" || true
|
||||
return 1
|
||||
fi
|
||||
|
||||
# optional demo files from container
|
||||
local demo_dir=""
|
||||
if ctr_running "$BANK_CTR"; then
|
||||
demo_dir="$WORKDIR/demo"
|
||||
mkdir -p "$demo_dir"
|
||||
podman cp "${BANK_CTR}:${BANK_LANDING_IN}/withdraw.uri" "$demo_dir/withdraw.uri" 2>/dev/null || true
|
||||
podman cp "${BANK_CTR}:${BANK_LANDING_IN}/withdraw.amount" "$demo_dir/withdraw.amount" 2>/dev/null || true
|
||||
podman cp "${BANK_CTR}:${BANK_LANDING_IN}/withdraw.created" "$demo_dir/withdraw.created" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
set +e
|
||||
BANK_URL="$BANK_URL" \
|
||||
BANK_PUBLIC_URL="$BANK_PUBLIC_URL" \
|
||||
EXCHANGE_CONFIG_URL="$EXCHANGE_CONFIG_URL" \
|
||||
BANK_ADMIN_PASS="$admin_pass" \
|
||||
BANK_EXPLORER_PASS="$explorer_pass" \
|
||||
DEMO_DIR="${demo_dir}" \
|
||||
SCAN_SKIP="${SCAN_SKIP:-exchange}" \
|
||||
TX_PAGE="${TX_PAGE:-500}" \
|
||||
MAX_TX_PAGES="${MAX_TX_PAGES:-0}" \
|
||||
ACCOUNTS_DELTA="${ACCOUNTS_DELTA:--10000}" \
|
||||
"$PY" "$LIB/collect_bank_stats.py" \
|
||||
--out "$WORKDIR/bank-stats.json" \
|
||||
--run-out "$WORKDIR/bank-stats-run.json" \
|
||||
>>"$LOG_DIR/bank.log" 2>&1
|
||||
ec_bank=$?
|
||||
set -e
|
||||
|
||||
if [ "$ec_bank" -eq 0 ] && [ -f "$WORKDIR/bank-stats.json" ]; then
|
||||
# Always attach in-container RSS/loadavg (not host /proc)
|
||||
merge_container_resources bank "$BANK_CTR" "$WORKDIR/bank-stats.json" || true
|
||||
publish_json "$BANK_CTR" "$BANK_LANDING_IN" \
|
||||
"$WORKDIR/bank-stats.json" "$WORKDIR/bank-stats-run.json"
|
||||
log "bank: OK → ${BANK_CTR}:${BANK_LANDING_IN}/stats.json"
|
||||
else
|
||||
log "ERROR: bank collect failed (ec=$ec_bank) — previous stats.json kept"
|
||||
[ -f "$WORKDIR/bank-stats-run.json" ] || \
|
||||
printf '%s\n' '{"ok":false,"error":"collect failed","at_human":"'"$(date +"%Y-%m-%d %H:%M %Z")"'"}' \
|
||||
>"$WORKDIR/bank-stats-run.json"
|
||||
publish_json "$BANK_CTR" "$BANK_LANDING_IN" "" "$WORKDIR/bank-stats-run.json" || true
|
||||
fi
|
||||
return "$ec_bank"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Exchange / merchant — in-container generators + host enrich
|
||||
# ---------------------------------------------------------------------------
|
||||
run_incontainer_stats() {
|
||||
local label="$1" ctr="$2" script_candidates="$3" landing="$4"
|
||||
local script="" s
|
||||
if ! ctr_running "$ctr"; then
|
||||
log "WARN: $label: $ctr not running"
|
||||
return 1
|
||||
fi
|
||||
# install latest script from admin-log if present
|
||||
local host_src=""
|
||||
case "$label" in
|
||||
exchange)
|
||||
host_src="$ADMIN_LOG/scripts/taler-exchange/landing-stats-exchange.sh"
|
||||
;;
|
||||
merchant)
|
||||
host_src="$ADMIN_LOG/scripts/taler-merchant/landing-stats-merchant.sh"
|
||||
;;
|
||||
esac
|
||||
if [ -n "$host_src" ] && [ -f "$host_src" ]; then
|
||||
podman cp "$host_src" "${ctr}:/usr/local/bin/$(basename "$host_src")"
|
||||
podman exec "$ctr" chmod 755 "/usr/local/bin/$(basename "$host_src")" 2>/dev/null || true
|
||||
script="/usr/local/bin/$(basename "$host_src")"
|
||||
else
|
||||
for s in $script_candidates; do
|
||||
if podman exec "$ctr" test -x "$s" 2>/dev/null || podman exec "$ctr" test -f "$s" 2>/dev/null; then
|
||||
script="$s"
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
if [ -z "$script" ]; then
|
||||
log "WARN: $label: no landing-stats script in $ctr"
|
||||
return 1
|
||||
fi
|
||||
log "$label: run $script inside $ctr"
|
||||
set +e
|
||||
podman exec \
|
||||
-e LANDING_DIR="$landing" \
|
||||
-e TZ=Europe/Zurich \
|
||||
-e PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
|
||||
"$ctr" bash "$script" >>"$LOG_DIR/${label}.log" 2>&1
|
||||
local ec=$?
|
||||
set -e
|
||||
if [ "$ec" -ne 0 ]; then
|
||||
log "ERROR: $label stats failed (ec=$ec)"
|
||||
return "$ec"
|
||||
fi
|
||||
# enrich with alt units + ensure resources (RSS/loadavg) on host
|
||||
podman cp "${ctr}:${landing}/stats.json" "$WORKDIR/${label}-stats.json" 2>/dev/null || return 0
|
||||
set +e
|
||||
"$PY" "$LIB/enrich_stats_alt.py" "$WORKDIR/${label}-stats.json" \
|
||||
--exchange-config "$EXCHANGE_CONFIG_URL" >>"$LOG_DIR/${label}.log" 2>&1
|
||||
set -e
|
||||
# Prefer fresh container snapshot (also fills gaps if in-container mem helper missing)
|
||||
merge_container_resources "$label" "$ctr" "$WORKDIR/${label}-stats.json" || true
|
||||
podman cp "$WORKDIR/${label}-stats.json" "${ctr}:${landing}/stats.json"
|
||||
log "$label: OK + alt + resources → ${ctr}:${landing}/stats.json"
|
||||
return 0
|
||||
}
|
||||
|
||||
collect_exchange() {
|
||||
run_incontainer_stats exchange "$EX_CTR" \
|
||||
"/usr/local/bin/landing-stats-exchange.sh /usr/local/bin/landing-stats.sh" \
|
||||
"$EX_LANDING_IN"
|
||||
}
|
||||
|
||||
collect_merchant() {
|
||||
run_incontainer_stats merchant "$MER_CTR" \
|
||||
"/usr/local/bin/landing-stats-merchant.sh /usr/local/bin/landing-stats.sh" \
|
||||
"$MER_LANDING_IN"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
main() {
|
||||
log "=== taler-landing-stats start (user=$(id -un) lib=$LIB) ==="
|
||||
collect_bank || ec_bank=$?
|
||||
collect_exchange || ec_ex=$?
|
||||
collect_merchant || ec_mer=$?
|
||||
log "=== done bank=$ec_bank exchange=$ec_ex merchant=$ec_mer ==="
|
||||
# non-zero if bank failed (primary public flow numbers); soft on ex/mer
|
||||
if [ "$ec_bank" -ne 0 ]; then
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
main "$@"
|
||||
813
scripts/taler-landing/collect_bank_stats.py
Executable file
813
scripts/taler-landing/collect_bank_stats.py
Executable file
|
|
@ -0,0 +1,813 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Full-scan bank landing stats (all accounts, all ledger money).
|
||||
|
||||
Run on koopa as hernani (or anywhere with admin API access). Writes stats.json
|
||||
compatible with bank.hacktivism.ch/intro/ plus amount_alt fields for compact UI.
|
||||
|
||||
Env (selected):
|
||||
BANK_URL default http://127.0.0.1:9012
|
||||
BANK_ADMIN_USER default admin
|
||||
BANK_ADMIN_PASS or BANK_ADMIN_PASS_FILE / pass via --admin-pass-file
|
||||
BANK_EXPLORER_USER default explorer
|
||||
BANK_EXPLORER_PASS optional (balance_explorer)
|
||||
EXCHANGE_CONFIG_URL for alt_unit_names (default https://exchange.hacktivism.ch/config)
|
||||
OUT output path (default stdout if -)
|
||||
SCAN_SKIP comma usernames excluded from flow (default: exchange)
|
||||
TX_PAGE page size for transactions delta (default 500)
|
||||
MAX_TX_PAGES 0 = unlimited pages per account (default 0)
|
||||
ACCOUNTS_DELTA GET /accounts?delta= (default -10000)
|
||||
RECENT_WD_N recent withdraws (default 10)
|
||||
TZ default Europe/Zurich
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Set, Tuple
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
# local import (same directory)
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from goa_amounts import ( # noqa: E402
|
||||
DEFAULT_ALT,
|
||||
enrich_stats_tree,
|
||||
format_amount_alt,
|
||||
load_alt_from_config,
|
||||
parse_amount,
|
||||
)
|
||||
|
||||
RESERVE_RE = re.compile(r"(?i)withdrawal\s+([A-Za-z0-9]+)")
|
||||
|
||||
|
||||
def env(name: str, default: str = "") -> str:
|
||||
return os.environ.get(name, default)
|
||||
|
||||
|
||||
def read_pass_file(path: str) -> str:
|
||||
p = Path(path)
|
||||
if p.is_file():
|
||||
return p.read_text(encoding="utf-8", errors="replace").strip()
|
||||
return ""
|
||||
|
||||
|
||||
def http_json(
|
||||
url: str,
|
||||
*,
|
||||
method: str = "GET",
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
data: Optional[bytes] = None,
|
||||
timeout: float = 30.0,
|
||||
auth: Optional[Tuple[str, str]] = None,
|
||||
) -> Tuple[int, Any, bytes]:
|
||||
h = dict(headers or {})
|
||||
if auth:
|
||||
import base64
|
||||
|
||||
token = base64.b64encode(f"{auth[0]}:{auth[1]}".encode()).decode("ascii")
|
||||
h["Authorization"] = f"Basic {token}"
|
||||
req = urllib.request.Request(url, data=data, headers=h, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
body = resp.read()
|
||||
code = getattr(resp, "status", 200) or 200
|
||||
except urllib.error.HTTPError as e:
|
||||
body = e.read() if e.fp else b""
|
||||
code = e.code
|
||||
except Exception as e:
|
||||
return 0, None, str(e).encode()
|
||||
|
||||
if not body:
|
||||
return code, None, body
|
||||
try:
|
||||
return code, json.loads(body.decode("utf-8", errors="replace")), body
|
||||
except Exception:
|
||||
return code, None, body
|
||||
|
||||
|
||||
def measure_ms(url: str, timeout: float = 8.0) -> Tuple[Optional[int], str]:
|
||||
t0 = time.perf_counter()
|
||||
code = "000"
|
||||
try:
|
||||
req = urllib.request.Request(url, method="GET")
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
resp.read()
|
||||
code = str(getattr(resp, "status", 200) or 200)
|
||||
except urllib.error.HTTPError as e:
|
||||
code = str(e.code)
|
||||
except Exception:
|
||||
return None, "000"
|
||||
ms = int(round((time.perf_counter() - t0) * 1000))
|
||||
if ms == 0:
|
||||
ms = 1
|
||||
return ms, code
|
||||
|
||||
|
||||
def get_token(bank: str, user: str, password: str) -> str:
|
||||
code, data, _ = http_json(
|
||||
f"{bank}/accounts/{user}/token",
|
||||
method="POST",
|
||||
headers={"Content-Type": "application/json"},
|
||||
data=b'{"scope":"readonly"}',
|
||||
auth=(user, password),
|
||||
timeout=15,
|
||||
)
|
||||
if code not in (200, 201) or not isinstance(data, dict):
|
||||
raise RuntimeError(f"token failed for {user}: HTTP {code}")
|
||||
tok = data.get("access_token") or data.get("token") or ""
|
||||
if not tok:
|
||||
raise RuntimeError(f"token empty for {user}")
|
||||
return str(tok)
|
||||
|
||||
|
||||
def list_all_accounts(bank: str, token: str, delta: int) -> List[str]:
|
||||
"""List accounts; page with start if needed."""
|
||||
names: List[str] = []
|
||||
seen: Set[str] = set()
|
||||
start: Optional[int] = None
|
||||
pages = 0
|
||||
max_pages = int(env("MAX_ACCOUNT_PAGES", "50") or "50")
|
||||
|
||||
while pages < max_pages:
|
||||
pages += 1
|
||||
q = f"delta={delta}"
|
||||
if start is not None:
|
||||
q += f"&start={start}"
|
||||
code, data, raw = http_json(
|
||||
f"{bank}/accounts?{q}",
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
timeout=45,
|
||||
)
|
||||
if code == 204 or not data:
|
||||
break
|
||||
if code != 200:
|
||||
raise RuntimeError(f"list accounts HTTP {code}: {raw[:200]!r}")
|
||||
|
||||
batch: List[Dict[str, Any]] = []
|
||||
if isinstance(data, dict):
|
||||
if isinstance(data.get("accounts"), list):
|
||||
batch = data["accounts"]
|
||||
elif isinstance(data.get("users"), list):
|
||||
batch = data["users"]
|
||||
elif isinstance(data, list):
|
||||
batch = data
|
||||
|
||||
if not batch:
|
||||
# flat object with username? try regex fallback
|
||||
text = raw.decode("utf-8", errors="replace")
|
||||
for m in re.finditer(r'"username"\s*:\s*"([^"]+)"', text):
|
||||
u = m.group(1)
|
||||
if u not in seen:
|
||||
seen.add(u)
|
||||
names.append(u)
|
||||
break
|
||||
|
||||
min_row = None
|
||||
for item in batch:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
u = item.get("username") or item.get("name") or ""
|
||||
if u and u not in seen:
|
||||
seen.add(u)
|
||||
names.append(str(u))
|
||||
rid = item.get("row_id") or item.get("rowId")
|
||||
if rid is not None:
|
||||
try:
|
||||
rid_i = int(rid)
|
||||
min_row = rid_i if min_row is None else min(min_row, rid_i)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if len(batch) < abs(delta):
|
||||
break
|
||||
if min_row is None:
|
||||
break
|
||||
# next older page
|
||||
start = min_row
|
||||
# avoid infinite loop on same start
|
||||
if pages > 1 and len(names) == len(seen):
|
||||
# if no growth and full page, still advance
|
||||
pass
|
||||
|
||||
return names
|
||||
|
||||
|
||||
def iter_transactions(
|
||||
bank: str,
|
||||
token: str,
|
||||
username: str,
|
||||
page: int,
|
||||
max_pages: int,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Return all transactions for account (paginated)."""
|
||||
out: List[Dict[str, Any]] = []
|
||||
start: Optional[int] = None
|
||||
pages = 0
|
||||
safety = max_pages if max_pages > 0 else 10_000
|
||||
|
||||
while pages < safety:
|
||||
pages += 1
|
||||
q = f"delta=-{abs(page)}"
|
||||
if start is not None:
|
||||
q += f"&start={start}"
|
||||
code, data, raw = http_json(
|
||||
f"{bank}/accounts/{urllib.parse.quote(username)}/transactions?{q}",
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
timeout=float(env("TX_CURL_TIMEOUT", "45") or "45"),
|
||||
)
|
||||
if code in (204, 404) or not data:
|
||||
break
|
||||
if code != 200:
|
||||
# soft-fail one account
|
||||
break
|
||||
|
||||
txs: List[Dict[str, Any]] = []
|
||||
if isinstance(data, dict) and isinstance(data.get("transactions"), list):
|
||||
txs = data["transactions"]
|
||||
elif isinstance(data, list):
|
||||
txs = data
|
||||
else:
|
||||
# tolerate raw array-ish
|
||||
break
|
||||
|
||||
if not txs:
|
||||
break
|
||||
|
||||
min_row = None
|
||||
for tx in txs:
|
||||
if not isinstance(tx, dict):
|
||||
continue
|
||||
out.append(tx)
|
||||
rid = tx.get("row_id") or tx.get("rowId")
|
||||
if rid is not None:
|
||||
try:
|
||||
rid_i = int(rid)
|
||||
min_row = rid_i if min_row is None else min(min_row, rid_i)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if len(txs) < abs(page):
|
||||
break
|
||||
if min_row is None:
|
||||
break
|
||||
start = min_row
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def tx_fields(tx: Dict[str, Any]) -> Tuple[str, str, int, str]:
|
||||
"""direction, amount, unix_ts, subject"""
|
||||
direction = str(tx.get("direction") or "").lower()
|
||||
amount = str(tx.get("amount") or "")
|
||||
subject = str(tx.get("subject") or tx.get("description") or "")
|
||||
ts = 0
|
||||
when = tx.get("when") or tx.get("date") or tx.get("timestamp")
|
||||
if isinstance(when, dict):
|
||||
# Taler AbsoluteTime: t_s seconds or t_ms
|
||||
if "t_s" in when:
|
||||
try:
|
||||
ts = int(when["t_s"])
|
||||
except Exception:
|
||||
ts = 0
|
||||
elif "t_ms" in when:
|
||||
try:
|
||||
ts = int(int(when["t_ms"]) / 1000)
|
||||
except Exception:
|
||||
ts = 0
|
||||
elif isinstance(when, (int, float)):
|
||||
ts = int(when)
|
||||
if ts > 10_000_000_000: # ms
|
||||
ts //= 1000
|
||||
elif tx.get("t_s") is not None:
|
||||
try:
|
||||
ts = int(tx["t_s"])
|
||||
except Exception:
|
||||
ts = 0
|
||||
return direction, amount, ts, subject
|
||||
|
||||
|
||||
def reserve_from_subject(subject: str) -> str:
|
||||
m = RESERVE_RE.search(subject or "")
|
||||
if m:
|
||||
return re.sub(r"[^A-Za-z0-9]", "", m.group(1))
|
||||
# fallback: last token
|
||||
parts = (subject or "").split()
|
||||
if parts:
|
||||
return re.sub(r"[^A-Za-z0-9]", "", parts[-1])
|
||||
return ""
|
||||
|
||||
|
||||
def now_parts(tz_name: str) -> Tuple[int, str, str]:
|
||||
tz = ZoneInfo(tz_name)
|
||||
dt = datetime.now(tz)
|
||||
unix = int(dt.timestamp())
|
||||
iso = dt.strftime("%Y-%m-%dT%H:%M%z")
|
||||
# +0200 → +02:00
|
||||
if len(iso) >= 5 and iso[-5] in "+-" and ":" not in iso[-5:]:
|
||||
iso = iso[:-2] + ":" + iso[-2:]
|
||||
human = dt.strftime("%Y-%m-%d %H:%M %Z")
|
||||
return unix, iso, human
|
||||
|
||||
|
||||
def human_from_unix(ts: int, tz_name: str) -> Tuple[str, str]:
|
||||
if not ts:
|
||||
return "", ""
|
||||
tz = ZoneInfo(tz_name)
|
||||
dt = datetime.fromtimestamp(ts, tz)
|
||||
iso = dt.strftime("%Y-%m-%dT%H:%M%z")
|
||||
if len(iso) >= 5 and iso[-5] in "+-" and ":" not in iso[-5:]:
|
||||
iso = iso[:-2] + ":" + iso[-2:]
|
||||
human = dt.strftime("%Y-%m-%d %H:%M %Z")
|
||||
return human, iso
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="Collect full bank landing stats")
|
||||
ap.add_argument("--bank", default=env("BANK_URL", "http://127.0.0.1:9012"))
|
||||
ap.add_argument("--admin-user", default=env("BANK_ADMIN_USER", "admin"))
|
||||
ap.add_argument("--admin-pass", default=env("BANK_ADMIN_PASS", ""))
|
||||
ap.add_argument("--admin-pass-file", default=env("BANK_ADMIN_PASS_FILE", ""))
|
||||
ap.add_argument("--explorer-user", default=env("BANK_EXPLORER_USER", "explorer"))
|
||||
ap.add_argument("--explorer-pass", default=env("BANK_EXPLORER_PASS", ""))
|
||||
ap.add_argument("--explorer-pass-file", default=env("BANK_EXPLORER_PASS_FILE", ""))
|
||||
ap.add_argument(
|
||||
"--exchange-config",
|
||||
default=env("EXCHANGE_CONFIG_URL", "https://exchange.hacktivism.ch/config"),
|
||||
)
|
||||
ap.add_argument("--out", default=env("OUT", "-"))
|
||||
ap.add_argument("--run-out", default=env("RUN_OUT", ""))
|
||||
ap.add_argument(
|
||||
"--skip",
|
||||
default=env("SCAN_SKIP", "exchange"),
|
||||
help="comma usernames excluded from flow scan (default: exchange)",
|
||||
)
|
||||
ap.add_argument("--tx-page", type=int, default=int(env("TX_PAGE", "500") or "500"))
|
||||
ap.add_argument(
|
||||
"--max-tx-pages",
|
||||
type=int,
|
||||
default=int(env("MAX_TX_PAGES", "0") or "0"),
|
||||
help="0 = unlimited pages per account",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--accounts-delta",
|
||||
type=int,
|
||||
default=int(env("ACCOUNTS_DELTA", "-10000") or "-10000"),
|
||||
)
|
||||
ap.add_argument("--recent", type=int, default=int(env("RECENT_WD_N", "10") or "10"))
|
||||
ap.add_argument("--public-base", default=env("BANK_PUBLIC_URL", "https://bank.hacktivism.ch"))
|
||||
args = ap.parse_args()
|
||||
|
||||
tz_name = env("TZ", "Europe/Zurich") or "Europe/Zurich"
|
||||
os.environ["TZ"] = tz_name
|
||||
|
||||
admin_pass = args.admin_pass or (
|
||||
read_pass_file(args.admin_pass_file) if args.admin_pass_file else ""
|
||||
)
|
||||
explorer_pass = args.explorer_pass or (
|
||||
read_pass_file(args.explorer_pass_file) if args.explorer_pass_file else ""
|
||||
)
|
||||
|
||||
bank = args.bank.rstrip("/")
|
||||
skip = {s.strip() for s in (args.skip or "").split(",") if s.strip()}
|
||||
|
||||
def write_run(ok: bool, err: Optional[str] = None) -> None:
|
||||
if not args.run_out:
|
||||
return
|
||||
unix, iso, human = now_parts(tz_name)
|
||||
payload = {
|
||||
"ok": ok,
|
||||
"at": iso,
|
||||
"at_human": human,
|
||||
"error": err,
|
||||
}
|
||||
Path(args.run_out).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.run_out).write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
try:
|
||||
if not admin_pass:
|
||||
raise RuntimeError("no admin password (BANK_ADMIN_PASS or --admin-pass-file)")
|
||||
token = get_token(bank, args.admin_user, admin_pass)
|
||||
alt = load_alt_from_config(args.exchange_config)
|
||||
if not alt:
|
||||
alt = dict(DEFAULT_ALT)
|
||||
|
||||
accounts = list_all_accounts(bank, token, args.accounts_delta)
|
||||
if not accounts:
|
||||
raise RuntimeError("accounts list empty")
|
||||
|
||||
# ALL accounts except skip set (default: only exchange — avoid double-count)
|
||||
scan_users = [u for u in accounts if u not in skip]
|
||||
# ensure explorer is included when present
|
||||
if args.explorer_user not in skip and args.explorer_user in accounts:
|
||||
if args.explorer_user not in scan_users:
|
||||
scan_users.append(args.explorer_user)
|
||||
|
||||
withdraws: List[Dict[str, Any]] = []
|
||||
incomings: List[Dict[str, Any]] = []
|
||||
total_in = Decimal(0)
|
||||
total_wd = Decimal(0)
|
||||
total_other = Decimal(0)
|
||||
n_incoming = 0
|
||||
scan_ok = 0
|
||||
scan_empty = 0
|
||||
scan_fail = 0
|
||||
|
||||
for uname in scan_users:
|
||||
try:
|
||||
txs = iter_transactions(
|
||||
bank, token, uname, args.tx_page, args.max_tx_pages
|
||||
)
|
||||
except Exception:
|
||||
scan_fail += 1
|
||||
continue
|
||||
if not txs:
|
||||
scan_empty += 1
|
||||
continue
|
||||
scan_ok += 1
|
||||
for tx in txs:
|
||||
direction, amount, ts, subject = tx_fields(tx)
|
||||
if not amount:
|
||||
continue
|
||||
try:
|
||||
_cur, n = parse_amount(amount)
|
||||
except Exception:
|
||||
continue
|
||||
low = (subject or "").lower()
|
||||
if direction == "credit":
|
||||
total_in += n
|
||||
n_incoming += 1
|
||||
incomings.append(
|
||||
{
|
||||
"kind": "incoming",
|
||||
"amount": amount if ":" in amount else f"GOA:{amount}",
|
||||
"at_unix": ts,
|
||||
"account": uname,
|
||||
"subject": subject,
|
||||
}
|
||||
)
|
||||
elif direction == "debit" and "withdraw" in low:
|
||||
total_wd += n
|
||||
res = reserve_from_subject(subject)
|
||||
withdraws.append(
|
||||
{
|
||||
"kind": "withdraw",
|
||||
"amount": amount if ":" in amount else f"GOA:{amount}",
|
||||
"at_unix": ts,
|
||||
"account": uname,
|
||||
"subject": subject,
|
||||
"reserve": res,
|
||||
}
|
||||
)
|
||||
elif direction == "debit":
|
||||
total_other += n
|
||||
|
||||
withdraws.sort(key=lambda r: r.get("at_unix") or 0, reverse=True)
|
||||
incomings.sort(key=lambda r: r.get("at_unix") or 0, reverse=True)
|
||||
|
||||
now_u, gen_iso, gen_human = now_parts(tz_name)
|
||||
day = now_u - 86400
|
||||
week = now_u - 7 * 86400
|
||||
|
||||
w24 = Decimal(0)
|
||||
w7 = Decimal(0)
|
||||
n24 = 0
|
||||
n7 = 0
|
||||
for w in withdraws:
|
||||
ts = int(w.get("at_unix") or 0)
|
||||
try:
|
||||
_c, n = parse_amount(w.get("amount"))
|
||||
except Exception:
|
||||
n = Decimal(0)
|
||||
if ts >= day:
|
||||
w24 += n
|
||||
n24 += 1
|
||||
if ts >= week:
|
||||
w7 += n
|
||||
n7 += 1
|
||||
|
||||
wallets = sorted(
|
||||
{w.get("reserve") for w in withdraws if w.get("reserve")}
|
||||
)
|
||||
accounts_with_wd = sorted(
|
||||
{w.get("account") for w in withdraws if w.get("account")}
|
||||
)
|
||||
users_n = sum(1 for u in accounts if u not in ("admin", "exchange"))
|
||||
|
||||
def pack_amt(d: Decimal) -> Dict[str, Any]:
|
||||
f = format_amount_alt(f"GOA:{d}", alt)
|
||||
return {
|
||||
"amount": f["amount"],
|
||||
"amount_alt": f["amount_alt"],
|
||||
"amount_full": f["amount_full"],
|
||||
"value": f["value"],
|
||||
"value_str": f["value_str"],
|
||||
}
|
||||
|
||||
fin = pack_amt(total_in)
|
||||
fwd = pack_amt(total_wd)
|
||||
foth = pack_amt(total_other)
|
||||
fout = pack_amt(total_wd + total_other)
|
||||
p24 = pack_amt(w24)
|
||||
p7 = pack_amt(w7)
|
||||
|
||||
last = withdraws[0] if withdraws else None
|
||||
last_amt = None
|
||||
last_at = last_iso = last_subj = None
|
||||
last_unix = None
|
||||
last_alt = None
|
||||
if last:
|
||||
lf = format_amount_alt(last["amount"], alt)
|
||||
last_amt = lf["amount"]
|
||||
last_alt = lf["amount_alt"]
|
||||
last_unix = last.get("at_unix")
|
||||
last_at, last_iso = human_from_unix(int(last_unix or 0), tz_name)
|
||||
last_subj = last.get("subject")
|
||||
|
||||
recent = []
|
||||
for w in withdraws[: max(0, args.recent)]:
|
||||
f = format_amount_alt(w["amount"], alt)
|
||||
at_h, at_iso = human_from_unix(int(w.get("at_unix") or 0), tz_name)
|
||||
recent.append(
|
||||
{
|
||||
"kind": "withdraw",
|
||||
"amount": f["amount"],
|
||||
"amount_alt": f["amount_alt"],
|
||||
"amount_full": f["amount_full"],
|
||||
"at": at_h,
|
||||
"at_iso": at_iso,
|
||||
"at_unix": w.get("at_unix") or None,
|
||||
"account": w.get("account"),
|
||||
"reserve": w.get("reserve") or "",
|
||||
}
|
||||
)
|
||||
|
||||
recent_in = []
|
||||
for row in incomings[:5]:
|
||||
f = format_amount_alt(row["amount"], alt)
|
||||
at_h, at_iso = human_from_unix(int(row.get("at_unix") or 0), tz_name)
|
||||
recent_in.append(
|
||||
{
|
||||
"kind": "incoming",
|
||||
"amount": f["amount"],
|
||||
"amount_alt": f["amount_alt"],
|
||||
"amount_full": f["amount_full"],
|
||||
"at": at_h,
|
||||
"at_iso": at_iso,
|
||||
"at_unix": row.get("at_unix") or None,
|
||||
"account": row.get("account"),
|
||||
}
|
||||
)
|
||||
|
||||
# explorer balance
|
||||
balance = "GOA:0"
|
||||
bal_alt = "GOA:0"
|
||||
bal_full = "GOA:0"
|
||||
if explorer_pass:
|
||||
try:
|
||||
etok = get_token(bank, args.explorer_user, explorer_pass)
|
||||
code, data, _ = http_json(
|
||||
f"{bank}/accounts/{args.explorer_user}",
|
||||
headers={"Authorization": f"Bearer {etok}"},
|
||||
timeout=12,
|
||||
)
|
||||
if code == 200 and isinstance(data, dict):
|
||||
balance = str(
|
||||
data.get("balance", {}).get("amount")
|
||||
if isinstance(data.get("balance"), dict)
|
||||
else data.get("amount") or data.get("balance") or "GOA:0"
|
||||
)
|
||||
# sometimes balance is object with amount
|
||||
if isinstance(data.get("balance"), dict) and data["balance"].get(
|
||||
"amount"
|
||||
):
|
||||
balance = str(data["balance"]["amount"])
|
||||
bf = format_amount_alt(balance, alt)
|
||||
balance, bal_alt, bal_full = (
|
||||
bf["amount"],
|
||||
bf["amount_alt"],
|
||||
bf["amount_full"],
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
# try admin-read of explorer account
|
||||
try:
|
||||
code, data, _ = http_json(
|
||||
f"{bank}/accounts/{args.explorer_user}",
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
timeout=12,
|
||||
)
|
||||
if code == 200 and isinstance(data, dict):
|
||||
if isinstance(data.get("balance"), dict):
|
||||
balance = str(data["balance"].get("amount") or "GOA:0")
|
||||
elif data.get("amount"):
|
||||
balance = str(data["amount"])
|
||||
bf = format_amount_alt(balance, alt)
|
||||
balance, bal_alt, bal_full = (
|
||||
bf["amount"],
|
||||
bf["amount_alt"],
|
||||
bf["amount_full"],
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
pub = args.public_base.rstrip("/")
|
||||
config_ms, config_http = measure_ms(f"{pub}/config")
|
||||
int_ms, int_http = measure_ms(f"{pub}/taler-integration/config")
|
||||
webui_ms, webui_http = measure_ms(f"{pub}/webui/")
|
||||
if config_http != "200":
|
||||
config_ms, config_http = measure_ms(f"{bank}/config")
|
||||
if int_http != "200":
|
||||
int_ms, int_http = measure_ms(f"{bank}/taler-integration/config")
|
||||
|
||||
# loadavg filled later by host merge from *inside* the bank container
|
||||
# (host /proc would be wrong when this script runs on koopa outside podman)
|
||||
loadavg = ""
|
||||
|
||||
total_pack = pack_amt(total_wd)
|
||||
|
||||
stats = {
|
||||
"ok": True,
|
||||
"currency": "GOA",
|
||||
"timezone": tz_name,
|
||||
"generated_at": gen_iso,
|
||||
"generated_at_human": gen_human,
|
||||
"generated_at_unix": now_u,
|
||||
"source": "host collect_bank_stats.py (hernani systemd)",
|
||||
"bank_url": bank,
|
||||
"scan": {
|
||||
"tx_page": args.tx_page,
|
||||
"max_tx_pages": args.max_tx_pages,
|
||||
"accounts_delta": args.accounts_delta,
|
||||
"skip_users": sorted(skip),
|
||||
"accounts_listed": len(accounts),
|
||||
"accounts_scanned_ok": scan_ok,
|
||||
"accounts_empty_tx": scan_empty,
|
||||
"accounts_scan_fail": scan_fail,
|
||||
"accounts_scanned": len(scan_users),
|
||||
"note": "all accounts except skip_users; full tx pagination; empty includes HTTP 204",
|
||||
},
|
||||
"bank_accounts": {
|
||||
"total": len(accounts),
|
||||
"users": users_n,
|
||||
"with_withdraws": len(accounts_with_wd),
|
||||
},
|
||||
"wallets": {
|
||||
"unique_reserves": len(wallets),
|
||||
"note": "unique reserve pubs from Taler withdrawals (one per wallet withdraw)",
|
||||
},
|
||||
"balance_explorer": balance,
|
||||
"balance_explorer_alt": bal_alt,
|
||||
"balance_explorer_full": bal_full,
|
||||
"flow": {
|
||||
"incoming": {
|
||||
"label": "Incoming bank credits",
|
||||
"count": n_incoming,
|
||||
"amount": fin["amount"],
|
||||
"amount_alt": fin["amount_alt"],
|
||||
"amount_full": fin["amount_full"],
|
||||
"value": fin["value"],
|
||||
"value_str": fin["value_str"],
|
||||
},
|
||||
"withdraw": {
|
||||
"label": "Taler withdrawals to wallets",
|
||||
"count": len(withdraws),
|
||||
"amount": fwd["amount"],
|
||||
"amount_alt": fwd["amount_alt"],
|
||||
"amount_full": fwd["amount_full"],
|
||||
"value": fwd["value"],
|
||||
"value_str": fwd["value_str"],
|
||||
},
|
||||
"other_out": {
|
||||
"label": "Other debits (non-withdraw)",
|
||||
"amount": foth["amount"],
|
||||
"amount_alt": foth["amount_alt"],
|
||||
"amount_full": foth["amount_full"],
|
||||
"value": foth["value"],
|
||||
"value_str": foth["value_str"],
|
||||
},
|
||||
"total_in": fin["amount"],
|
||||
"total_in_alt": fin["amount_alt"],
|
||||
"total_in_value": fin["value"],
|
||||
"total_out": fout["amount"],
|
||||
"total_out_alt": fout["amount_alt"],
|
||||
"total_out_value": fout["value"],
|
||||
"note": "incoming=credits; withdraw=Taler withdrawal debits; skip_users excluded (default exchange)",
|
||||
},
|
||||
"withdraws": {
|
||||
"count": len(withdraws),
|
||||
"total_amount": total_pack["amount"],
|
||||
"total_amount_alt": total_pack["amount_alt"],
|
||||
"total_amount_full": total_pack["amount_full"],
|
||||
"total_value": total_pack["value"],
|
||||
"total_value_str": total_pack["value_str"],
|
||||
"last_amount": last_amt,
|
||||
"last_amount_alt": last_alt,
|
||||
"last_at": last_at,
|
||||
"last_at_iso": last_iso,
|
||||
"last_at_unix": last_unix,
|
||||
"last_subject": last_subj,
|
||||
"last_24h": {
|
||||
"count": n24,
|
||||
"amount": p24["amount"],
|
||||
"amount_alt": p24["amount_alt"],
|
||||
"amount_full": p24["amount_full"],
|
||||
"value": p24["value"],
|
||||
},
|
||||
"last_7d": {
|
||||
"count": n7,
|
||||
"amount": p7["amount"],
|
||||
"amount_alt": p7["amount_alt"],
|
||||
"amount_full": p7["amount_full"],
|
||||
"value": p7["value"],
|
||||
},
|
||||
},
|
||||
"recent_withdraws": recent,
|
||||
"recent_incoming": recent_in,
|
||||
"demo": {
|
||||
"uri": None,
|
||||
"amount": None,
|
||||
"created": None,
|
||||
"withdrawal_id": None,
|
||||
"status": None,
|
||||
"ready": False,
|
||||
},
|
||||
"performance": {
|
||||
"config_http": config_http,
|
||||
"config_ms": config_ms,
|
||||
"integration_http": int_http,
|
||||
"integration_ms": int_ms,
|
||||
"webui_http": webui_http,
|
||||
"webui_ms": webui_ms,
|
||||
"loadavg": loadavg,
|
||||
"memory": {
|
||||
"container_rss_human": "—",
|
||||
"container_rss_label": "—",
|
||||
"note": "filled by collect_container_resources.sh (in-container /proc + cgroup)",
|
||||
},
|
||||
},
|
||||
"alt_unit_names": alt,
|
||||
}
|
||||
|
||||
# pull demo withdraw files + memory from bank container if OUT is path and caller merged
|
||||
# demo files optional via DEMO_DIR
|
||||
demo_dir = env("DEMO_DIR", "")
|
||||
if demo_dir:
|
||||
dpath = Path(demo_dir)
|
||||
uri = (dpath / "withdraw.uri").read_text().strip() if (dpath / "withdraw.uri").is_file() else ""
|
||||
amt = (
|
||||
(dpath / "withdraw.amount").read_text().strip()
|
||||
if (dpath / "withdraw.amount").is_file()
|
||||
else ""
|
||||
)
|
||||
created = (
|
||||
(dpath / "withdraw.created").read_text().strip()
|
||||
if (dpath / "withdraw.created").is_file()
|
||||
else ""
|
||||
)
|
||||
if uri:
|
||||
wid = uri.rstrip("/").split("/")[-1]
|
||||
stats["demo"] = {
|
||||
"uri": uri,
|
||||
"amount": amt or None,
|
||||
"created": created or None,
|
||||
"withdrawal_id": wid,
|
||||
"status": None,
|
||||
"ready": True,
|
||||
}
|
||||
|
||||
stats = enrich_stats_tree(stats, alt)
|
||||
|
||||
text = json.dumps(stats, indent=2, ensure_ascii=False) + "\n"
|
||||
if args.out == "-" or not args.out:
|
||||
sys.stdout.write(text)
|
||||
else:
|
||||
outp = Path(args.out)
|
||||
outp.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = outp.with_suffix(outp.suffix + f".tmp.{os.getpid()}")
|
||||
tmp.write_text(text, encoding="utf-8")
|
||||
tmp.replace(outp)
|
||||
write_run(True, None)
|
||||
print(
|
||||
f"ok accounts={len(accounts)} scanned={scan_ok} withdraws={len(withdraws)} "
|
||||
f"total={total_pack['amount']} alt={total_pack['amount_alt']}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0
|
||||
except Exception as e:
|
||||
write_run(False, str(e))
|
||||
print(f"error: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
70
scripts/taler-landing/collect_container_resources.sh
Executable file
70
scripts/taler-landing/collect_container_resources.sh
Executable file
|
|
@ -0,0 +1,70 @@
|
|||
#!/usr/bin/env bash
|
||||
# Snapshot loadavg + RSS groups inside a podman container (for landing stats).
|
||||
# Usage: collect_container_resources.sh CONTAINER [OUT.json]
|
||||
# Prints JSON to stdout (and writes OUT when given).
|
||||
set -euo pipefail
|
||||
|
||||
CTR="${1:-}"
|
||||
OUT="${2:-}"
|
||||
if [ -z "$CTR" ]; then
|
||||
echo "usage: $0 CONTAINER [OUT.json]" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
ADMIN_LOG="${ADMIN_LOG:-${HOME}/src/koopa/koopa-admin-log}"
|
||||
MEM_SRC="${MEM_SNAPSHOT_SRC:-$ADMIN_LOG/scripts/taler-shared/mem-snapshot.sh}"
|
||||
MEM_DST="${MEM_SNAPSHOT_DST:-/usr/local/lib/landing-mem-snapshot.sh}"
|
||||
|
||||
if ! podman inspect -f '{{.State.Running}}' "$CTR" 2>/dev/null | grep -qx true; then
|
||||
echo "{\"ok\":false,\"error\":\"container not running: $CTR\"}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -f "$MEM_SRC" ]; then
|
||||
podman exec "$CTR" mkdir -p "$(dirname "$MEM_DST")" 2>/dev/null || true
|
||||
podman cp "$MEM_SRC" "${CTR}:${MEM_DST}"
|
||||
fi
|
||||
|
||||
# Run emit inside container (needs /proc + cgroup of that container)
|
||||
set +e
|
||||
raw=$(podman exec "$CTR" bash -c '
|
||||
set -e
|
||||
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
if [ ! -f /usr/local/lib/landing-mem-snapshot.sh ]; then
|
||||
echo "{\"ok\":false,\"error\":\"mem-snapshot helper missing\"}"
|
||||
exit 1
|
||||
fi
|
||||
# shellcheck disable=SC1091
|
||||
. /usr/local/lib/landing-mem-snapshot.sh
|
||||
if ! declare -F mem_snapshot_emit >/dev/null 2>&1; then
|
||||
# older helper without emit — synthesize
|
||||
mem_snapshot_json
|
||||
loadavg=""
|
||||
[ -r /proc/loadavg ] && loadavg=$(awk "{print \$1\",\"\$2\",\"\$3}" /proc/loadavg)
|
||||
printf "{\"ok\":true,\"source\":\"mem-snapshot-legacy\",\"loadavg\":%s,\"memory\":{%s}}\n" \
|
||||
"\"$loadavg\"" "$MEM_JSON"
|
||||
else
|
||||
mem_snapshot_emit
|
||||
fi
|
||||
' 2>/tmp/landing-mem-err.$$)
|
||||
ec=$?
|
||||
set -e
|
||||
|
||||
if [ "$ec" -ne 0 ] || [ -z "$raw" ]; then
|
||||
err=$(tr '\n' ' ' </tmp/landing-mem-err.$$ 2>/dev/null | head -c 200 || true)
|
||||
rm -f /tmp/landing-mem-err.$$
|
||||
printf '{"ok":false,"error":"podman exec failed: %s"}\n' "${err//\"/\'}"
|
||||
exit 1
|
||||
fi
|
||||
rm -f /tmp/landing-mem-err.$$
|
||||
|
||||
# Validate JSON
|
||||
if ! printf '%s' "$raw" | python3 -c 'import json,sys; json.load(sys.stdin)' 2>/dev/null; then
|
||||
printf '{"ok":false,"error":"invalid json from container"}\n'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -n "$OUT" ]; then
|
||||
printf '%s\n' "$raw" >"$OUT"
|
||||
fi
|
||||
printf '%s\n' "$raw"
|
||||
39
scripts/taler-landing/deploy-landings.sh
Normal file → Executable file
39
scripts/taler-landing/deploy-landings.sh
Normal file → Executable file
|
|
@ -14,10 +14,47 @@ ROOT=$(cd "$(dirname "$0")/../.." && pwd)
|
|||
# allow override when files already on host /tmp
|
||||
SRC_EX="${SRC_EX:-$ROOT/configs/exchange-landing}"
|
||||
SRC_MER="${SRC_MER:-$ROOT/configs/merchant-landing}"
|
||||
SRC_BANK="${SRC_BANK:-$ROOT/configs/bank-landing}"
|
||||
SRC_QR="${SRC_QR:-$ROOT/configs/bank-landing/qrcode.min.js}"
|
||||
SRC_GOA_AMT="${SRC_GOA_AMT:-$ROOT/configs/shared/goa-amount.js}"
|
||||
|
||||
C_EX=taler-hacktivism-exchange-ansible
|
||||
C_MER=taler-hacktivism
|
||||
C_BANK="${C_BANK:-taler-hacktivism-bank}"
|
||||
|
||||
# Prefer per-landing copy, then shared (keeps bank/exchange/merchant in sync)
|
||||
resolve_goa_amount() {
|
||||
local local_copy="$1"
|
||||
if [ -f "$local_copy" ]; then
|
||||
printf '%s' "$local_copy"
|
||||
elif [ -f "$SRC_GOA_AMT" ]; then
|
||||
printf '%s' "$SRC_GOA_AMT"
|
||||
else
|
||||
printf ''
|
||||
fi
|
||||
}
|
||||
|
||||
copy_goa_amount() {
|
||||
local ctr="$1" dest="$2" src="$3"
|
||||
src=$(resolve_goa_amount "$src")
|
||||
if [ -n "$src" ] && [ -f "$src" ]; then
|
||||
podman cp "$src" "${ctr}:${dest}/goa-amount.js"
|
||||
echo " goa-amount.js ← $src"
|
||||
else
|
||||
echo " WARN: goa-amount.js missing for $ctr"
|
||||
fi
|
||||
}
|
||||
|
||||
echo "=== bank landing → $C_BANK :9013 (html + goa-amount) ==="
|
||||
if podman inspect -f '{{.State.Running}}' "$C_BANK" 2>/dev/null | grep -qx true; then
|
||||
podman exec "$C_BANK" mkdir -p /var/www/bank-landing
|
||||
if [ -f "$SRC_BANK/index.html" ]; then
|
||||
podman cp "$SRC_BANK/index.html" "$C_BANK:/var/www/bank-landing/index.html"
|
||||
fi
|
||||
copy_goa_amount "$C_BANK" /var/www/bank-landing "$SRC_BANK/goa-amount.js"
|
||||
else
|
||||
echo "WARN: $C_BANK not running — skip bank landing files"
|
||||
fi
|
||||
|
||||
echo "=== exchange landing → $C_EX :9014 ==="
|
||||
# nginx may be missing on exchange image
|
||||
|
|
@ -30,6 +67,7 @@ podman cp "$SRC_EX/index.html" "$C_EX:/var/www/exchange-landing/index.html"
|
|||
if [ -f "$SRC_QR" ]; then
|
||||
podman cp "$SRC_QR" "$C_EX:/var/www/exchange-landing/qrcode.min.js"
|
||||
fi
|
||||
copy_goa_amount "$C_EX" /var/www/exchange-landing "$SRC_EX/goa-amount.js"
|
||||
podman cp "$SRC_EX/nginx-landing.conf" "$C_EX:/etc/nginx/sites-available/exchange-landing"
|
||||
podman exec "$C_EX" bash -c '
|
||||
ln -sfn /etc/nginx/sites-available/exchange-landing /etc/nginx/sites-enabled/exchange-landing
|
||||
|
|
@ -50,6 +88,7 @@ podman exec "$C_EX" bash -c '
|
|||
echo "=== merchant landing → $C_MER :9015 ==="
|
||||
podman exec "$C_MER" mkdir -p /var/www/merchant-landing
|
||||
podman cp "$SRC_MER/index.html" "$C_MER:/var/www/merchant-landing/index.html"
|
||||
copy_goa_amount "$C_MER" /var/www/merchant-landing "$SRC_MER/goa-amount.js"
|
||||
podman cp "$SRC_MER/nginx-landing.conf" "$C_MER:/etc/nginx/sites-available/merchant-landing"
|
||||
podman exec "$C_MER" bash -c '
|
||||
ln -sfn /etc/nginx/sites-available/merchant-landing /etc/nginx/sites-enabled/merchant-landing
|
||||
|
|
|
|||
40
scripts/taler-landing/enrich_stats_alt.py
Executable file
40
scripts/taler-landing/enrich_stats_alt.py
Executable file
|
|
@ -0,0 +1,40 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Add amount_alt / amount_full fields to a landing stats.json in place."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from goa_amounts import DEFAULT_ALT, enrich_stats_tree, load_alt_from_config # noqa: E402
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("path", help="stats.json path")
|
||||
ap.add_argument(
|
||||
"--exchange-config",
|
||||
default="https://exchange.hacktivism.ch/config",
|
||||
)
|
||||
ap.add_argument("-o", "--out", default="", help="default: overwrite path")
|
||||
args = ap.parse_args()
|
||||
p = Path(args.path)
|
||||
data = json.loads(p.read_text(encoding="utf-8"))
|
||||
if not isinstance(data, dict) or not data.get("ok"):
|
||||
print("skip: not ok stats", file=sys.stderr)
|
||||
return 0
|
||||
alt = load_alt_from_config(args.exchange_config) or dict(DEFAULT_ALT)
|
||||
enrich_stats_tree(data, alt)
|
||||
out = Path(args.out) if args.out else p
|
||||
tmp = out.with_suffix(out.suffix + f".tmp.{os.getpid()}")
|
||||
tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
tmp.replace(out)
|
||||
print(f"enriched {out}", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
318
scripts/taler-landing/goa_amounts.py
Executable file
318
scripts/taler-landing/goa_amounts.py
Executable file
|
|
@ -0,0 +1,318 @@
|
|||
#!/usr/bin/env python3
|
||||
"""GOA (and generic CUR:amount) alt-unit formatting for landing stats.
|
||||
|
||||
Matches exchange currency_specification.alt_unit_names:
|
||||
0→GOA, 3→Kilo-GOA, 6→Mega-GOA, … and fractional scales.
|
||||
Display prefers compact alt names for large values so landing tiles fit.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
# Default GOA ladder (same shape as exchange /config)
|
||||
DEFAULT_ALT: Dict[str, str] = {
|
||||
"24": "Yotta-GOA",
|
||||
"21": "Zetta-GOA",
|
||||
"18": "Exa-GOA",
|
||||
"15": "Peta-GOA",
|
||||
"12": "Tera-GOA",
|
||||
"9": "Giga-GOA",
|
||||
"6": "Mega-GOA",
|
||||
"3": "Kilo-GOA",
|
||||
"0": "GOA",
|
||||
"-1": "Deci-GOA",
|
||||
"-2": "Centi-GOA",
|
||||
"-3": "Milli-GOA",
|
||||
"-6": "Micro-GOA",
|
||||
"-7": "Deci-Micro-GOA",
|
||||
"-8": "Atomic-GOA",
|
||||
}
|
||||
|
||||
# Use alt unit when |value| >= this (base units). Below: keep compact CUR:n.
|
||||
ALT_THRESHOLD = Decimal("1000")
|
||||
|
||||
|
||||
def parse_amount(s: Any) -> Tuple[str, Decimal]:
|
||||
"""Parse 'GOA:12.5' or bare number → (currency, value)."""
|
||||
if s is None:
|
||||
return "GOA", Decimal(0)
|
||||
if isinstance(s, (int, float, Decimal)):
|
||||
return "GOA", Decimal(str(s))
|
||||
text = str(s).strip()
|
||||
if not text:
|
||||
return "GOA", Decimal(0)
|
||||
if ":" in text:
|
||||
cur, rest = text.split(":", 1)
|
||||
return (cur or "GOA").strip(), Decimal(rest.strip() or "0")
|
||||
return "GOA", Decimal(text)
|
||||
|
||||
|
||||
def fmt_coeff(v: Decimal) -> str:
|
||||
if v == v.to_integral_value():
|
||||
return format(int(v), "d")
|
||||
# up to 4 significant fractional digits, strip trailing zeros
|
||||
q = v.quantize(Decimal("0.0001"), rounding=ROUND_HALF_UP)
|
||||
s = format(q, "f").rstrip("0").rstrip(".")
|
||||
return s or "0"
|
||||
|
||||
|
||||
def fmt_base(cur: str, val: Decimal) -> str:
|
||||
if val == val.to_integral_value():
|
||||
return f"{cur}:{int(val)}"
|
||||
# preserve up to 8 fractional digits (Taler style)
|
||||
s = format(val, "f").rstrip("0").rstrip(".")
|
||||
return f"{cur}:{s}"
|
||||
|
||||
|
||||
def format_amount_alt(
|
||||
amount: Any,
|
||||
alt: Optional[Dict[str, str]] = None,
|
||||
*,
|
||||
threshold: Decimal = ALT_THRESHOLD,
|
||||
with_base: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
"""Return display fields for one amount.
|
||||
|
||||
Keys:
|
||||
amount canonical CUR:n
|
||||
amount_alt short display (e.g. "1.23 Mega-GOA")
|
||||
amount_full "1.23 Mega-GOA (GOA:1230000)" when alt used
|
||||
value float for JSON (may lose precision above 2^53 — also value_str)
|
||||
value_str exact decimal string
|
||||
"""
|
||||
alt = alt or DEFAULT_ALT
|
||||
try:
|
||||
cur, val = parse_amount(amount)
|
||||
except (InvalidOperation, ValueError, ArithmeticError):
|
||||
raw = str(amount or "")
|
||||
return {
|
||||
"amount": raw,
|
||||
"amount_alt": raw or "—",
|
||||
"amount_full": raw or "—",
|
||||
"value": 0.0,
|
||||
"value_str": "0",
|
||||
}
|
||||
|
||||
base = fmt_base(cur, val)
|
||||
value_str = format(val, "f").rstrip("0").rstrip(".") if val != val.to_integral_value() else str(int(val))
|
||||
try:
|
||||
value_f = float(val)
|
||||
except Exception:
|
||||
value_f = 0.0
|
||||
|
||||
base_name = alt.get("0") or cur
|
||||
# Never rename foreign currencies with a GOA alt map (merchant dual CHF+GOA).
|
||||
cur_u = (cur or "").upper()
|
||||
base_u = str(base_name).upper()
|
||||
if cur_u and cur_u != "GOA" and (base_u == "GOA" or base_u.endswith("-GOA") or "GOA" in base_u):
|
||||
return {
|
||||
"amount": base,
|
||||
"amount_alt": base,
|
||||
"amount_full": base,
|
||||
"value": value_f,
|
||||
"value_str": value_str,
|
||||
}
|
||||
|
||||
if val == 0:
|
||||
# Keep canonical CUR:0 (avoids "0 GOA" for empty foreign balances)
|
||||
return {
|
||||
"amount": base,
|
||||
"amount_alt": base,
|
||||
"amount_full": base,
|
||||
"value": 0.0,
|
||||
"value_str": "0",
|
||||
}
|
||||
|
||||
scales = []
|
||||
for k, name in alt.items():
|
||||
try:
|
||||
scales.append((int(k), str(name)))
|
||||
except Exception:
|
||||
continue
|
||||
scales.sort(key=lambda x: -x[0])
|
||||
|
||||
absval = abs(val)
|
||||
# Below threshold: short base form (saves noise on small demo amounts)
|
||||
if absval < threshold:
|
||||
# Prefer "GOA:12.5" style (matches existing landings) when small
|
||||
return {
|
||||
"amount": base,
|
||||
"amount_alt": base,
|
||||
"amount_full": base,
|
||||
"value": value_f,
|
||||
"value_str": value_str,
|
||||
}
|
||||
|
||||
chosen = None
|
||||
for sc, name in scales:
|
||||
unit = Decimal(10) ** sc
|
||||
if unit <= 0:
|
||||
continue
|
||||
coeff = absval / unit
|
||||
if coeff >= 1:
|
||||
chosen = (sc, name, coeff if val >= 0 else -coeff)
|
||||
break
|
||||
|
||||
if chosen is None or chosen[0] == 0:
|
||||
return {
|
||||
"amount": base,
|
||||
"amount_alt": base,
|
||||
"amount_full": base,
|
||||
"value": value_f,
|
||||
"value_str": value_str,
|
||||
}
|
||||
|
||||
sc, name, coeff = chosen
|
||||
short = f"{fmt_coeff(coeff)} {name}"
|
||||
full = f"{short} ({base})" if with_base or True else short
|
||||
return {
|
||||
"amount": base,
|
||||
"amount_alt": short,
|
||||
"amount_full": full,
|
||||
"value": value_f,
|
||||
"value_str": value_str,
|
||||
}
|
||||
|
||||
|
||||
def attach_alt(obj: Dict[str, Any], amount_key: str = "amount", alt: Optional[Dict[str, str]] = None) -> Dict[str, Any]:
|
||||
"""Mutate obj: ensure amount_alt / amount_full from amount_key."""
|
||||
if not isinstance(obj, dict):
|
||||
return obj
|
||||
src = obj.get(amount_key)
|
||||
if src is None:
|
||||
return obj
|
||||
f = format_amount_alt(src, alt)
|
||||
obj[amount_key] = f["amount"]
|
||||
obj["amount_alt"] = f["amount_alt"]
|
||||
obj["amount_full"] = f["amount_full"]
|
||||
if "value" not in obj:
|
||||
obj["value"] = f["value"]
|
||||
obj["value_str"] = f["value_str"]
|
||||
return obj
|
||||
|
||||
|
||||
def load_alt_from_config(url: str, timeout: float = 8.0) -> Dict[str, str]:
|
||||
"""GET exchange/bank /config → alt_unit_names (fallback DEFAULT_ALT)."""
|
||||
import json
|
||||
import urllib.request
|
||||
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"Accept": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8", errors="replace"))
|
||||
except Exception:
|
||||
return dict(DEFAULT_ALT)
|
||||
|
||||
au = None
|
||||
cs = data.get("currency_specification")
|
||||
if isinstance(cs, dict):
|
||||
au = cs.get("alt_unit_names")
|
||||
if not au and isinstance(data.get("currencies"), dict):
|
||||
for _code, spec in data["currencies"].items():
|
||||
if isinstance(spec, dict) and spec.get("alt_unit_names"):
|
||||
au = spec["alt_unit_names"]
|
||||
break
|
||||
if not isinstance(au, dict) or "0" not in au:
|
||||
return dict(DEFAULT_ALT)
|
||||
return {str(k): str(v) for k, v in au.items()}
|
||||
|
||||
|
||||
def enrich_stats_tree(data: Dict[str, Any], alt: Optional[Dict[str, str]] = None) -> Dict[str, Any]:
|
||||
"""Walk common landing stats.json shapes and add amount_alt fields."""
|
||||
alt = alt or DEFAULT_ALT
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
|
||||
# bank-style
|
||||
for path in (
|
||||
("flow", "incoming"),
|
||||
("flow", "withdraw"),
|
||||
("flow", "other_out"),
|
||||
("withdraws",),
|
||||
("withdraws", "last_24h"),
|
||||
("withdraws", "last_7d"),
|
||||
):
|
||||
cur: Any = data
|
||||
ok = True
|
||||
for p in path:
|
||||
if not isinstance(cur, dict) or p not in cur:
|
||||
ok = False
|
||||
break
|
||||
cur = cur[p]
|
||||
if ok and isinstance(cur, dict):
|
||||
if "amount" in cur:
|
||||
attach_alt(cur, "amount", alt)
|
||||
if "total_amount" in cur:
|
||||
f = format_amount_alt(cur["total_amount"], alt)
|
||||
cur["total_amount"] = f["amount"]
|
||||
cur["total_amount_alt"] = f["amount_alt"]
|
||||
cur["total_amount_full"] = f["amount_full"]
|
||||
if "last_amount" in cur and cur["last_amount"]:
|
||||
f = format_amount_alt(cur["last_amount"], alt)
|
||||
cur["last_amount"] = f["amount"]
|
||||
cur["last_amount_alt"] = f["amount_alt"]
|
||||
|
||||
if isinstance(data.get("flow"), dict):
|
||||
fl = data["flow"]
|
||||
for k in ("total_in", "total_out"):
|
||||
if fl.get(k):
|
||||
f = format_amount_alt(fl[k], alt)
|
||||
fl[k] = f["amount"]
|
||||
fl[f"{k}_alt"] = f["amount_alt"]
|
||||
fl[f"{k}_full"] = f["amount_full"]
|
||||
|
||||
if data.get("balance_explorer"):
|
||||
f = format_amount_alt(data["balance_explorer"], alt)
|
||||
data["balance_explorer"] = f["amount"]
|
||||
data["balance_explorer_alt"] = f["amount_alt"]
|
||||
data["balance_explorer_full"] = f["amount_full"]
|
||||
|
||||
for key in ("recent_withdraws", "recent_incoming", "recent_activity"):
|
||||
rows = data.get(key)
|
||||
if isinstance(rows, list):
|
||||
for row in rows:
|
||||
if isinstance(row, dict) and row.get("amount"):
|
||||
attach_alt(row, "amount", alt)
|
||||
|
||||
# exchange-style top-level amounts
|
||||
for k in (
|
||||
"wire_in_amount",
|
||||
"withdraw_amount",
|
||||
"coins_remaining_amount",
|
||||
):
|
||||
if data.get(k):
|
||||
f = format_amount_alt(data[k], alt)
|
||||
data[k] = f["amount"]
|
||||
data[f"{k}_alt"] = f["amount_alt"]
|
||||
data[f"{k}_full"] = f["amount_full"]
|
||||
|
||||
for key in ("by_denom", "denom_ladder"):
|
||||
rows = data.get(key)
|
||||
if isinstance(rows, list):
|
||||
for row in rows:
|
||||
if isinstance(row, dict) and row.get("value"):
|
||||
f = format_amount_alt(row["value"], alt)
|
||||
row["value"] = f["amount"]
|
||||
row["value_alt"] = f["amount_alt"]
|
||||
|
||||
# merchant dual currency
|
||||
for block in data.get("by_currency") or []:
|
||||
if not isinstance(block, dict):
|
||||
continue
|
||||
for k in ("amount_sum", "amount_paid_sum"):
|
||||
if block.get(k):
|
||||
f = format_amount_alt(block[k], alt)
|
||||
block[k] = f["amount"]
|
||||
block[f"{k}_alt"] = f["amount_alt"]
|
||||
block[f"{k}_full"] = f["amount_full"]
|
||||
|
||||
for block in data.get("recent_activity_by_currency") or []:
|
||||
if not isinstance(block, dict):
|
||||
continue
|
||||
for row in block.get("items") or []:
|
||||
if isinstance(row, dict) and row.get("amount"):
|
||||
attach_alt(row, "amount", alt)
|
||||
|
||||
data["alt_unit_names"] = alt
|
||||
return data
|
||||
68
scripts/taler-landing/install-landing-stats-host.sh
Executable file
68
scripts/taler-landing/install-landing-stats-host.sh
Executable file
|
|
@ -0,0 +1,68 @@
|
|||
#!/usr/bin/env bash
|
||||
# Install central landing-stats collector as hernani user systemd timer.
|
||||
#
|
||||
# On koopa:
|
||||
# cd ~/src/koopa/koopa-admin-log
|
||||
# ./scripts/taler-landing/install-landing-stats-host.sh
|
||||
# systemctl --user start taler-landing-stats.service # one-shot now
|
||||
# systemctl --user status taler-landing-stats.timer
|
||||
#
|
||||
# Requires: linger enabled for hernani (loginctl enable-linger hernani)
|
||||
# so the timer runs without an interactive login.
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
ADMIN_LOG="${ADMIN_LOG:-$ROOT}"
|
||||
SRC="$ADMIN_LOG/scripts/taler-landing"
|
||||
UNIT_SRC="$ADMIN_LOG/configs/systemd/user"
|
||||
|
||||
BIN_DST="${HOME}/.local/bin"
|
||||
LIB_DST="${HOME}/.local/lib/taler-landing"
|
||||
UNIT_DST="${HOME}/.config/systemd/user"
|
||||
STATE_DST="${HOME}/.local/state/taler-landing-stats"
|
||||
|
||||
mkdir -p "$BIN_DST" "$LIB_DST" "$UNIT_DST" "$STATE_DST"
|
||||
|
||||
install -m 0755 "$SRC/collect-landing-stats.sh" "$BIN_DST/collect-landing-stats.sh"
|
||||
install -m 0755 "$SRC/collect_bank_stats.py" "$LIB_DST/collect_bank_stats.py"
|
||||
install -m 0755 "$SRC/enrich_stats_alt.py" "$LIB_DST/enrich_stats_alt.py"
|
||||
install -m 0755 "$SRC/merge_resources.py" "$LIB_DST/merge_resources.py"
|
||||
install -m 0755 "$SRC/collect_container_resources.sh" "$LIB_DST/collect_container_resources.sh"
|
||||
install -m 0755 "$SRC/test-landing-stats.sh" "$BIN_DST/test-landing-stats.sh" 2>/dev/null || \
|
||||
install -m 0755 "$SRC/test-landing-stats.sh" "$LIB_DST/test-landing-stats.sh"
|
||||
install -m 0644 "$SRC/goa_amounts.py" "$LIB_DST/goa_amounts.py"
|
||||
# mem-snapshot helper lives in taler-shared (copied into containers at collect time)
|
||||
if [ -f "$ADMIN_LOG/scripts/taler-shared/mem-snapshot.sh" ]; then
|
||||
install -m 0644 "$ADMIN_LOG/scripts/taler-shared/mem-snapshot.sh" \
|
||||
"$LIB_DST/mem-snapshot.sh"
|
||||
fi
|
||||
|
||||
# Wrapper always uses installed lib next to itself when LIB discovery works;
|
||||
# also point ADMIN_LOG for in-container script refresh.
|
||||
install -m 0644 "$UNIT_SRC/taler-landing-stats.service" "$UNIT_DST/"
|
||||
install -m 0644 "$UNIT_SRC/taler-landing-stats.timer" "$UNIT_DST/"
|
||||
|
||||
# Rewrite ExecStart to installed binary if unit uses %h path — units already do.
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user enable --now taler-landing-stats.timer
|
||||
|
||||
echo "Installed (user=$(id -un)):"
|
||||
echo " $BIN_DST/collect-landing-stats.sh"
|
||||
echo " $LIB_DST/{collect_bank_stats,enrich_stats_alt,goa_amounts}.py"
|
||||
echo " $UNIT_DST/taler-landing-stats.{service,timer} (timer enabled)"
|
||||
echo " logs: $STATE_DST/"
|
||||
echo
|
||||
if ! loginctl show-user "$(id -un)" -p Linger 2>/dev/null | grep -q 'Linger=yes'; then
|
||||
echo "NOTE: enable linger so the timer survives logout:"
|
||||
echo " sudo loginctl enable-linger $(id -un)"
|
||||
echo
|
||||
fi
|
||||
echo "Run once now:"
|
||||
echo " systemctl --user start taler-landing-stats.service"
|
||||
echo " journalctl --user -u taler-landing-stats.service -n 40 --no-pager"
|
||||
echo
|
||||
echo "Optional secrets (if not readable via podman exec bank /root/…):"
|
||||
echo " mkdir -p ~/.config/taler-landing"
|
||||
echo " cp …/bank-admin-password.txt ~/.config/taler-landing/"
|
||||
echo " cp …/bank-explorer-password.txt ~/.config/taler-landing/"
|
||||
echo " chmod 600 ~/.config/taler-landing/*"
|
||||
106
scripts/taler-landing/merge_resources.py
Executable file
106
scripts/taler-landing/merge_resources.py
Executable file
|
|
@ -0,0 +1,106 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Merge container resource snapshot into landing stats.json performance block.
|
||||
|
||||
Usage:
|
||||
merge_resources.py STATS.json RESOURCES.json [-o OUT.json]
|
||||
|
||||
RESOURCES shape (from mem_snapshot_emit / collect_container_resources.sh):
|
||||
{ "ok": true, "loadavg": "0.1,0.2,0.3", "memory": { ... } }
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
|
||||
|
||||
def compact_memory(mem: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Ensure human labels + short top cmd for UI width."""
|
||||
if not isinstance(mem, dict):
|
||||
return mem
|
||||
# Prefer explicit label (with limit); else human
|
||||
if not mem.get("container_rss_label"):
|
||||
h = mem.get("container_rss_human") or "—"
|
||||
lim = mem.get("cgroup_limit_human")
|
||||
if lim:
|
||||
mem["container_rss_label"] = f"{h} / {lim}"
|
||||
else:
|
||||
mem["container_rss_label"] = h
|
||||
tops = mem.get("top")
|
||||
if isinstance(tops, list):
|
||||
for t in tops:
|
||||
if not isinstance(t, dict):
|
||||
continue
|
||||
cmd = str(t.get("cmd") or "")
|
||||
if len(cmd) > 72:
|
||||
t["cmd_full"] = cmd
|
||||
t["cmd"] = cmd[:69] + "…"
|
||||
# ensure human present
|
||||
if not t.get("rss_human") and t.get("rss_bytes") is not None:
|
||||
try:
|
||||
b = int(t["rss_bytes"])
|
||||
if b < 1024:
|
||||
t["rss_human"] = f"{b} B"
|
||||
elif b < 1048576:
|
||||
t["rss_human"] = f"{b/1024:.1f} KiB"
|
||||
elif b < 1073741824:
|
||||
t["rss_human"] = f"{b/1048576:.1f} MiB"
|
||||
else:
|
||||
t["rss_human"] = f"{b/1073741824:.2f} GiB"
|
||||
except Exception:
|
||||
pass
|
||||
return mem
|
||||
|
||||
|
||||
def merge(stats: Dict[str, Any], resources: Dict[str, Any]) -> Dict[str, Any]:
|
||||
perf = stats.setdefault("performance", {})
|
||||
if not isinstance(perf, dict):
|
||||
perf = {}
|
||||
stats["performance"] = perf
|
||||
|
||||
if resources.get("loadavg"):
|
||||
perf["loadavg"] = resources["loadavg"]
|
||||
perf["loadavg_source"] = "container"
|
||||
|
||||
mem = resources.get("memory")
|
||||
if isinstance(mem, dict) and mem:
|
||||
perf["memory"] = compact_memory(dict(mem))
|
||||
perf["memory"]["source"] = resources.get("source") or "mem-snapshot"
|
||||
return stats
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("stats")
|
||||
ap.add_argument("resources")
|
||||
ap.add_argument("-o", "--out", default="")
|
||||
args = ap.parse_args()
|
||||
|
||||
stats = json.loads(Path(args.stats).read_text(encoding="utf-8"))
|
||||
resources = json.loads(Path(args.resources).read_text(encoding="utf-8"))
|
||||
if not isinstance(stats, dict) or not stats.get("ok"):
|
||||
print("skip: stats not ok", file=sys.stderr)
|
||||
return 1
|
||||
if not isinstance(resources, dict) or resources.get("ok") is False:
|
||||
print("skip: resources not ok", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
merge(stats, resources)
|
||||
out = Path(args.out) if args.out else Path(args.stats)
|
||||
tmp = out.with_suffix(out.suffix + f".tmp.{os.getpid()}")
|
||||
tmp.write_text(json.dumps(stats, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
tmp.replace(out)
|
||||
mem = (stats.get("performance") or {}).get("memory") or {}
|
||||
print(
|
||||
f"merged memory container={mem.get('container_rss_human')} "
|
||||
f"pg={mem.get('postgres_rss_human')} loadavg={stats.get('performance', {}).get('loadavg')}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
249
scripts/taler-landing/test-landing-stats.sh
Executable file
249
scripts/taler-landing/test-landing-stats.sh
Executable file
|
|
@ -0,0 +1,249 @@
|
|||
#!/usr/bin/env bash
|
||||
# Outside-in checks for bank / exchange / merchant landing stats + alt UI assets.
|
||||
# Usage: ./test-landing-stats.sh
|
||||
# Exit 0 only if all critical checks pass.
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
PY="${PYTHON:-python3}"
|
||||
BANK="${BANK_PUBLIC:-https://bank.hacktivism.ch}"
|
||||
EX="${EXCHANGE_PUBLIC:-https://exchange.hacktivism.ch}"
|
||||
MER="${MERCHANT_PUBLIC:-https://taler.hacktivism.ch}"
|
||||
|
||||
fail=0
|
||||
pass=0
|
||||
note() { printf ' %s\n' "$*"; }
|
||||
ok() { pass=$((pass + 1)); printf ' [OK] %s\n' "$*"; }
|
||||
bad() { fail=$((fail + 1)); printf ' [FAIL] %s\n' "$*"; }
|
||||
|
||||
http_code() {
|
||||
curl -skS -m 15 -o /tmp/landing-test.body -w '%{http_code}' "$1" 2>/dev/null || echo 000
|
||||
}
|
||||
|
||||
echo "=== local repo checks ==="
|
||||
for f in \
|
||||
configs/shared/goa-amount.js \
|
||||
configs/bank-landing/goa-amount.js \
|
||||
configs/exchange-landing/goa-amount.js \
|
||||
configs/merchant-landing/goa-amount.js \
|
||||
configs/bank-landing/index.html \
|
||||
configs/exchange-landing/index.html \
|
||||
configs/merchant-landing/index.html \
|
||||
scripts/taler-landing/goa_amounts.py \
|
||||
scripts/taler-landing/collect_bank_stats.py
|
||||
do
|
||||
if [ -f "$ROOT/$f" ]; then ok "file $f"
|
||||
else bad "missing $f"
|
||||
fi
|
||||
done
|
||||
|
||||
for html in bank-landing exchange-landing merchant-landing; do
|
||||
if grep -q 'goa-amount.js' "$ROOT/configs/$html/index.html" \
|
||||
&& grep -q 'GoaAmount' "$ROOT/configs/$html/index.html"; then
|
||||
ok "$html index wires GoaAmount"
|
||||
else
|
||||
bad "$html index missing goa-amount / GoaAmount"
|
||||
fi
|
||||
if grep -q 'Fallback if goa-amount.js' "$ROOT/configs/$html/index.html"; then
|
||||
ok "$html has offline fallback"
|
||||
else
|
||||
bad "$html missing offline GoaAmount fallback"
|
||||
fi
|
||||
done
|
||||
|
||||
echo
|
||||
echo "=== unit: goa_amounts enrich (all three shapes) ==="
|
||||
if "$PY" - <<PY
|
||||
import sys
|
||||
sys.path.insert(0, "$ROOT/scripts/taler-landing")
|
||||
from goa_amounts import enrich_stats_tree, DEFAULT_ALT
|
||||
|
||||
bank = {
|
||||
"ok": True,
|
||||
"balance_explorer": "GOA:40614.67",
|
||||
"withdraws": {"total_amount": "GOA:120944.66", "last_24h": {"amount": "GOA:9494.66"}},
|
||||
"flow": {"withdraw": {"amount": "GOA:120944.66"}, "incoming": {"amount": "GOA:1000"}},
|
||||
"recent_withdraws": [{"amount": "GOA:4503599627370495"}],
|
||||
}
|
||||
enrich_stats_tree(bank, DEFAULT_ALT)
|
||||
assert "Kilo-GOA" in bank["withdraws"]["total_amount_alt"]
|
||||
assert "Peta-GOA" in bank["recent_withdraws"][0]["amount_alt"]
|
||||
|
||||
ex = {
|
||||
"ok": True,
|
||||
"wire_in_amount": "GOA:1267008731299764.85",
|
||||
"withdraw_amount": "GOA:1783661.57",
|
||||
"coins_remaining_amount": "GOA:0",
|
||||
"by_denom": [{"value": "GOA:1000"}],
|
||||
}
|
||||
enrich_stats_tree(ex, DEFAULT_ALT)
|
||||
assert "Peta-GOA" in ex["wire_in_amount_alt"]
|
||||
assert "Mega-GOA" in ex["withdraw_amount_alt"]
|
||||
assert ex["coins_remaining_amount_alt"] in ("GOA:0", "0 GOA")
|
||||
|
||||
mer = {
|
||||
"ok": True,
|
||||
"by_currency": [
|
||||
{"currency": "GOA", "amount_sum": "GOA:294.05", "amount_paid_sum": "GOA:294.05"},
|
||||
{"currency": "CHF", "amount_sum": "CHF:64000", "amount_paid_sum": "CHF:64000"},
|
||||
],
|
||||
"recent_activity_by_currency": [
|
||||
{"currency": "GOA", "items": [{"amount": "GOA:2500"}]},
|
||||
],
|
||||
}
|
||||
enrich_stats_tree(mer, DEFAULT_ALT)
|
||||
# CHF must NOT become Kilo-GOA
|
||||
assert "GOA" not in mer["by_currency"][1]["amount_paid_sum_alt"] or mer["by_currency"][1]["amount_paid_sum_alt"].startswith("CHF")
|
||||
assert mer["by_currency"][1]["amount_paid_sum_alt"].startswith("CHF")
|
||||
assert "Kilo-GOA" in mer["recent_activity_by_currency"][0]["items"][0]["amount_alt"]
|
||||
print("unit ok")
|
||||
PY
|
||||
then ok "enrich shapes bank/exchange/merchant"
|
||||
else bad "enrich unit failed"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "=== unit: merge_resources + mem labels ==="
|
||||
if "$PY" - <<PY
|
||||
import json, sys, tempfile, os
|
||||
sys.path.insert(0, "$ROOT/scripts/taler-landing")
|
||||
# import as script module path
|
||||
import importlib.util
|
||||
spec = importlib.util.spec_from_file_location("merge_resources", "$ROOT/scripts/taler-landing/merge_resources.py")
|
||||
mr = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mr)
|
||||
|
||||
stats = {"ok": True, "performance": {"config_ms": 12, "memory": {"container_rss_human": "—"}}}
|
||||
res = {
|
||||
"ok": True,
|
||||
"loadavg": "0.5,0.4,0.3",
|
||||
"source": "mem-snapshot",
|
||||
"memory": {
|
||||
"container_rss_bytes": 726663168,
|
||||
"container_rss_human": "693.4 MiB",
|
||||
"cgroup_limit_bytes": 2147483648,
|
||||
"cgroup_limit_human": "2.00 GiB",
|
||||
"postgres_rss_human": "281.1 MiB",
|
||||
"postgres_n": 17,
|
||||
"java_rss_human": "353.7 MiB",
|
||||
"java_n": 2,
|
||||
"top": [{"rss_bytes": 367312896, "rss_human": "350.3 MiB", "comm": "java",
|
||||
"cmd": "java -classpath /usr/lib/libeufin-bank-all.jar " + ("x" * 120)}],
|
||||
},
|
||||
}
|
||||
mr.merge(stats, res)
|
||||
mem = stats["performance"]["memory"]
|
||||
assert mem["container_rss_label"].startswith("693.4 MiB")
|
||||
assert "/ 2.00 GiB" in mem["container_rss_label"]
|
||||
assert stats["performance"]["loadavg"] == "0.5,0.4,0.3"
|
||||
assert len(mem["top"][0]["cmd"]) <= 75
|
||||
assert mem["top"][0].get("cmd_full")
|
||||
print("merge ok", mem["container_rss_label"])
|
||||
PY
|
||||
then ok "merge_resources compact labels"
|
||||
else bad "merge_resources unit failed"
|
||||
fi
|
||||
|
||||
# mem-snapshot syntax
|
||||
if bash -n "$ROOT/scripts/taler-shared/mem-snapshot.sh" \
|
||||
&& bash -n "$ROOT/scripts/taler-landing/collect_container_resources.sh"; then
|
||||
ok "mem-snapshot + collect_container_resources bash -n"
|
||||
else
|
||||
bad "bash -n resources helpers"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "=== public HTTPS (live stack) ==="
|
||||
for name_base in "bank|$BANK" "exchange|$EX" "merchant|$MER"; do
|
||||
name="${name_base%%|*}"
|
||||
base="${name_base#*|}"
|
||||
echo "-- $name ($base) --"
|
||||
code=$(http_code "$base/intro/")
|
||||
if [ "$code" = "200" ]; then ok "$name /intro/ HTTP 200"
|
||||
else bad "$name /intro/ HTTP $code"
|
||||
fi
|
||||
# live HTML may lag deploy — warn only if missing script after we expect it
|
||||
if grep -q 'goa-amount.js' /tmp/landing-test.body 2>/dev/null; then
|
||||
ok "$name live HTML references goa-amount.js"
|
||||
else
|
||||
note "[WARN] $name live HTML has no goa-amount.js yet (deploy pending)"
|
||||
fi
|
||||
|
||||
code=$(http_code "$base/intro/stats.json")
|
||||
if [ "$code" = "200" ] && "$PY" -c 'import json,sys; d=json.load(open("/tmp/landing-test.body")); sys.exit(0 if d.get("ok") else 1)'; then
|
||||
gen=$("$PY" -c 'import json; d=json.load(open("/tmp/landing-test.body")); print(d.get("generated_at_human") or d.get("generated_at") or "?")')
|
||||
ok "$name stats.json ok (gen=$gen)"
|
||||
# enrich live payload
|
||||
if "$PY" "$ROOT/scripts/taler-landing/enrich_stats_alt.py" /tmp/landing-test.body -o /tmp/landing-test.enriched.json >/dev/null 2>&1; then
|
||||
ok "$name live stats enrichable with amount_alt"
|
||||
else
|
||||
bad "$name live stats enrich failed"
|
||||
fi
|
||||
# resources / memory block
|
||||
if "$PY" - <<'PY'
|
||||
import json, sys
|
||||
d = json.load(open("/tmp/landing-test.body"))
|
||||
p = d.get("performance") or {}
|
||||
mem = p.get("memory") or {}
|
||||
fail = []
|
||||
if not p:
|
||||
fail.append("no performance")
|
||||
# latency probes differ per site but config_ms is common
|
||||
if p.get("config_ms") is None and p.get("keys_ms") is None:
|
||||
fail.append("no latency ms")
|
||||
ctr = mem.get("container_rss_human") or ""
|
||||
if not ctr or ctr in ("—", "-", "0"):
|
||||
fail.append("container_rss_human missing")
|
||||
if not isinstance(mem.get("top"), list) or len(mem.get("top") or []) < 1:
|
||||
fail.append("top processes empty")
|
||||
# role groups should exist as keys
|
||||
for k in ("postgres_rss_human", "nginx_rss_human"):
|
||||
if k not in mem:
|
||||
fail.append("missing " + k)
|
||||
if fail:
|
||||
print("resources issues:", ", ".join(fail))
|
||||
sys.exit(1)
|
||||
print("resources ok container=", ctr, "top_n=", len(mem.get("top") or []),
|
||||
"loadavg=", p.get("loadavg"))
|
||||
sys.exit(0)
|
||||
PY
|
||||
then ok "$name performance.memory present (RSS + top)"
|
||||
else bad "$name performance.memory incomplete"
|
||||
fi
|
||||
# merge dry-run on live stats (re-apply compact labels)
|
||||
if "$PY" "$ROOT/scripts/taler-landing/merge_resources.py" /tmp/landing-test.body \
|
||||
<("$PY" -c 'import json; d=json.load(open("/tmp/landing-test.body")); p=d.get("performance") or {}; print(json.dumps({"ok":True,"loadavg":p.get("loadavg") or "","memory":p.get("memory") or {}}))') \
|
||||
-o /tmp/landing-test.resmerged.json 2>/dev/null; then
|
||||
ok "$name resource merge on live payload"
|
||||
else
|
||||
# fallback without process substitution for macOS bash
|
||||
"$PY" -c 'import json; d=json.load(open("/tmp/landing-test.body")); p=d.get("performance") or {}; json.dump({"ok":True,"loadavg":p.get("loadavg") or "","memory":p.get("memory") or {}}, open("/tmp/landing-test.res.json","w"))'
|
||||
if "$PY" "$ROOT/scripts/taler-landing/merge_resources.py" /tmp/landing-test.body /tmp/landing-test.res.json -o /tmp/landing-test.resmerged.json 2>/dev/null; then
|
||||
ok "$name resource merge on live payload"
|
||||
else
|
||||
bad "$name resource merge failed"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
bad "$name stats.json HTTP $code / not ok"
|
||||
fi
|
||||
|
||||
code=$(http_code "$base/intro/stats-run.json")
|
||||
if [ "$code" = "200" ]; then ok "$name stats-run.json HTTP 200"
|
||||
else bad "$name stats-run.json HTTP $code"
|
||||
fi
|
||||
|
||||
code=$(http_code "$base/intro/goa-amount.js")
|
||||
if [ "$code" = "200" ] && grep -q 'GoaAmount' /tmp/landing-test.body; then
|
||||
ok "$name goa-amount.js live"
|
||||
else
|
||||
note "[WARN] $name goa-amount.js not live yet (HTTP $code) — deploy-landings.sh needed"
|
||||
fi
|
||||
done
|
||||
|
||||
echo
|
||||
echo "=== result: pass=$pass fail=$fail ==="
|
||||
if [ "$fail" -gt 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
|
|
@ -135,6 +135,7 @@ E2E_VARIABLE=0 WITHDRAW_AMT=GOA:50 PAY_AMT=GOA:1 ./taler-monitoring.sh e2e # s
|
|||
| **versions** | `deb.taler.net` reachable (InRelease/Packages/pool `.deb`); containers can reach it + have apt source; installed Taler packages vs **trixie** |
|
||||
| **sanity** | bank · exchange · merchant sections (public + server) |
|
||||
| **e2e** | account → credit → withdraw → wallet → confirm → order → pay |
|
||||
| **ladder** | GOA withdraw: fixed **0** + low bands + **random high ranges** + fixed **max** |
|
||||
|
||||
```bash
|
||||
# package suite (default trixie on deb.taler.net)
|
||||
|
|
@ -150,8 +151,86 @@ E2E maps failures to blockers, e.g.:
|
|||
- `Alarm clock` during pay → usually missing `handle-uri --yes` or wrong pay URI (must be `…/instances/{inst}/{oid}/?c={token}` from merchant `taler_pay_uri`)
|
||||
- `insufficient balance` → withdraw incomplete
|
||||
|
||||
## Performance (urls phase)
|
||||
|
||||
Outside-in HTTPS RTT for bank / exchange / merchant critical paths. Each probe prints
|
||||
`HTTP … · N ms` on the `[OK]`/`[WARN]`/`[ERROR]` line; end of the block prints
|
||||
`perf summary` (n / min / p50 / avg / max).
|
||||
|
||||
| Env | Default | Meaning |
|
||||
|-----|---------|---------|
|
||||
| `PERF_WARN_MS` | `8000` | WARN if latency ≥ this |
|
||||
| `PERF_FAIL_MS` | `20000` | ERROR if latency ≥ this |
|
||||
| `PERF_CURL_TIMEOUT` | `25` | curl max seconds per probe |
|
||||
|
||||
```bash
|
||||
PERF_WARN_MS=3000 PERF_FAIL_MS=10000 ./taler-monitoring.sh urls
|
||||
```
|
||||
|
||||
## Wallet coins (e2e · ladder)
|
||||
|
||||
At every relevant withdraw/pay step the wallet is snapped via `advanced dump-coins`.
|
||||
Amounts use exchange **`alt_unit_names`** (Kilo-GOA, Mega-GOA, …) with the base
|
||||
**GOA value in parentheses**:
|
||||
|
||||
```text
|
||||
[INFO] coins after-ATM — amount_circ=2 Kilo-GOA (GOA:2000)
|
||||
[INFO] coins after-ATM — denoms_in_circ 1 Kilo-GOA (GOA:1000)×2 (=2 Kilo-GOA (GOA:2000))
|
||||
== ladder · rung 12 GOA:1000000 · 1 Mega-GOA (GOA:1000000) ==
|
||||
```
|
||||
|
||||
Also: coin counts (in circulation / spent), status histogram, Δ vs previous snap.
|
||||
History TSV: `$METRICS_DIR/coins-history.tsv`. Map loaded from
|
||||
`${EXCHANGE_PUBLIC}/config` → `currency_specification.alt_unit_names`.
|
||||
|
||||
### Final statistics dashboard (e2e · ladder)
|
||||
|
||||
At the end of **e2e** and **ladder**, a boxed **FINAL STATISTICS** block prints:
|
||||
|
||||
| Section | Content |
|
||||
|---------|---------|
|
||||
| **Coins** | in circulation / spent counts + amounts + denoms (alt names) |
|
||||
| **Money flow** | withdrawn events/amount, spent (paid), net (wd − spent) |
|
||||
| **Tendency** | coin/amount trend over snaps (↑↓→), slope, last history rows |
|
||||
| **Performance** | min / p50 / avg / max per phase; first-half vs second-half latency |
|
||||
| **Load** | host loadavg + mem before/after; per-container RSS/CPU; tendencies |
|
||||
|
||||
Successful withdraw/pay steps append to `flow-withdrawn.txt` / `flow-spent.txt` (ladder also uses its TSVs).
|
||||
|
||||
## Load / memory (inside · e2e · ladder)
|
||||
|
||||
Host **loadavg**, **RAM used/avail**, and per-container **RSS / CPU / block I/O**
|
||||
(plus postgres DB sizes and role RSS: libeufin, exchange, merchant, …) are
|
||||
snapshotted via SSH whenever the stack is under load:
|
||||
|
||||
| Phase | When |
|
||||
|-------|------|
|
||||
| **inside** | end of container status |
|
||||
| **e2e** | before work · after ATM withdraws · after payments · after shop · e2e-end (+ delta) |
|
||||
| **ladder** | before first rung · after last rung (+ delta + timing overall) |
|
||||
|
||||
```bash
|
||||
# LAN down? force WAN DNAT host
|
||||
KOOPA_SSH=koopa-external ./taler-monitoring.sh inside
|
||||
# or rely on automatic fallback (default KOOPA_SSH_FALLBACKS=koopa-external)
|
||||
|
||||
METRICS_LOAD=0 ./taler-monitoring.sh e2e # skip host probes
|
||||
LADDER_LOAD=0 ./taler-monitoring.sh ladder
|
||||
```
|
||||
|
||||
| Env | Default | Meaning |
|
||||
|-----|---------|---------|
|
||||
| `KOOPA_SSH` | `koopa` | primary SSH host |
|
||||
| `KOOPA_SSH_FALLBACKS` | `koopa-external` | tried if primary fails |
|
||||
| `METRICS_LOAD` | `1` | `0` = skip all host/container load probes |
|
||||
| `LADDER_LOAD` | `1` | `0` = skip load in ladder only |
|
||||
| `LADDER_PAY` | `1` | `0` = skip payment ladder after withdraws |
|
||||
| `LADDER_WITHDRAW_SCALE` | `1.5` | withdraw mids / pay mids ratio (funding headroom) |
|
||||
| `LADDER_STEPS` | `23` | rungs for both withdraw and pay (0 + mids + max−1 + max) |
|
||||
| `METRICS_LOAD_SSH_TIMEOUT` | `90` | seconds for remote load python |
|
||||
|
||||
## Needs
|
||||
|
||||
- SSH `koopa` (inside/sanity server bits)
|
||||
- SSH `koopa` or `koopa-external` (inside / load / sanity server bits)
|
||||
- secrets under `koopa-admin-secrets/...` for e2e
|
||||
- `taler-wallet-cli` for e2e
|
||||
|
|
|
|||
|
|
@ -47,15 +47,19 @@ IDs are assigned **in run order** within the area (`set_area` resets the counter
|
|||
| www-… | cross-links: one line (4 intros) on local stack |
|
||||
| www-… | bank demo-withdraw + shop assets: one line (+ soft shop-pay if missing) |
|
||||
| www-… | **bank `/intro/demo-withdraw.json`** → `taler://withdraw/HOST:PORT/taler-integration/…` + integration op HTTP 200 |
|
||||
| www-… | **landing exposed links** (bank / merchant / exchange): parse each `/intro/` HTML, probe every own-stack `https://` + root-relative `href`/`src`/`content`, soft-check external stores/docs |
|
||||
| www-… | landing static: `qrcode.min.js`, `og-goa-shop.png`, `qr-logo.png`, shop-pay.js/css |
|
||||
| www-… | cross-links between bank ↔ merchant ↔ exchange intros (local stack) |
|
||||
| www-… | **bank `/intro/demo-withdraw.json`** → `taler://withdraw/HOST/taler-integration/…` (no default `:443`/`:80`; non-default port OK) + integration op HTTP 200 |
|
||||
| www-… | bank `/intro/auto-account.json` (earlier) → same withdraw shape, **no payto_uri**, login at `/webui/` |
|
||||
| www-… | **performance** (outside-in RTT): bank/exchange/merchant `/config`, keys, webui, intro — ms; WARN ≥ `PERF_WARN_MS` (8000); ERROR ≥ `PERF_FAIL_MS` (20000) |
|
||||
| www-… | **landing load stats**: `/intro/stats.json` → loadavg, container RSS, in-container probe ms (public; SSH container cat fallback); warn if stale/missing |
|
||||
|
||||
**Legal docs rule:** HTTP 200, non-empty body, not plain `not configured`, not merchant API JSON `code:21`. On local stack, optional content needle (terms/privacy/FADP/GOA…).
|
||||
|
||||
**Performance rule:** Measured from the **monitoring runner** (public URLs via Caddy), not container loopback. HTTP must match expect (usually 200); latency is reported on the OK line. Slow ≥ `PERF_WARN_MS` → WARN only (no ERROR on slowness alone).
|
||||
**Performance rule:** Measured from the **monitoring runner** (public URLs via Caddy), not container loopback. HTTP must match expect (usually 200); **latency (ms) is on every perf OK/WARN/ERROR line**, plus a **perf summary** (n / min / p50 / avg / max). Slow ≥ `PERF_WARN_MS` (default 8000) → WARN; ≥ `PERF_FAIL_MS` (default 20000) → ERROR.
|
||||
|
||||
**Landing links rule:** Own-stack (bank/exchange/taler.\* + page host) must be HTTP 200 (or redirect→200). External (App Store, Play, F-Droid, wallet.taler.net, docs/git.taler.net, …) soft WARN if down. Auto-account wallet link must be `taler://withdraw/…:port/taler-integration/…`, never payto.
|
||||
**Landing links rule:** Own-stack (bank/exchange/taler.\* + page host) must be HTTP 200 (or redirect→200). External (App Store, Play, F-Droid, wallet.taler.net, docs/git.taler.net, …) soft WARN if down. Auto-account wallet link must be `taler://withdraw/HOST/taler-integration/…` (default ports stripped for mobile wallets), never payto.
|
||||
|
||||
**alt_unit_names rule:** wallet codec requires a non-empty map including scale key `"0"`. For multi-currency merchant, also follow every entry in `exchanges[]` and check that exchange’s public `/config`.
|
||||
|
||||
|
|
@ -67,11 +71,14 @@ IDs are assigned **in run order** within the area (`set_area` resets the counter
|
|||
|
||||
| ID | Check (typical order) |
|
||||
|----|------------------------|
|
||||
| inside-001 | ssh koopa |
|
||||
| inside-001 | ssh koopa (or **koopa-external** fallback) |
|
||||
| inside-002+ | per-component emit: container, ports, libeufin/httpd, postgres, local `/config`/`/keys`, wirewatch, DNS pin, caddy |
|
||||
| inside-… | **load / memory**: host loadavg + RAM; bank/exchange/merchant podman CPU/mem/block + process RSS by role + DB sizes |
|
||||
|
||||
Remote lines `E|comp|LEVEL|key|detail` each become one numbered result.
|
||||
|
||||
SSH: `KOOPA_SSH` (default `koopa`), then `KOOPA_SSH_FALLBACKS` (default `koopa-external`) when LAN is unreachable.
|
||||
|
||||
---
|
||||
|
||||
## sanity — bank · exchange · merchant (`./taler-monitoring.sh sanity`)
|
||||
|
|
@ -145,8 +152,10 @@ Local GOA also: **ATM includes GOA:4200**, then **paivana** (HTTP 302 on `PAIVAN
|
|||
| e2e-003 | mode / currency info |
|
||||
| e2e-004… | secrets, reachability gates |
|
||||
| e2e-… | account, credit, withdraw, confirm, coins, order, pay ladder |
|
||||
| e2e-… | **load snapshots** at e2e-start, after-withdraw, after-pay, after-shop, e2e-end (host loadavg/RAM + container RSS/CPU/DB) |
|
||||
| e2e-… | **coins** before/after each ATM withdraw, settle, pay, shop: count in circulation / spent, amount + denoms using **`alt_unit_names`** (e.g. `5 Kilo-GOA (GOA:5000)`), Δ vs previous snap |
|
||||
| e2e-… | **GOA shop products** — full catalog list; **random pick of 2** (override `E2E_SHOP_PICK_N`) |
|
||||
| e2e-… | balances, dig on failure |
|
||||
| e2e-… | balances, dig on failure, load delta + overall metrics |
|
||||
|
||||
Shop product pays use instance `goa-shop` (default) and catalog `E2E_SHOP_PRODUCTS`
|
||||
(`id|Product name|amount` lines). Each e2e run **shuffles** the catalog and pays
|
||||
|
|
@ -157,6 +166,45 @@ Blockers keep the same ID prefix: `[BLOCKER] e2e-0NN step: message`.
|
|||
|
||||
---
|
||||
|
||||
## ladder — GOA withdraw ranges (`./taler-monitoring.sh ladder`)
|
||||
|
||||
| Step | What |
|
||||
|------|------|
|
||||
| ladder-001 | `GET /intro/auto-account.json` (personal account, GOA:0) |
|
||||
| ladder-… | **load snapshot** before withdraws (host + bank/exchange/merchant mem/CPU) |
|
||||
| ladder-… | **Phase A withdraw** (23): mint explorer pool — 0 → random → max−1 → max into **one cumulative wallet** (mids × `LADDER_WITHDRAW_SCALE`, default 1.5) |
|
||||
| ladder-… | wallet-cli accept-uri + explorer confirm when `selected`; settle; coins snap |
|
||||
| ladder-… | **Phase B pay** (23): same shape 0 → random → max−1 → max via merchant orders + `handle-uri --yes` |
|
||||
| ladder-… | **load snapshot** after + delta + overall timing (withdraw + pay) |
|
||||
| report | withdraw TSV + pay TSV + JSON |
|
||||
|
||||
Default amounts (`build_ladder_pair`, strictly increasing):
|
||||
|
||||
```text
|
||||
withdraw: [0] + random mids + [max−1] + [max]
|
||||
pay: [0] + (withdraw_mid / scale) + [max−1] + [max] # always ≤ matching withdraw mid
|
||||
```
|
||||
|
||||
TSV `range`: `pin:0` | `random` | `pin:max-1` | `pin:max`.
|
||||
|
||||
Soft: absolute **max** mint/pay may `CEILING_REJECT` / skip (WARN). **max-1** is a hard pin.
|
||||
`LADDER_PAY=0` skips phase B.
|
||||
|
||||
| Env | Default | Meaning |
|
||||
|-----|---------|---------|
|
||||
| `LADDER_STEPS` | `23` | total rungs (0 + mids + max) |
|
||||
| `LADDER_MAX_AMOUNT` | `4503599627370496` | absolute last pin |
|
||||
| `LADDER_TIMEOUT_S` | `3600` | large rungs need time |
|
||||
| `LADDER_MAX_RUNGS` | `99` | cap list length after build |
|
||||
|
||||
```bash
|
||||
./taler-monitoring.sh ladder
|
||||
LADDER_STEPS=10 ./taler-monitoring.sh ladder
|
||||
LADDER_REPORT_DIR=/tmp/my-ladder ./taler-monitoring.sh ladder
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Run one area
|
||||
|
||||
```bash
|
||||
|
|
@ -164,5 +212,6 @@ Blockers keep the same ID prefix: `[BLOCKER] e2e-0NN step: message`.
|
|||
./taler-monitoring.sh inside # inside only
|
||||
./taler-monitoring.sh versions # deb.taler.net + package drift
|
||||
./taler-monitoring.sh e2e # e2e only
|
||||
./taler-monitoring.sh ladder # GOA amount ladder + timings
|
||||
./taler-monitoring.sh -d taler.net urls
|
||||
```
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ set -euo pipefail
|
|||
ROOT=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=lib.sh
|
||||
source "$ROOT/lib.sh"
|
||||
# shellcheck source=metrics.sh
|
||||
source "$ROOT/metrics.sh"
|
||||
|
||||
: "${E2E_TIMEOUT:=180}" # ATM ladder + wirewatch lag needs room (was 55 — too tight)
|
||||
: "${E2E_PAY_SECS:=22}" # hard reserve for handle-uri + settle (do not starve pay)
|
||||
|
|
@ -150,6 +152,15 @@ e2e_finish() {
|
|||
fi
|
||||
E2E_REPORTED=1
|
||||
print_balances
|
||||
# Final coin inventory + host/container load + statistics dashboard
|
||||
if [ -n "${METRICS_DIR:-}" ]; then
|
||||
section "metrics · e2e coins final"
|
||||
metrics_report_coins "e2e-end" || true
|
||||
if [ "${METRICS_LOAD:-1}" != "0" ]; then
|
||||
metrics_report_load "${METRICS_DIR}/load-after.json" "e2e-end" || true
|
||||
fi
|
||||
metrics_print_overall "e2e final" || true
|
||||
fi
|
||||
summary || true
|
||||
return "$code"
|
||||
}
|
||||
|
|
@ -288,6 +299,9 @@ BANK_HOST=$(python3 -c 'from urllib.parse import urlparse; print(urlparse("'"$BA
|
|||
|
||||
SCRATCH=$(mktemp -d)
|
||||
WDB="$SCRATCH/wallet.sqlite3"
|
||||
METRICS_DIR="${METRICS_DIR:-$SCRATCH/metrics}"
|
||||
mkdir -p "$METRICS_DIR"
|
||||
export METRICS_DIR CUR="${CUR:-${EXPECT_CURRENCY:-GOA}}" WDB
|
||||
export PATH="${HOME}/.local/bin:${HOME}/taler/opt/taler-wallet-cli/usr/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"
|
||||
# Always dump balances on any exit (timeout, blocker, signal, success)
|
||||
trap 'ec=$?; e2e_finish "$ec"; rm -rf "$SCRATCH"; exit "$ec"' EXIT
|
||||
|
|
@ -358,6 +372,19 @@ info "ATM withdraw ladder" "$WITHDRAW_LIST (credit $CREDIT_AMT)"
|
|||
info "pay ladder" "$PAY_LIST"
|
||||
info "e2e budget" "${E2E_TIMEOUT}s"
|
||||
|
||||
# alt_unit_names for human amounts (Kilo-GOA … + base GOA in parentheses)
|
||||
ALT_UNITS_FILE="${METRICS_DIR}/alt_unit_names.json"
|
||||
export ALT_UNITS_FILE
|
||||
if metrics_load_alt_units "${EXCHANGE_PUBLIC}/config"; then
|
||||
info "alt_unit_names" "from ${EXCHANGE_PUBLIC}/config"
|
||||
else
|
||||
warn "alt_unit_names" "using built-in SI fallback"
|
||||
fi
|
||||
|
||||
# Host + bank/exchange/merchant RAM/CPU/load (SSH koopa or koopa-external)
|
||||
section "e2e · load snapshot (before withdraw/pay)"
|
||||
metrics_report_load "${METRICS_DIR}/load-before.json" "e2e-start" || true
|
||||
|
||||
# Local stack: koopa secrets. Remote: only explicit env (never leak local passwords to foreign banks).
|
||||
if [ "$E2E_REMOTE" = "1" ]; then
|
||||
ADMIN_PASS="${E2E_BANK_ADMIN_PASS:-}"
|
||||
|
|
@ -672,6 +699,8 @@ else
|
|||
warn "wallet accept ToS" "$(tail -c 80 "$SCRATCH/ex-tos.out" | tr '\n' ' ')"
|
||||
fi
|
||||
fi
|
||||
# Empty wallet baseline before ATM / pay load
|
||||
metrics_report_coins "wallet-ready" || true
|
||||
|
||||
wd_status() {
|
||||
curl -sS -m 8 -o "$SCRATCH/wd-st.json" -w '' \
|
||||
|
|
@ -714,8 +743,9 @@ e2e_one_withdraw() {
|
|||
WITHDRAW_AMT="$1"
|
||||
local tag
|
||||
tag=$(printf '%s' "$WITHDRAW_AMT" | tr '.:' '__')
|
||||
section "e2e · ATM withdraw $WITHDRAW_AMT"
|
||||
section "e2e · ATM withdraw $WITHDRAW_AMT · $(format_amount_alt "$WITHDRAW_AMT")"
|
||||
e2e_over && { warn "ATM withdraw" "budget exhausted — skip $WITHDRAW_AMT"; return 1; }
|
||||
metrics_report_coins "before-ATM-${tag}" || true
|
||||
|
||||
curl -sS -m 15 -o "$SCRATCH/wd-$tag.json" -H "Authorization: Bearer $UT" \
|
||||
-H 'Content-Type: application/json' \
|
||||
|
|
@ -808,7 +838,9 @@ else:
|
|||
fi
|
||||
sleep 2
|
||||
done
|
||||
metrics_report_coins "after-ATM-${tag}" || true
|
||||
if [ "$ok_bal" = "1" ]; then
|
||||
metrics_record_flow withdrawn "$WITHDRAW_AMT" || true
|
||||
return 0
|
||||
fi
|
||||
# Bank side often already confirmed — treat as timing lag, not hard fail
|
||||
|
|
@ -832,8 +864,9 @@ e2e_one_pay() {
|
|||
fi
|
||||
# shell-safe tag for files
|
||||
tag=$(printf '%s' "$tag" | tr -c 'A-Za-z0-9._-' '_')
|
||||
section "e2e · pay $PAY_AMT · $PAY_SUM"
|
||||
section "e2e · pay $PAY_AMT · $(format_amount_alt "$PAY_AMT") · $PAY_SUM"
|
||||
e2e_over && { warn "pay" "budget exhausted — skip $PAY_AMT ($PAY_SUM)"; return 1; }
|
||||
metrics_report_coins "before-pay-${tag}" || true
|
||||
if [ -z "${MPW:-}" ]; then
|
||||
warn "pay $PAY_AMT" "no merchant token"
|
||||
return 1
|
||||
|
|
@ -906,9 +939,12 @@ d=json.load(open(sys.argv[1]))
|
|||
sys.exit(0 if d.get("paid") is True or str(d.get("order_status","")).lower()=="paid" else 1)
|
||||
' "$SCRATCH/ord-paid-$tag.json" 2>/dev/null; then
|
||||
ok "payment settled $PAY_AMT ($PAY_SUM · order $OID)"
|
||||
metrics_report_coins "after-pay-${tag}" || true
|
||||
metrics_record_flow spent "$PAY_AMT" || true
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
metrics_report_coins "after-pay-fail-${tag}" || true
|
||||
warn "pay $PAY_AMT ($PAY_SUM)" "not settled for order $OID"
|
||||
return 1
|
||||
}
|
||||
|
|
@ -945,8 +981,9 @@ e2e_one_pay_public_template() {
|
|||
local pamt="$3"
|
||||
local tag
|
||||
tag=$(printf 'shop_%s' "$pid" | tr -c 'A-Za-z0-9._-' '_')
|
||||
section "e2e · shop product · $pname ($pid) · $pamt"
|
||||
section "e2e · shop product · $pname ($pid) · $pamt · $(format_amount_alt "$pamt")"
|
||||
e2e_over && { warn "shop $pname" "budget exhausted — skip"; return 1; }
|
||||
metrics_report_coins "before-shop-${tag}" || true
|
||||
|
||||
local MH
|
||||
MH=$(python3 -c 'from urllib.parse import urlparse; print(urlparse("'"$MERCHANT_PUBLIC"'").hostname or "taler.hacktivism.ch")' 2>/dev/null || echo "taler.hacktivism.ch")
|
||||
|
|
@ -1023,15 +1060,20 @@ d=json.load(open(sys.argv[1]))
|
|||
sys.exit(0 if d.get("paid") is True or str(d.get("order_status","")).lower()=="paid" else 1)
|
||||
' "$SCRATCH/ord-paid-$tag.json" 2>/dev/null; then
|
||||
ok "shop $pname" "payment settled ($pamt · order $OID)"
|
||||
metrics_report_coins "after-shop-${tag}" || true
|
||||
metrics_record_flow spent "$pamt" || true
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
if grep -qiE 'payment|paid|Payment' "$SCRATCH/tx-$tag.out" 2>/dev/null \
|
||||
|| grep -qiE 'done|paid|success|Payment' "$SCRATCH/pay-$tag.out" 2>/dev/null; then
|
||||
ok "shop $pname" "payment settled via wallet tx ($pamt · order $OID)"
|
||||
metrics_report_coins "after-shop-${tag}" || true
|
||||
metrics_record_flow spent "$pamt" || true
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
metrics_report_coins "after-shop-fail-${tag}" || true
|
||||
warn "shop $pname ($pid)" "not settled for order $OID"
|
||||
return 1
|
||||
}
|
||||
|
|
@ -1067,6 +1109,10 @@ for WITHDRAW_AMT in $WITHDRAW_LIST; do
|
|||
esac
|
||||
done
|
||||
info "ATM withdraw summary" "$WITHDRAW_REPORT (ok=$WITHDRAW_OK_N lag=$WITHDRAW_LAG_N fail=$WITHDRAW_FAIL_N)"
|
||||
section "e2e · load snapshot (after ATM withdraws)"
|
||||
metrics_report_load "${METRICS_DIR}/load-after-withdraw.json" "after-withdraw" || true
|
||||
cp -f "${METRICS_DIR}/load-after-withdraw.json" "${METRICS_DIR}/load-after.json" 2>/dev/null || true
|
||||
metrics_report_coins "after-ATM-ladder" || true
|
||||
|
||||
# Settlement catch-up: bank may have confirmed while wallet was still empty
|
||||
section "e2e · wallet settlement (timing)"
|
||||
|
|
@ -1083,6 +1129,7 @@ if python3 -c "import sys; sys.exit(0 if float(sys.argv[1]) > 0 else 1)" "$av_no
|
|||
WITHDRAW_REPORT="${WITHDRAW_REPORT} → late-OK avail=${CUR}:${av_now}"
|
||||
fi
|
||||
ok "spendable balance for payments" "${CUR}:${av_now}"
|
||||
metrics_report_coins "after-settle" || true
|
||||
else
|
||||
if [ "$WITHDRAW_OK" != "1" ]; then
|
||||
warn "withdraw" "still no spendable ${CUR} after settle wait"
|
||||
|
|
@ -1146,6 +1193,10 @@ for PAY_AMT in $PAY_LIST; do
|
|||
fi
|
||||
done
|
||||
info "pay summary" "$PAY_REPORT (ok=$PAY_OK_N fail=$PAY_FAIL_N skip=$PAY_SKIP_N)"
|
||||
section "e2e · load snapshot (after payments)"
|
||||
metrics_report_load "${METRICS_DIR}/load-after-pay.json" "after-pay" || true
|
||||
cp -f "${METRICS_DIR}/load-after-pay.json" "${METRICS_DIR}/load-after.json" 2>/dev/null || true
|
||||
metrics_report_coins "after-pay-ladder" || true
|
||||
if [ "$PAY_OK" != "1" ]; then
|
||||
av_now=$(wallet_avail_num)
|
||||
if python3 -c "import sys; sys.exit(0 if float(sys.argv[1]) > 0 else 1)" "$av_now" 2>/dev/null; then
|
||||
|
|
@ -1239,6 +1290,9 @@ EOF
|
|||
if [ "$SHOP_OK" != "1" ] && [ "$SHOP_FAIL_N" -gt 0 ]; then
|
||||
warn "goa-shop" "no sample product payment succeeded ($SHOP_REPORT)"
|
||||
fi
|
||||
section "e2e · load snapshot (after shop pays)"
|
||||
metrics_report_load "${METRICS_DIR}/load-after-shop.json" "after-shop" || true
|
||||
cp -f "${METRICS_DIR}/load-after-shop.json" "${METRICS_DIR}/load-after.json" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
|
||||
|
|
|
|||
1174
scripts/taler-monitoring/check_goa_ladder.sh
Executable file
1174
scripts/taler-monitoring/check_goa_ladder.sh
Executable file
File diff suppressed because it is too large
Load diff
|
|
@ -4,6 +4,8 @@ set -euo pipefail
|
|||
ROOT=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=lib.sh
|
||||
source "$ROOT/lib.sh"
|
||||
# shellcheck source=metrics.sh
|
||||
source "$ROOT/metrics.sh"
|
||||
|
||||
# Area inside-### — container / process state on koopa (SSH)
|
||||
set_area inside
|
||||
|
|
@ -35,8 +37,10 @@ hasp() { podman exec "$1" pgrep -f "$2" >/dev/null 2>&1; }
|
|||
BANK=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -iE 'hacktivism-bank|taler-bank' | head -1)
|
||||
[ -z "$BANK" ] && BANK=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -i bank | head -1)
|
||||
EX=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -i exchange | head -1)
|
||||
# Exact merchant container name first — never fall through to *-bank
|
||||
MER=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -E '^taler-hacktivism$' | head -1)
|
||||
[ -z "$MER" ] && MER=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -iE 'merchant|hacktivism' | grep -viE 'bank|exchange' | head -1)
|
||||
[ -z "$MER" ] && MER=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -i merchant | grep -viE 'bank|exchange' | head -1)
|
||||
[ -z "$MER" ] && MER=$(podman ps --format '{{.Names}}' 2>/dev/null | grep -E 'hacktivism' | grep -viE 'bank|exchange' | head -1)
|
||||
|
||||
# Domain resolve inside container (wirewatch needs bank.hacktivism.ch → real IP)
|
||||
# emit: comp LEVEL dns "host → ip" or fail
|
||||
|
|
@ -138,4 +142,10 @@ while IFS= read -r line; do
|
|||
esac
|
||||
done <<<"$RAW"
|
||||
|
||||
# Host loadavg + RAM + per-container RSS/CPU (same probe as e2e/ladder)
|
||||
section "inside · load / memory"
|
||||
METRICS_DIR="${METRICS_DIR:-$(mktemp -d)}"
|
||||
export METRICS_DIR
|
||||
metrics_report_load "${METRICS_DIR}/load-inside.json" "inside" || true
|
||||
|
||||
summary
|
||||
|
|
|
|||
|
|
@ -98,6 +98,77 @@ check_legal_doc() {
|
|||
rm -f "$f"
|
||||
}
|
||||
|
||||
# Shared shape for bank mint JSON (demo-withdraw + auto-account).
|
||||
# Args: $1=json file $2=auto|demo
|
||||
# stdout (ok): one detail line; for demo, optional "\twithdrawal_id" suffix.
|
||||
# stderr (fail): short reason. Exit 0/1.
|
||||
validate_bank_withdraw_json() {
|
||||
python3 - "$1" "$2" <<'PY'
|
||||
import json, re, sys
|
||||
from urllib.parse import urlparse
|
||||
|
||||
def check_withdraw_uri(wuri: str):
|
||||
"""HOST or HOST:non-default-port; default :443/:80 must be stripped (wallet fix)."""
|
||||
m = re.match(
|
||||
r"^taler://withdraw/([^/]+)/taler-integration/([0-9a-fA-F-]+)$",
|
||||
wuri or "",
|
||||
)
|
||||
if not m:
|
||||
print(
|
||||
"need taler://withdraw/HOST/taler-integration/ID:",
|
||||
(wuri or "")[:120],
|
||||
file=sys.stderr,
|
||||
)
|
||||
return None
|
||||
host = m.group(1)
|
||||
if not host or host.startswith(":"):
|
||||
print("bad withdraw host:", host, file=sys.stderr)
|
||||
return None
|
||||
if ":" in host:
|
||||
h, _, p = host.rpartition(":")
|
||||
if not h or not p.isdigit():
|
||||
print("bad withdraw host:port:", host, file=sys.stderr)
|
||||
return None
|
||||
if p in ("443", "80"):
|
||||
print("withdraw must strip default port :%s:" % p, host, file=sys.stderr)
|
||||
return None
|
||||
return host, m.group(2), wuri
|
||||
|
||||
path, mode = sys.argv[1], sys.argv[2]
|
||||
d = json.load(open(path))
|
||||
wuri = d.get("taler_withdraw_uri") or (
|
||||
d.get("qr_payload") if mode == "auto" else ""
|
||||
) or ""
|
||||
|
||||
if mode == "auto":
|
||||
if not d.get("ok"):
|
||||
print("ok!=true", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if d.get("payto_uri"):
|
||||
print("payto_uri must not be present", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if not check_withdraw_uri(wuri):
|
||||
sys.exit(1)
|
||||
webui = d.get("login_url") or d.get("webui") or d.get("account_url") or ""
|
||||
u = urlparse(webui)
|
||||
if u.scheme not in ("http", "https") or "webui" not in (u.path or ""):
|
||||
print("login webui missing:", webui[:80], file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print("%s · %s" % (d.get("username", ""), wuri[:72]))
|
||||
elif mode == "demo":
|
||||
if not d.get("ok", True) and "taler_withdraw_uri" not in d:
|
||||
print("not ok", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if not check_withdraw_uri(wuri):
|
||||
sys.exit(1)
|
||||
print("%s\t%s" % (wuri[:80], d.get("withdrawal_id") or ""))
|
||||
else:
|
||||
print("bad mode:", mode, file=sys.stderr)
|
||||
sys.exit(2)
|
||||
sys.exit(0)
|
||||
PY
|
||||
}
|
||||
|
||||
# --- exchange (core; always required) --- www-001 …
|
||||
check_url "exchange /config" 200 "$EXCHANGE_PUBLIC/config"
|
||||
code=$(http_body "$EXCHANGE_PUBLIC/config" "$tmp/ec.json")
|
||||
|
|
@ -154,6 +225,8 @@ section "www · performance · public HTTPS latency (outside-in)"
|
|||
# ≥ PERF_WARN_MS → WARN. ≥ PERF_FAIL_MS → ERROR (suite fails).
|
||||
PERF_WARN_MS="${PERF_WARN_MS:-8000}"
|
||||
PERF_FAIL_MS="${PERF_FAIL_MS:-20000}"
|
||||
PERF_TSV="$tmp/perf.tsv"
|
||||
: >"$PERF_TSV"
|
||||
|
||||
# Measure one URL: require HTTP expect (default 200), report time_total in ms.
|
||||
# $1=label $2=url $3=optional expected codes (default 200)
|
||||
|
|
@ -169,6 +242,8 @@ check_perf() {
|
|||
if (ms>0 && ms<1) ms=1
|
||||
printf "%d", int(ms+0.5)
|
||||
}')
|
||||
# Always record sample for rollup (label, ms, http, url)
|
||||
printf '%s\t%s\t%s\t%s\n' "$label" "$ms" "$code" "$url" >>"$PERF_TSV"
|
||||
case ",$expect," in
|
||||
*",$code,"*)
|
||||
if [ "$ms" -ge "${PERF_FAIL_MS}" ] 2>/dev/null; then
|
||||
|
|
@ -323,6 +398,76 @@ if [ "${CHECK_LANDING:-1}" = "1" ] || [ "${LOCAL_STACK:-0}" = "1" ]; then
|
|||
report_landing_load_stats "merchant" "$MERCHANT_PUBLIC"
|
||||
info "perf landing-stats note" "from /intro/stats.json (public); SSH container fallback if needed"
|
||||
fi
|
||||
# Rollup + optional JSON for metrics_print_overall
|
||||
if [ -s "$PERF_TSV" ]; then
|
||||
PERF_JSON="$tmp/perf-summary.json"
|
||||
perf_line=$(python3 - "$PERF_TSV" "$PERF_JSON" "$PERF_WARN_MS" "$PERF_FAIL_MS" <<'PY'
|
||||
import json, sys
|
||||
rows = []
|
||||
for line in open(sys.argv[1]):
|
||||
parts = line.rstrip("\n").split("\t")
|
||||
if len(parts) < 3:
|
||||
continue
|
||||
label, ms_s, code = parts[0], parts[1], parts[2]
|
||||
try:
|
||||
ms = int(float(ms_s))
|
||||
except Exception:
|
||||
continue
|
||||
rows.append({"label": label, "ms": ms, "http": code})
|
||||
vals = sorted(r["ms"] for r in rows)
|
||||
n = len(vals)
|
||||
if n == 0:
|
||||
print("n=0")
|
||||
json.dump({"n": 0}, open(sys.argv[2], "w"))
|
||||
raise SystemExit(0)
|
||||
def pct(p):
|
||||
if n == 1:
|
||||
return vals[0]
|
||||
i = min(n - 1, max(0, int(round((p / 100.0) * (n - 1)))))
|
||||
return vals[i]
|
||||
avg = int(round(sum(vals) / n))
|
||||
warn_ms = int(sys.argv[3])
|
||||
fail_ms = int(sys.argv[4])
|
||||
slow = [r for r in rows if r["ms"] >= warn_ms]
|
||||
rep = {
|
||||
"n": n,
|
||||
"min_ms": vals[0],
|
||||
"p50_ms": pct(50),
|
||||
"avg_ms": avg,
|
||||
"max_ms": vals[-1],
|
||||
"warn_ms": warn_ms,
|
||||
"fail_ms": fail_ms,
|
||||
"slow": [{"label": r["label"], "ms": r["ms"]} for r in slow],
|
||||
"samples": rows,
|
||||
}
|
||||
# metrics_print_overall expects named buckets with n/min/p50/avg/max
|
||||
out = {
|
||||
"www_public_https": {
|
||||
"n": n,
|
||||
"min_ms": vals[0],
|
||||
"p50_ms": pct(50),
|
||||
"avg_ms": avg,
|
||||
"max_ms": vals[-1],
|
||||
},
|
||||
**{r["label"].replace(" ", "_"): {"n": 1, "min_ms": r["ms"], "p50_ms": r["ms"], "avg_ms": r["ms"], "max_ms": r["ms"]} for r in rows},
|
||||
}
|
||||
json.dump(out, open(sys.argv[2], "w"), indent=2)
|
||||
extra = ""
|
||||
if slow:
|
||||
extra = " slow: " + ", ".join("%s=%dms" % (r["label"].replace("perf ", ""), r["ms"]) for r in slow)
|
||||
print(
|
||||
"n=%d min=%dms p50=%dms avg=%dms max=%dms (warn≥%dms fail≥%dms)%s"
|
||||
% (n, vals[0], pct(50), avg, vals[-1], warn_ms, fail_ms, extra)
|
||||
)
|
||||
PY
|
||||
)
|
||||
info "perf summary" "$perf_line"
|
||||
# Keep a copy if METRICS_DIR is set (e2e/ladder overall stats)
|
||||
if [ -n "${METRICS_DIR:-}" ] && [ -d "${METRICS_DIR}" ]; then
|
||||
cp -f "$PERF_JSON" "${METRICS_DIR}/perf-summary.json" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
info "perf note" "measured from this host (outside-in); thresholds PERF_WARN_MS=${PERF_WARN_MS} PERF_FAIL_MS=${PERF_FAIL_MS}"
|
||||
|
||||
|
||||
# Terms + privacy (legal docs)
|
||||
|
|
@ -361,48 +506,25 @@ if [ "$code" = "200" ]; then
|
|||
check_url_soft "bank /taler-integration/config" 200 "$BANK_PUBLIC/taler-integration/config"
|
||||
fi
|
||||
check_url_soft "bank /webui/" 200 "$BANK_PUBLIC/webui/"
|
||||
if [ "${CHECK_LANDING:-1}" = "1" ]; then
|
||||
check_url_soft "bank /intro/" 200 "$BANK_PUBLIC/intro/"
|
||||
check_url_soft "bank /" 302,301,200 "$BANK_PUBLIC/"
|
||||
# Auto-account: credentials + shared-pool taler://withdraw (like step 2)
|
||||
aa_code=$(http_body "$BANK_PUBLIC/intro/auto-account.json" "$tmp/aa.json")
|
||||
case "$aa_code" in
|
||||
200)
|
||||
if python3 - "$tmp/aa.json" <<'PY'
|
||||
import json, re, sys
|
||||
from urllib.parse import urlparse
|
||||
d = json.load(open(sys.argv[1]))
|
||||
if not d.get("ok"):
|
||||
print("ok!=true"); sys.exit(1)
|
||||
if "payto_uri" in d and d.get("payto_uri"):
|
||||
print("payto_uri must not be present"); sys.exit(1)
|
||||
wuri = d.get("taler_withdraw_uri") or d.get("qr_payload") or ""
|
||||
wm = re.match(r"^taler://withdraw/([^/]+)/taler-integration/([0-9a-fA-F-]+)$", wuri)
|
||||
if not wm:
|
||||
print("need taler://withdraw/HOST:PORT/taler-integration/ID:", wuri[:120]); sys.exit(1)
|
||||
if ":" not in wm.group(1):
|
||||
print("withdraw missing port:", wm.group(1)); sys.exit(1)
|
||||
webui = d.get("login_url") or d.get("webui") or d.get("account_url") or ""
|
||||
u = urlparse(webui)
|
||||
if u.scheme not in ("http", "https") or "webui" not in (u.path or ""):
|
||||
print("login webui missing:", webui[:80]); sys.exit(1)
|
||||
print("user=%s withdraw=%s login=%s" % (d.get("username"), wm.group(1), webui))
|
||||
sys.exit(0)
|
||||
PY
|
||||
then
|
||||
ok "bank /intro/auto-account.json" "$(python3 -c 'import json;d=json.load(open("'"$tmp/aa.json"'"));print(d.get("username",""),"·",(d.get("taler_withdraw_uri") or "")[:72])' 2>/dev/null || true)"
|
||||
else
|
||||
fail "bank /intro/auto-account.json" "invalid withdraw/login (HTTP body bad)"
|
||||
fi
|
||||
;;
|
||||
405|501|404|502|503|000)
|
||||
fail "bank /intro/auto-account.json" "HTTP $aa_code (want 200; 405/501 = broken)"
|
||||
;;
|
||||
*)
|
||||
fail "bank /intro/auto-account.json" "HTTP $aa_code want 200"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
check_url_soft "bank /intro/" 200 "$BANK_PUBLIC/intro/"
|
||||
check_url_soft "bank /" 302,301,200 "$BANK_PUBLIC/"
|
||||
# Auto-account: credentials + shared-pool taler://withdraw (like step 2)
|
||||
aa_code=$(http_body "$BANK_PUBLIC/intro/auto-account.json" "$tmp/aa.json")
|
||||
case "$aa_code" in
|
||||
200)
|
||||
if aa_detail=$(validate_bank_withdraw_json "$tmp/aa.json" auto 2>"$tmp/aa-val"); then
|
||||
ok "bank /intro/auto-account.json" "$aa_detail"
|
||||
else
|
||||
fail "bank /intro/auto-account.json" "invalid withdraw/login ($(tr '\n' ' ' <"$tmp/aa-val" | sed 's/[[:space:]]*$//'))"
|
||||
fi
|
||||
;;
|
||||
405|501|404|502|503|000)
|
||||
fail "bank /intro/auto-account.json" "HTTP $aa_code (want 200; 405/501 = broken)"
|
||||
;;
|
||||
*)
|
||||
fail "bank /intro/auto-account.json" "HTTP $aa_code want 200"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# Bank legal docs (landing nginx via Caddy /terms* /privacy* or /intro/*)
|
||||
|
|
@ -802,23 +924,11 @@ if [ "${LOCAL_STACK:-1}" = "1" ] || [ -n "${BANK_PUBLIC:-}" ]; then
|
|||
dw_code=$(http_body "$BANK_PUBLIC/intro/demo-withdraw.json" "$tmp/dw.json")
|
||||
case "$dw_code" in
|
||||
200)
|
||||
if python3 - "$tmp/dw.json" <<'PY'
|
||||
import json, re, sys
|
||||
d = json.load(open(sys.argv[1]))
|
||||
if not d.get("ok", True) and "taler_withdraw_uri" not in d:
|
||||
print("not ok"); sys.exit(1)
|
||||
u = d.get("taler_withdraw_uri") or ""
|
||||
m = re.match(r"^taler://withdraw/([^/]+)/taler-integration/([0-9a-fA-F-]+)$", u)
|
||||
if not m:
|
||||
print("bad uri:", u[:120]); sys.exit(1)
|
||||
if ":" not in m.group(1):
|
||||
print("missing port:", m.group(1)); sys.exit(1)
|
||||
print(u[:88])
|
||||
sys.exit(0)
|
||||
PY
|
||||
then
|
||||
_ba_ok=$((_ba_ok + 1))
|
||||
wid=$(python3 -c 'import json;print(json.load(open("'"$tmp/dw.json"'")).get("withdrawal_id",""))' 2>/dev/null || true)
|
||||
if dw_out=$(validate_bank_withdraw_json "$tmp/dw.json" demo 2>"$tmp/dw-val"); then
|
||||
dw_detail=${dw_out%%$'\t'*}
|
||||
wid=${dw_out#*$'\t'}
|
||||
[ "$wid" = "$dw_out" ] && wid=
|
||||
ok "bank /intro/demo-withdraw.json" "$dw_detail"
|
||||
if [ -n "$wid" ]; then
|
||||
if _landing_probe "$BANK_PUBLIC/taler-integration/withdrawal-operation/${wid}"; then
|
||||
_ba_ok=$((_ba_ok + 1))
|
||||
|
|
@ -829,6 +939,7 @@ PY
|
|||
ok "landing bank withdraw/shop" "demo-withdraw + shop assets ok (${_ba_ok} checks)"
|
||||
else
|
||||
fail "landing bank demo-withdraw" "invalid taler://withdraw shape"
|
||||
fail "bank /intro/demo-withdraw.json" "invalid taler://withdraw ($(tr '\n' ' ' <"$tmp/dw-val" | sed 's/[[:space:]]*$//'))"
|
||||
fi
|
||||
;;
|
||||
405|501|404|502|503|000)
|
||||
|
|
|
|||
|
|
@ -11,6 +11,8 @@
|
|||
: "${MERCHANT_LOCAL:=https://127.0.0.1:9010}"
|
||||
: "${LANDING_LOCAL:=http://127.0.0.1:9013}"
|
||||
: "${KOOPA_SSH:=koopa}"
|
||||
# When LAN Host "koopa" is unreachable, try WAN DNAT (see ~/.ssh/config Host koopa-external).
|
||||
: "${KOOPA_SSH_FALLBACKS:=koopa-external}"
|
||||
: "${MERCHANT_INSTANCE:=goa-demo-cp4zqk}"
|
||||
: "${WITHDRAW_AMT:=GOA:20}" # single-shot fallback; e2e ladder uses ATM notes
|
||||
: "${PAY_AMT:=GOA:0.01}"
|
||||
|
|
@ -289,11 +291,42 @@ with_timeout() {
|
|||
' "$secs" "$@"
|
||||
}
|
||||
|
||||
# Probe: 0 if koopa SSH works quickly
|
||||
# Pick a working SSH host: KOOPA_SSH first, then KOOPA_SSH_FALLBACKS (koopa-external).
|
||||
# Sets KOOPA_SSH to the first host that answers. 0 = ok, 1 = none.
|
||||
KOOPA_SSH_RESOLVED=0
|
||||
resolve_koopa_ssh() {
|
||||
[ "${SKIP_SSH:-0}" = "1" ] && return 1
|
||||
if [ "${KOOPA_SSH_RESOLVED}" = "1" ]; then
|
||||
return 0
|
||||
fi
|
||||
local cands=() c f seen=" "
|
||||
cands+=("${KOOPA_SSH}")
|
||||
# shellcheck disable=SC2086
|
||||
for f in ${KOOPA_SSH_FALLBACKS}; do
|
||||
case "$seen" in *" $f "*) continue ;; esac
|
||||
cands+=("$f")
|
||||
seen="$seen$f "
|
||||
done
|
||||
for c in "${cands[@]}"; do
|
||||
if with_timeout $((SSH_CONNECT_TIMEOUT + 5)) \
|
||||
ssh "${SSH_BASE_OPTS[@]}" "$c" 'echo ok' >/dev/null 2>&1; then
|
||||
if [ "$c" != "${KOOPA_SSH}" ]; then
|
||||
# surface once so operators know we used WAN jump
|
||||
printf '[INFO] SSH host %s unreachable — using %s\n' "${KOOPA_SSH}" "$c" >&2 || true
|
||||
fi
|
||||
KOOPA_SSH="$c"
|
||||
KOOPA_SSH_RESOLVED=1
|
||||
export KOOPA_SSH
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# Probe: 0 if any koopa SSH host works quickly
|
||||
koopa_ssh_ok() {
|
||||
[ "${SKIP_SSH}" = "1" ] && return 1
|
||||
with_timeout $((SSH_CONNECT_TIMEOUT + 3)) \
|
||||
ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" 'echo ok' >/dev/null 2>&1
|
||||
resolve_koopa_ssh
|
||||
}
|
||||
|
||||
# Run remote bash -s with optional stdin script; hard-capped
|
||||
|
|
@ -302,17 +335,25 @@ koopa_ssh_ok() {
|
|||
koopa_ssh_run() {
|
||||
local t="${1:-$SSH_CMD_TIMEOUT}"
|
||||
shift
|
||||
resolve_koopa_ssh || return 1
|
||||
with_timeout "$t" ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" "$@"
|
||||
}
|
||||
|
||||
koopa_ssh_bash() {
|
||||
local t="${1:-$SSH_CMD_TIMEOUT}"
|
||||
resolve_koopa_ssh || return 1
|
||||
with_timeout "$t" ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" 'bash -s'
|
||||
}
|
||||
|
||||
# stdin → remote python3 - (for metrics load probe)
|
||||
koopa_ssh_python() {
|
||||
local t="${1:-60}"
|
||||
resolve_koopa_ssh || return 1
|
||||
with_timeout "$t" ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" python3 -
|
||||
}
|
||||
|
||||
# ANSI colours for tags (green OK / yellow WARN / red ERROR·BLOCKER / cyan INFO).
|
||||
# Default: always on so pasted ```bash``` logs keep visible tags when the terminal
|
||||
# supports colour. Disable: NO_COLOR=1. Force off for dumb pipes: CLICOLOR=0.
|
||||
# Default: always on so pasted logs keep visible tags. Disable: NO_COLOR=1 or CLICOLOR=0.
|
||||
if [ "${NO_COLOR:-0}" = "1" ] || [ "${CLICOLOR:-1}" = "0" ]; then
|
||||
G= R= Y= C= N= B=
|
||||
else
|
||||
|
|
@ -359,6 +400,9 @@ _fmt_tid() {
|
|||
}
|
||||
|
||||
ok() {
|
||||
# Forms (same idea as info/warn):
|
||||
# ok "what passed"
|
||||
# ok "what passed" "detail / ms / bytes / …"
|
||||
local label="$1" detail="${2:-}"
|
||||
_take_tid
|
||||
if [ -n "$detail" ]; then
|
||||
|
|
@ -385,9 +429,28 @@ fail() {
|
|||
ERRORS+=("${LAST_TID:+$LAST_TID }$label${detail:+ — $detail}")
|
||||
}
|
||||
warn() {
|
||||
local label="$1" detail="${2:-}"
|
||||
# Forms (same idea as err; always try to state the problem):
|
||||
# warn "problem"
|
||||
# warn "problem" "why / context"
|
||||
# warn component "problem" "why / context"
|
||||
local a1="${1:-}" a2="${2:-}" a3="${3:-}"
|
||||
local head detail
|
||||
_take_tid
|
||||
printf '%s[WARN]%s %s%s%s\n' "$Y" "$N" "$(_fmt_tid)" "$label" "${detail:+ — $detail}"
|
||||
if [ -n "$a3" ]; then
|
||||
head="${a1}: ${a2}"
|
||||
detail="$a3"
|
||||
elif [ -n "$a2" ]; then
|
||||
head="$a1"
|
||||
detail="$a2"
|
||||
else
|
||||
head="$a1"
|
||||
detail=""
|
||||
fi
|
||||
if [ -n "$detail" ]; then
|
||||
printf '%s[WARN]%s %s%s — %s\n' "$Y" "$N" "$(_fmt_tid)" "$head" "$detail"
|
||||
else
|
||||
printf '%s[WARN]%s %s%s\n' "$Y" "$N" "$(_fmt_tid)" "$head"
|
||||
fi
|
||||
WARN_N=$((WARN_N + 1))
|
||||
}
|
||||
info() {
|
||||
|
|
|
|||
1408
scripts/taler-monitoring/metrics.sh
Normal file
1408
scripts/taler-monitoring/metrics.sh
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -26,6 +26,7 @@ Phases:
|
|||
sanity public + optional server
|
||||
server server-side only (SSH)
|
||||
e2e withdraw + pay (small amounts; remote aborts on login/KYC)
|
||||
ladder GOA withdraw ladder (auto-account + explorer pool + timings)
|
||||
all urls + inside + versions + sanity + e2e (SSH phases only on koopa)
|
||||
|
||||
Options:
|
||||
|
|
@ -56,6 +57,10 @@ Env (same meaning):
|
|||
TALER_DOMAIN BANK_PUBLIC EXCHANGE_PUBLIC MERCHANT_PUBLIC EXPECT_CURRENCY
|
||||
TALER_DOMAINS_CONF SKIP_SSH=1
|
||||
NO_COLOR=1 / CLICOLOR=0 disable green/yellow/red tags (default: coloured)
|
||||
SKIP_SSH=1 NO_COLOR=1
|
||||
PERF_WARN_MS PERF_FAIL_MS (urls latency; default 8000 / 20000)
|
||||
KOOPA_SSH KOOPA_SSH_FALLBACKS (default koopa → koopa-external)
|
||||
METRICS_LOAD=0 skip host/container RAM/CPU probes (e2e/ladder/inside)
|
||||
EOF
|
||||
}
|
||||
|
||||
|
|
@ -96,7 +101,7 @@ while [ $# -gt 0 ]; do
|
|||
CURRENCY_OVERRIDE="$2"; shift 2
|
||||
;;
|
||||
--no-probe) NO_PROBE=1; shift ;;
|
||||
urls|inside|versions|sanity|server|e2e|all) PHASES+=("$1"); shift ;;
|
||||
urls|inside|versions|sanity|server|e2e|ladder|goa-ladder|all) PHASES+=("$1"); shift ;;
|
||||
*)
|
||||
# bare domain shorthand: ./taler-monitoring.sh taler.net
|
||||
if [[ "$1" == *.* && "$1" != *://* && "$1" != -* ]]; then
|
||||
|
|
@ -140,6 +145,30 @@ export WITHDRAW_AMT PAY_AMT CREDIT_AMT MERCHANT_INSTANCE
|
|||
export E2E_FAKE_INCOMING E2E_REMOTE E2E_VARIABLE E2E_ATM_MAX
|
||||
export E2E_WITHDRAW_VALUES E2E_PAY_VALUES
|
||||
export PAIVANA_PUBLIC E2E_PAIVANA E2E_PAIVANA_TEMPLATE E2E_PAIVANA_AMOUNT E2E_PAIVANA_INSTANCE
|
||||
# Ladder: withdraw then pay — 0 + random mids + max-1 + max (see check_goa_ladder.sh).
|
||||
# Defaults so set -u export is safe when vars were never set by caller.
|
||||
: "${LADDER_STEPS:=23}"
|
||||
: "${LADDER_MIN_AMOUNT:=0.000001}"
|
||||
: "${LADDER_CONFIRM_POLLS:=40}"
|
||||
: "${LADDER_MAX_RUNGS:=99}"
|
||||
: "${LADDER_TIMEOUT_S:=3600}"
|
||||
: "${LADDER_REPORT_DIR:=}"
|
||||
: "${LADDER_SETTLE_ROUNDS:=18}"
|
||||
: "${LADDER_SETTLE_SLEEP:=2}"
|
||||
: "${LADDER_MAX_AMOUNT:=4503599627370496}"
|
||||
: "${LADDER_INCLUDE_ZERO:=1}"
|
||||
: "${LADDER_INCLUDE_MAX:=1}"
|
||||
: "${LADDER_HIGH_FROM:=1000000}"
|
||||
: "${LADDER_HIGH_RUNGS:=12}"
|
||||
: "${LADDER_LOAD:=1}"
|
||||
: "${LADDER_PAY:=1}"
|
||||
: "${LADDER_WITHDRAW_SCALE:=1.5}"
|
||||
: "${LADDER_PAY_SETTLE_ROUNDS:=6}"
|
||||
export LADDER_STEPS LADDER_MIN_AMOUNT LADDER_CONFIRM_POLLS LADDER_MAX_RUNGS LADDER_TIMEOUT_S LADDER_REPORT_DIR
|
||||
export LADDER_SETTLE_ROUNDS LADDER_SETTLE_SLEEP EXP_PW_FILE EXP_USER
|
||||
export LADDER_MAX_AMOUNT LADDER_INCLUDE_ZERO LADDER_INCLUDE_MAX
|
||||
export LADDER_HIGH_FROM LADDER_HIGH_RUNGS LADDER_LOAD
|
||||
export LADDER_PAY LADDER_WITHDRAW_SCALE LADDER_PAY_SETTLE_ROUNDS
|
||||
export TALER_DOMAIN_APPLIED=1
|
||||
|
||||
# Default phases
|
||||
|
|
@ -200,6 +229,7 @@ for p in "${PHASES[@]}"; do
|
|||
sanity) "$ROOT/check_sanity.sh" || ec=1 ;;
|
||||
server) "$ROOT/check_server.sh" || ec=1 ;;
|
||||
e2e) "$ROOT/check_e2e.sh" || ec=1 ;;
|
||||
ladder|goa-ladder) "$ROOT/check_goa_ladder.sh" || ec=1 ;;
|
||||
esac
|
||||
done
|
||||
exit "$ec"
|
||||
|
|
|
|||
179
scripts/taler-shared/ensure-taler-apps.sh
Normal file
179
scripts/taler-shared/ensure-taler-apps.sh
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
#!/usr/bin/env bash
|
||||
# Host (hernani@koopa): start Taler *apps inside* merchant/bank containers.
|
||||
#
|
||||
# Containers use CMD sleep infinity — podman start alone is not enough.
|
||||
# Called by user systemd units after container-taler-*.service.
|
||||
#
|
||||
# Usage:
|
||||
# ensure-taler-apps.sh # merchant + bank + helpers
|
||||
# ensure-taler-apps.sh merchant
|
||||
# ensure-taler-apps.sh bank
|
||||
# ensure-taler-apps.sh status
|
||||
set -euo pipefail
|
||||
|
||||
MER_CTR="${MER_CTR:-taler-hacktivism}"
|
||||
BANK_CTR="${BANK_CTR:-taler-hacktivism-bank}"
|
||||
AUTO_LOOP_SECS="${AUTO_CONFIRM_LOOP_SECS:-2}"
|
||||
WAIT_SECS="${WAIT_SECS:-90}"
|
||||
|
||||
log() { printf '%s %s\n' "$(date -Iseconds)" "$*"; }
|
||||
|
||||
# Oneshot container-*.service stays "active" after podman stop — always start here.
|
||||
ensure_container() {
|
||||
local name="$1"
|
||||
if podman inspect -f '{{.State.Running}}' "$name" 2>/dev/null | grep -qx true; then
|
||||
return 0
|
||||
fi
|
||||
log "container $name: not running — podman start"
|
||||
podman start "$name" >/dev/null
|
||||
}
|
||||
|
||||
wait_running() {
|
||||
local name="$1" i
|
||||
ensure_container "$name"
|
||||
for i in $(seq 1 "$WAIT_SECS"); do
|
||||
if podman inspect -f '{{.State.Running}}' "$name" 2>/dev/null | grep -qx true; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
log "ERROR: container $name not running after ${WAIT_SECS}s"
|
||||
return 1
|
||||
}
|
||||
|
||||
ctr_has_proc() {
|
||||
local ctr="$1" pattern="$2"
|
||||
podman exec "$ctr" bash -lc "ps -eo args= | grep -F -- '$pattern' | grep -v grep" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
ensure_merchant_loopback_helper() {
|
||||
# Optional loopback helper on :19097 if present in the container image
|
||||
if podman exec "$MER_CTR" bash -lc "ss -tln 2>/dev/null | grep -q ':19097'" 2>/dev/null; then
|
||||
log "merchant: loopback :19097 already listening"
|
||||
return 0
|
||||
fi
|
||||
if ! podman exec "$MER_CTR" test -f /usr/local/bin/ui-unlock-api.py 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
log "merchant: start loopback helper :19097…"
|
||||
podman exec -u root -d "$MER_CTR" python3 /usr/local/bin/ui-unlock-api.py \
|
||||
|| log "WARN: loopback helper start failed"
|
||||
}
|
||||
|
||||
ensure_merchant() {
|
||||
wait_running "$MER_CTR"
|
||||
if ctr_has_proc "$MER_CTR" 'taler-merchant-httpd'; then
|
||||
log "merchant: httpd already up"
|
||||
ensure_merchant_loopback_helper
|
||||
return 0
|
||||
fi
|
||||
log "merchant: start base (postgres/nginx)…"
|
||||
podman exec -u root "$MER_CTR" \
|
||||
/root/start_base_services_for_taler.sh --no-shell
|
||||
log "merchant: start_merchant.sh…"
|
||||
podman exec -u root "$MER_CTR" \
|
||||
runuser -u taler-merchant-httpd -- /usr/local/bin/start_merchant.sh
|
||||
ensure_merchant_loopback_helper
|
||||
log "merchant: done"
|
||||
}
|
||||
|
||||
ensure_bank() {
|
||||
wait_running "$BANK_CTR"
|
||||
if ! ctr_has_proc "$BANK_CTR" 'libeufin-bank'; then
|
||||
log "bank: start base + libeufin-bank…"
|
||||
podman exec -u root "$BANK_CTR" \
|
||||
/root/start_base_services_for_taler_bank.sh --no-shell --start-bank
|
||||
else
|
||||
log "bank: libeufin-bank already up"
|
||||
fi
|
||||
|
||||
# Landing nginx on :9013 (separate from bank API :9012)
|
||||
if ! podman exec "$BANK_CTR" bash -lc "ss -tln 2>/dev/null | grep -q ':9013 '" 2>/dev/null; then
|
||||
log "bank: start landing nginx :9013…"
|
||||
podman exec -u root "$BANK_CTR" bash -lc '
|
||||
if [ -x /etc/init.d/nginx ]; then /etc/init.d/nginx start || true
|
||||
else nginx || true
|
||||
fi
|
||||
'
|
||||
else
|
||||
log "bank: nginx :9013 already listening"
|
||||
fi
|
||||
|
||||
ensure_bank_helpers
|
||||
log "bank: done"
|
||||
}
|
||||
|
||||
# Single auto-confirm loop + demo-withdraw API (idempotent)
|
||||
ensure_bank_helpers() {
|
||||
local n
|
||||
# Collapse duplicate loops (common after manual restarts)
|
||||
n=$(podman exec "$BANK_CTR" bash -lc \
|
||||
"ps -eo pid=,args= | awk '/auto-confirm-withdrawals\\.sh --loop/ {print \$1}'" 2>/dev/null || true)
|
||||
set -- $n
|
||||
if [ "$#" -gt 1 ]; then
|
||||
log "bank: stop $(($# - 1)) extra auto-confirm pid(s)…"
|
||||
shift # keep first
|
||||
for pid in "$@"; do
|
||||
podman exec -u root "$BANK_CTR" kill "$pid" 2>/dev/null || true
|
||||
done
|
||||
sleep 0.3
|
||||
fi
|
||||
if [ "$#" -eq 0 ]; then
|
||||
log "bank: start auto-confirm --loop ${AUTO_LOOP_SECS}…"
|
||||
podman exec -u root -d "$BANK_CTR" \
|
||||
/usr/local/bin/auto-confirm-withdrawals.sh --loop "$AUTO_LOOP_SECS" \
|
||||
|| log "WARN: auto-confirm start failed (script missing?)"
|
||||
else
|
||||
log "bank: auto-confirm already running"
|
||||
fi
|
||||
|
||||
if podman exec "$BANK_CTR" test -f /usr/local/bin/demo-withdraw-api.py 2>/dev/null; then
|
||||
if ! ctr_has_proc "$BANK_CTR" 'demo-withdraw-api.py'; then
|
||||
log "bank: start demo-withdraw-api…"
|
||||
podman exec -u root -d "$BANK_CTR" \
|
||||
python3 /usr/local/bin/demo-withdraw-api.py \
|
||||
|| log "WARN: demo-withdraw-api start failed"
|
||||
else
|
||||
log "bank: demo-withdraw-api already running"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
status() {
|
||||
echo "=== containers ==="
|
||||
podman ps -a --filter name='taler-hacktivism' --format '{{.Names}} {{.Status}}'
|
||||
echo "=== merchant procs (sample) ==="
|
||||
podman exec "$MER_CTR" bash -lc "ps -eo args= | grep -E 'taler-merchant-httpd|postgres -D|nginx: master' | grep -v grep || true" 2>/dev/null || echo "(container down)"
|
||||
echo "=== bank procs (sample) ==="
|
||||
podman exec "$BANK_CTR" bash -lc "ps -eo args= | grep -E 'libeufin-bank|auto-confirm|demo-withdraw|nginx: master' | grep -v grep || true" 2>/dev/null || echo "(container down)"
|
||||
echo "=== public ==="
|
||||
for u in \
|
||||
https://taler.hacktivism.ch/config \
|
||||
https://taler.hacktivism.ch/intro/ \
|
||||
https://bank.hacktivism.ch/config \
|
||||
https://bank.hacktivism.ch/intro/
|
||||
do
|
||||
code=$(curl -sk -o /dev/null -w '%{http_code}' --connect-timeout 4 --max-redirs 0 "$u" || echo err)
|
||||
printf ' %s %s\n' "$code" "$u"
|
||||
done
|
||||
}
|
||||
|
||||
cmd="${1:-all}"
|
||||
case "$cmd" in
|
||||
all|"")
|
||||
ensure_merchant
|
||||
ensure_bank
|
||||
;;
|
||||
merchant) ensure_merchant ;;
|
||||
bank) ensure_bank ;;
|
||||
helpers) ensure_bank_helpers ;;
|
||||
status) status ;;
|
||||
-h|--help)
|
||||
sed -n '2,16p' "$0"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "unknown: $cmd (all|merchant|bank|helpers|status)" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
43
scripts/taler-shared/install-ensure-taler-apps.sh
Normal file
43
scripts/taler-shared/install-ensure-taler-apps.sh
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
#!/usr/bin/env bash
|
||||
# Install ensure-taler-apps + user systemd units on this host (hernani@koopa).
|
||||
# Run from admin-log checkout or with ADMIN_LOG set.
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
ADMIN_LOG="${ADMIN_LOG:-$ROOT}"
|
||||
UNIT_SRC="$ADMIN_LOG/configs/systemd/user"
|
||||
BIN_DST="${HOME}/.local/bin"
|
||||
UNIT_DST="${HOME}/.config/systemd/user"
|
||||
|
||||
mkdir -p "$BIN_DST" "$UNIT_DST"
|
||||
|
||||
install -m 0755 "$ADMIN_LOG/scripts/taler-shared/ensure-taler-apps.sh" \
|
||||
"$BIN_DST/ensure-taler-apps.sh"
|
||||
|
||||
install -m 0644 "$UNIT_SRC/taler-merchant-apps.service" "$UNIT_DST/"
|
||||
install -m 0644 "$UNIT_SRC/taler-bank-apps.service" "$UNIT_DST/"
|
||||
|
||||
mkdir -p \
|
||||
"$UNIT_DST/container-taler-hacktivism.service.d" \
|
||||
"$UNIT_DST/container-taler-hacktivism-bank.service.d" \
|
||||
"$UNIT_DST/container-koopa-paivana.service.d"
|
||||
|
||||
install -m 0644 "$UNIT_SRC/container-taler-hacktivism.service.d/apps.conf" \
|
||||
"$UNIT_DST/container-taler-hacktivism.service.d/apps.conf"
|
||||
install -m 0644 "$UNIT_SRC/container-taler-hacktivism-bank.service.d/apps.conf" \
|
||||
"$UNIT_DST/container-taler-hacktivism-bank.service.d/apps.conf"
|
||||
install -m 0644 "$UNIT_SRC/container-koopa-paivana.service.d/order.conf" \
|
||||
"$UNIT_DST/container-koopa-paivana.service.d/order.conf"
|
||||
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user enable taler-merchant-apps.service taler-bank-apps.service
|
||||
|
||||
echo "Installed:"
|
||||
echo " $BIN_DST/ensure-taler-apps.sh"
|
||||
echo " $UNIT_DST/taler-{merchant,bank}-apps.service (enabled)"
|
||||
echo " drop-ins: container-taler-hacktivism{,-bank}.service.d/apps.conf"
|
||||
echo " drop-in: container-koopa-paivana.service.d/order.conf"
|
||||
echo
|
||||
echo "Start now (if containers already up):"
|
||||
echo " systemctl --user start taler-merchant-apps.service taler-bank-apps.service"
|
||||
echo " $BIN_DST/ensure-taler-apps.sh status"
|
||||
|
|
@ -1,7 +1,17 @@
|
|||
#!/bin/bash
|
||||
# Memory snapshot for landing-stats (source after json_str is defined).
|
||||
# Memory snapshot for landing-stats.
|
||||
# Sets MEM_JSON (fields to embed inside performance.memory).
|
||||
# Also: mem_snapshot_emit → full JSON object on stdout (host collector).
|
||||
# IMPORTANT: no pipelines around the /proc loop (bash subshell loses counters).
|
||||
#
|
||||
# json_str may already be defined by the caller (landing-stats.sh); provide a
|
||||
# safe default so this file is usable stand-alone via podman exec.
|
||||
|
||||
if ! declare -F json_str >/dev/null 2>&1; then
|
||||
json_str() {
|
||||
printf '"%s"' "$(printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g' | tr '\n\r\t' ' ')"
|
||||
}
|
||||
fi
|
||||
|
||||
mem_fmt_bytes() {
|
||||
awk -v b="${1:-0}" 'BEGIN{
|
||||
|
|
@ -92,21 +102,39 @@ mem_snapshot_json() {
|
|||
rm -f "$topf"
|
||||
|
||||
lim_json="null"
|
||||
lim_human_json="null"
|
||||
if [[ "$limit_b" =~ ^[0-9]+$ ]] && [ "$limit_b" -gt 0 ]; then
|
||||
lim_json=$limit_b
|
||||
lim_human_json=$(json_str "$(mem_fmt_bytes "$limit_b")")
|
||||
fi
|
||||
cg_json="null"
|
||||
if [[ "$cgroup_b" =~ ^[0-9]+$ ]]; then
|
||||
cg_json=$cgroup_b
|
||||
fi
|
||||
|
||||
# Compact container label: "693.4 MiB" or "693.4 MiB / 2.00 GiB" when capped
|
||||
local ctr_human
|
||||
ctr_human=$(mem_fmt_bytes "$total_b")
|
||||
if [ "$lim_human_json" != "null" ]; then
|
||||
# lim_human_json is a quoted string already
|
||||
:
|
||||
fi
|
||||
|
||||
MEM_JSON="
|
||||
\"container_rss_bytes\": ${total_b},
|
||||
\"container_rss_human\": $(json_str "$(mem_fmt_bytes "$total_b")"),
|
||||
\"container_rss_human\": $(json_str "$ctr_human"),
|
||||
\"container_rss_label\": $(json_str "$(
|
||||
if [ "$lim_json" != "null" ]; then
|
||||
printf '%s / %s' "$ctr_human" "$(mem_fmt_bytes "$limit_b")"
|
||||
else
|
||||
printf '%s' "$ctr_human"
|
||||
fi
|
||||
)"),
|
||||
\"proc_sum_rss_bytes\": ${sum_b},
|
||||
\"proc_sum_rss_human\": $(json_str "$(mem_fmt_bytes "$sum_b")"),
|
||||
\"cgroup_bytes\": ${cg_json},
|
||||
\"cgroup_limit_bytes\": ${lim_json},
|
||||
\"cgroup_limit_human\": ${lim_human_json},
|
||||
\"postgres_rss_bytes\": ${postgres_b},
|
||||
\"postgres_rss_human\": $(json_str "$(mem_fmt_bytes "$postgres_b")"),
|
||||
\"postgres_n\": ${n_pg},
|
||||
|
|
@ -128,3 +156,15 @@ mem_snapshot_json() {
|
|||
\"top\": ${top_json}
|
||||
"
|
||||
}
|
||||
|
||||
# Full JSON for host collector: { ok, loadavg, memory: {…} }
|
||||
mem_snapshot_emit() {
|
||||
mem_snapshot_json || return 1
|
||||
local loadavg=""
|
||||
if [ -r /proc/loadavg ]; then
|
||||
loadavg=$(awk '{print $1","$2","$3}' /proc/loadavg)
|
||||
fi
|
||||
printf '{\n "ok": true,\n "source": "mem-snapshot",\n "loadavg": %s,\n "memory": {\n%s\n }\n}\n' \
|
||||
"$(json_str "$loadavg")" \
|
||||
"$MEM_JSON"
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue