scripts/taler-landing: full-scan bank collector with GOA alt units
Add a host-side Python collector that lists every bank account, pages through all ledger rows, and emits stats.json with amount_alt fields (Kilo/Mega/Peta-GOA) so large ladder withdrawals fit on the landing tiles.
This commit is contained in:
Hernâni Marques
parent
51785031d6
commit
25695805b4
5 changed files with 1528 additions and 0 deletions
260
scripts/taler-landing/collect-landing-stats.sh
Executable file
260
scripts/taler-landing/collect-landing-stats.sh
Executable file
|
|
@ -0,0 +1,260 @@
|
|||
#!/usr/bin/env bash
|
||||
# Central landing-stats collector for hernani@koopa (user systemd timer).
|
||||
#
|
||||
# - Bank: full account+ledger scan via collect_bank_stats.py → bank container
|
||||
# - Exchange / merchant: existing in-container scripts, then amount_alt enrich
|
||||
# - Never wipes a good stats.json on failure (writes stats-run.json only)
|
||||
#
|
||||
# Install: scripts/taler-landing/install-landing-stats-host.sh
|
||||
set -euo pipefail
|
||||
|
||||
export TZ="${TZ:-Europe/Zurich}"
|
||||
export PATH="${HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin${PATH:+:$PATH}"
|
||||
|
||||
log() { printf '%s %s\n' "$(date -Iseconds)" "$*"; }
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")" && pwd)"
|
||||
# When installed to ~/.local/bin, libs live in ~/.local/lib/taler-landing
|
||||
if [ -f "$ROOT/collect_bank_stats.py" ]; then
|
||||
LIB="$ROOT"
|
||||
elif [ -f "${HOME}/.local/lib/taler-landing/collect_bank_stats.py" ]; then
|
||||
LIB="${HOME}/.local/lib/taler-landing"
|
||||
elif [ -f "${HOME}/src/koopa/koopa-admin-log/scripts/taler-landing/collect_bank_stats.py" ]; then
|
||||
LIB="${HOME}/src/koopa/koopa-admin-log/scripts/taler-landing"
|
||||
else
|
||||
LIB="$ROOT"
|
||||
fi
|
||||
|
||||
ADMIN_LOG="${ADMIN_LOG:-${HOME}/src/koopa/koopa-admin-log}"
|
||||
SECRETS_BANK="${SECRETS_BANK:-${HOME}/src/koopa/koopa-admin-secrets/koopa/host-root/taler-bank}"
|
||||
|
||||
BANK_CTR="${BANK_CTR:-taler-hacktivism-bank}"
|
||||
EX_CTR="${EX_CTR:-taler-hacktivism-exchange-ansible}"
|
||||
MER_CTR="${MER_CTR:-taler-hacktivism}"
|
||||
|
||||
BANK_URL="${BANK_URL:-http://127.0.0.1:9012}"
|
||||
BANK_PUBLIC_URL="${BANK_PUBLIC_URL:-https://bank.hacktivism.ch}"
|
||||
EXCHANGE_CONFIG_URL="${EXCHANGE_CONFIG_URL:-https://exchange.hacktivism.ch/config}"
|
||||
|
||||
BANK_LANDING_IN="${BANK_LANDING_IN:-/var/www/bank-landing}"
|
||||
EX_LANDING_IN="${EX_LANDING_IN:-/var/www/exchange-landing}"
|
||||
MER_LANDING_IN="${MER_LANDING_IN:-/var/www/merchant-landing}"
|
||||
|
||||
WORKDIR="${LANDING_STATS_WORKDIR:-${XDG_RUNTIME_DIR:-/tmp}/taler-landing-stats}"
|
||||
mkdir -p "$WORKDIR"
|
||||
LOG_DIR="${LANDING_STATS_LOGDIR:-${HOME}/.local/state/taler-landing-stats}"
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
PY="${PYTHON:-python3}"
|
||||
ec_bank=0
|
||||
ec_ex=0
|
||||
ec_mer=0
|
||||
|
||||
read_pass() {
|
||||
local name="$1" f
|
||||
for f in \
|
||||
"${SECRETS_BANK}/${name}" \
|
||||
"${HOME}/.config/taler-landing/${name}" \
|
||||
"/run/user/$(id -u)/taler-landing/${name}"
|
||||
do
|
||||
if [ -r "$f" ]; then
|
||||
tr -d '\n' <"$f"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
# container (hernani can podman exec root files inside owned containers)
|
||||
if podman inspect -f '{{.State.Running}}' "$BANK_CTR" 2>/dev/null | grep -qx true; then
|
||||
if podman exec "$BANK_CTR" test -r "/root/${name}" 2>/dev/null; then
|
||||
podman exec "$BANK_CTR" cat "/root/${name}" 2>/dev/null | tr -d '\n'
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
ctr_running() {
|
||||
podman inspect -f '{{.State.Running}}' "$1" 2>/dev/null | grep -qx true
|
||||
}
|
||||
|
||||
publish_json() {
|
||||
local ctr="$1" dest_dir="$2" src_stats="$3" src_run="$4"
|
||||
if ! ctr_running "$ctr"; then
|
||||
log "WARN: $ctr not running — skip publish $dest_dir"
|
||||
return 1
|
||||
fi
|
||||
podman exec "$ctr" mkdir -p "$dest_dir" 2>/dev/null || true
|
||||
if [ -f "$src_stats" ]; then
|
||||
podman cp "$src_stats" "${ctr}:${dest_dir}/stats.json"
|
||||
fi
|
||||
if [ -f "$src_run" ]; then
|
||||
podman cp "$src_run" "${ctr}:${dest_dir}/stats-run.json"
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bank — full scan (all accounts / all money except SCAN_SKIP)
|
||||
# ---------------------------------------------------------------------------
|
||||
collect_bank() {
|
||||
log "bank: collect full stats via $LIB/collect_bank_stats.py"
|
||||
local admin_pass explorer_pass
|
||||
admin_pass="$(read_pass bank-admin-password.txt || true)"
|
||||
explorer_pass="$(read_pass bank-explorer-password.txt || true)"
|
||||
if [ -z "$admin_pass" ]; then
|
||||
log "ERROR: bank admin password not found (secrets or container)"
|
||||
printf '%s\n' '{"ok":false,"error":"no admin password","at_human":"'"$(date +"%Y-%m-%d %H:%M %Z")"'"}' \
|
||||
>"$WORKDIR/bank-stats-run.json"
|
||||
publish_json "$BANK_CTR" "$BANK_LANDING_IN" "" "$WORKDIR/bank-stats-run.json" || true
|
||||
return 1
|
||||
fi
|
||||
|
||||
# optional demo files from container
|
||||
local demo_dir=""
|
||||
if ctr_running "$BANK_CTR"; then
|
||||
demo_dir="$WORKDIR/demo"
|
||||
mkdir -p "$demo_dir"
|
||||
podman cp "${BANK_CTR}:${BANK_LANDING_IN}/withdraw.uri" "$demo_dir/withdraw.uri" 2>/dev/null || true
|
||||
podman cp "${BANK_CTR}:${BANK_LANDING_IN}/withdraw.amount" "$demo_dir/withdraw.amount" 2>/dev/null || true
|
||||
podman cp "${BANK_CTR}:${BANK_LANDING_IN}/withdraw.created" "$demo_dir/withdraw.created" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
set +e
|
||||
BANK_URL="$BANK_URL" \
|
||||
BANK_PUBLIC_URL="$BANK_PUBLIC_URL" \
|
||||
EXCHANGE_CONFIG_URL="$EXCHANGE_CONFIG_URL" \
|
||||
BANK_ADMIN_PASS="$admin_pass" \
|
||||
BANK_EXPLORER_PASS="$explorer_pass" \
|
||||
DEMO_DIR="${demo_dir}" \
|
||||
SCAN_SKIP="${SCAN_SKIP:-exchange}" \
|
||||
TX_PAGE="${TX_PAGE:-500}" \
|
||||
MAX_TX_PAGES="${MAX_TX_PAGES:-0}" \
|
||||
ACCOUNTS_DELTA="${ACCOUNTS_DELTA:--10000}" \
|
||||
"$PY" "$LIB/collect_bank_stats.py" \
|
||||
--out "$WORKDIR/bank-stats.json" \
|
||||
--run-out "$WORKDIR/bank-stats-run.json" \
|
||||
>>"$LOG_DIR/bank.log" 2>&1
|
||||
ec_bank=$?
|
||||
set -e
|
||||
|
||||
if [ "$ec_bank" -eq 0 ] && [ -f "$WORKDIR/bank-stats.json" ]; then
|
||||
# merge in-container memory snapshot when helper exists
|
||||
if ctr_running "$BANK_CTR" && podman exec "$BANK_CTR" test -f /usr/local/lib/landing-mem-snapshot.sh 2>/dev/null; then
|
||||
set +e
|
||||
mem_json=$(podman exec "$BANK_CTR" bash -c '
|
||||
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
MEM_JSON=""
|
||||
# shellcheck disable=SC1091
|
||||
. /usr/local/lib/landing-mem-snapshot.sh
|
||||
mem_snapshot_json 2>/dev/null || true
|
||||
# print MEM_JSON lines only if function set them via echo — fallback empty
|
||||
if [ -n "${MEM_JSON:-}" ]; then printf "%s" "$MEM_JSON"; fi
|
||||
' 2>/dev/null)
|
||||
set -e
|
||||
if [ -n "${mem_json:-}" ]; then
|
||||
"$PY" - "$WORKDIR/bank-stats.json" <<'PY' || true
|
||||
import json, sys
|
||||
path = sys.argv[1]
|
||||
# optional: leave memory as-is if merge fails
|
||||
print("mem merge skipped (structured merge via podman helper optional)", file=sys.stderr)
|
||||
PY
|
||||
fi
|
||||
fi
|
||||
publish_json "$BANK_CTR" "$BANK_LANDING_IN" \
|
||||
"$WORKDIR/bank-stats.json" "$WORKDIR/bank-stats-run.json"
|
||||
log "bank: OK → ${BANK_CTR}:${BANK_LANDING_IN}/stats.json"
|
||||
else
|
||||
log "ERROR: bank collect failed (ec=$ec_bank) — previous stats.json kept"
|
||||
[ -f "$WORKDIR/bank-stats-run.json" ] || \
|
||||
printf '%s\n' '{"ok":false,"error":"collect failed","at_human":"'"$(date +"%Y-%m-%d %H:%M %Z")"'"}' \
|
||||
>"$WORKDIR/bank-stats-run.json"
|
||||
publish_json "$BANK_CTR" "$BANK_LANDING_IN" "" "$WORKDIR/bank-stats-run.json" || true
|
||||
fi
|
||||
return "$ec_bank"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Exchange / merchant — in-container generators + host enrich
|
||||
# ---------------------------------------------------------------------------
|
||||
run_incontainer_stats() {
|
||||
local label="$1" ctr="$2" script_candidates="$3" landing="$4"
|
||||
local script="" s
|
||||
if ! ctr_running "$ctr"; then
|
||||
log "WARN: $label: $ctr not running"
|
||||
return 1
|
||||
fi
|
||||
# install latest script from admin-log if present
|
||||
local host_src=""
|
||||
case "$label" in
|
||||
exchange)
|
||||
host_src="$ADMIN_LOG/scripts/taler-exchange/landing-stats-exchange.sh"
|
||||
;;
|
||||
merchant)
|
||||
host_src="$ADMIN_LOG/scripts/taler-merchant/landing-stats-merchant.sh"
|
||||
;;
|
||||
esac
|
||||
if [ -n "$host_src" ] && [ -f "$host_src" ]; then
|
||||
podman cp "$host_src" "${ctr}:/usr/local/bin/$(basename "$host_src")"
|
||||
podman exec "$ctr" chmod 755 "/usr/local/bin/$(basename "$host_src")" 2>/dev/null || true
|
||||
script="/usr/local/bin/$(basename "$host_src")"
|
||||
else
|
||||
for s in $script_candidates; do
|
||||
if podman exec "$ctr" test -x "$s" 2>/dev/null || podman exec "$ctr" test -f "$s" 2>/dev/null; then
|
||||
script="$s"
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
if [ -z "$script" ]; then
|
||||
log "WARN: $label: no landing-stats script in $ctr"
|
||||
return 1
|
||||
fi
|
||||
log "$label: run $script inside $ctr"
|
||||
set +e
|
||||
podman exec \
|
||||
-e LANDING_DIR="$landing" \
|
||||
-e TZ=Europe/Zurich \
|
||||
-e PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
|
||||
"$ctr" bash "$script" >>"$LOG_DIR/${label}.log" 2>&1
|
||||
local ec=$?
|
||||
set -e
|
||||
if [ "$ec" -ne 0 ]; then
|
||||
log "ERROR: $label stats failed (ec=$ec)"
|
||||
return "$ec"
|
||||
fi
|
||||
# enrich with alt units on host
|
||||
podman cp "${ctr}:${landing}/stats.json" "$WORKDIR/${label}-stats.json" 2>/dev/null || return 0
|
||||
set +e
|
||||
"$PY" "$LIB/enrich_stats_alt.py" "$WORKDIR/${label}-stats.json" \
|
||||
--exchange-config "$EXCHANGE_CONFIG_URL" >>"$LOG_DIR/${label}.log" 2>&1
|
||||
set -e
|
||||
podman cp "$WORKDIR/${label}-stats.json" "${ctr}:${landing}/stats.json"
|
||||
log "$label: OK + alt enrich → ${ctr}:${landing}/stats.json"
|
||||
return 0
|
||||
}
|
||||
|
||||
collect_exchange() {
|
||||
run_incontainer_stats exchange "$EX_CTR" \
|
||||
"/usr/local/bin/landing-stats-exchange.sh /usr/local/bin/landing-stats.sh" \
|
||||
"$EX_LANDING_IN"
|
||||
}
|
||||
|
||||
collect_merchant() {
|
||||
run_incontainer_stats merchant "$MER_CTR" \
|
||||
"/usr/local/bin/landing-stats-merchant.sh /usr/local/bin/landing-stats.sh" \
|
||||
"$MER_LANDING_IN"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
main() {
|
||||
log "=== taler-landing-stats start (user=$(id -un) lib=$LIB) ==="
|
||||
collect_bank || ec_bank=$?
|
||||
collect_exchange || ec_ex=$?
|
||||
collect_merchant || ec_mer=$?
|
||||
log "=== done bank=$ec_bank exchange=$ec_ex merchant=$ec_mer ==="
|
||||
# non-zero if bank failed (primary public flow numbers); soft on ex/mer
|
||||
if [ "$ec_bank" -ne 0 ]; then
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Loading…
Add table
Add a link
Reference in a new issue