388 lines
14 KiB
Bash
Executable file
388 lines
14 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Generic landing-stats orchestrator (any Taler stack).
|
|
#
|
|
# Defaults = GOA / hacktivism on koopa. Override via env or EnvironmentFile
|
|
# (see profiles: goa | stage-testpaysan).
|
|
#
|
|
# - Bank: collect_bank_stats.py (currency-agnostic) → stats.json
|
|
# - Optional: exchange / merchant in-container generators
|
|
# - Optional: container RSS/loadavg merge
|
|
# - Publish: podman into container landings and/or host dirs (shared bank stats)
|
|
# - Never wipes a good stats.json on failure (stats-run.json only)
|
|
#
|
|
# Install: scripts/taler-landing/install-landing-stats-host.sh
|
|
set -euo pipefail
|
|
|
|
export TZ="${TZ:-Europe/Zurich}"
|
|
export PATH="${HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin${PATH:+:$PATH}"
|
|
|
|
# Optional stack profile: ~/.config/taler-landing/stack.env or STACK_ENV_FILE
|
|
if [ -n "${STACK_ENV_FILE:-}" ] && [ -f "${STACK_ENV_FILE}" ]; then
|
|
# shellcheck disable=SC1090
|
|
set -a
|
|
# shellcheck source=/dev/null
|
|
. "${STACK_ENV_FILE}"
|
|
set +a
|
|
elif [ -f "${HOME}/.config/taler-landing/stack.env" ]; then
|
|
set -a
|
|
# shellcheck source=/dev/null
|
|
. "${HOME}/.config/taler-landing/stack.env"
|
|
set +a
|
|
fi
|
|
|
|
log() { printf '%s %s\n' "$(date -Iseconds)" "$*"; }
|
|
|
|
ROOT="$(cd "$(dirname "$0")" && pwd)"
|
|
# When installed to ~/.local/bin, libs live in ~/.local/lib/taler-landing
|
|
if [ -f "$ROOT/collect_bank_stats.py" ]; then
|
|
LIB="$ROOT"
|
|
elif [ -f "${HOME}/.local/lib/taler-landing/collect_bank_stats.py" ]; then
|
|
LIB="${HOME}/.local/lib/taler-landing"
|
|
elif [ -f "${HOME}/src/koopa/koopa-admin-log/scripts/taler-landing/collect_bank_stats.py" ]; then
|
|
LIB="${HOME}/src/koopa/koopa-admin-log/scripts/taler-landing"
|
|
elif [ -f "${HOME}/taler/src/koopa-admin-log/scripts/taler-landing/collect_bank_stats.py" ]; then
|
|
LIB="${HOME}/taler/src/koopa-admin-log/scripts/taler-landing"
|
|
else
|
|
LIB="$ROOT"
|
|
fi
|
|
|
|
ADMIN_LOG="${ADMIN_LOG:-${HOME}/src/koopa/koopa-admin-log}"
|
|
SECRETS_BANK="${SECRETS_BANK:-${HOME}/src/koopa/koopa-admin-secrets/koopa/host-root/taler-bank}"
|
|
|
|
# Which phases to run (1/0)
|
|
COLLECT_BANK="${COLLECT_BANK:-1}"
|
|
COLLECT_EXCHANGE="${COLLECT_EXCHANGE:-1}"
|
|
COLLECT_MERCHANT="${COLLECT_MERCHANT:-1}"
|
|
COLLECT_RESOURCES="${COLLECT_RESOURCES:-1}"
|
|
|
|
# Containers (empty = skip podman publish for that role)
|
|
BANK_CTR="${BANK_CTR:-taler-hacktivism-bank}"
|
|
EX_CTR="${EX_CTR:-taler-hacktivism-exchange-ansible}"
|
|
MER_CTR="${MER_CTR:-taler-hacktivism}"
|
|
|
|
BANK_URL="${BANK_URL:-http://127.0.0.1:9012}"
|
|
BANK_PUBLIC_URL="${BANK_PUBLIC_URL:-https://bank.hacktivism.ch}"
|
|
EXCHANGE_CONFIG_URL="${EXCHANGE_CONFIG_URL:-https://exchange.hacktivism.ch/config}"
|
|
BANK_CURRENCY="${BANK_CURRENCY:-}" # empty → auto from bank/exchange /config
|
|
STATS_SOURCE_LABEL="${STATS_SOURCE_LABEL:-host collect_bank_stats.py}"
|
|
|
|
# In-container landing roots (podman publish)
|
|
BANK_LANDING_IN="${BANK_LANDING_IN:-/var/www/bank-landing}"
|
|
EX_LANDING_IN="${EX_LANDING_IN:-/var/www/exchange-landing}"
|
|
MER_LANDING_IN="${MER_LANDING_IN:-/var/www/merchant-landing}"
|
|
PUBLISH_PODMAN="${PUBLISH_PODMAN:-1}"
|
|
|
|
# Host dirs that should get the *same* bank stats.json (shared public feed).
|
|
# Colon-separated (systemd EnvironmentFile-safe). Example stage:
|
|
# HOST_STATS_DIRS=/var/www/lfp-stage/bank:/var/www/lfp-stage/exchange:/var/www/lfp-stage/merchant
|
|
HOST_STATS_DIRS="${HOST_STATS_DIRS:-}"
|
|
|
|
WORKDIR="${LANDING_STATS_WORKDIR:-${XDG_RUNTIME_DIR:-/tmp}/taler-landing-stats}"
|
|
mkdir -p "$WORKDIR"
|
|
LOG_DIR="${LANDING_STATS_LOGDIR:-${HOME}/.local/state/taler-landing-stats}"
|
|
mkdir -p "$LOG_DIR"
|
|
|
|
PY="${PYTHON:-python3}"
|
|
ec_bank=0
|
|
ec_ex=0
|
|
ec_mer=0
|
|
|
|
# Resolve helper scripts (checkout or ~/.local/lib)
|
|
COLLECT_RES="$LIB/collect_container_resources.sh"
|
|
MERGE_RES="$LIB/merge_resources.py"
|
|
if [ ! -f "$COLLECT_RES" ] && [ -f "$ROOT/collect_container_resources.sh" ]; then
|
|
COLLECT_RES="$ROOT/collect_container_resources.sh"
|
|
fi
|
|
if [ ! -f "$MERGE_RES" ] && [ -f "$ROOT/merge_resources.py" ]; then
|
|
MERGE_RES="$ROOT/merge_resources.py"
|
|
fi
|
|
MEM_SRC="${MEM_SNAPSHOT_SRC:-$ADMIN_LOG/scripts/taler-shared/mem-snapshot.sh}"
|
|
|
|
merge_container_resources() {
|
|
local label="$1" ctr="$2" stats_file="$3"
|
|
[ "${COLLECT_RESOURCES}" = "1" ] || return 0
|
|
[ -f "$stats_file" ] || return 0
|
|
[ -n "$ctr" ] || return 0
|
|
if ! ctr_running "$ctr"; then
|
|
log "WARN: $label resources: $ctr not running"
|
|
return 1
|
|
fi
|
|
if [ ! -x "$COLLECT_RES" ] && [ -f "$COLLECT_RES" ]; then
|
|
chmod +x "$COLLECT_RES" 2>/dev/null || true
|
|
fi
|
|
if [ ! -f "$COLLECT_RES" ] || [ ! -f "$MERGE_RES" ]; then
|
|
log "WARN: $label resources: helpers missing ($COLLECT_RES / $MERGE_RES)"
|
|
return 1
|
|
fi
|
|
local resf="$WORKDIR/${label}-resources.json"
|
|
set +e
|
|
ADMIN_LOG="$ADMIN_LOG" MEM_SNAPSHOT_SRC="$MEM_SRC" \
|
|
bash "$COLLECT_RES" "$ctr" "$resf" >>"$LOG_DIR/${label}-resources.log" 2>&1
|
|
local ec=$?
|
|
set -e
|
|
if [ "$ec" -ne 0 ] || [ ! -s "$resf" ]; then
|
|
log "WARN: $label resources: collect failed (ec=$ec)"
|
|
return 1
|
|
fi
|
|
if ! "$PY" -c 'import json,sys; d=json.load(open(sys.argv[1])); sys.exit(0 if d.get("ok") and d.get("memory") else 1)' "$resf" 2>/dev/null; then
|
|
log "WARN: $label resources: bad payload"
|
|
return 1
|
|
fi
|
|
"$PY" "$MERGE_RES" "$stats_file" "$resf" >>"$LOG_DIR/${label}-resources.log" 2>&1
|
|
log "$label: resources merged (RSS + loadavg from $ctr)"
|
|
return 0
|
|
}
|
|
|
|
# Write stats.json into host directories (shared feed: same file everywhere).
|
|
publish_host_stats() {
|
|
local src_stats="$1" src_run="${2:-}"
|
|
[ -n "${HOST_STATS_DIRS:-}" ] || return 0
|
|
[ -f "$src_stats" ] || return 0
|
|
local d paths
|
|
# Accept colon or space separators
|
|
paths=$(printf '%s' "$HOST_STATS_DIRS" | tr ': ' '\n' | sed '/^$/d')
|
|
while IFS= read -r d; do
|
|
[ -n "$d" ] || continue
|
|
if [ -d "$d" ] || mkdir -p "$d" 2>/dev/null; then
|
|
if cp -f "$src_stats" "${d}/stats.json" 2>/dev/null; then
|
|
chmod a+r "${d}/stats.json" 2>/dev/null || true
|
|
[ -n "$src_run" ] && [ -f "$src_run" ] && cp -f "$src_run" "${d}/stats-run.json" 2>/dev/null || true
|
|
log "host: published ${d}/stats.json"
|
|
continue
|
|
fi
|
|
fi
|
|
if command -v sudo >/dev/null 2>&1 && sudo -n true 2>/dev/null; then
|
|
sudo mkdir -p "$d" 2>/dev/null || true
|
|
sudo cp -f "$src_stats" "${d}/stats.json"
|
|
sudo chmod a+r "${d}/stats.json" 2>/dev/null || true
|
|
[ -n "$src_run" ] && [ -f "$src_run" ] && sudo cp -f "$src_run" "${d}/stats-run.json" 2>/dev/null || true
|
|
log "host: published ${d}/stats.json (sudo)"
|
|
else
|
|
log "WARN: cannot write ${d}/stats.json (need write or passwordless sudo)"
|
|
fi
|
|
done <<<"$paths"
|
|
}
|
|
|
|
read_pass() {
|
|
local name="$1" f
|
|
for f in \
|
|
"${SECRETS_BANK}/${name}" \
|
|
"${HOME}/.config/taler-landing/${name}" \
|
|
"/run/user/$(id -u)/taler-landing/${name}"
|
|
do
|
|
if [ -r "$f" ]; then
|
|
tr -d '\n' <"$f"
|
|
return 0
|
|
fi
|
|
done
|
|
# container secrets: /root/… (GOA) or /data/secrets/… (FrancPaysan stage)
|
|
if [ -n "${BANK_CTR:-}" ] && podman inspect -f '{{.State.Running}}' "$BANK_CTR" 2>/dev/null | grep -qx true; then
|
|
for path in "/root/${name}" "/data/secrets/${name}" "/data/secrets/bank-admin-password.txt"; do
|
|
case "$path" in
|
|
*bank-admin-password.txt)
|
|
[ "$name" = "bank-admin-password.txt" ] || continue
|
|
;;
|
|
esac
|
|
if podman exec "$BANK_CTR" test -r "$path" 2>/dev/null; then
|
|
podman exec "$BANK_CTR" cat "$path" 2>/dev/null | tr -d '\n\r'
|
|
return 0
|
|
fi
|
|
done
|
|
# stage explorer secret naming
|
|
if [ "$name" = "bank-explorer-password.txt" ]; then
|
|
if podman exec "$BANK_CTR" test -r /data/secrets/bank-explorer-password.txt 2>/dev/null; then
|
|
podman exec "$BANK_CTR" cat /data/secrets/bank-explorer-password.txt 2>/dev/null | tr -d '\n\r'
|
|
return 0
|
|
fi
|
|
fi
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
ctr_running() {
|
|
podman inspect -f '{{.State.Running}}' "$1" 2>/dev/null | grep -qx true
|
|
}
|
|
|
|
publish_json() {
|
|
local ctr="$1" dest_dir="$2" src_stats="$3" src_run="$4"
|
|
[ "${PUBLISH_PODMAN}" = "1" ] || return 0
|
|
[ -n "$ctr" ] || return 0
|
|
if ! ctr_running "$ctr"; then
|
|
log "WARN: $ctr not running — skip publish $dest_dir"
|
|
return 1
|
|
fi
|
|
podman exec "$ctr" mkdir -p "$dest_dir" 2>/dev/null || true
|
|
if [ -f "$src_stats" ]; then
|
|
podman cp "$src_stats" "${ctr}:${dest_dir}/stats.json"
|
|
fi
|
|
if [ -f "$src_run" ]; then
|
|
podman cp "$src_run" "${ctr}:${dest_dir}/stats-run.json"
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Bank — full scan (all accounts / all money except SCAN_SKIP)
|
|
# ---------------------------------------------------------------------------
|
|
collect_bank() {
|
|
log "bank: collect full stats via $LIB/collect_bank_stats.py"
|
|
local admin_pass explorer_pass
|
|
admin_pass="$(read_pass bank-admin-password.txt || true)"
|
|
explorer_pass="$(read_pass bank-explorer-password.txt || true)"
|
|
if [ -z "$admin_pass" ]; then
|
|
log "ERROR: bank admin password not found (secrets or container)"
|
|
printf '%s\n' '{"ok":false,"error":"no admin password","at_human":"'"$(date +"%Y-%m-%d %H:%M %Z")"'"}' \
|
|
>"$WORKDIR/bank-stats-run.json"
|
|
publish_json "$BANK_CTR" "$BANK_LANDING_IN" "" "$WORKDIR/bank-stats-run.json" || true
|
|
return 1
|
|
fi
|
|
|
|
# optional demo files from container
|
|
local demo_dir=""
|
|
if ctr_running "$BANK_CTR"; then
|
|
demo_dir="$WORKDIR/demo"
|
|
mkdir -p "$demo_dir"
|
|
podman cp "${BANK_CTR}:${BANK_LANDING_IN}/withdraw.uri" "$demo_dir/withdraw.uri" 2>/dev/null || true
|
|
podman cp "${BANK_CTR}:${BANK_LANDING_IN}/withdraw.amount" "$demo_dir/withdraw.amount" 2>/dev/null || true
|
|
podman cp "${BANK_CTR}:${BANK_LANDING_IN}/withdraw.created" "$demo_dir/withdraw.created" 2>/dev/null || true
|
|
fi
|
|
|
|
set +e
|
|
BANK_URL="$BANK_URL" \
|
|
BANK_PUBLIC_URL="$BANK_PUBLIC_URL" \
|
|
EXCHANGE_CONFIG_URL="$EXCHANGE_CONFIG_URL" \
|
|
BANK_CURRENCY="${BANK_CURRENCY:-}" \
|
|
BANK_ADMIN_PASS="$admin_pass" \
|
|
BANK_EXPLORER_PASS="$explorer_pass" \
|
|
DEMO_DIR="${demo_dir}" \
|
|
SCAN_SKIP="${SCAN_SKIP:-exchange}" \
|
|
TX_PAGE="${TX_PAGE:-500}" \
|
|
MAX_TX_PAGES="${MAX_TX_PAGES:-0}" \
|
|
ACCOUNTS_DELTA="${ACCOUNTS_DELTA:--10000}" \
|
|
STATS_SOURCE_LABEL="$STATS_SOURCE_LABEL" \
|
|
"$PY" "$LIB/collect_bank_stats.py" \
|
|
--out "$WORKDIR/bank-stats.json" \
|
|
--run-out "$WORKDIR/bank-stats-run.json" \
|
|
>>"$LOG_DIR/bank.log" 2>&1
|
|
ec_bank=$?
|
|
set -e
|
|
|
|
if [ "$ec_bank" -eq 0 ] && [ -f "$WORKDIR/bank-stats.json" ]; then
|
|
# Optional: attach in-container RSS/loadavg (not host /proc)
|
|
merge_container_resources bank "$BANK_CTR" "$WORKDIR/bank-stats.json" || true
|
|
publish_json "$BANK_CTR" "$BANK_LANDING_IN" \
|
|
"$WORKDIR/bank-stats.json" "$WORKDIR/bank-stats-run.json"
|
|
# Shared public feed: same bank stats on all HOST_STATS_DIRS (bank/ex/merchant landings)
|
|
publish_host_stats "$WORKDIR/bank-stats.json" "$WORKDIR/bank-stats-run.json"
|
|
log "bank: OK (podman=${BANK_CTR:-none} host_dirs=${HOST_STATS_DIRS:-none})"
|
|
else
|
|
log "ERROR: bank collect failed (ec=$ec_bank) — previous stats.json kept"
|
|
[ -f "$WORKDIR/bank-stats-run.json" ] || \
|
|
printf '%s\n' '{"ok":false,"error":"collect failed","at_human":"'"$(date +"%Y-%m-%d %H:%M %Z")"'"}' \
|
|
>"$WORKDIR/bank-stats-run.json"
|
|
publish_json "$BANK_CTR" "$BANK_LANDING_IN" "" "$WORKDIR/bank-stats-run.json" || true
|
|
fi
|
|
return "$ec_bank"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Exchange / merchant — in-container generators + host enrich
|
|
# ---------------------------------------------------------------------------
|
|
run_incontainer_stats() {
|
|
local label="$1" ctr="$2" script_candidates="$3" landing="$4"
|
|
local script="" s
|
|
if ! ctr_running "$ctr"; then
|
|
log "WARN: $label: $ctr not running"
|
|
return 1
|
|
fi
|
|
# install latest script from admin-log if present
|
|
local host_src=""
|
|
case "$label" in
|
|
exchange)
|
|
host_src="$ADMIN_LOG/scripts/taler-exchange/landing-stats-exchange.sh"
|
|
;;
|
|
merchant)
|
|
host_src="$ADMIN_LOG/scripts/taler-merchant/landing-stats-merchant.sh"
|
|
;;
|
|
esac
|
|
if [ -n "$host_src" ] && [ -f "$host_src" ]; then
|
|
podman cp "$host_src" "${ctr}:/usr/local/bin/$(basename "$host_src")"
|
|
podman exec "$ctr" chmod 755 "/usr/local/bin/$(basename "$host_src")" 2>/dev/null || true
|
|
script="/usr/local/bin/$(basename "$host_src")"
|
|
else
|
|
for s in $script_candidates; do
|
|
if podman exec "$ctr" test -x "$s" 2>/dev/null || podman exec "$ctr" test -f "$s" 2>/dev/null; then
|
|
script="$s"
|
|
break
|
|
fi
|
|
done
|
|
fi
|
|
if [ -z "$script" ]; then
|
|
log "WARN: $label: no landing-stats script in $ctr"
|
|
return 1
|
|
fi
|
|
log "$label: run $script inside $ctr"
|
|
set +e
|
|
podman exec \
|
|
-e LANDING_DIR="$landing" \
|
|
-e TZ=Europe/Zurich \
|
|
-e PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
|
|
"$ctr" bash "$script" >>"$LOG_DIR/${label}.log" 2>&1
|
|
local ec=$?
|
|
set -e
|
|
if [ "$ec" -ne 0 ]; then
|
|
log "ERROR: $label stats failed (ec=$ec)"
|
|
return "$ec"
|
|
fi
|
|
# enrich with alt units + ensure resources (RSS/loadavg) on host
|
|
podman cp "${ctr}:${landing}/stats.json" "$WORKDIR/${label}-stats.json" 2>/dev/null || return 0
|
|
set +e
|
|
"$PY" "$LIB/enrich_stats_alt.py" "$WORKDIR/${label}-stats.json" \
|
|
--exchange-config "$EXCHANGE_CONFIG_URL" >>"$LOG_DIR/${label}.log" 2>&1
|
|
set -e
|
|
# Prefer fresh container snapshot (also fills gaps if in-container mem helper missing)
|
|
merge_container_resources "$label" "$ctr" "$WORKDIR/${label}-stats.json" || true
|
|
podman cp "$WORKDIR/${label}-stats.json" "${ctr}:${landing}/stats.json"
|
|
log "$label: OK + alt + resources → ${ctr}:${landing}/stats.json"
|
|
return 0
|
|
}
|
|
|
|
collect_exchange() {
|
|
run_incontainer_stats exchange "$EX_CTR" \
|
|
"/usr/local/bin/landing-stats-exchange.sh /usr/local/bin/landing-stats.sh" \
|
|
"$EX_LANDING_IN"
|
|
}
|
|
|
|
collect_merchant() {
|
|
run_incontainer_stats merchant "$MER_CTR" \
|
|
"/usr/local/bin/landing-stats-merchant.sh /usr/local/bin/landing-stats.sh" \
|
|
"$MER_LANDING_IN"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
main() {
|
|
log "=== taler-landing-stats start (user=$(id -un) lib=$LIB currency=${BANK_CURRENCY:-auto}) ==="
|
|
log "phases bank=${COLLECT_BANK} exchange=${COLLECT_EXCHANGE} merchant=${COLLECT_MERCHANT} resources=${COLLECT_RESOURCES}"
|
|
if [ "${COLLECT_BANK}" = "1" ]; then
|
|
collect_bank || ec_bank=$?
|
|
else
|
|
log "skip bank collect"
|
|
fi
|
|
if [ "${COLLECT_EXCHANGE}" = "1" ]; then
|
|
collect_exchange || ec_ex=$?
|
|
else
|
|
log "skip exchange collect"
|
|
fi
|
|
if [ "${COLLECT_MERCHANT}" = "1" ]; then
|
|
collect_merchant || ec_mer=$?
|
|
else
|
|
log "skip merchant collect"
|
|
fi
|
|
log "=== done bank=$ec_bank exchange=$ec_ex merchant=$ec_mer ==="
|
|
# non-zero if bank failed (primary public flow numbers); soft on ex/mer
|
|
if [ "${COLLECT_BANK}" = "1" ] && [ "$ec_bank" -ne 0 ]; then
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
main "$@"
|