78bbd80d1f
Collect RSS groups, top processes, and loadavg from inside each podman container (not host /proc), merge into performance.memory, show compact labels with cgroup limits and byte tooltips, and cover this in the test suite.
287 lines
10 KiB
Bash
Executable file
287 lines
10 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Central landing-stats collector for hernani@koopa (user systemd timer).
|
|
#
|
|
# - Bank: full account+ledger scan via collect_bank_stats.py → bank container
|
|
# - Exchange / merchant: existing in-container scripts, then amount_alt enrich
|
|
# - All three: container RSS / loadavg / top procs via mem-snapshot (podman exec)
|
|
# - Never wipes a good stats.json on failure (writes stats-run.json only)
|
|
#
|
|
# Install: scripts/taler-landing/install-landing-stats-host.sh
|
|
set -euo pipefail
|
|
|
|
export TZ="${TZ:-Europe/Zurich}"
|
|
export PATH="${HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin${PATH:+:$PATH}"
|
|
|
|
log() { printf '%s %s\n' "$(date -Iseconds)" "$*"; }
|
|
|
|
ROOT="$(cd "$(dirname "$0")" && pwd)"
|
|
# When installed to ~/.local/bin, libs live in ~/.local/lib/taler-landing
|
|
if [ -f "$ROOT/collect_bank_stats.py" ]; then
|
|
LIB="$ROOT"
|
|
elif [ -f "${HOME}/.local/lib/taler-landing/collect_bank_stats.py" ]; then
|
|
LIB="${HOME}/.local/lib/taler-landing"
|
|
elif [ -f "${HOME}/src/koopa/koopa-admin-log/scripts/taler-landing/collect_bank_stats.py" ]; then
|
|
LIB="${HOME}/src/koopa/koopa-admin-log/scripts/taler-landing"
|
|
else
|
|
LIB="$ROOT"
|
|
fi
|
|
|
|
ADMIN_LOG="${ADMIN_LOG:-${HOME}/src/koopa/koopa-admin-log}"
|
|
SECRETS_BANK="${SECRETS_BANK:-${HOME}/src/koopa/koopa-admin-secrets/koopa/host-root/taler-bank}"
|
|
|
|
BANK_CTR="${BANK_CTR:-taler-hacktivism-bank}"
|
|
EX_CTR="${EX_CTR:-taler-hacktivism-exchange-ansible}"
|
|
MER_CTR="${MER_CTR:-taler-hacktivism}"
|
|
|
|
BANK_URL="${BANK_URL:-http://127.0.0.1:9012}"
|
|
BANK_PUBLIC_URL="${BANK_PUBLIC_URL:-https://bank.hacktivism.ch}"
|
|
EXCHANGE_CONFIG_URL="${EXCHANGE_CONFIG_URL:-https://exchange.hacktivism.ch/config}"
|
|
|
|
BANK_LANDING_IN="${BANK_LANDING_IN:-/var/www/bank-landing}"
|
|
EX_LANDING_IN="${EX_LANDING_IN:-/var/www/exchange-landing}"
|
|
MER_LANDING_IN="${MER_LANDING_IN:-/var/www/merchant-landing}"
|
|
|
|
WORKDIR="${LANDING_STATS_WORKDIR:-${XDG_RUNTIME_DIR:-/tmp}/taler-landing-stats}"
|
|
mkdir -p "$WORKDIR"
|
|
LOG_DIR="${LANDING_STATS_LOGDIR:-${HOME}/.local/state/taler-landing-stats}"
|
|
mkdir -p "$LOG_DIR"
|
|
|
|
PY="${PYTHON:-python3}"
|
|
ec_bank=0
|
|
ec_ex=0
|
|
ec_mer=0
|
|
|
|
# Resolve helper scripts (checkout or ~/.local/lib)
|
|
COLLECT_RES="$LIB/collect_container_resources.sh"
|
|
MERGE_RES="$LIB/merge_resources.py"
|
|
if [ ! -f "$COLLECT_RES" ] && [ -f "$ROOT/collect_container_resources.sh" ]; then
|
|
COLLECT_RES="$ROOT/collect_container_resources.sh"
|
|
fi
|
|
if [ ! -f "$MERGE_RES" ] && [ -f "$ROOT/merge_resources.py" ]; then
|
|
MERGE_RES="$ROOT/merge_resources.py"
|
|
fi
|
|
MEM_SRC="${MEM_SNAPSHOT_SRC:-$ADMIN_LOG/scripts/taler-shared/mem-snapshot.sh}"
|
|
|
|
merge_container_resources() {
|
|
local label="$1" ctr="$2" stats_file="$3"
|
|
[ -f "$stats_file" ] || return 0
|
|
if ! ctr_running "$ctr"; then
|
|
log "WARN: $label resources: $ctr not running"
|
|
return 1
|
|
fi
|
|
if [ ! -x "$COLLECT_RES" ] && [ -f "$COLLECT_RES" ]; then
|
|
chmod +x "$COLLECT_RES" 2>/dev/null || true
|
|
fi
|
|
if [ ! -f "$COLLECT_RES" ] || [ ! -f "$MERGE_RES" ]; then
|
|
log "WARN: $label resources: helpers missing ($COLLECT_RES / $MERGE_RES)"
|
|
return 1
|
|
fi
|
|
local resf="$WORKDIR/${label}-resources.json"
|
|
set +e
|
|
ADMIN_LOG="$ADMIN_LOG" MEM_SNAPSHOT_SRC="$MEM_SRC" \
|
|
bash "$COLLECT_RES" "$ctr" "$resf" >>"$LOG_DIR/${label}-resources.log" 2>&1
|
|
local ec=$?
|
|
set -e
|
|
if [ "$ec" -ne 0 ] || [ ! -s "$resf" ]; then
|
|
log "WARN: $label resources: collect failed (ec=$ec)"
|
|
return 1
|
|
fi
|
|
if ! "$PY" -c 'import json,sys; d=json.load(open(sys.argv[1])); sys.exit(0 if d.get("ok") and d.get("memory") else 1)' "$resf" 2>/dev/null; then
|
|
log "WARN: $label resources: bad payload"
|
|
return 1
|
|
fi
|
|
"$PY" "$MERGE_RES" "$stats_file" "$resf" >>"$LOG_DIR/${label}-resources.log" 2>&1
|
|
log "$label: resources merged (RSS + loadavg from $ctr)"
|
|
return 0
|
|
}
|
|
|
|
read_pass() {
|
|
local name="$1" f
|
|
for f in \
|
|
"${SECRETS_BANK}/${name}" \
|
|
"${HOME}/.config/taler-landing/${name}" \
|
|
"/run/user/$(id -u)/taler-landing/${name}"
|
|
do
|
|
if [ -r "$f" ]; then
|
|
tr -d '\n' <"$f"
|
|
return 0
|
|
fi
|
|
done
|
|
# container (hernani can podman exec root files inside owned containers)
|
|
if podman inspect -f '{{.State.Running}}' "$BANK_CTR" 2>/dev/null | grep -qx true; then
|
|
if podman exec "$BANK_CTR" test -r "/root/${name}" 2>/dev/null; then
|
|
podman exec "$BANK_CTR" cat "/root/${name}" 2>/dev/null | tr -d '\n'
|
|
return 0
|
|
fi
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
ctr_running() {
|
|
podman inspect -f '{{.State.Running}}' "$1" 2>/dev/null | grep -qx true
|
|
}
|
|
|
|
publish_json() {
|
|
local ctr="$1" dest_dir="$2" src_stats="$3" src_run="$4"
|
|
if ! ctr_running "$ctr"; then
|
|
log "WARN: $ctr not running — skip publish $dest_dir"
|
|
return 1
|
|
fi
|
|
podman exec "$ctr" mkdir -p "$dest_dir" 2>/dev/null || true
|
|
if [ -f "$src_stats" ]; then
|
|
podman cp "$src_stats" "${ctr}:${dest_dir}/stats.json"
|
|
fi
|
|
if [ -f "$src_run" ]; then
|
|
podman cp "$src_run" "${ctr}:${dest_dir}/stats-run.json"
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Bank — full scan (all accounts / all money except SCAN_SKIP)
|
|
# ---------------------------------------------------------------------------
|
|
collect_bank() {
|
|
log "bank: collect full stats via $LIB/collect_bank_stats.py"
|
|
local admin_pass explorer_pass
|
|
admin_pass="$(read_pass bank-admin-password.txt || true)"
|
|
explorer_pass="$(read_pass bank-explorer-password.txt || true)"
|
|
if [ -z "$admin_pass" ]; then
|
|
log "ERROR: bank admin password not found (secrets or container)"
|
|
printf '%s\n' '{"ok":false,"error":"no admin password","at_human":"'"$(date +"%Y-%m-%d %H:%M %Z")"'"}' \
|
|
>"$WORKDIR/bank-stats-run.json"
|
|
publish_json "$BANK_CTR" "$BANK_LANDING_IN" "" "$WORKDIR/bank-stats-run.json" || true
|
|
return 1
|
|
fi
|
|
|
|
# optional demo files from container
|
|
local demo_dir=""
|
|
if ctr_running "$BANK_CTR"; then
|
|
demo_dir="$WORKDIR/demo"
|
|
mkdir -p "$demo_dir"
|
|
podman cp "${BANK_CTR}:${BANK_LANDING_IN}/withdraw.uri" "$demo_dir/withdraw.uri" 2>/dev/null || true
|
|
podman cp "${BANK_CTR}:${BANK_LANDING_IN}/withdraw.amount" "$demo_dir/withdraw.amount" 2>/dev/null || true
|
|
podman cp "${BANK_CTR}:${BANK_LANDING_IN}/withdraw.created" "$demo_dir/withdraw.created" 2>/dev/null || true
|
|
fi
|
|
|
|
set +e
|
|
BANK_URL="$BANK_URL" \
|
|
BANK_PUBLIC_URL="$BANK_PUBLIC_URL" \
|
|
EXCHANGE_CONFIG_URL="$EXCHANGE_CONFIG_URL" \
|
|
BANK_ADMIN_PASS="$admin_pass" \
|
|
BANK_EXPLORER_PASS="$explorer_pass" \
|
|
DEMO_DIR="${demo_dir}" \
|
|
SCAN_SKIP="${SCAN_SKIP:-exchange}" \
|
|
TX_PAGE="${TX_PAGE:-500}" \
|
|
MAX_TX_PAGES="${MAX_TX_PAGES:-0}" \
|
|
ACCOUNTS_DELTA="${ACCOUNTS_DELTA:--10000}" \
|
|
"$PY" "$LIB/collect_bank_stats.py" \
|
|
--out "$WORKDIR/bank-stats.json" \
|
|
--run-out "$WORKDIR/bank-stats-run.json" \
|
|
>>"$LOG_DIR/bank.log" 2>&1
|
|
ec_bank=$?
|
|
set -e
|
|
|
|
if [ "$ec_bank" -eq 0 ] && [ -f "$WORKDIR/bank-stats.json" ]; then
|
|
# Always attach in-container RSS/loadavg (not host /proc)
|
|
merge_container_resources bank "$BANK_CTR" "$WORKDIR/bank-stats.json" || true
|
|
publish_json "$BANK_CTR" "$BANK_LANDING_IN" \
|
|
"$WORKDIR/bank-stats.json" "$WORKDIR/bank-stats-run.json"
|
|
log "bank: OK → ${BANK_CTR}:${BANK_LANDING_IN}/stats.json"
|
|
else
|
|
log "ERROR: bank collect failed (ec=$ec_bank) — previous stats.json kept"
|
|
[ -f "$WORKDIR/bank-stats-run.json" ] || \
|
|
printf '%s\n' '{"ok":false,"error":"collect failed","at_human":"'"$(date +"%Y-%m-%d %H:%M %Z")"'"}' \
|
|
>"$WORKDIR/bank-stats-run.json"
|
|
publish_json "$BANK_CTR" "$BANK_LANDING_IN" "" "$WORKDIR/bank-stats-run.json" || true
|
|
fi
|
|
return "$ec_bank"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Exchange / merchant — in-container generators + host enrich
|
|
# ---------------------------------------------------------------------------
|
|
run_incontainer_stats() {
|
|
local label="$1" ctr="$2" script_candidates="$3" landing="$4"
|
|
local script="" s
|
|
if ! ctr_running "$ctr"; then
|
|
log "WARN: $label: $ctr not running"
|
|
return 1
|
|
fi
|
|
# install latest script from admin-log if present
|
|
local host_src=""
|
|
case "$label" in
|
|
exchange)
|
|
host_src="$ADMIN_LOG/scripts/taler-exchange/landing-stats-exchange.sh"
|
|
;;
|
|
merchant)
|
|
host_src="$ADMIN_LOG/scripts/taler-merchant/landing-stats-merchant.sh"
|
|
;;
|
|
esac
|
|
if [ -n "$host_src" ] && [ -f "$host_src" ]; then
|
|
podman cp "$host_src" "${ctr}:/usr/local/bin/$(basename "$host_src")"
|
|
podman exec "$ctr" chmod 755 "/usr/local/bin/$(basename "$host_src")" 2>/dev/null || true
|
|
script="/usr/local/bin/$(basename "$host_src")"
|
|
else
|
|
for s in $script_candidates; do
|
|
if podman exec "$ctr" test -x "$s" 2>/dev/null || podman exec "$ctr" test -f "$s" 2>/dev/null; then
|
|
script="$s"
|
|
break
|
|
fi
|
|
done
|
|
fi
|
|
if [ -z "$script" ]; then
|
|
log "WARN: $label: no landing-stats script in $ctr"
|
|
return 1
|
|
fi
|
|
log "$label: run $script inside $ctr"
|
|
set +e
|
|
podman exec \
|
|
-e LANDING_DIR="$landing" \
|
|
-e TZ=Europe/Zurich \
|
|
-e PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
|
|
"$ctr" bash "$script" >>"$LOG_DIR/${label}.log" 2>&1
|
|
local ec=$?
|
|
set -e
|
|
if [ "$ec" -ne 0 ]; then
|
|
log "ERROR: $label stats failed (ec=$ec)"
|
|
return "$ec"
|
|
fi
|
|
# enrich with alt units + ensure resources (RSS/loadavg) on host
|
|
podman cp "${ctr}:${landing}/stats.json" "$WORKDIR/${label}-stats.json" 2>/dev/null || return 0
|
|
set +e
|
|
"$PY" "$LIB/enrich_stats_alt.py" "$WORKDIR/${label}-stats.json" \
|
|
--exchange-config "$EXCHANGE_CONFIG_URL" >>"$LOG_DIR/${label}.log" 2>&1
|
|
set -e
|
|
# Prefer fresh container snapshot (also fills gaps if in-container mem helper missing)
|
|
merge_container_resources "$label" "$ctr" "$WORKDIR/${label}-stats.json" || true
|
|
podman cp "$WORKDIR/${label}-stats.json" "${ctr}:${landing}/stats.json"
|
|
log "$label: OK + alt + resources → ${ctr}:${landing}/stats.json"
|
|
return 0
|
|
}
|
|
|
|
collect_exchange() {
|
|
run_incontainer_stats exchange "$EX_CTR" \
|
|
"/usr/local/bin/landing-stats-exchange.sh /usr/local/bin/landing-stats.sh" \
|
|
"$EX_LANDING_IN"
|
|
}
|
|
|
|
collect_merchant() {
|
|
run_incontainer_stats merchant "$MER_CTR" \
|
|
"/usr/local/bin/landing-stats-merchant.sh /usr/local/bin/landing-stats.sh" \
|
|
"$MER_LANDING_IN"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
main() {
|
|
log "=== taler-landing-stats start (user=$(id -un) lib=$LIB) ==="
|
|
collect_bank || ec_bank=$?
|
|
collect_exchange || ec_ex=$?
|
|
collect_merchant || ec_mer=$?
|
|
log "=== done bank=$ec_bank exchange=$ec_ex merchant=$ec_mer ==="
|
|
# non-zero if bank failed (primary public flow numbers); soft on ex/mer
|
|
if [ "$ec_bank" -ne 0 ]; then
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
main "$@"
|