koopa-admin-log/scripts/taler-merchant/landing-stats-merchant.sh
Hernâni Marques 458472e177
merchant landing: recent activity split GOA/CHF, 5 each
Stats script emits recent_activity_by_currency (limit 5); UI shows
side-by-side columns instead of one long mixed list.
2026-07-10 20:19:59 +02:00

439 lines
15 KiB
Bash
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/bin/bash
# Run INSIDE taler-hacktivism. Writes /var/www/merchant-landing/stats.json
#
# Current taler-merchant schema:
# merchant.merchant_instances → merchant_serial, merchant_id
# merchant_instance_<serial>.* → per-instance tables
#
# IMPORTANT:
# - cron uses a minimal PATH. Without /usr/sbin, runuser is missing and
# every query used to fail silently → all zeros (and overwrote good data).
# - Never write stats.json on failure: leave the previous good file in place.
# - Avoid: psql -F$'\t' -v ON_ERROR_STOP=… (if -F loses the tab arg, -v is
# eaten as fieldsep and ON_ERROR_STOP becomes the *username*).
set -euo pipefail
export TZ="${TZ:-Europe/Zurich}"
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin${PATH:+:$PATH}"
LANDING_DIR="${LANDING_DIR:-/var/www/merchant-landing}"
OUT="$LANDING_DIR/stats.json"
RUN="$LANDING_DIR/stats-run.json"
TMP="${OUT}.tmp.$$"
DB="${MERCHANT_DB:-taler-merchant}"
ACTIVITY_LIMIT="${ACTIVITY_LIMIT:-12}"
ERRLOG="${LANDING_STATS_ERRLOG:-/var/log/landing-stats-merchant.err}"
mkdir -p "$LANDING_DIR"
now_iso() { date +%Y-%m-%dT%H:%M%z | sed -E 's/([+-][0-9]{2})([0-9]{2})$/\1:\2/'; }
now_human() { date +"%Y-%m-%d %H:%M %Z"; }
json_str() {
printf '"%s"' "$(printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g' | tr '\n\r\t' ' ' | sed 's/ */ /g')"
}
# Public run status for the intro page (never wipe stats.json on failure).
write_run() {
local ok_json="$1" msg="${2:-}"
cat >"$RUN" <<EOF
{
"ok": ${ok_json},
"at": $(json_str "$(now_iso)"),
"at_human": $(json_str "$(now_human)"),
"error": $( [ -n "$msg" ] && json_str "$msg" || echo null )
}
EOF
}
abort() {
# Do NOT touch $OUT — previous good stats stay live; site shows run note.
write_run false "$*"
printf '%s\n' "abort merchant-stats: $*" | tee -a "$ERRLOG" >&2
rm -f "$TMP" /tmp/merch_cur_$$.tsv /tmp/merch_act_$$.tsv 2>/dev/null || true
exit 1
}
as_postgres() {
if command -v runuser >/dev/null 2>&1; then
runuser -u postgres -- "$@"
elif command -v su >/dev/null 2>&1; then
su -s /bin/bash postgres -c "$*"
else
return 127
fi
}
# pipe SQL on stdin — never -f on root-owned temps (postgres cannot read them)
psql_pipe() {
local fs=$'\t' ec
set +e
as_postgres psql -d "$DB" -At -F"$fs" 2>>"$ERRLOG"
ec=$?
set -e
return "$ec"
}
# Required single-value query. Empty/fail → abort (no site write).
psqlq() {
local sql="$1" out ec
set +e
out=$(as_postgres psql -d "$DB" -At -c "$sql" 2>>"$ERRLOG")
ec=$?
set -e
if [ "$ec" -ne 0 ]; then
abort "psql failed (ec=$ec): ${sql:0:120}"
fi
# strip trailing newlines only
printf '%s' "$out" | tr -d '\r'
}
# "64.03" or "64.03000001" → clean CUR:value using decimal string (no binary float)
fmt_cur_num() {
local c="$1" n="$2"
awk -v c="$c" -v n="$n" 'BEGIN{
gsub(/ /,"",n)
if (n == "" || n+0 == 0 && n !~ /[1-9]/) { printf "%s:0", c; exit }
if (n ~ /[eE]/) {
x = n+0
s = sprintf("%.8f", x)
} else {
s = n
}
if (s ~ /\./) {
split(s, a, ".")
whole = a[1]; frac = substr(a[2] "00000000", 1, 8)
extra = substr(a[2], 9, 1)
if (extra != "" && extra+0 >= 5) {
f = frac+0 + 1
if (f >= 100000000) { whole = whole+1; f = f - 100000000 }
frac = sprintf("%08d", f)
}
sub(/0+$/, "", frac)
if (frac == "") printf "%s:%s", c, whole
else printf "%s:%s.%s", c, whole, frac
} else {
printf "%s:%s", c, s
}
}'
}
# --- preflight: tools + DB reachability (fail closed) ---
command -v psql >/dev/null 2>&1 || abort "psql not in PATH ($PATH)"
command -v runuser >/dev/null 2>&1 || command -v su >/dev/null 2>&1 || abort "neither runuser nor su in PATH"
PROBE=$(psqlq "SELECT 1;")
[ "$PROBE" = "1" ] || abort "postgres not reachable (SELECT 1 → '$PROBE')"
SERIALS=$(psqlq "SELECT merchant_serial::text FROM merchant.merchant_instances ORDER BY merchant_serial;")
# If table exists but we got nothing, still ok (empty install). If table missing, psqlq aborted.
# Build list of existing instance schemas
SCHEMAS=""
INSTANCES=0
while read -r serial; do
[ -z "$serial" ] && continue
# only pure integers
[[ "$serial" =~ ^[0-9]+$ ]] || abort "bad merchant_serial='$serial'"
sch="merchant_instance_${serial}"
exists=$(psqlq "SELECT 1 FROM pg_namespace WHERE nspname = '${sch}';")
if [ "$exists" != "1" ]; then
# schema lag — skip, do not abort (instance row without schema yet)
continue
fi
INSTANCES=$((INSTANCES + 1))
SCHEMAS="${SCHEMAS}${SCHEMAS:+ }$sch:$serial"
done <<<"$SERIALS"
# Sanity: if instance schemas exist in PG but SERIALS empty → query path broken
NS_COUNT=$(psqlq "SELECT count(*)::text FROM pg_namespace WHERE nspname LIKE 'merchant_instance_%';")
NS_COUNT=$(printf '%s' "$NS_COUNT" | tr -d '[:space:]')
if [ "${NS_COUNT:-0}" -gt 0 ] && [ "$INSTANCES" -eq 0 ]; then
abort "pg has ${NS_COUNT} merchant_instance_* schemas but resolved INSTANCES=0 (query/PATH bug)"
fi
# --- dual-currency aggregation entirely in PostgreSQL (numeric) ---
{
echo "SELECT"
echo " coalesce(nullif(split_part(c.contract_terms->>'amount', ':', 1), ''), '?') AS currency,"
echo " count(*)::bigint,"
echo " count(*) FILTER (WHERE c.paid)::bigint,"
echo " count(*) FILTER (WHERE NOT c.paid)::bigint,"
echo " count(*) FILTER (WHERE c.wired)::bigint,"
echo " round(coalesce(sum(NULLIF(split_part(c.contract_terms->>'amount', ':', 2), '')::numeric), 0), 8)::text,"
echo " round(coalesce(sum(NULLIF(split_part(c.contract_terms->>'amount', ':', 2), '')::numeric) FILTER (WHERE c.paid), 0), 8)::text"
echo "FROM ("
first=1
for item in $SCHEMAS; do
sch=${item%%:*}
if [ "$first" = 1 ]; then first=0; else echo " UNION ALL "; fi
echo "SELECT contract_terms, paid, wired FROM ${sch}.merchant_contract_terms"
done
if [ "$first" = 1 ]; then
echo "SELECT NULL::jsonb AS contract_terms, false AS paid, false AS wired WHERE false"
fi
echo ") c"
echo "WHERE c.contract_terms ? 'amount'"
echo "GROUP BY 1 ORDER BY 1;"
} | psql_pipe >"/tmp/merch_cur_$$.tsv" || abort "currency aggregate query failed"
CUR_TSV=$(cat "/tmp/merch_cur_$$.tsv" 2>/dev/null || true)
rm -f "/tmp/merch_cur_$$.tsv"
TOTAL_CONTRACTS=0; TOTAL_PAID=0; TOTAL_WIRED=0; TOTAL_UNPAID=0
declare -A C_CONTRACTS C_PAID C_UNPAID C_WIRED C_AMT C_AMT_PAID
while IFS=$'\t' read -r cur contracts paid unpaid wired amt amt_paid; do
[ -z "${cur:-}" ] && continue
TOTAL_CONTRACTS=$((TOTAL_CONTRACTS + contracts))
TOTAL_PAID=$((TOTAL_PAID + paid))
TOTAL_UNPAID=$((TOTAL_UNPAID + unpaid))
TOTAL_WIRED=$((TOTAL_WIRED + wired))
C_CONTRACTS[$cur]=$contracts
C_PAID[$cur]=$paid
C_UNPAID[$cur]=$unpaid
C_WIRED[$cur]=$wired
C_AMT[$cur]=$amt
C_AMT_PAID[$cur]=$amt_paid
done <<<"$CUR_TSV"
# deposits / refunds
DEPOSITS=0; REFUNDS=0
for item in $SCHEMAS; do
sch=${item%%:*}
d=$(psqlq "SELECT count(*)::text FROM ${sch}.merchant_deposits;")
r=$(psqlq "SELECT count(*)::text FROM ${sch}.merchant_refunds;")
d=$(printf '%s' "$d" | tr -d '[:space:]')
r=$(printf '%s' "$r" | tr -d '[:space:]')
[[ "$d" =~ ^[0-9]+$ ]] || abort "bad deposits count for $sch: '$d'"
[[ "$r" =~ ^[0-9]+$ ]] || abort "bad refunds count for $sch: '$r'"
DEPOSITS=$((DEPOSITS + d))
REFUNDS=$((REFUNDS + r))
done
# --- recent activity: 5 latest events per currency (GOA + CHF), payments + refunds ---
ACT_PER_CURRENCY="${ACT_PER_CURRENCY:-5}"
# Query latest ACT_PER_CURRENCY events for one currency code (payments refunds).
# Writes TSV rows: ts kind order_id amount summary status
query_activity_for_currency() {
local cur="$1"
local out="$2"
{
echo "SELECT * FROM ("
echo "SELECT * FROM ("
first=1
for item in $SCHEMAS; do
sch=${item%%:*}
if [ "$first" = 1 ]; then first=0; else echo " UNION ALL "; fi
cat <<SQL
SELECT creation_time AS ts,
'payment'::text AS kind,
order_id,
coalesce(contract_terms->>'amount','') AS amount,
left(translate(coalesce(contract_terms->>'summary',''), E'\t\n\r', ' '), 64) AS summary,
CASE WHEN wired THEN 'wired' ELSE 'paid' END AS status
FROM ${sch}.merchant_contract_terms
WHERE paid
AND upper(split_part(coalesce(contract_terms->>'amount',''), ':', 1)) = upper('${cur}')
SQL
done
if [ "$first" = 1 ]; then
echo "SELECT 0::bigint AS ts, ''::text AS kind, ''::text AS order_id, ''::text AS amount, ''::text AS summary, ''::text AS status WHERE false"
fi
echo " UNION ALL "
first=1
for item in $SCHEMAS; do
sch=${item%%:*}
if [ "$first" = 1 ]; then first=0; else echo " UNION ALL "; fi
cat <<SQL
SELECT r.refund_timestamp AS ts,
'refund'::text AS kind,
ct.order_id,
(r.refund_amount).curr || ':' ||
CASE WHEN (r.refund_amount).frac = 0
THEN (r.refund_amount).val::text
ELSE trim(trailing '0' FROM trim(trailing '.' FROM (
((r.refund_amount).val + (r.refund_amount).frac::numeric / 100000000)::numeric(32,8)
)::text))
END AS amount,
left(translate(coalesce(r.reason,''), E'\t\n\r', ' '), 64) AS summary,
'refunded'::text AS status
FROM ${sch}.merchant_refunds r
JOIN ${sch}.merchant_contract_terms ct ON ct.order_serial = r.order_serial
WHERE upper((r.refund_amount).curr) = upper('${cur}')
SQL
done
if [ "$first" = 1 ]; then
echo "SELECT 0::bigint AS ts, ''::text AS kind, ''::text AS order_id, ''::text AS amount, ''::text AS summary, ''::text AS status WHERE false"
fi
echo ") u ORDER BY ts DESC LIMIT ${ACT_PER_CURRENCY}"
echo ") act;"
} | psql_pipe >"$out" || abort "activity query failed for $cur"
}
# Build JSON array of activity items from a TSV file
activity_tsv_to_json() {
local tsv_file="$1"
local json="["
local af=1
local ts kind oid amt sum st sec human_fmt iso
while IFS=$'\t' read -r ts kind oid amt sum st; do
[ -z "${ts:-}" ] && continue
[ "$ts" = "0" ] && [ -z "$kind" ] && continue
sec=$(awk -v t="$ts" 'BEGIN{printf "%d", int(t/1000000)}')
human_fmt=$(date -d "@${sec}" +"%Y-%m-%d %H:%M %Z" 2>/dev/null || echo "$sec")
iso=$(date -d "@${sec}" +"%Y-%m-%dT%H:%M:%S%z" 2>/dev/null | sed -E 's/([+-][0-9]{2})([0-9]{2})$/\1:\2/' || true)
if [ "$af" = 1 ]; then af=0; else json="${json},"; fi
json="${json}
{
\"ts_us\": ${ts:-0},
\"ts\": $(json_str "$iso"),
\"ts_human\": $(json_str "$human_fmt"),
\"kind\": $(json_str "$kind"),
\"order_id\": $(json_str "$oid"),
\"amount\": $(json_str "$amt"),
\"summary\": $(json_str "$sum"),
\"status\": $(json_str "$st")
}"
done <"$tsv_file"
json="${json}
]"
printf '%s' "$json"
}
query_activity_for_currency GOA "/tmp/merch_act_goa_$$.tsv"
query_activity_for_currency CHF "/tmp/merch_act_chf_$$.tsv"
ACT_GOA_JSON=$(activity_tsv_to_json "/tmp/merch_act_goa_$$.tsv")
ACT_CHF_JSON=$(activity_tsv_to_json "/tmp/merch_act_chf_$$.tsv")
rm -f "/tmp/merch_act_goa_$$.tsv" "/tmp/merch_act_chf_$$.tsv"
# Flat recent_activity: GOA then CHF (compat; landing prefers by_currency)
ACT_JSON="["
af=1
for block in "$ACT_GOA_JSON" "$ACT_CHF_JSON"; do
# strip outer [ ] and inject if non-empty
inner=$(printf '%s' "$block" | sed '1s/^\s*\[//; $s/\]\s*$//')
# empty array → skip
if printf '%s' "$inner" | grep -q '"kind"'; then
if [ "$af" = 1 ]; then af=0; else ACT_JSON="${ACT_JSON},"; fi
ACT_JSON="${ACT_JSON}${inner}"
fi
done
ACT_JSON="${ACT_JSON}
]"
ACT_BY_CUR_JSON="[
{
\"currency\": \"GOA\",
\"limit\": ${ACT_PER_CURRENCY},
\"items\": ${ACT_GOA_JSON}
},
{
\"currency\": \"CHF\",
\"limit\": ${ACT_PER_CURRENCY},
\"items\": ${ACT_CHF_JSON}
}
]"
# by_currency JSON — GOA then CHF then rest
CUR_JSON="["
cf=1
emit_cur() {
local cur="$1"
[ -n "${C_CONTRACTS[$cur]+x}" ] || return 1
local amt_fmt paid_fmt
amt_fmt=$(fmt_cur_num "$cur" "${C_AMT[$cur]}")
paid_fmt=$(fmt_cur_num "$cur" "${C_AMT_PAID[$cur]}")
if [ "$cf" = 1 ]; then cf=0; else CUR_JSON="${CUR_JSON},"; fi
CUR_JSON="${CUR_JSON}
{
\"currency\": $(json_str "$cur"),
\"contracts\": ${C_CONTRACTS[$cur]:-0},
\"paid\": ${C_PAID[$cur]:-0},
\"unpaid\": ${C_UNPAID[$cur]:-0},
\"wired\": ${C_WIRED[$cur]:-0},
\"amount_sum\": $(json_str "$amt_fmt"),
\"amount_paid_sum\": $(json_str "$paid_fmt")
}"
}
emit_cur GOA || true
emit_cur CHF || true
for cur in "${!C_CONTRACTS[@]}"; do
case "$cur" in GOA|CHF) continue ;; esac
emit_cur "$cur" || true
done
CUR_JSON="${CUR_JSON}
]"
GEN=$(now_iso)
HUMAN=$(now_human)
# Live performance probes (exchange-style)
measure_ms() {
local url="$1" t
t=$(curl -skS -o /dev/null -m 8 -w '%{time_total}' "$url" 2>/dev/null || echo "")
[ -z "$t" ] && { echo "null"; return; }
awk -v t="$t" 'BEGIN{printf "%d", (t+0)*1000}'
}
num_or_null() { case "${1:-}" in ''|null) echo null ;; *) echo "$1" ;; esac; }
MERCHANT_PUBLIC_BASE="${MERCHANT_PUBLIC_URL:-https://taler.hacktivism.ch}"
MERCHANT_LOCAL="${MERCHANT_LOCAL_URL:-https://127.0.0.1:9010}"
CONFIG_MS=$(measure_ms "${MERCHANT_PUBLIC_BASE}/config")
CONFIG_HTTP=$(curl -skS -o /dev/null -m 8 -w '%{http_code}' "${MERCHANT_PUBLIC_BASE}/config" 2>/dev/null || echo "000")
if [ "$CONFIG_HTTP" != "200" ]; then
CONFIG_MS=$(measure_ms "${MERCHANT_LOCAL}/config")
CONFIG_HTTP=$(curl -skS -o /dev/null -m 8 -w '%{http_code}' "${MERCHANT_LOCAL}/config" 2>/dev/null || echo "000")
fi
TERMS_MS=$(measure_ms "${MERCHANT_PUBLIC_BASE}/terms")
TERMS_HTTP=$(curl -skS -o /dev/null -m 8 -w '%{http_code}' -H "Accept: text/html" "${MERCHANT_PUBLIC_BASE}/terms" 2>/dev/null || echo "000")
WEBUI_MS=$(measure_ms "${MERCHANT_PUBLIC_BASE}/webui/")
WEBUI_HTTP=$(curl -skS -o /dev/null -m 8 -w '%{http_code}' "${MERCHANT_PUBLIC_BASE}/webui/" 2>/dev/null || echo "000")
LOADAVG=""
[ -r /proc/loadavg ] && LOADAVG=$(awk '{print $1","$2","$3}' /proc/loadavg)
MEM_JSON='"container_rss_human": "—"'
MEM_HELPER="${MEM_HELPER:-/usr/local/lib/landing-mem-snapshot.sh}"
if [ -f "$MEM_HELPER" ]; then
# shellcheck disable=SC1090
. "$MEM_HELPER"
mem_snapshot_json || true
fi
# Atomic write only after full success
cat >"$TMP" <<EOF
{
"ok": true,
"source": "merchant-db",
"schema": "merchant.merchant_instances + merchant_instance_<serial>",
"dual_currency": true,
"currencies_note": "CHF (taler-ops) + GOA (hacktivism)",
"timezone": $(json_str "$TZ"),
"generated_at": $(json_str "$GEN"),
"generated_at_human": $(json_str "$HUMAN"),
"instances": ${INSTANCES:-0},
"orders": ${TOTAL_CONTRACTS:-0},
"paid": ${TOTAL_PAID:-0},
"unpaid": ${TOTAL_UNPAID:-0},
"wired": ${TOTAL_WIRED:-0},
"deposits": ${DEPOSITS:-0},
"refunds": ${REFUNDS:-0},
"by_currency": $CUR_JSON,
"recent_activity_limit_per_currency": ${ACT_PER_CURRENCY},
"recent_activity_by_currency": $ACT_BY_CUR_JSON,
"recent_activity": $ACT_JSON,
"performance": {
"config_http": $(json_str "$CONFIG_HTTP"),
"config_ms": $(num_or_null "$CONFIG_MS"),
"terms_http": $(json_str "$TERMS_HTTP"),
"terms_ms": $(num_or_null "$TERMS_MS"),
"webui_http": $(json_str "$WEBUI_HTTP"),
"webui_ms": $(num_or_null "$WEBUI_MS"),
"loadavg": $(json_str "${LOADAVG:-}"),
"memory": {
${MEM_JSON}
}
}
}
EOF
# basic JSON sanity before publish
grep -q '"ok": true' "$TMP" || abort "tmp json missing ok:true"
mv -f "$TMP" "$OUT"
write_run true
echo "ok merchant instances=$INSTANCES orders=$TOTAL_CONTRACTS paid=$TOTAL_PAID refunds=$REFUNDS -> $OUT"