koopa-admin-log/scripts/taler-shared/goa-regio-ng/goa-vanilla-restore-do.sh

#!/usr/bin/env bash
# Restore bak dumps + master/secmod into goa-regio-ng (vanilla 901x).
# No hacktivism CTR dependency. Freigabe: live cutover restore.
set -euo pipefail

CTR="${CTR:-goa-regio-ng}"
DUMP_ROOT="${DUMP_ROOT:-/tmp/goa-restore}"
SECMOD_SRC="${SECMOD_SRC:-/tmp/goa-restore-staging/secmod-from-live}"
MASTER_BAK="${MASTER_BAK:-$DUMP_ROOT/exchange/offline/master.priv}"
TS="$(date -u +%Y%m%dT%H%MZ)"
LOG="/tmp/goa-vanilla-restore-run-${TS}.log"
exec > >(tee -a "$LOG") 2>&1

echo "==== VANILLA RESTORE START $(date -u +%Y-%m-%dT%H:%M:%SZ) ===="
echo "CTR=$CTR DUMP_ROOT=$DUMP_ROOT SECMOD_SRC=$SECMOD_SRC LOG=$LOG"

need() { [ -f "$1" ] || { echo "FAIL missing $1"; exit 1; }; }
need "$DUMP_ROOT/bank/libeufin.dump"
need "$DUMP_ROOT/exchange/taler-exchange.dump"
need "$DUMP_ROOT/merchant/taler-merchant.dump"
need "$MASTER_BAK"
need "$SECMOD_SRC/master.priv"
[ -d "$SECMOD_SRC/secmod-rsa" ] || { echo "FAIL missing secmod-rsa"; exit 1; }

BAK_SHA="$(sha256sum "$MASTER_BAK" | awk '{print $1}')"
STG_SHA="$(sha256sum "$SECMOD_SRC/master.priv" | awk '{print $1}')"
echo "MASTER_BAK_SHA=$BAK_SHA"
echo "MASTER_STG_SHA=$STG_SHA"
[ "$BAK_SHA" = "$STG_SHA" ] || { echo "FAIL master.priv bak≠staging"; exit 1; }

podman container exists "$CTR" || { echo "FAIL CTR missing $CTR"; exit 1; }

wait_pg() {
  # EMPTY_IMG may need long crash-recovery after unclean shutdown.
  local max="${1:-600}" i=0 st
  echo "==== WAIT PG accept (max ${max}s) ===="
  podman exec -u root "$CTR" bash -c '
set +e
systemctl start postgresql 2>/dev/null || true
systemctl start postgresql@17-main 2>/dev/null || true
'
  while [ "$i" -lt "$max" ]; do
    st=$(podman exec -u root "$CTR" bash -c '
set +e
systemctl is-active postgresql@17-main 2>/dev/null
runuser -u postgres -- psql -d postgres -v ON_ERROR_STOP=1 -tAc "SELECT 1" 2>/dev/null
' 2>/dev/null | tr '\n' ' ')
    if echo "$st" | grep -q 'active' && echo "$st" | grep -qw '1'; then
      echo "PG_ACCEPT after ${i}s ($st)"
      return 0
    fi
    if [ $((i % 30)) -eq 0 ]; then
      echo "PG_WAIT ${i}s state=[$st]"
      podman exec -u root "$CTR" bash -c 'tail -n 3 /var/log/postgresql/postgresql-17-main.log 2>/dev/null || journalctl -u postgresql@17-main -n 3 --no-pager 2>/dev/null | tail -n 3' || true
    fi
    sleep 2
    i=$((i + 2))
  done
  echo "FAIL PG not accepting after ${max}s"
  podman exec -u root "$CTR" bash -c 'systemctl status postgresql@17-main --no-pager -l 2>&1 | head -40; journalctl -u postgresql@17-main -n 40 --no-pager 2>&1 | tail -40' || true
  return 1
}

wait_unit_active() {
  local unit="$1" max="${2:-120}" i=0 st
  while [ "$i" -lt "$max" ]; do
    st=$(podman exec -u root "$CTR" systemctl is-active "$unit" 2>/dev/null || echo missing)
    if [ "$st" = "active" ]; then
      echo "UNIT_ACTIVE $unit after ${i}s"
      return 0
    fi
    if [ "$st" = "failed" ]; then
      echo "UNIT_FAILED $unit after ${i}s"
      podman exec -u root "$CTR" bash -c "systemctl status $unit --no-pager -l 2>&1 | head -50; journalctl -u $unit -n 80 --no-pager 2>&1 | tail -80" || true
      return 1
    fi
    sleep 2
    i=$((i + 2))
  done
  echo "UNIT_TIMEOUT $unit last=$st after ${max}s"
  podman exec -u root "$CTR" bash -c "systemctl status $unit --no-pager -l 2>&1 | head -50; journalctl -u $unit -n 80 --no-pager 2>&1 | tail -80" || true
  return 1
}

echo "==== STOP app units (keep postgresql) ===="
podman exec -u root "$CTR" bash -c '
set -e
for u in \
  goa-demo-withdraw-api \
  taler-merchant-httpd \
  taler-exchange-httpd \
  taler-exchange-wirewatch \
  taler-exchange-aggregator \
  taler-exchange-closer \
  taler-exchange-expire \
  taler-exchange-transfer \
  taler-exchange-secmod-rsa \
  taler-exchange-secmod-eddsa \
  taler-exchange-secmod-cs \
  libeufin-bank \
  libeufin-bank-nginx \
  nginx
do
  systemctl stop "$u" 2>/dev/null || true
done
systemctl reset-failed 2>/dev/null || true
'

wait_pg 600 || exit 1

echo "==== COPY dumps into CTR ===="
podman cp "$DUMP_ROOT/bank/libeufin.dump" "${CTR}:/tmp/libeufin.dump"
podman cp "$DUMP_ROOT/exchange/taler-exchange.dump" "${CTR}:/tmp/taler-exchange.dump"
podman cp "$DUMP_ROOT/merchant/taler-merchant.dump" "${CTR}:/tmp/taler-merchant.dump"
# ACL helper must be in CTR before restore_one (post pg_restore --no-acl)
if [ ! -f /tmp/goa-pg-acl-fixup.sh ]; then
  echo "FAIL missing /tmp/goa-pg-acl-fixup.sh on host"
  exit 2
fi
podman cp /tmp/goa-pg-acl-fixup.sh "${CTR}:/tmp/goa-pg-acl-fixup.sh"

echo "==== DROP+CREATE+RESTORE DBs ===="
# Live vanilla CONFIG=postgres:///exchange (bak dump dbname was taler-exchange).
podman exec -u root "$CTR" bash -c '
set -euo pipefail
restore_one() {
  local db="$1" dump="$2" owner="$3"
  echo "--- restore $db from $dump owner=$owner ---"
  runuser -u postgres -- psql -d postgres -c \
    "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '"'"'$db'"'"' AND pid <> pg_backend_pid();" \
    || true
  runuser -u postgres -- dropdb --if-exists --force "$db" || true
  runuser -u postgres -- createdb -O "$owner" "$db"
  set +e
  runuser -u postgres -- pg_restore --no-owner --no-acl --dbname="$db" "$dump"
  RC=$?
  set -e
  # pg_restore exits 1 on warnings; 0 ok; >=2 hard fail
  echo "PG_RESTORE_$db RC=$RC"
  if [ "$RC" -ge 2 ]; then
    echo "FAIL pg_restore $db"
    exit 2
  fi
  rm -f "$dump"
  # --no-owner --no-acl leaves schemas owned by postgres with no grants →
  # app roles hit "permission denied for schema _v". Fix ownership + GRANTs.
  # Skip ALTER SEQUENCE OWNER for linked serials; ALTER TABLE OWNER covers them.
  bash /tmp/goa-pg-acl-fixup.sh "$db" "$owner"
  runuser -u postgres -- psql -d "$db" -v ON_ERROR_STOP=1 -c "SELECT current_database() AS db;"
  runuser -u postgres -- psql -d "$db" -tAc "SELECT pg_size_pretty(pg_database_size(current_database()));"
  runuser -u postgres -- psql -d "$db" -tAc "SELECT count(*) FROM information_schema.tables WHERE table_schema='"'"'public'"'"';"
  # also count exchange schema (bak dumps use schema exchange, not public)
  runuser -u postgres -- psql -d "$db" -tAc "SELECT '"'"'schema_exchange_tables='"'"'||count(*)::text FROM information_schema.tables WHERE table_schema='"'"'exchange'"'"';" || true
}

restore_one libeufin /tmp/libeufin.dump libeufin-bank
restore_one exchange /tmp/taler-exchange.dump taler-exchange-httpd
restore_one taler-merchant /tmp/taler-merchant.dump taler-merchant-httpd

# drop leftover bak-named DB if any
runuser -u postgres -- dropdb --if-exists --force taler-exchange || true
runuser -u postgres -- psql -d postgres -c "\l"
'

echo "==== INSTALL master.priv + secmod from staging ===="
HOST_STG="/tmp/goa-restore-host-secmod-$$"
rm -rf "$HOST_STG"
mkdir -p "$HOST_STG"
cp -a "$SECMOD_SRC/." "$HOST_STG/"
cp -a "$MASTER_BAK" "$HOST_STG/master.priv"
# also secm_tofus.pub from bak offline if present
if [ -f "$DUMP_ROOT/exchange/offline/secm_tofus.pub" ]; then
  # staging copy is often mode 0400; plain cp -a cannot overwrite it
  rm -f "$HOST_STG/secm_tofus.pub"
  cp -a "$DUMP_ROOT/exchange/offline/secm_tofus.pub" "$HOST_STG/secm_tofus.pub"
fi

podman exec -u root "$CTR" bash -c '
set -euo pipefail
rm -rf /tmp/secmod-in
mkdir -p /tmp/secmod-in
'
podman cp "$HOST_STG/." "${CTR}:/tmp/secmod-in/"

podman exec -u root "$CTR" bash -c '
set -euo pipefail
OFF=/var/lib/taler-exchange/offline
mkdir -p "$OFF"
install -o taler-exchange-offline -g taler-exchange-offline -m 400 /tmp/secmod-in/master.priv "$OFF/master.priv"
if [ -f /tmp/secmod-in/secm_tofus.pub ]; then
  install -o taler-exchange-offline -g taler-exchange-offline -m 400 /tmp/secmod-in/secm_tofus.pub "$OFF/secm_tofus.pub"
fi
# offline-keys home path (also used live)
HOME_OFF=/home/taler-exchange-offline/.local/share/taler/exchange/offline-keys
mkdir -p "$HOME_OFF"
install -o taler-exchange-offline -g taler-exchange-offline -m 400 /tmp/secmod-in/master.priv "$HOME_OFF/master.priv"

# secmod trees
for kind in rsa eddsa cs; do
  src="/tmp/secmod-in/secmod-$kind"
  dst="/var/lib/taler-exchange/secmod-$kind"
  [ -d "$src" ] || { echo "FAIL missing $src"; exit 1; }
  rm -rf "$dst"
  mkdir -p "$dst"
  cp -a "$src/." "$dst/"
  # ownership: taler-exchange-secmod-<kind> : taler-exchange-secmod
  u="taler-exchange-secmod-$kind"
  chown -R "$u:taler-exchange-secmod" "$dst"
  chmod -R u=rwX,g=rX,o= "$dst"
  find "$dst" -type f -name "*private*" -exec chmod 400 {} \; || true
done
sha256sum "$OFF/master.priv"
rm -rf /tmp/secmod-in
echo SECMOD_INSTALL_OK
'
rm -rf "$HOST_STG"

echo "==== START units ===="
podman exec -u root "$CTR" bash -c '
set -e
systemctl start postgresql
systemctl start postgresql@17-main || true
systemctl start taler-exchange-secmod-rsa taler-exchange-secmod-eddsa taler-exchange-secmod-cs
sleep 2
systemctl start taler-exchange-httpd
systemctl start libeufin-bank
systemctl start taler-merchant-httpd
systemctl start nginx || true
systemctl start goa-demo-withdraw-api || true
# helpers
for u in taler-exchange-wirewatch taler-exchange-aggregator taler-exchange-closer taler-exchange-expire taler-exchange-transfer; do
  systemctl start "$u" 2>/dev/null || true
done
'

UNIT_FAIL=0
for u in taler-exchange-secmod-rsa taler-exchange-secmod-eddsa taler-exchange-secmod-cs \
         taler-exchange-httpd libeufin-bank taler-merchant-httpd nginx; do
  if ! wait_unit_active "$u" 180; then
    UNIT_FAIL=1
  fi
done
# demo-withdraw optional
wait_unit_active goa-demo-withdraw-api 60 || echo "WARN goa-demo-withdraw-api not active (optional)"

echo "==== UNIT STATUS SNAPSHOT ===="
podman exec -u root "$CTR" bash -c '
for u in postgresql postgresql@17-main taler-exchange-httpd libeufin-bank taler-merchant-httpd \
         taler-exchange-secmod-rsa taler-exchange-secmod-eddsa taler-exchange-secmod-cs \
         nginx goa-demo-withdraw-api; do
  printf "%s=%s\n" "$u" "$(systemctl is-active "$u" 2>/dev/null || echo missing)"
done
'

if [ "$UNIT_FAIL" -ne 0 ]; then
  echo "FAIL required units not active"
  exit 1
fi

echo "==== WIRE POST-RESTORE (hosts + nginx 9012 + dbconfig + -a drop-ins) ===="
if [ ! -f /tmp/goa-wire-post-restore.sh ]; then
  echo "FAIL missing /tmp/goa-wire-post-restore.sh on host"
  exit 2
fi
podman cp /tmp/goa-wire-post-restore.sh "${CTR}:/tmp/goa-wire-post-restore.sh"
podman exec -u root "$CTR" bash /tmp/goa-wire-post-restore.sh

echo "==== LOCAL SMOKE (scheme-aware) ===="
SMOKE_FAIL=0
code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 http://127.0.0.1:9012/config 2>/dev/null || echo ERR)
echo "bank_http_9012=$code"
case "$code" in 200|204) ;; *) SMOKE_FAIL=1 ;; esac

code=$(curl -skS -o /dev/null -w "%{http_code}" --max-time 10 https://127.0.0.1:9010/config 2>/dev/null || echo ERR)
echo "merchant_https_9010=$code"
case "$code" in 200|204) ;; *) SMOKE_FAIL=1 ;; esac

code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 http://127.0.0.1:9011/config 2>/dev/null || echo ERR)
echo "exchange_http_9011=$code"
case "$code" in 200|204) ;; *)
  # fallback https in case nginx wraps it
  code2=$(curl -skS -o /dev/null -w "%{http_code}" --max-time 10 https://127.0.0.1:9011/config 2>/dev/null || echo ERR)
  echo "exchange_https_9011=$code2"
  case "$code2" in 200|204) ;; *) SMOKE_FAIL=1 ;; esac
  ;;
esac

ww=$(podman exec -u root "$CTR" systemctl is-active taler-exchange-wirewatch.service 2>/dev/null || echo missing)
echo "wirewatch=$ww"
[ "$ww" = "active" ] || SMOKE_FAIL=1

if [ "$SMOKE_FAIL" -ne 0 ]; then
  echo "FAIL local smoke probes"
  podman exec -u root "$CTR" bash -c '
ss -lntp 2>/dev/null | grep -E ":901[0-5]|:80 " || netstat -lntp 2>/dev/null | grep -E ":901[0-5]|:80 " || true
for u in taler-exchange-httpd libeufin-bank taler-merchant-httpd nginx taler-exchange-wirewatch; do
  echo "--- journal $u ---"
  journalctl -u "$u" -n 40 --no-pager 2>&1 | tail -40
done
' || true
  exit 1
fi

echo "==== PUBLIC SMOKE ===="
for u in \
  https://exchange.hacktivism.ch/config \
  https://bank.hacktivism.ch/config \
  https://backend.hacktivism.ch/config
do
  code=$(curl -skS -o /dev/null -w "%{http_code}" --max-time 15 "$u" || echo ERR)
  echo "PUB $code $u"
done

echo "==== POST DB SIZES ===="
podman exec -u root "$CTR" runuser -u postgres -- psql -d postgres -tAc \
  "SELECT datname||'='||pg_size_pretty(pg_database_size(datname)) FROM pg_database WHERE datname IN ('exchange','libeufin','taler-merchant') ORDER BY 1;"
podman exec -u root "$CTR" runuser -u postgres -- psql -d exchange -tAc \
  "SELECT 'exchange_schema_tables='||count(*)::text FROM information_schema.tables WHERE table_schema='exchange';"

echo "RESTORE_SCRIPT_OK $(date -u +%Y-%m-%dT%H:%M:%SZ) LOG=$LOG"