#!/usr/bin/env bash # Restore bak dumps + master/secmod into goa-regio-ng (vanilla 901x). # No hacktivism CTR dependency. Freigabe: live cutover restore. set -euo pipefail CTR="${CTR:-goa-regio-ng}" DUMP_ROOT="${DUMP_ROOT:-/tmp/goa-restore}" SECMOD_SRC="${SECMOD_SRC:-/tmp/goa-restore-staging/secmod-from-live}" MASTER_BAK="${MASTER_BAK:-$DUMP_ROOT/exchange/offline/master.priv}" TS="$(date -u +%Y%m%dT%H%MZ)" LOG="/tmp/goa-vanilla-restore-run-${TS}.log" exec > >(tee -a "$LOG") 2>&1 echo "==== VANILLA RESTORE START $(date -u +%Y-%m-%dT%H:%M:%SZ) ====" echo "CTR=$CTR DUMP_ROOT=$DUMP_ROOT SECMOD_SRC=$SECMOD_SRC LOG=$LOG" need() { [ -f "$1" ] || { echo "FAIL missing $1"; exit 1; }; } need "$DUMP_ROOT/bank/libeufin.dump" need "$DUMP_ROOT/exchange/taler-exchange.dump" need "$DUMP_ROOT/merchant/taler-merchant.dump" need "$MASTER_BAK" need "$SECMOD_SRC/master.priv" [ -d "$SECMOD_SRC/secmod-rsa" ] || { echo "FAIL missing secmod-rsa"; exit 1; } BAK_SHA="$(sha256sum "$MASTER_BAK" | awk '{print $1}')" STG_SHA="$(sha256sum "$SECMOD_SRC/master.priv" | awk '{print $1}')" echo "MASTER_BAK_SHA=$BAK_SHA" echo "MASTER_STG_SHA=$STG_SHA" [ "$BAK_SHA" = "$STG_SHA" ] || { echo "FAIL master.priv bak≠staging"; exit 1; } podman container exists "$CTR" || { echo "FAIL CTR missing $CTR"; exit 1; } wait_pg() { # EMPTY_IMG may need long crash-recovery after unclean shutdown. local max="${1:-600}" i=0 st echo "==== WAIT PG accept (max ${max}s) ====" podman exec -u root "$CTR" bash -c ' set +e systemctl start postgresql 2>/dev/null || true systemctl start postgresql@17-main 2>/dev/null || true ' while [ "$i" -lt "$max" ]; do st=$(podman exec -u root "$CTR" bash -c ' set +e systemctl is-active postgresql@17-main 2>/dev/null runuser -u postgres -- psql -d postgres -v ON_ERROR_STOP=1 -tAc "SELECT 1" 2>/dev/null ' 2>/dev/null | tr '\n' ' ') if echo "$st" | grep -q 'active' && echo "$st" | grep -qw '1'; then echo "PG_ACCEPT after ${i}s ($st)" return 0 fi if [ $((i % 30)) -eq 0 ]; then echo "PG_WAIT ${i}s state=[$st]" podman exec -u root "$CTR" bash -c 'tail -n 3 /var/log/postgresql/postgresql-17-main.log 2>/dev/null || journalctl -u postgresql@17-main -n 3 --no-pager 2>/dev/null | tail -n 3' || true fi sleep 2 i=$((i + 2)) done echo "FAIL PG not accepting after ${max}s" podman exec -u root "$CTR" bash -c 'systemctl status postgresql@17-main --no-pager -l 2>&1 | head -40; journalctl -u postgresql@17-main -n 40 --no-pager 2>&1 | tail -40' || true return 1 } wait_unit_active() { local unit="$1" max="${2:-120}" i=0 st while [ "$i" -lt "$max" ]; do st=$(podman exec -u root "$CTR" systemctl is-active "$unit" 2>/dev/null || echo missing) if [ "$st" = "active" ]; then echo "UNIT_ACTIVE $unit after ${i}s" return 0 fi if [ "$st" = "failed" ]; then echo "UNIT_FAILED $unit after ${i}s" podman exec -u root "$CTR" bash -c "systemctl status $unit --no-pager -l 2>&1 | head -50; journalctl -u $unit -n 80 --no-pager 2>&1 | tail -80" || true return 1 fi sleep 2 i=$((i + 2)) done echo "UNIT_TIMEOUT $unit last=$st after ${max}s" podman exec -u root "$CTR" bash -c "systemctl status $unit --no-pager -l 2>&1 | head -50; journalctl -u $unit -n 80 --no-pager 2>&1 | tail -80" || true return 1 } echo "==== STOP app units (keep postgresql) ====" podman exec -u root "$CTR" bash -c ' set -e for u in \ goa-demo-withdraw-api \ taler-merchant-httpd \ taler-exchange-httpd \ taler-exchange-wirewatch \ taler-exchange-aggregator \ taler-exchange-closer \ taler-exchange-expire \ taler-exchange-transfer \ taler-exchange-secmod-rsa \ taler-exchange-secmod-eddsa \ taler-exchange-secmod-cs \ libeufin-bank \ libeufin-bank-nginx \ nginx do systemctl stop "$u" 2>/dev/null || true done systemctl reset-failed 2>/dev/null || true ' wait_pg 600 || exit 1 echo "==== COPY dumps into CTR ====" podman cp "$DUMP_ROOT/bank/libeufin.dump" "${CTR}:/tmp/libeufin.dump" podman cp "$DUMP_ROOT/exchange/taler-exchange.dump" "${CTR}:/tmp/taler-exchange.dump" podman cp "$DUMP_ROOT/merchant/taler-merchant.dump" "${CTR}:/tmp/taler-merchant.dump" # ACL helper must be in CTR before restore_one (post pg_restore --no-acl) if [ ! -f /tmp/goa-pg-acl-fixup.sh ]; then echo "FAIL missing /tmp/goa-pg-acl-fixup.sh on host" exit 2 fi podman cp /tmp/goa-pg-acl-fixup.sh "${CTR}:/tmp/goa-pg-acl-fixup.sh" echo "==== DROP+CREATE+RESTORE DBs ====" # Live vanilla CONFIG=postgres:///exchange (bak dump dbname was taler-exchange). podman exec -u root "$CTR" bash -c ' set -euo pipefail restore_one() { local db="$1" dump="$2" owner="$3" echo "--- restore $db from $dump owner=$owner ---" runuser -u postgres -- psql -d postgres -c \ "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '"'"'$db'"'"' AND pid <> pg_backend_pid();" \ || true runuser -u postgres -- dropdb --if-exists --force "$db" || true runuser -u postgres -- createdb -O "$owner" "$db" set +e runuser -u postgres -- pg_restore --no-owner --no-acl --dbname="$db" "$dump" RC=$? set -e # pg_restore exits 1 on warnings; 0 ok; >=2 hard fail echo "PG_RESTORE_$db RC=$RC" if [ "$RC" -ge 2 ]; then echo "FAIL pg_restore $db" exit 2 fi rm -f "$dump" # --no-owner --no-acl leaves schemas owned by postgres with no grants → # app roles hit "permission denied for schema _v". Fix ownership + GRANTs. # Skip ALTER SEQUENCE OWNER for linked serials; ALTER TABLE OWNER covers them. bash /tmp/goa-pg-acl-fixup.sh "$db" "$owner" runuser -u postgres -- psql -d "$db" -v ON_ERROR_STOP=1 -c "SELECT current_database() AS db;" runuser -u postgres -- psql -d "$db" -tAc "SELECT pg_size_pretty(pg_database_size(current_database()));" runuser -u postgres -- psql -d "$db" -tAc "SELECT count(*) FROM information_schema.tables WHERE table_schema='"'"'public'"'"';" # also count exchange schema (bak dumps use schema exchange, not public) runuser -u postgres -- psql -d "$db" -tAc "SELECT '"'"'schema_exchange_tables='"'"'||count(*)::text FROM information_schema.tables WHERE table_schema='"'"'exchange'"'"';" || true } restore_one libeufin /tmp/libeufin.dump libeufin-bank restore_one exchange /tmp/taler-exchange.dump taler-exchange-httpd restore_one taler-merchant /tmp/taler-merchant.dump taler-merchant-httpd # drop leftover bak-named DB if any runuser -u postgres -- dropdb --if-exists --force taler-exchange || true runuser -u postgres -- psql -d postgres -c "\l" ' echo "==== INSTALL master.priv + secmod from staging ====" HOST_STG="/tmp/goa-restore-host-secmod-$$" rm -rf "$HOST_STG" mkdir -p "$HOST_STG" cp -a "$SECMOD_SRC/." "$HOST_STG/" cp -a "$MASTER_BAK" "$HOST_STG/master.priv" # also secm_tofus.pub from bak offline if present if [ -f "$DUMP_ROOT/exchange/offline/secm_tofus.pub" ]; then # staging copy is often mode 0400; plain cp -a cannot overwrite it rm -f "$HOST_STG/secm_tofus.pub" cp -a "$DUMP_ROOT/exchange/offline/secm_tofus.pub" "$HOST_STG/secm_tofus.pub" fi podman exec -u root "$CTR" bash -c ' set -euo pipefail rm -rf /tmp/secmod-in mkdir -p /tmp/secmod-in ' podman cp "$HOST_STG/." "${CTR}:/tmp/secmod-in/" podman exec -u root "$CTR" bash -c ' set -euo pipefail OFF=/var/lib/taler-exchange/offline mkdir -p "$OFF" install -o taler-exchange-offline -g taler-exchange-offline -m 400 /tmp/secmod-in/master.priv "$OFF/master.priv" if [ -f /tmp/secmod-in/secm_tofus.pub ]; then install -o taler-exchange-offline -g taler-exchange-offline -m 400 /tmp/secmod-in/secm_tofus.pub "$OFF/secm_tofus.pub" fi # offline-keys home path (also used live) HOME_OFF=/home/taler-exchange-offline/.local/share/taler/exchange/offline-keys mkdir -p "$HOME_OFF" install -o taler-exchange-offline -g taler-exchange-offline -m 400 /tmp/secmod-in/master.priv "$HOME_OFF/master.priv" # secmod trees for kind in rsa eddsa cs; do src="/tmp/secmod-in/secmod-$kind" dst="/var/lib/taler-exchange/secmod-$kind" [ -d "$src" ] || { echo "FAIL missing $src"; exit 1; } rm -rf "$dst" mkdir -p "$dst" cp -a "$src/." "$dst/" # ownership: taler-exchange-secmod- : taler-exchange-secmod u="taler-exchange-secmod-$kind" chown -R "$u:taler-exchange-secmod" "$dst" chmod -R u=rwX,g=rX,o= "$dst" find "$dst" -type f -name "*private*" -exec chmod 400 {} \; || true done sha256sum "$OFF/master.priv" rm -rf /tmp/secmod-in echo SECMOD_INSTALL_OK ' rm -rf "$HOST_STG" echo "==== START units ====" podman exec -u root "$CTR" bash -c ' set -e systemctl start postgresql systemctl start postgresql@17-main || true systemctl start taler-exchange-secmod-rsa taler-exchange-secmod-eddsa taler-exchange-secmod-cs sleep 2 systemctl start taler-exchange-httpd systemctl start libeufin-bank systemctl start taler-merchant-httpd systemctl start nginx || true systemctl start goa-demo-withdraw-api || true # helpers for u in taler-exchange-wirewatch taler-exchange-aggregator taler-exchange-closer taler-exchange-expire taler-exchange-transfer; do systemctl start "$u" 2>/dev/null || true done ' UNIT_FAIL=0 for u in taler-exchange-secmod-rsa taler-exchange-secmod-eddsa taler-exchange-secmod-cs \ taler-exchange-httpd libeufin-bank taler-merchant-httpd nginx; do if ! wait_unit_active "$u" 180; then UNIT_FAIL=1 fi done # demo-withdraw optional wait_unit_active goa-demo-withdraw-api 60 || echo "WARN goa-demo-withdraw-api not active (optional)" echo "==== UNIT STATUS SNAPSHOT ====" podman exec -u root "$CTR" bash -c ' for u in postgresql postgresql@17-main taler-exchange-httpd libeufin-bank taler-merchant-httpd \ taler-exchange-secmod-rsa taler-exchange-secmod-eddsa taler-exchange-secmod-cs \ nginx goa-demo-withdraw-api; do printf "%s=%s\n" "$u" "$(systemctl is-active "$u" 2>/dev/null || echo missing)" done ' if [ "$UNIT_FAIL" -ne 0 ]; then echo "FAIL required units not active" exit 1 fi echo "==== WIRE POST-RESTORE (hosts + nginx 9012 + dbconfig + -a drop-ins) ====" if [ ! -f /tmp/goa-wire-post-restore.sh ]; then echo "FAIL missing /tmp/goa-wire-post-restore.sh on host" exit 2 fi podman cp /tmp/goa-wire-post-restore.sh "${CTR}:/tmp/goa-wire-post-restore.sh" podman exec -u root "$CTR" bash /tmp/goa-wire-post-restore.sh echo "==== LOCAL SMOKE (scheme-aware) ====" SMOKE_FAIL=0 code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 http://127.0.0.1:9012/config 2>/dev/null || echo ERR) echo "bank_http_9012=$code" case "$code" in 200|204) ;; *) SMOKE_FAIL=1 ;; esac code=$(curl -skS -o /dev/null -w "%{http_code}" --max-time 10 https://127.0.0.1:9010/config 2>/dev/null || echo ERR) echo "merchant_https_9010=$code" case "$code" in 200|204) ;; *) SMOKE_FAIL=1 ;; esac code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 http://127.0.0.1:9011/config 2>/dev/null || echo ERR) echo "exchange_http_9011=$code" case "$code" in 200|204) ;; *) # fallback https in case nginx wraps it code2=$(curl -skS -o /dev/null -w "%{http_code}" --max-time 10 https://127.0.0.1:9011/config 2>/dev/null || echo ERR) echo "exchange_https_9011=$code2" case "$code2" in 200|204) ;; *) SMOKE_FAIL=1 ;; esac ;; esac ww=$(podman exec -u root "$CTR" systemctl is-active taler-exchange-wirewatch.service 2>/dev/null || echo missing) echo "wirewatch=$ww" [ "$ww" = "active" ] || SMOKE_FAIL=1 if [ "$SMOKE_FAIL" -ne 0 ]; then echo "FAIL local smoke probes" podman exec -u root "$CTR" bash -c ' ss -lntp 2>/dev/null | grep -E ":901[0-5]|:80 " || netstat -lntp 2>/dev/null | grep -E ":901[0-5]|:80 " || true for u in taler-exchange-httpd libeufin-bank taler-merchant-httpd nginx taler-exchange-wirewatch; do echo "--- journal $u ---" journalctl -u "$u" -n 40 --no-pager 2>&1 | tail -40 done ' || true exit 1 fi echo "==== PUBLIC SMOKE ====" for u in \ https://exchange.hacktivism.ch/config \ https://bank.hacktivism.ch/config \ https://backend.hacktivism.ch/config do code=$(curl -skS -o /dev/null -w "%{http_code}" --max-time 15 "$u" || echo ERR) echo "PUB $code $u" done echo "==== POST DB SIZES ====" podman exec -u root "$CTR" runuser -u postgres -- psql -d postgres -tAc \ "SELECT datname||'='||pg_size_pretty(pg_database_size(datname)) FROM pg_database WHERE datname IN ('exchange','libeufin','taler-merchant') ORDER BY 1;" podman exec -u root "$CTR" runuser -u postgres -- psql -d exchange -tAc \ "SELECT 'exchange_schema_tables='||count(*)::text FROM information_schema.tables WHERE table_schema='exchange';" echo "RESTORE_SCRIPT_OK $(date -u +%Y-%m-%dT%H:%M:%SZ) LOG=$LOG"