ops: GOA regio-ng AIO restore scripts + wire post-restore helper
This commit is contained in:
Hernâni Marques
parent
337d253691
commit
83b1616ed6
7 changed files with 864 additions and 0 deletions
320
scripts/taler-shared/goa-regio-ng/goa-vanilla-restore-do.sh
Executable file
320
scripts/taler-shared/goa-regio-ng/goa-vanilla-restore-do.sh
Executable file
|
|
@ -0,0 +1,320 @@
|
|||
#!/usr/bin/env bash
|
||||
# Restore bak dumps + master/secmod into goa-regio-ng (vanilla 901x).
|
||||
# No hacktivism CTR dependency. Freigabe: live cutover restore.
|
||||
set -euo pipefail
|
||||
|
||||
CTR="${CTR:-goa-regio-ng}"
|
||||
DUMP_ROOT="${DUMP_ROOT:-/tmp/goa-restore}"
|
||||
SECMOD_SRC="${SECMOD_SRC:-/tmp/goa-restore-staging/secmod-from-live}"
|
||||
MASTER_BAK="${MASTER_BAK:-$DUMP_ROOT/exchange/offline/master.priv}"
|
||||
TS="$(date -u +%Y%m%dT%H%MZ)"
|
||||
LOG="/tmp/goa-vanilla-restore-run-${TS}.log"
|
||||
exec > >(tee -a "$LOG") 2>&1
|
||||
|
||||
echo "==== VANILLA RESTORE START $(date -u +%Y-%m-%dT%H:%M:%SZ) ===="
|
||||
echo "CTR=$CTR DUMP_ROOT=$DUMP_ROOT SECMOD_SRC=$SECMOD_SRC LOG=$LOG"
|
||||
|
||||
need() { [ -f "$1" ] || { echo "FAIL missing $1"; exit 1; }; }
|
||||
need "$DUMP_ROOT/bank/libeufin.dump"
|
||||
need "$DUMP_ROOT/exchange/taler-exchange.dump"
|
||||
need "$DUMP_ROOT/merchant/taler-merchant.dump"
|
||||
need "$MASTER_BAK"
|
||||
need "$SECMOD_SRC/master.priv"
|
||||
[ -d "$SECMOD_SRC/secmod-rsa" ] || { echo "FAIL missing secmod-rsa"; exit 1; }
|
||||
|
||||
BAK_SHA="$(sha256sum "$MASTER_BAK" | awk '{print $1}')"
|
||||
STG_SHA="$(sha256sum "$SECMOD_SRC/master.priv" | awk '{print $1}')"
|
||||
echo "MASTER_BAK_SHA=$BAK_SHA"
|
||||
echo "MASTER_STG_SHA=$STG_SHA"
|
||||
[ "$BAK_SHA" = "$STG_SHA" ] || { echo "FAIL master.priv bak≠staging"; exit 1; }
|
||||
|
||||
podman container exists "$CTR" || { echo "FAIL CTR missing $CTR"; exit 1; }
|
||||
|
||||
wait_pg() {
|
||||
# EMPTY_IMG may need long crash-recovery after unclean shutdown.
|
||||
local max="${1:-600}" i=0 st
|
||||
echo "==== WAIT PG accept (max ${max}s) ===="
|
||||
podman exec -u root "$CTR" bash -c '
|
||||
set +e
|
||||
systemctl start postgresql 2>/dev/null || true
|
||||
systemctl start postgresql@17-main 2>/dev/null || true
|
||||
'
|
||||
while [ "$i" -lt "$max" ]; do
|
||||
st=$(podman exec -u root "$CTR" bash -c '
|
||||
set +e
|
||||
systemctl is-active postgresql@17-main 2>/dev/null
|
||||
runuser -u postgres -- psql -d postgres -v ON_ERROR_STOP=1 -tAc "SELECT 1" 2>/dev/null
|
||||
' 2>/dev/null | tr '\n' ' ')
|
||||
if echo "$st" | grep -q 'active' && echo "$st" | grep -qw '1'; then
|
||||
echo "PG_ACCEPT after ${i}s ($st)"
|
||||
return 0
|
||||
fi
|
||||
if [ $((i % 30)) -eq 0 ]; then
|
||||
echo "PG_WAIT ${i}s state=[$st]"
|
||||
podman exec -u root "$CTR" bash -c 'tail -n 3 /var/log/postgresql/postgresql-17-main.log 2>/dev/null || journalctl -u postgresql@17-main -n 3 --no-pager 2>/dev/null | tail -n 3' || true
|
||||
fi
|
||||
sleep 2
|
||||
i=$((i + 2))
|
||||
done
|
||||
echo "FAIL PG not accepting after ${max}s"
|
||||
podman exec -u root "$CTR" bash -c 'systemctl status postgresql@17-main --no-pager -l 2>&1 | head -40; journalctl -u postgresql@17-main -n 40 --no-pager 2>&1 | tail -40' || true
|
||||
return 1
|
||||
}
|
||||
|
||||
wait_unit_active() {
|
||||
local unit="$1" max="${2:-120}" i=0 st
|
||||
while [ "$i" -lt "$max" ]; do
|
||||
st=$(podman exec -u root "$CTR" systemctl is-active "$unit" 2>/dev/null || echo missing)
|
||||
if [ "$st" = "active" ]; then
|
||||
echo "UNIT_ACTIVE $unit after ${i}s"
|
||||
return 0
|
||||
fi
|
||||
if [ "$st" = "failed" ]; then
|
||||
echo "UNIT_FAILED $unit after ${i}s"
|
||||
podman exec -u root "$CTR" bash -c "systemctl status $unit --no-pager -l 2>&1 | head -50; journalctl -u $unit -n 80 --no-pager 2>&1 | tail -80" || true
|
||||
return 1
|
||||
fi
|
||||
sleep 2
|
||||
i=$((i + 2))
|
||||
done
|
||||
echo "UNIT_TIMEOUT $unit last=$st after ${max}s"
|
||||
podman exec -u root "$CTR" bash -c "systemctl status $unit --no-pager -l 2>&1 | head -50; journalctl -u $unit -n 80 --no-pager 2>&1 | tail -80" || true
|
||||
return 1
|
||||
}
|
||||
|
||||
echo "==== STOP app units (keep postgresql) ===="
|
||||
podman exec -u root "$CTR" bash -c '
|
||||
set -e
|
||||
for u in \
|
||||
goa-demo-withdraw-api \
|
||||
taler-merchant-httpd \
|
||||
taler-exchange-httpd \
|
||||
taler-exchange-wirewatch \
|
||||
taler-exchange-aggregator \
|
||||
taler-exchange-closer \
|
||||
taler-exchange-expire \
|
||||
taler-exchange-transfer \
|
||||
taler-exchange-secmod-rsa \
|
||||
taler-exchange-secmod-eddsa \
|
||||
taler-exchange-secmod-cs \
|
||||
libeufin-bank \
|
||||
libeufin-bank-nginx \
|
||||
nginx
|
||||
do
|
||||
systemctl stop "$u" 2>/dev/null || true
|
||||
done
|
||||
systemctl reset-failed 2>/dev/null || true
|
||||
'
|
||||
|
||||
wait_pg 600 || exit 1
|
||||
|
||||
echo "==== COPY dumps into CTR ===="
|
||||
podman cp "$DUMP_ROOT/bank/libeufin.dump" "${CTR}:/tmp/libeufin.dump"
|
||||
podman cp "$DUMP_ROOT/exchange/taler-exchange.dump" "${CTR}:/tmp/taler-exchange.dump"
|
||||
podman cp "$DUMP_ROOT/merchant/taler-merchant.dump" "${CTR}:/tmp/taler-merchant.dump"
|
||||
# ACL helper must be in CTR before restore_one (post pg_restore --no-acl)
|
||||
if [ ! -f /tmp/goa-pg-acl-fixup.sh ]; then
|
||||
echo "FAIL missing /tmp/goa-pg-acl-fixup.sh on host"
|
||||
exit 2
|
||||
fi
|
||||
podman cp /tmp/goa-pg-acl-fixup.sh "${CTR}:/tmp/goa-pg-acl-fixup.sh"
|
||||
|
||||
echo "==== DROP+CREATE+RESTORE DBs ===="
|
||||
# Live vanilla CONFIG=postgres:///exchange (bak dump dbname was taler-exchange).
|
||||
podman exec -u root "$CTR" bash -c '
|
||||
set -euo pipefail
|
||||
restore_one() {
|
||||
local db="$1" dump="$2" owner="$3"
|
||||
echo "--- restore $db from $dump owner=$owner ---"
|
||||
runuser -u postgres -- psql -d postgres -c \
|
||||
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '"'"'$db'"'"' AND pid <> pg_backend_pid();" \
|
||||
|| true
|
||||
runuser -u postgres -- dropdb --if-exists --force "$db" || true
|
||||
runuser -u postgres -- createdb -O "$owner" "$db"
|
||||
set +e
|
||||
runuser -u postgres -- pg_restore --no-owner --no-acl --dbname="$db" "$dump"
|
||||
RC=$?
|
||||
set -e
|
||||
# pg_restore exits 1 on warnings; 0 ok; >=2 hard fail
|
||||
echo "PG_RESTORE_$db RC=$RC"
|
||||
if [ "$RC" -ge 2 ]; then
|
||||
echo "FAIL pg_restore $db"
|
||||
exit 2
|
||||
fi
|
||||
rm -f "$dump"
|
||||
# --no-owner --no-acl leaves schemas owned by postgres with no grants →
|
||||
# app roles hit "permission denied for schema _v". Fix ownership + GRANTs.
|
||||
# Skip ALTER SEQUENCE OWNER for linked serials; ALTER TABLE OWNER covers them.
|
||||
bash /tmp/goa-pg-acl-fixup.sh "$db" "$owner"
|
||||
runuser -u postgres -- psql -d "$db" -v ON_ERROR_STOP=1 -c "SELECT current_database() AS db;"
|
||||
runuser -u postgres -- psql -d "$db" -tAc "SELECT pg_size_pretty(pg_database_size(current_database()));"
|
||||
runuser -u postgres -- psql -d "$db" -tAc "SELECT count(*) FROM information_schema.tables WHERE table_schema='"'"'public'"'"';"
|
||||
# also count exchange schema (bak dumps use schema exchange, not public)
|
||||
runuser -u postgres -- psql -d "$db" -tAc "SELECT '"'"'schema_exchange_tables='"'"'||count(*)::text FROM information_schema.tables WHERE table_schema='"'"'exchange'"'"';" || true
|
||||
}
|
||||
|
||||
restore_one libeufin /tmp/libeufin.dump libeufin-bank
|
||||
restore_one exchange /tmp/taler-exchange.dump taler-exchange-httpd
|
||||
restore_one taler-merchant /tmp/taler-merchant.dump taler-merchant-httpd
|
||||
|
||||
# drop leftover bak-named DB if any
|
||||
runuser -u postgres -- dropdb --if-exists --force taler-exchange || true
|
||||
runuser -u postgres -- psql -d postgres -c "\l"
|
||||
'
|
||||
|
||||
echo "==== INSTALL master.priv + secmod from staging ===="
|
||||
HOST_STG="/tmp/goa-restore-host-secmod-$$"
|
||||
rm -rf "$HOST_STG"
|
||||
mkdir -p "$HOST_STG"
|
||||
cp -a "$SECMOD_SRC/." "$HOST_STG/"
|
||||
cp -a "$MASTER_BAK" "$HOST_STG/master.priv"
|
||||
# also secm_tofus.pub from bak offline if present
|
||||
if [ -f "$DUMP_ROOT/exchange/offline/secm_tofus.pub" ]; then
|
||||
# staging copy is often mode 0400; plain cp -a cannot overwrite it
|
||||
rm -f "$HOST_STG/secm_tofus.pub"
|
||||
cp -a "$DUMP_ROOT/exchange/offline/secm_tofus.pub" "$HOST_STG/secm_tofus.pub"
|
||||
fi
|
||||
|
||||
podman exec -u root "$CTR" bash -c '
|
||||
set -euo pipefail
|
||||
rm -rf /tmp/secmod-in
|
||||
mkdir -p /tmp/secmod-in
|
||||
'
|
||||
podman cp "$HOST_STG/." "${CTR}:/tmp/secmod-in/"
|
||||
|
||||
podman exec -u root "$CTR" bash -c '
|
||||
set -euo pipefail
|
||||
OFF=/var/lib/taler-exchange/offline
|
||||
mkdir -p "$OFF"
|
||||
install -o taler-exchange-offline -g taler-exchange-offline -m 400 /tmp/secmod-in/master.priv "$OFF/master.priv"
|
||||
if [ -f /tmp/secmod-in/secm_tofus.pub ]; then
|
||||
install -o taler-exchange-offline -g taler-exchange-offline -m 400 /tmp/secmod-in/secm_tofus.pub "$OFF/secm_tofus.pub"
|
||||
fi
|
||||
# offline-keys home path (also used live)
|
||||
HOME_OFF=/home/taler-exchange-offline/.local/share/taler/exchange/offline-keys
|
||||
mkdir -p "$HOME_OFF"
|
||||
install -o taler-exchange-offline -g taler-exchange-offline -m 400 /tmp/secmod-in/master.priv "$HOME_OFF/master.priv"
|
||||
|
||||
# secmod trees
|
||||
for kind in rsa eddsa cs; do
|
||||
src="/tmp/secmod-in/secmod-$kind"
|
||||
dst="/var/lib/taler-exchange/secmod-$kind"
|
||||
[ -d "$src" ] || { echo "FAIL missing $src"; exit 1; }
|
||||
rm -rf "$dst"
|
||||
mkdir -p "$dst"
|
||||
cp -a "$src/." "$dst/"
|
||||
# ownership: taler-exchange-secmod-<kind> : taler-exchange-secmod
|
||||
u="taler-exchange-secmod-$kind"
|
||||
chown -R "$u:taler-exchange-secmod" "$dst"
|
||||
chmod -R u=rwX,g=rX,o= "$dst"
|
||||
find "$dst" -type f -name "*private*" -exec chmod 400 {} \; || true
|
||||
done
|
||||
sha256sum "$OFF/master.priv"
|
||||
rm -rf /tmp/secmod-in
|
||||
echo SECMOD_INSTALL_OK
|
||||
'
|
||||
rm -rf "$HOST_STG"
|
||||
|
||||
echo "==== START units ===="
|
||||
podman exec -u root "$CTR" bash -c '
|
||||
set -e
|
||||
systemctl start postgresql
|
||||
systemctl start postgresql@17-main || true
|
||||
systemctl start taler-exchange-secmod-rsa taler-exchange-secmod-eddsa taler-exchange-secmod-cs
|
||||
sleep 2
|
||||
systemctl start taler-exchange-httpd
|
||||
systemctl start libeufin-bank
|
||||
systemctl start taler-merchant-httpd
|
||||
systemctl start nginx || true
|
||||
systemctl start goa-demo-withdraw-api || true
|
||||
# helpers
|
||||
for u in taler-exchange-wirewatch taler-exchange-aggregator taler-exchange-closer taler-exchange-expire taler-exchange-transfer; do
|
||||
systemctl start "$u" 2>/dev/null || true
|
||||
done
|
||||
'
|
||||
|
||||
UNIT_FAIL=0
|
||||
for u in taler-exchange-secmod-rsa taler-exchange-secmod-eddsa taler-exchange-secmod-cs \
|
||||
taler-exchange-httpd libeufin-bank taler-merchant-httpd nginx; do
|
||||
if ! wait_unit_active "$u" 180; then
|
||||
UNIT_FAIL=1
|
||||
fi
|
||||
done
|
||||
# demo-withdraw optional
|
||||
wait_unit_active goa-demo-withdraw-api 60 || echo "WARN goa-demo-withdraw-api not active (optional)"
|
||||
|
||||
echo "==== UNIT STATUS SNAPSHOT ===="
|
||||
podman exec -u root "$CTR" bash -c '
|
||||
for u in postgresql postgresql@17-main taler-exchange-httpd libeufin-bank taler-merchant-httpd \
|
||||
taler-exchange-secmod-rsa taler-exchange-secmod-eddsa taler-exchange-secmod-cs \
|
||||
nginx goa-demo-withdraw-api; do
|
||||
printf "%s=%s\n" "$u" "$(systemctl is-active "$u" 2>/dev/null || echo missing)"
|
||||
done
|
||||
'
|
||||
|
||||
if [ "$UNIT_FAIL" -ne 0 ]; then
|
||||
echo "FAIL required units not active"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "==== WIRE POST-RESTORE (hosts + nginx 9012 + dbconfig + -a drop-ins) ===="
|
||||
if [ ! -f /tmp/goa-wire-post-restore.sh ]; then
|
||||
echo "FAIL missing /tmp/goa-wire-post-restore.sh on host"
|
||||
exit 2
|
||||
fi
|
||||
podman cp /tmp/goa-wire-post-restore.sh "${CTR}:/tmp/goa-wire-post-restore.sh"
|
||||
podman exec -u root "$CTR" bash /tmp/goa-wire-post-restore.sh
|
||||
|
||||
echo "==== LOCAL SMOKE (scheme-aware) ===="
|
||||
SMOKE_FAIL=0
|
||||
code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 http://127.0.0.1:9012/config 2>/dev/null || echo ERR)
|
||||
echo "bank_http_9012=$code"
|
||||
case "$code" in 200|204) ;; *) SMOKE_FAIL=1 ;; esac
|
||||
|
||||
code=$(curl -skS -o /dev/null -w "%{http_code}" --max-time 10 https://127.0.0.1:9010/config 2>/dev/null || echo ERR)
|
||||
echo "merchant_https_9010=$code"
|
||||
case "$code" in 200|204) ;; *) SMOKE_FAIL=1 ;; esac
|
||||
|
||||
code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 http://127.0.0.1:9011/config 2>/dev/null || echo ERR)
|
||||
echo "exchange_http_9011=$code"
|
||||
case "$code" in 200|204) ;; *)
|
||||
# fallback https in case nginx wraps it
|
||||
code2=$(curl -skS -o /dev/null -w "%{http_code}" --max-time 10 https://127.0.0.1:9011/config 2>/dev/null || echo ERR)
|
||||
echo "exchange_https_9011=$code2"
|
||||
case "$code2" in 200|204) ;; *) SMOKE_FAIL=1 ;; esac
|
||||
;;
|
||||
esac
|
||||
|
||||
ww=$(podman exec -u root "$CTR" systemctl is-active taler-exchange-wirewatch.service 2>/dev/null || echo missing)
|
||||
echo "wirewatch=$ww"
|
||||
[ "$ww" = "active" ] || SMOKE_FAIL=1
|
||||
|
||||
if [ "$SMOKE_FAIL" -ne 0 ]; then
|
||||
echo "FAIL local smoke probes"
|
||||
podman exec -u root "$CTR" bash -c '
|
||||
ss -lntp 2>/dev/null | grep -E ":901[0-5]|:80 " || netstat -lntp 2>/dev/null | grep -E ":901[0-5]|:80 " || true
|
||||
for u in taler-exchange-httpd libeufin-bank taler-merchant-httpd nginx taler-exchange-wirewatch; do
|
||||
echo "--- journal $u ---"
|
||||
journalctl -u "$u" -n 40 --no-pager 2>&1 | tail -40
|
||||
done
|
||||
' || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "==== PUBLIC SMOKE ===="
|
||||
for u in \
|
||||
https://exchange.hacktivism.ch/config \
|
||||
https://bank.hacktivism.ch/config \
|
||||
https://backend.hacktivism.ch/config
|
||||
do
|
||||
code=$(curl -skS -o /dev/null -w "%{http_code}" --max-time 15 "$u" || echo ERR)
|
||||
echo "PUB $code $u"
|
||||
done
|
||||
|
||||
echo "==== POST DB SIZES ===="
|
||||
podman exec -u root "$CTR" runuser -u postgres -- psql -d postgres -tAc \
|
||||
"SELECT datname||'='||pg_size_pretty(pg_database_size(datname)) FROM pg_database WHERE datname IN ('exchange','libeufin','taler-merchant') ORDER BY 1;"
|
||||
podman exec -u root "$CTR" runuser -u postgres -- psql -d exchange -tAc \
|
||||
"SELECT 'exchange_schema_tables='||count(*)::text FROM information_schema.tables WHERE table_schema='exchange';"
|
||||
|
||||
echo "RESTORE_SCRIPT_OK $(date -u +%Y-%m-%dT%H:%M:%SZ) LOG=$LOG"
|
||||
Loading…
Add table
Add a link
Reference in a new issue