ops: GOA regio-ng AIO restore scripts + wire post-restore helper
This commit is contained in:
Hernâni Marques
parent
337d253691
commit
83b1616ed6
7 changed files with 864 additions and 0 deletions
269
scripts/taler-shared/goa-regio-ng/goa-all-in-one.sh
Executable file
269
scripts/taler-shared/goa-regio-ng/goa-all-in-one.sh
Executable file
|
|
@ -0,0 +1,269 @@
|
|||
#!/usr/bin/env bash
|
||||
# All-in-one: restore empty → verify → recreate from EMPTY_IMG → restore →
|
||||
# hacktivism boot off → stack.env → probes. Landings stamp/deploy from laptop after.
|
||||
set -uo pipefail
|
||||
# intentionally NOT set -e: settle/grep and optional probes must not abort
|
||||
|
||||
CTR=goa-regio-ng
|
||||
EMPTY_IMG=${EMPTY_IMG:-localhost/goa-regio-ng:empty-current}
|
||||
EXPECT_MASTER=6a1cececf164063ecd48172c21d56061ce7881fbd7a88d4ffd25504552508706
|
||||
TS=$(date -u +%Y%m%dT%H%MZ)
|
||||
LOG=/tmp/goa-all-in-one-${TS}.log
|
||||
FAIL=0
|
||||
exec > >(tee -a "$LOG") 2>&1
|
||||
|
||||
echo "==== ALL-IN-ONE START $(date -u +%Y-%m-%dT%H:%M:%SZ) LOG=$LOG ===="
|
||||
echo "EMPTY_IMG=$EMPTY_IMG CTR=$CTR"
|
||||
|
||||
die() { echo "FAIL: $*"; FAIL=1; }
|
||||
ok() { echo "OK: $*"; }
|
||||
|
||||
settle_ctr() {
|
||||
local i st
|
||||
for i in $(seq 1 90); do
|
||||
st=$(podman exec -u root "$CTR" systemctl is-system-running 2>/dev/null || true)
|
||||
case "$st" in
|
||||
running|degraded) echo "SYSTEMD_SETTLE=$st after ${i}s"; return 0 ;;
|
||||
esac
|
||||
sleep 2
|
||||
done
|
||||
echo "SYSTEMD_SETTLE=timeout last=${st:-none}"
|
||||
return 1
|
||||
}
|
||||
|
||||
wait_pg_host() {
|
||||
local max="${1:-600}" i=0 st
|
||||
echo "==== WAIT PG (host-side, max ${max}s) ===="
|
||||
podman exec -u root "$CTR" bash -c 'systemctl start postgresql 2>/dev/null; systemctl start postgresql@17-main 2>/dev/null; true' || true
|
||||
while [ "$i" -lt "$max" ]; do
|
||||
st=$(podman exec -u root "$CTR" bash -c '
|
||||
set +e
|
||||
systemctl is-active postgresql@17-main 2>/dev/null
|
||||
runuser -u postgres -- psql -d postgres -v ON_ERROR_STOP=1 -tAc "SELECT 1" 2>/dev/null
|
||||
' 2>/dev/null | tr '\n' ' ')
|
||||
if echo "$st" | grep -q 'active' && echo "$st" | grep -qw '1'; then
|
||||
echo "PG_ACCEPT after ${i}s"
|
||||
return 0
|
||||
fi
|
||||
[ $((i % 30)) -eq 0 ] && echo "PG_WAIT ${i}s [$st]"
|
||||
sleep 2
|
||||
i=$((i + 2))
|
||||
done
|
||||
echo "PG_TIMEOUT after ${max}s"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Scheme-aware probes; hard-fail on bad codes.
|
||||
# 9010 merchant HTTPS; 9011 exchange HTTP; 9012 bank HTTP; 9013-15 landings 302 OK.
|
||||
probe_ports() {
|
||||
local tag=$1 hard=${2:-0} p code okc failc=0
|
||||
echo "=== probes $tag (hard=$hard) ==="
|
||||
|
||||
code=$(curl -skS -o /dev/null -w '%{http_code}' --connect-timeout 3 --max-time 10 "https://127.0.0.1:9010/config" 2>/dev/null || echo ERR)
|
||||
echo "MERCHANT_HTTPS_9010_CONFIG=${code}"
|
||||
case "$code" in 200|204) ;; *) failc=$((failc+1)) ;; esac
|
||||
|
||||
code=$(curl -sS -o /dev/null -w '%{http_code}' --connect-timeout 3 --max-time 10 "http://127.0.0.1:9011/config" 2>/dev/null || echo ERR)
|
||||
echo "EXCHANGE_HTTP_9011_CONFIG=${code}"
|
||||
case "$code" in 200|204) ;; *)
|
||||
code=$(curl -skS -o /dev/null -w '%{http_code}' --connect-timeout 3 --max-time 10 "https://127.0.0.1:9011/config" 2>/dev/null || echo ERR)
|
||||
echo "EXCHANGE_HTTPS_9011_CONFIG=${code}"
|
||||
case "$code" in 200|204) ;; *) failc=$((failc+1)) ;; esac
|
||||
;;
|
||||
esac
|
||||
|
||||
code=$(curl -sS -o /dev/null -w '%{http_code}' --connect-timeout 3 --max-time 10 "http://127.0.0.1:9012/config" 2>/dev/null || echo ERR)
|
||||
echo "BANK_HTTP_9012_CONFIG=${code}"
|
||||
case "$code" in 200|204) ;; *) failc=$((failc+1)) ;; esac
|
||||
|
||||
for p in 9013 9014 9015; do
|
||||
code=$(curl -sS -o /dev/null -w '%{http_code}' --connect-timeout 3 --max-time 8 "http://127.0.0.1:${p}/" 2>/dev/null || echo ERR)
|
||||
echo "LANDING_${p}=${code}"
|
||||
case "$code" in 200|301|302|303|307|308) ;; *) failc=$((failc+1)) ;; esac
|
||||
done
|
||||
|
||||
echo "PROBE_FAILS_${tag}=${failc}"
|
||||
if [ "$hard" = "1" ] && [ "$failc" -gt 0 ]; then
|
||||
die "probes $tag had $failc failures"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
smoke_inside() {
|
||||
local tag=$1 ww
|
||||
echo "=== inside smoke $tag ==="
|
||||
podman exec -u root "$CTR" bash -lc '
|
||||
set +e
|
||||
echo MASTER_SHA=$(sha256sum /var/lib/taler-exchange/offline/master.priv 2>/dev/null | awk "{print \$1}")
|
||||
echo EXPECT='"$EXPECT_MASTER"'
|
||||
for u in postgresql postgresql@17-main nginx taler-exchange-httpd taler-merchant-httpd libeufin-bank goa-demo-withdraw-api taler-exchange-wirewatch taler-exchange-transfer; do
|
||||
printf "%s=%s\n" "$u" "$(systemctl is-active "$u" 2>/dev/null || echo missing)"
|
||||
done
|
||||
sudo -u postgres psql -Atc "SELECT datname||\"=\"||pg_size_pretty(pg_database_size(datname)) FROM pg_database WHERE datname ~ '"'"'exchange|libeufin|merchant'"'"' ORDER BY 1;"
|
||||
'
|
||||
ww=$(podman exec -u root "$CTR" systemctl is-active taler-exchange-wirewatch.service 2>/dev/null || echo missing)
|
||||
if [ "$ww" = "active" ]; then
|
||||
ok "wirewatch active ($tag)"
|
||||
else
|
||||
die "wirewatch not active ($tag)=$ww"
|
||||
fi
|
||||
}
|
||||
|
||||
run_restore() {
|
||||
local tag=$1
|
||||
echo "==== RESTORE $tag ===="
|
||||
if ! bash /tmp/goa-vanilla-restore-do.sh; then
|
||||
die "restore $tag failed"
|
||||
return 1
|
||||
fi
|
||||
ok "restore $tag"
|
||||
return 0
|
||||
}
|
||||
|
||||
recreate_from_empty() {
|
||||
echo "==== RECREATE from EMPTY_IMG (no commit of restored) ===="
|
||||
podman image exists "$EMPTY_IMG" || { die "missing EMPTY_IMG $EMPTY_IMG"; return 1; }
|
||||
podman stop -t 30 "$CTR" || true
|
||||
podman rm "$CTR" || true
|
||||
podman run -d \
|
||||
--name "$CTR" \
|
||||
--privileged \
|
||||
--systemd=always \
|
||||
--network pasta \
|
||||
-v /sys/fs/cgroup:/sys/fs/cgroup:rw \
|
||||
--tmpfs /run \
|
||||
--tmpfs /run/lock \
|
||||
-p 127.0.0.1:9322:22 \
|
||||
-p 127.0.0.1:9180:80 \
|
||||
-p 0.0.0.0:9010-9015:9010-9015 \
|
||||
--hostname "$CTR" \
|
||||
"$EMPTY_IMG" \
|
||||
/sbin/init
|
||||
sleep 8
|
||||
settle_ctr || { die "settle after recreate"; return 1; }
|
||||
wait_pg_host 600 || { die "PG not ready after recreate"; return 1; }
|
||||
podman ps --filter "name=^${CTR}$" --format '{{.Names}} {{.Status}} {{.Image}} {{.Ports}}'
|
||||
echo "=== pre-restore DBs (expect empty/minimal) ==="
|
||||
podman exec -u root "$CTR" bash -lc 'sudo -u postgres psql -Atc "SELECT datname FROM pg_database WHERE datname ~ '"'"'exchange|libeufin|merchant'"'"' ORDER BY 1;"' || true
|
||||
ok "recreate from empty"
|
||||
}
|
||||
|
||||
disable_hacktivism_boot() {
|
||||
echo "==== DISABLE hacktivism boot units ===="
|
||||
local u
|
||||
for u in \
|
||||
container-taler-hacktivism-bank.service \
|
||||
container-taler-hacktivism-exchange-ansible.service \
|
||||
container-taler-hacktivism.service \
|
||||
taler-monitoring-hacktivism.timer \
|
||||
taler-monitoring-hacktivism.path
|
||||
do
|
||||
if systemctl --user list-unit-files "$u" >/dev/null 2>&1; then
|
||||
systemctl --user disable --now "$u" 2>/dev/null || systemctl --user disable "$u" 2>/dev/null || true
|
||||
echo "disabled $u"
|
||||
else
|
||||
echo "skip missing $u"
|
||||
fi
|
||||
done
|
||||
systemctl --user daemon-reload || true
|
||||
echo "=== enabled leftover hacktivism? ==="
|
||||
systemctl --user list-unit-files 2>/dev/null | grep -i hacktivism || echo '(none listed)'
|
||||
ok "hacktivism boot disable attempted"
|
||||
}
|
||||
|
||||
install_stack_env() {
|
||||
echo "==== INSTALL stack.env → goa-regio-ng ===="
|
||||
mkdir -p "$HOME/.config/taler-landing"
|
||||
cat > "$HOME/.config/taler-landing/stack.env" <<'ENV'
|
||||
# Stack profile: GOA / goa-regio-ng vanilla (9010-9015)
|
||||
# systemd EnvironmentFile: KEY=value only (no export)
|
||||
BANK_CTR=goa-regio-ng
|
||||
EX_CTR=goa-regio-ng
|
||||
MER_CTR=goa-regio-ng
|
||||
BANK_URL=http://127.0.0.1:9012
|
||||
BANK_PUBLIC_URL=https://bank.hacktivism.ch
|
||||
EXCHANGE_CONFIG_URL=https://exchange.hacktivism.ch/config
|
||||
BANK_CURRENCY=GOA
|
||||
BANK_LANDING_IN=/var/www/bank-landing
|
||||
EX_LANDING_IN=/var/www/exchange-landing
|
||||
MER_LANDING_IN=/var/www/merchant-landing
|
||||
COLLECT_BANK=1
|
||||
COLLECT_EXCHANGE=1
|
||||
COLLECT_MERCHANT=1
|
||||
COLLECT_RESOURCES=1
|
||||
PUBLISH_PODMAN=1
|
||||
STATS_SOURCE_LABEL="host collect_bank_stats.py goa regio-ng vanilla"
|
||||
ENV
|
||||
echo "--- stack.env ---"
|
||||
cat "$HOME/.config/taler-landing/stack.env"
|
||||
local unit="$HOME/.config/systemd/user/taler-landing-stats.service"
|
||||
if [ -f "$unit" ]; then
|
||||
if grep -q hacktivism "$unit"; then
|
||||
cp -a "$unit" "${unit}.bak-${TS}"
|
||||
sed -i \
|
||||
-e 's/container-taler-hacktivism[^ ]*\.service/goa-regio-ng.service/g' \
|
||||
-e 's/After=.*/After=goa-regio-ng.service/' \
|
||||
"$unit" || true
|
||||
if ! systemctl --user cat goa-regio-ng.service >/dev/null 2>&1; then
|
||||
sed -i 's/^After=.*/# After= (no user unit for goa-regio-ng; started via podman)/' "$unit" || true
|
||||
fi
|
||||
systemctl --user daemon-reload || true
|
||||
echo "patched $unit"
|
||||
grep -E 'After=|Requires=|EnvironmentFile' "$unit" || true
|
||||
else
|
||||
echo "stats unit already non-hacktivism"
|
||||
fi
|
||||
else
|
||||
echo "no taler-landing-stats.service yet"
|
||||
fi
|
||||
ok "stack.env installed"
|
||||
}
|
||||
|
||||
check_master() {
|
||||
local sha
|
||||
sha=$(podman exec -u root "$CTR" sha256sum /var/lib/taler-exchange/offline/master.priv 2>/dev/null | awk '{print $1}')
|
||||
echo "MASTER_LIVE=$sha"
|
||||
if [ "$sha" = "$EXPECT_MASTER" ]; then
|
||||
ok "master.priv matches bak"
|
||||
else
|
||||
die "master.priv mismatch got=$sha expect=$EXPECT_MASTER"
|
||||
fi
|
||||
}
|
||||
|
||||
# -------- Phase 1: restore current empty CTR --------
|
||||
echo "==== PHASE1: restore current CTR (expect empty image) ===="
|
||||
podman ps --filter "name=^${CTR}$" --format '{{.Names}} {{.Status}} {{.Image}}'
|
||||
settle_ctr || die "settle before phase1 restore"
|
||||
wait_pg_host 600 || die "PG not ready before phase1"
|
||||
run_restore phase1 || true
|
||||
if [ "$FAIL" -eq 0 ]; then
|
||||
smoke_inside phase1
|
||||
check_master
|
||||
probe_ports phase1 1 || true
|
||||
fi
|
||||
|
||||
# -------- Phase 2: recreate from EMPTY + restore again --------
|
||||
echo "==== PHASE2: recreate-from-EMPTY + restore again ===="
|
||||
if [ "$FAIL" -eq 0 ]; then
|
||||
recreate_from_empty || true
|
||||
fi
|
||||
if [ "$FAIL" -eq 0 ]; then
|
||||
run_restore phase2 || true
|
||||
smoke_inside phase2
|
||||
check_master
|
||||
probe_ports phase2 1 || true
|
||||
fi
|
||||
|
||||
# -------- Phase 3: hacktivism off + stack.env --------
|
||||
disable_hacktivism_boot
|
||||
install_stack_env
|
||||
|
||||
probe_ports final 1 || true
|
||||
echo "==== ALL-IN-ONE END $(date -u +%Y-%m-%dT%H:%M:%SZ) FAIL=$FAIL LOG=$LOG ===="
|
||||
if [ "$FAIL" -ne 0 ]; then
|
||||
echo "ALL_IN_ONE_FAILED"
|
||||
exit 1
|
||||
fi
|
||||
echo "ALL_IN_ONE_OK"
|
||||
exit 0
|
||||
Loading…
Add table
Add a link
Reference in a new issue