269 lines
9.3 KiB
Bash
Executable file
269 lines
9.3 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# All-in-one: restore empty → verify → recreate from EMPTY_IMG → restore →
|
|
# hacktivism boot off → stack.env → probes. Landings stamp/deploy from laptop after.
|
|
set -uo pipefail
|
|
# intentionally NOT set -e: settle/grep and optional probes must not abort
|
|
|
|
CTR=goa-regio-ng
|
|
EMPTY_IMG=${EMPTY_IMG:-localhost/goa-regio-ng:empty-current}
|
|
EXPECT_MASTER=6a1cececf164063ecd48172c21d56061ce7881fbd7a88d4ffd25504552508706
|
|
TS=$(date -u +%Y%m%dT%H%MZ)
|
|
LOG=/tmp/goa-all-in-one-${TS}.log
|
|
FAIL=0
|
|
exec > >(tee -a "$LOG") 2>&1
|
|
|
|
echo "==== ALL-IN-ONE START $(date -u +%Y-%m-%dT%H:%M:%SZ) LOG=$LOG ===="
|
|
echo "EMPTY_IMG=$EMPTY_IMG CTR=$CTR"
|
|
|
|
die() { echo "FAIL: $*"; FAIL=1; }
|
|
ok() { echo "OK: $*"; }
|
|
|
|
settle_ctr() {
|
|
local i st
|
|
for i in $(seq 1 90); do
|
|
st=$(podman exec -u root "$CTR" systemctl is-system-running 2>/dev/null || true)
|
|
case "$st" in
|
|
running|degraded) echo "SYSTEMD_SETTLE=$st after ${i}s"; return 0 ;;
|
|
esac
|
|
sleep 2
|
|
done
|
|
echo "SYSTEMD_SETTLE=timeout last=${st:-none}"
|
|
return 1
|
|
}
|
|
|
|
wait_pg_host() {
|
|
local max="${1:-600}" i=0 st
|
|
echo "==== WAIT PG (host-side, max ${max}s) ===="
|
|
podman exec -u root "$CTR" bash -c 'systemctl start postgresql 2>/dev/null; systemctl start postgresql@17-main 2>/dev/null; true' || true
|
|
while [ "$i" -lt "$max" ]; do
|
|
st=$(podman exec -u root "$CTR" bash -c '
|
|
set +e
|
|
systemctl is-active postgresql@17-main 2>/dev/null
|
|
runuser -u postgres -- psql -d postgres -v ON_ERROR_STOP=1 -tAc "SELECT 1" 2>/dev/null
|
|
' 2>/dev/null | tr '\n' ' ')
|
|
if echo "$st" | grep -q 'active' && echo "$st" | grep -qw '1'; then
|
|
echo "PG_ACCEPT after ${i}s"
|
|
return 0
|
|
fi
|
|
[ $((i % 30)) -eq 0 ] && echo "PG_WAIT ${i}s [$st]"
|
|
sleep 2
|
|
i=$((i + 2))
|
|
done
|
|
echo "PG_TIMEOUT after ${max}s"
|
|
return 1
|
|
}
|
|
|
|
# Scheme-aware probes; hard-fail on bad codes.
|
|
# 9010 merchant HTTPS; 9011 exchange HTTP; 9012 bank HTTP; 9013-15 landings 302 OK.
|
|
probe_ports() {
|
|
local tag=$1 hard=${2:-0} p code okc failc=0
|
|
echo "=== probes $tag (hard=$hard) ==="
|
|
|
|
code=$(curl -skS -o /dev/null -w '%{http_code}' --connect-timeout 3 --max-time 10 "https://127.0.0.1:9010/config" 2>/dev/null || echo ERR)
|
|
echo "MERCHANT_HTTPS_9010_CONFIG=${code}"
|
|
case "$code" in 200|204) ;; *) failc=$((failc+1)) ;; esac
|
|
|
|
code=$(curl -sS -o /dev/null -w '%{http_code}' --connect-timeout 3 --max-time 10 "http://127.0.0.1:9011/config" 2>/dev/null || echo ERR)
|
|
echo "EXCHANGE_HTTP_9011_CONFIG=${code}"
|
|
case "$code" in 200|204) ;; *)
|
|
code=$(curl -skS -o /dev/null -w '%{http_code}' --connect-timeout 3 --max-time 10 "https://127.0.0.1:9011/config" 2>/dev/null || echo ERR)
|
|
echo "EXCHANGE_HTTPS_9011_CONFIG=${code}"
|
|
case "$code" in 200|204) ;; *) failc=$((failc+1)) ;; esac
|
|
;;
|
|
esac
|
|
|
|
code=$(curl -sS -o /dev/null -w '%{http_code}' --connect-timeout 3 --max-time 10 "http://127.0.0.1:9012/config" 2>/dev/null || echo ERR)
|
|
echo "BANK_HTTP_9012_CONFIG=${code}"
|
|
case "$code" in 200|204) ;; *) failc=$((failc+1)) ;; esac
|
|
|
|
for p in 9013 9014 9015; do
|
|
code=$(curl -sS -o /dev/null -w '%{http_code}' --connect-timeout 3 --max-time 8 "http://127.0.0.1:${p}/" 2>/dev/null || echo ERR)
|
|
echo "LANDING_${p}=${code}"
|
|
case "$code" in 200|301|302|303|307|308) ;; *) failc=$((failc+1)) ;; esac
|
|
done
|
|
|
|
echo "PROBE_FAILS_${tag}=${failc}"
|
|
if [ "$hard" = "1" ] && [ "$failc" -gt 0 ]; then
|
|
die "probes $tag had $failc failures"
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
smoke_inside() {
|
|
local tag=$1 ww
|
|
echo "=== inside smoke $tag ==="
|
|
podman exec -u root "$CTR" bash -lc '
|
|
set +e
|
|
echo MASTER_SHA=$(sha256sum /var/lib/taler-exchange/offline/master.priv 2>/dev/null | awk "{print \$1}")
|
|
echo EXPECT='"$EXPECT_MASTER"'
|
|
for u in postgresql postgresql@17-main nginx taler-exchange-httpd taler-merchant-httpd libeufin-bank goa-demo-withdraw-api taler-exchange-wirewatch taler-exchange-transfer; do
|
|
printf "%s=%s\n" "$u" "$(systemctl is-active "$u" 2>/dev/null || echo missing)"
|
|
done
|
|
sudo -u postgres psql -Atc "SELECT datname||\"=\"||pg_size_pretty(pg_database_size(datname)) FROM pg_database WHERE datname ~ '"'"'exchange|libeufin|merchant'"'"' ORDER BY 1;"
|
|
'
|
|
ww=$(podman exec -u root "$CTR" systemctl is-active taler-exchange-wirewatch.service 2>/dev/null || echo missing)
|
|
if [ "$ww" = "active" ]; then
|
|
ok "wirewatch active ($tag)"
|
|
else
|
|
die "wirewatch not active ($tag)=$ww"
|
|
fi
|
|
}
|
|
|
|
run_restore() {
|
|
local tag=$1
|
|
echo "==== RESTORE $tag ===="
|
|
if ! bash /tmp/goa-vanilla-restore-do.sh; then
|
|
die "restore $tag failed"
|
|
return 1
|
|
fi
|
|
ok "restore $tag"
|
|
return 0
|
|
}
|
|
|
|
recreate_from_empty() {
|
|
echo "==== RECREATE from EMPTY_IMG (no commit of restored) ===="
|
|
podman image exists "$EMPTY_IMG" || { die "missing EMPTY_IMG $EMPTY_IMG"; return 1; }
|
|
podman stop -t 30 "$CTR" || true
|
|
podman rm "$CTR" || true
|
|
podman run -d \
|
|
--name "$CTR" \
|
|
--privileged \
|
|
--systemd=always \
|
|
--network pasta \
|
|
-v /sys/fs/cgroup:/sys/fs/cgroup:rw \
|
|
--tmpfs /run \
|
|
--tmpfs /run/lock \
|
|
-p 127.0.0.1:9322:22 \
|
|
-p 127.0.0.1:9180:80 \
|
|
-p 0.0.0.0:9010-9015:9010-9015 \
|
|
--hostname "$CTR" \
|
|
"$EMPTY_IMG" \
|
|
/sbin/init
|
|
sleep 8
|
|
settle_ctr || { die "settle after recreate"; return 1; }
|
|
wait_pg_host 600 || { die "PG not ready after recreate"; return 1; }
|
|
podman ps --filter "name=^${CTR}$" --format '{{.Names}} {{.Status}} {{.Image}} {{.Ports}}'
|
|
echo "=== pre-restore DBs (expect empty/minimal) ==="
|
|
podman exec -u root "$CTR" bash -lc 'sudo -u postgres psql -Atc "SELECT datname FROM pg_database WHERE datname ~ '"'"'exchange|libeufin|merchant'"'"' ORDER BY 1;"' || true
|
|
ok "recreate from empty"
|
|
}
|
|
|
|
disable_hacktivism_boot() {
|
|
echo "==== DISABLE hacktivism boot units ===="
|
|
local u
|
|
for u in \
|
|
container-taler-hacktivism-bank.service \
|
|
container-taler-hacktivism-exchange-ansible.service \
|
|
container-taler-hacktivism.service \
|
|
taler-monitoring-hacktivism.timer \
|
|
taler-monitoring-hacktivism.path
|
|
do
|
|
if systemctl --user list-unit-files "$u" >/dev/null 2>&1; then
|
|
systemctl --user disable --now "$u" 2>/dev/null || systemctl --user disable "$u" 2>/dev/null || true
|
|
echo "disabled $u"
|
|
else
|
|
echo "skip missing $u"
|
|
fi
|
|
done
|
|
systemctl --user daemon-reload || true
|
|
echo "=== enabled leftover hacktivism? ==="
|
|
systemctl --user list-unit-files 2>/dev/null | grep -i hacktivism || echo '(none listed)'
|
|
ok "hacktivism boot disable attempted"
|
|
}
|
|
|
|
install_stack_env() {
|
|
echo "==== INSTALL stack.env → goa-regio-ng ===="
|
|
mkdir -p "$HOME/.config/taler-landing"
|
|
cat > "$HOME/.config/taler-landing/stack.env" <<'ENV'
|
|
# Stack profile: GOA / goa-regio-ng vanilla (9010-9015)
|
|
# systemd EnvironmentFile: KEY=value only (no export)
|
|
BANK_CTR=goa-regio-ng
|
|
EX_CTR=goa-regio-ng
|
|
MER_CTR=goa-regio-ng
|
|
BANK_URL=http://127.0.0.1:9012
|
|
BANK_PUBLIC_URL=https://bank.hacktivism.ch
|
|
EXCHANGE_CONFIG_URL=https://exchange.hacktivism.ch/config
|
|
BANK_CURRENCY=GOA
|
|
BANK_LANDING_IN=/var/www/bank-landing
|
|
EX_LANDING_IN=/var/www/exchange-landing
|
|
MER_LANDING_IN=/var/www/merchant-landing
|
|
COLLECT_BANK=1
|
|
COLLECT_EXCHANGE=1
|
|
COLLECT_MERCHANT=1
|
|
COLLECT_RESOURCES=1
|
|
PUBLISH_PODMAN=1
|
|
STATS_SOURCE_LABEL="host collect_bank_stats.py goa regio-ng vanilla"
|
|
ENV
|
|
echo "--- stack.env ---"
|
|
cat "$HOME/.config/taler-landing/stack.env"
|
|
local unit="$HOME/.config/systemd/user/taler-landing-stats.service"
|
|
if [ -f "$unit" ]; then
|
|
if grep -q hacktivism "$unit"; then
|
|
cp -a "$unit" "${unit}.bak-${TS}"
|
|
sed -i \
|
|
-e 's/container-taler-hacktivism[^ ]*\.service/goa-regio-ng.service/g' \
|
|
-e 's/After=.*/After=goa-regio-ng.service/' \
|
|
"$unit" || true
|
|
if ! systemctl --user cat goa-regio-ng.service >/dev/null 2>&1; then
|
|
sed -i 's/^After=.*/# After= (no user unit for goa-regio-ng; started via podman)/' "$unit" || true
|
|
fi
|
|
systemctl --user daemon-reload || true
|
|
echo "patched $unit"
|
|
grep -E 'After=|Requires=|EnvironmentFile' "$unit" || true
|
|
else
|
|
echo "stats unit already non-hacktivism"
|
|
fi
|
|
else
|
|
echo "no taler-landing-stats.service yet"
|
|
fi
|
|
ok "stack.env installed"
|
|
}
|
|
|
|
check_master() {
|
|
local sha
|
|
sha=$(podman exec -u root "$CTR" sha256sum /var/lib/taler-exchange/offline/master.priv 2>/dev/null | awk '{print $1}')
|
|
echo "MASTER_LIVE=$sha"
|
|
if [ "$sha" = "$EXPECT_MASTER" ]; then
|
|
ok "master.priv matches bak"
|
|
else
|
|
die "master.priv mismatch got=$sha expect=$EXPECT_MASTER"
|
|
fi
|
|
}
|
|
|
|
# -------- Phase 1: restore current empty CTR --------
|
|
echo "==== PHASE1: restore current CTR (expect empty image) ===="
|
|
podman ps --filter "name=^${CTR}$" --format '{{.Names}} {{.Status}} {{.Image}}'
|
|
settle_ctr || die "settle before phase1 restore"
|
|
wait_pg_host 600 || die "PG not ready before phase1"
|
|
run_restore phase1 || true
|
|
if [ "$FAIL" -eq 0 ]; then
|
|
smoke_inside phase1
|
|
check_master
|
|
probe_ports phase1 1 || true
|
|
fi
|
|
|
|
# -------- Phase 2: recreate from EMPTY + restore again --------
|
|
echo "==== PHASE2: recreate-from-EMPTY + restore again ===="
|
|
if [ "$FAIL" -eq 0 ]; then
|
|
recreate_from_empty || true
|
|
fi
|
|
if [ "$FAIL" -eq 0 ]; then
|
|
run_restore phase2 || true
|
|
smoke_inside phase2
|
|
check_master
|
|
probe_ports phase2 1 || true
|
|
fi
|
|
|
|
# -------- Phase 3: hacktivism off + stack.env --------
|
|
disable_hacktivism_boot
|
|
install_stack_env
|
|
|
|
probe_ports final 1 || true
|
|
echo "==== ALL-IN-ONE END $(date -u +%Y-%m-%dT%H:%M:%SZ) FAIL=$FAIL LOG=$LOG ===="
|
|
if [ "$FAIL" -ne 0 ]; then
|
|
echo "ALL_IN_ONE_FAILED"
|
|
exit 1
|
|
fi
|
|
echo "ALL_IN_ONE_OK"
|
|
exit 0
|