diff --git a/scripts/taler-shared/goa-regio-ng/README.md b/scripts/taler-shared/goa-regio-ng/README.md new file mode 100644 index 0000000..49c224a --- /dev/null +++ b/scripts/taler-shared/goa-regio-ng/README.md @@ -0,0 +1,12 @@ +# GOA regio-ng lab (koopa) — empty → restore + +Operator path for live GOA on `goa-regio-ng` (ports 9010–9015 via pasta). + +1. Empty image SoT: `localhost/goa-regio-ng:empty-current` (do **not** commit a restored CTR as empty). +2. Bak: `~/backups/goa/current` on koopa. +3. Copy helpers to `/tmp/` on koopa (or use `~/goa-regio-ng-scripts/`), then run `./goa-all-in-one.sh` (recreate from empty → restore → probes). +4. Helpers: `goa-vanilla-restore-do.sh`, `goa-pg-acl-fixup.sh`, `goa-wire-post-restore.sh` (hosts `*.goa.local`, nginx bank proxy → `:9012`, `taler-exchange-dbconfig -p -s`, wirewatch/transfer `-a exchange-account-default` drop-ins). Restore calls the wire helper after core units are up and requires wirewatch active. +5. Ansible SoT is laptop `taler-deployment` branch `dev/hernani/goa-regio-ng-vanilla` (lab inventory: `configs/lab-local/goa-regio-ng-koopa.yml`). koopa has no git.taler.net deploy key — rsync/copy tip when needed. +6. Host landings 9013–9015 + Caddy stay outside the CTR; deploy from `~/goa-landing-deploy`. + +Greenfield Ansible `site.yml` on a fresh host is separate; this tree covers the verified restore bring-up. diff --git a/scripts/taler-shared/goa-regio-ng/goa-901x-recreate.sh b/scripts/taler-shared/goa-regio-ng/goa-901x-recreate.sh new file mode 100755 index 0000000..5fa3ca1 --- /dev/null +++ b/scripts/taler-shared/goa-regio-ng/goa-901x-recreate.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# Recreate goa-regio-ng publishing 0.0.0.0:9010-9015 (host Caddy untouched). +# Freigabe: live cutover restore / 901x replace. Rollback hacktivism = recreate from existing images. +set -euo pipefail + +TS="$(date -u +%Y%m%dT%H%MZ)" +IMG="localhost/goa-regio-ng:901x-wired-${TS}" +NAME="goa-regio-ng" +LOG="/tmp/goa-901x-recreate-${TS}.log" +exec > >(tee -a "$LOG") 2>&1 + +echo "TS=$TS IMG=$IMG LOG=$LOG" + +echo "=== pre: CTR/ports ===" +podman ps -a --format '{{.Names}}\t{{.Status}}\t{{.Ports}}' | grep -E 'goa-regio|hacktivism' || true +ss -lntp 2>/dev/null | grep -E ':901[0-5]\b' || echo '(host 901x none)' + +echo "=== save hacktivism CreateCommand (rollback recipe) ===" +ROLLBACK="/tmp/hacktivism-rollback-create-${TS}.txt" +: >"$ROLLBACK" +for c in taler-hacktivism-bank taler-hacktivism taler-hacktivism-exchange-ansible; do + if podman container exists "$c" 2>/dev/null; then + echo "--- $c ---" >>"$ROLLBACK" + podman inspect "$c" --format '{{json .Config.CreateCommand}}' >>"$ROLLBACK" + echo >>"$ROLLBACK" + podman inspect "$c" --format 'ImageName={{.ImageName}} ImageID={{.Image}}' >>"$ROLLBACK" + echo >>"$ROLLBACK" + fi +done +echo "ROLLBACK_FILE=$ROLLBACK" + +echo "=== commit $NAME -> $IMG ===" +podman commit --pause=false "$NAME" "$IMG" +podman images "$IMG" --format '{{.Repository}}:{{.Tag}}\t{{.ID}}\t{{.Size}}' + +echo "=== free 901x: rm exited hacktivism CTRs (images kept) ===" +for c in taler-hacktivism-bank taler-hacktivism taler-hacktivism-exchange-ansible; do + if podman container exists "$c" 2>/dev/null; then + st="$(podman inspect "$c" --format '{{.State.Status}}')" + echo "rm $c (was $st)" + podman rm "$c" + else + echo "skip missing $c" + fi +done + +echo "=== stop+rm old $NAME ===" +podman stop -t 30 "$NAME" || true +podman rm "$NAME" + +echo "=== run $NAME from $IMG publishing 9010-9015 + lab 9180/9322 ===" +podman run -d \ + --name "$NAME" \ + --privileged \ + --systemd=always \ + --network pasta \ + -v /sys/fs/cgroup:/sys/fs/cgroup:rw \ + --tmpfs /run \ + --tmpfs /run/lock \ + -p 127.0.0.1:9322:22 \ + -p 127.0.0.1:9180:80 \ + -p 0.0.0.0:9010-9015:9010-9015 \ + --hostname "$NAME" \ + "$IMG" \ + /sbin/init + +echo "=== wait systemd + services ===" +sleep 8 +for i in 1 2 3 4 5 6 7 8 9 10 11 12; do + if podman exec -u root "$NAME" systemctl is-system-running --wait 2>/dev/null | grep -qE 'running|degraded'; then + break + fi + sleep 2 +done + +echo "=== host ports ===" +podman ps --filter "name=^${NAME}$" --format '{{.Names}}\t{{.Status}}\t{{.Ports}}' +ss -lntp 2>/dev/null | grep -E ':901[0-5]\b' || echo '(WARN host 901x none)' + +echo "=== in-CTR listeners + units ===" +podman exec -u root "$NAME" bash -c ' +ss -lntp 2>/dev/null | grep -E ":901[0-5]\b|:19096\b" || true +for u in nginx taler-exchange-httpd libeufin-bank goa-demo-withdraw-api postgresql; do + printf "%s=%s\n" "$u" "$(systemctl is-active "$u" 2>/dev/null || echo missing)" +done +' + +echo "RECREATE_OK IMG=$IMG LOG=$LOG ROLLBACK=$ROLLBACK" diff --git a/scripts/taler-shared/goa-regio-ng/goa-all-in-one.sh b/scripts/taler-shared/goa-regio-ng/goa-all-in-one.sh new file mode 100755 index 0000000..52bed90 --- /dev/null +++ b/scripts/taler-shared/goa-regio-ng/goa-all-in-one.sh @@ -0,0 +1,269 @@ +#!/usr/bin/env bash +# All-in-one: restore empty → verify → recreate from EMPTY_IMG → restore → +# hacktivism boot off → stack.env → probes. Landings stamp/deploy from laptop after. +set -uo pipefail +# intentionally NOT set -e: settle/grep and optional probes must not abort + +CTR=goa-regio-ng +EMPTY_IMG=${EMPTY_IMG:-localhost/goa-regio-ng:empty-current} +EXPECT_MASTER=6a1cececf164063ecd48172c21d56061ce7881fbd7a88d4ffd25504552508706 +TS=$(date -u +%Y%m%dT%H%MZ) +LOG=/tmp/goa-all-in-one-${TS}.log +FAIL=0 +exec > >(tee -a "$LOG") 2>&1 + +echo "==== ALL-IN-ONE START $(date -u +%Y-%m-%dT%H:%M:%SZ) LOG=$LOG ====" +echo "EMPTY_IMG=$EMPTY_IMG CTR=$CTR" + +die() { echo "FAIL: $*"; FAIL=1; } +ok() { echo "OK: $*"; } + +settle_ctr() { + local i st + for i in $(seq 1 90); do + st=$(podman exec -u root "$CTR" systemctl is-system-running 2>/dev/null || true) + case "$st" in + running|degraded) echo "SYSTEMD_SETTLE=$st after ${i}s"; return 0 ;; + esac + sleep 2 + done + echo "SYSTEMD_SETTLE=timeout last=${st:-none}" + return 1 +} + +wait_pg_host() { + local max="${1:-600}" i=0 st + echo "==== WAIT PG (host-side, max ${max}s) ====" + podman exec -u root "$CTR" bash -c 'systemctl start postgresql 2>/dev/null; systemctl start postgresql@17-main 2>/dev/null; true' || true + while [ "$i" -lt "$max" ]; do + st=$(podman exec -u root "$CTR" bash -c ' +set +e +systemctl is-active postgresql@17-main 2>/dev/null +runuser -u postgres -- psql -d postgres -v ON_ERROR_STOP=1 -tAc "SELECT 1" 2>/dev/null +' 2>/dev/null | tr '\n' ' ') + if echo "$st" | grep -q 'active' && echo "$st" | grep -qw '1'; then + echo "PG_ACCEPT after ${i}s" + return 0 + fi + [ $((i % 30)) -eq 0 ] && echo "PG_WAIT ${i}s [$st]" + sleep 2 + i=$((i + 2)) + done + echo "PG_TIMEOUT after ${max}s" + return 1 +} + +# Scheme-aware probes; hard-fail on bad codes. +# 9010 merchant HTTPS; 9011 exchange HTTP; 9012 bank HTTP; 9013-15 landings 302 OK. +probe_ports() { + local tag=$1 hard=${2:-0} p code okc failc=0 + echo "=== probes $tag (hard=$hard) ===" + + code=$(curl -skS -o /dev/null -w '%{http_code}' --connect-timeout 3 --max-time 10 "https://127.0.0.1:9010/config" 2>/dev/null || echo ERR) + echo "MERCHANT_HTTPS_9010_CONFIG=${code}" + case "$code" in 200|204) ;; *) failc=$((failc+1)) ;; esac + + code=$(curl -sS -o /dev/null -w '%{http_code}' --connect-timeout 3 --max-time 10 "http://127.0.0.1:9011/config" 2>/dev/null || echo ERR) + echo "EXCHANGE_HTTP_9011_CONFIG=${code}" + case "$code" in 200|204) ;; *) + code=$(curl -skS -o /dev/null -w '%{http_code}' --connect-timeout 3 --max-time 10 "https://127.0.0.1:9011/config" 2>/dev/null || echo ERR) + echo "EXCHANGE_HTTPS_9011_CONFIG=${code}" + case "$code" in 200|204) ;; *) failc=$((failc+1)) ;; esac + ;; + esac + + code=$(curl -sS -o /dev/null -w '%{http_code}' --connect-timeout 3 --max-time 10 "http://127.0.0.1:9012/config" 2>/dev/null || echo ERR) + echo "BANK_HTTP_9012_CONFIG=${code}" + case "$code" in 200|204) ;; *) failc=$((failc+1)) ;; esac + + for p in 9013 9014 9015; do + code=$(curl -sS -o /dev/null -w '%{http_code}' --connect-timeout 3 --max-time 8 "http://127.0.0.1:${p}/" 2>/dev/null || echo ERR) + echo "LANDING_${p}=${code}" + case "$code" in 200|301|302|303|307|308) ;; *) failc=$((failc+1)) ;; esac + done + + echo "PROBE_FAILS_${tag}=${failc}" + if [ "$hard" = "1" ] && [ "$failc" -gt 0 ]; then + die "probes $tag had $failc failures" + return 1 + fi + return 0 +} + +smoke_inside() { + local tag=$1 ww + echo "=== inside smoke $tag ===" + podman exec -u root "$CTR" bash -lc ' +set +e +echo MASTER_SHA=$(sha256sum /var/lib/taler-exchange/offline/master.priv 2>/dev/null | awk "{print \$1}") +echo EXPECT='"$EXPECT_MASTER"' +for u in postgresql postgresql@17-main nginx taler-exchange-httpd taler-merchant-httpd libeufin-bank goa-demo-withdraw-api taler-exchange-wirewatch taler-exchange-transfer; do + printf "%s=%s\n" "$u" "$(systemctl is-active "$u" 2>/dev/null || echo missing)" +done +sudo -u postgres psql -Atc "SELECT datname||\"=\"||pg_size_pretty(pg_database_size(datname)) FROM pg_database WHERE datname ~ '"'"'exchange|libeufin|merchant'"'"' ORDER BY 1;" +' + ww=$(podman exec -u root "$CTR" systemctl is-active taler-exchange-wirewatch.service 2>/dev/null || echo missing) + if [ "$ww" = "active" ]; then + ok "wirewatch active ($tag)" + else + die "wirewatch not active ($tag)=$ww" + fi +} + +run_restore() { + local tag=$1 + echo "==== RESTORE $tag ====" + if ! bash /tmp/goa-vanilla-restore-do.sh; then + die "restore $tag failed" + return 1 + fi + ok "restore $tag" + return 0 +} + +recreate_from_empty() { + echo "==== RECREATE from EMPTY_IMG (no commit of restored) ====" + podman image exists "$EMPTY_IMG" || { die "missing EMPTY_IMG $EMPTY_IMG"; return 1; } + podman stop -t 30 "$CTR" || true + podman rm "$CTR" || true + podman run -d \ + --name "$CTR" \ + --privileged \ + --systemd=always \ + --network pasta \ + -v /sys/fs/cgroup:/sys/fs/cgroup:rw \ + --tmpfs /run \ + --tmpfs /run/lock \ + -p 127.0.0.1:9322:22 \ + -p 127.0.0.1:9180:80 \ + -p 0.0.0.0:9010-9015:9010-9015 \ + --hostname "$CTR" \ + "$EMPTY_IMG" \ + /sbin/init + sleep 8 + settle_ctr || { die "settle after recreate"; return 1; } + wait_pg_host 600 || { die "PG not ready after recreate"; return 1; } + podman ps --filter "name=^${CTR}$" --format '{{.Names}} {{.Status}} {{.Image}} {{.Ports}}' + echo "=== pre-restore DBs (expect empty/minimal) ===" + podman exec -u root "$CTR" bash -lc 'sudo -u postgres psql -Atc "SELECT datname FROM pg_database WHERE datname ~ '"'"'exchange|libeufin|merchant'"'"' ORDER BY 1;"' || true + ok "recreate from empty" +} + +disable_hacktivism_boot() { + echo "==== DISABLE hacktivism boot units ====" + local u + for u in \ + container-taler-hacktivism-bank.service \ + container-taler-hacktivism-exchange-ansible.service \ + container-taler-hacktivism.service \ + taler-monitoring-hacktivism.timer \ + taler-monitoring-hacktivism.path + do + if systemctl --user list-unit-files "$u" >/dev/null 2>&1; then + systemctl --user disable --now "$u" 2>/dev/null || systemctl --user disable "$u" 2>/dev/null || true + echo "disabled $u" + else + echo "skip missing $u" + fi + done + systemctl --user daemon-reload || true + echo "=== enabled leftover hacktivism? ===" + systemctl --user list-unit-files 2>/dev/null | grep -i hacktivism || echo '(none listed)' + ok "hacktivism boot disable attempted" +} + +install_stack_env() { + echo "==== INSTALL stack.env → goa-regio-ng ====" + mkdir -p "$HOME/.config/taler-landing" + cat > "$HOME/.config/taler-landing/stack.env" <<'ENV' +# Stack profile: GOA / goa-regio-ng vanilla (9010-9015) +# systemd EnvironmentFile: KEY=value only (no export) +BANK_CTR=goa-regio-ng +EX_CTR=goa-regio-ng +MER_CTR=goa-regio-ng +BANK_URL=http://127.0.0.1:9012 +BANK_PUBLIC_URL=https://bank.hacktivism.ch +EXCHANGE_CONFIG_URL=https://exchange.hacktivism.ch/config +BANK_CURRENCY=GOA +BANK_LANDING_IN=/var/www/bank-landing +EX_LANDING_IN=/var/www/exchange-landing +MER_LANDING_IN=/var/www/merchant-landing +COLLECT_BANK=1 +COLLECT_EXCHANGE=1 +COLLECT_MERCHANT=1 +COLLECT_RESOURCES=1 +PUBLISH_PODMAN=1 +STATS_SOURCE_LABEL="host collect_bank_stats.py goa regio-ng vanilla" +ENV + echo "--- stack.env ---" + cat "$HOME/.config/taler-landing/stack.env" + local unit="$HOME/.config/systemd/user/taler-landing-stats.service" + if [ -f "$unit" ]; then + if grep -q hacktivism "$unit"; then + cp -a "$unit" "${unit}.bak-${TS}" + sed -i \ + -e 's/container-taler-hacktivism[^ ]*\.service/goa-regio-ng.service/g' \ + -e 's/After=.*/After=goa-regio-ng.service/' \ + "$unit" || true + if ! systemctl --user cat goa-regio-ng.service >/dev/null 2>&1; then + sed -i 's/^After=.*/# After= (no user unit for goa-regio-ng; started via podman)/' "$unit" || true + fi + systemctl --user daemon-reload || true + echo "patched $unit" + grep -E 'After=|Requires=|EnvironmentFile' "$unit" || true + else + echo "stats unit already non-hacktivism" + fi + else + echo "no taler-landing-stats.service yet" + fi + ok "stack.env installed" +} + +check_master() { + local sha + sha=$(podman exec -u root "$CTR" sha256sum /var/lib/taler-exchange/offline/master.priv 2>/dev/null | awk '{print $1}') + echo "MASTER_LIVE=$sha" + if [ "$sha" = "$EXPECT_MASTER" ]; then + ok "master.priv matches bak" + else + die "master.priv mismatch got=$sha expect=$EXPECT_MASTER" + fi +} + +# -------- Phase 1: restore current empty CTR -------- +echo "==== PHASE1: restore current CTR (expect empty image) ====" +podman ps --filter "name=^${CTR}$" --format '{{.Names}} {{.Status}} {{.Image}}' +settle_ctr || die "settle before phase1 restore" +wait_pg_host 600 || die "PG not ready before phase1" +run_restore phase1 || true +if [ "$FAIL" -eq 0 ]; then + smoke_inside phase1 + check_master + probe_ports phase1 1 || true +fi + +# -------- Phase 2: recreate from EMPTY + restore again -------- +echo "==== PHASE2: recreate-from-EMPTY + restore again ====" +if [ "$FAIL" -eq 0 ]; then + recreate_from_empty || true +fi +if [ "$FAIL" -eq 0 ]; then + run_restore phase2 || true + smoke_inside phase2 + check_master + probe_ports phase2 1 || true +fi + +# -------- Phase 3: hacktivism off + stack.env -------- +disable_hacktivism_boot +install_stack_env + +probe_ports final 1 || true +echo "==== ALL-IN-ONE END $(date -u +%Y-%m-%dT%H:%M:%SZ) FAIL=$FAIL LOG=$LOG ====" +if [ "$FAIL" -ne 0 ]; then + echo "ALL_IN_ONE_FAILED" + exit 1 +fi +echo "ALL_IN_ONE_OK" +exit 0 diff --git a/scripts/taler-shared/goa-regio-ng/goa-e2e-recreate-restore.sh b/scripts/taler-shared/goa-regio-ng/goa-e2e-recreate-restore.sh new file mode 100755 index 0000000..23fa267 --- /dev/null +++ b/scripts/taler-shared/goa-regio-ng/goa-e2e-recreate-restore.sh @@ -0,0 +1,53 @@ +#!/bin/bash +set -euo pipefail +CTR=goa-regio-ng +LOG=/tmp/goa-e2e-recreate-restore.log +exec > >(tee -a "$LOG") 2>&1 +echo "=== START $(date -u +%Y-%m-%dT%H:%M:%SZ) ===" + +echo "=== 1) RECREATE (empty CTR) ===" +bash /tmp/goa-901x-recreate.sh +echo "RECREATE_RC=$?" + +echo "=== wait ports / systemd settle ===" +for i in $(seq 1 60); do + if podman exec "$CTR" systemctl is-system-running >/dev/null 2>&1 \ + || podman exec "$CTR" systemctl is-system-running 2>&1 | grep -qE 'running|degraded'; then + echo "SYSTEMD_OK after ${i}s" + break + fi + sleep 5 +done +podman ps --filter name="$CTR" --format '{{.Names}} {{.Status}}' +ss -lntp 2>/dev/null | grep -E '901[0-5]|9180|9322' || true + +echo "=== pre-restore: expect empty/minimal DBs ===" +podman exec "$CTR" bash -lc 'sudo -u postgres psql -Atc "SELECT datname FROM pg_database WHERE datname ~ '"'"'exchange|libeufin|merchant'"'"' ORDER BY 1;"' || true + +echo "=== 2) RESTORE from bak via script ===" +bash /tmp/goa-vanilla-restore-do.sh +echo "RESTORE_RC=$?" + +echo "=== 3) post-restore smoke inside CTR ===" +podman exec "$CTR" bash -lc ' +set +e +echo MASTER_SHA=$(sha256sum /var/lib/taler-exchange/offline/master.priv 2>/dev/null | awk "{print \$1}") +echo EXPECT=6a1cececf164063ecd48172c21d56061ce7881fbd7a88d4ffd25504552508706 +for u in taler-exchange-httpd taler-merchant-httpd libeufin-bank libeufin-nexus; do + echo -n "$u: "; systemctl is-active $u 2>/dev/null || systemctl is-active ${u}.service 2>/dev/null +done +sudo -u postgres psql -Atc "SELECT datname FROM pg_database WHERE datname ~ '"'"'exchange|libeufin|merchant'"'"' ORDER BY 1;" +' + +echo "=== 4) host probes 9010-9015 ===" +for p in 9010 9011 9012 9013 9014 9015; do + code=$(curl -sS -o /dev/null -w '%{http_code}' --connect-timeout 3 "http://127.0.0.1:${p}/" || echo ERR) + echo "PORT_${p}=${code}" +done + +echo "=== 5) leftover hacktivism / failed units ===" +systemctl list-units --all --no-pager 'container-taler-hacktivism*' 2>/dev/null || true +systemctl list-units --failed --no-pager 2>/dev/null | head -40 || true +podman ps -a --format '{{.Names}} {{.Status}}' | grep -iE 'hacktivism|goa|taler' || true + +echo "=== END $(date -u +%Y-%m-%dT%H:%M:%SZ) ===" diff --git a/scripts/taler-shared/goa-regio-ng/goa-pg-acl-fixup.sh b/scripts/taler-shared/goa-regio-ng/goa-pg-acl-fixup.sh new file mode 100755 index 0000000..d4d6f88 --- /dev/null +++ b/scripts/taler-shared/goa-regio-ng/goa-pg-acl-fixup.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# After pg_restore --no-owner --no-acl: give app role full rights on non-system schemas. +# Do NOT ALTER SEQUENCE OWNER for serials linked to tables (Postgres rejects that); +# ALTER TABLE OWNER covers owned serials. +# Usage inside CTR: goa-pg-acl-fixup.sh +set -euo pipefail +db="${1:?db}" +owner="${2:?owner}" +# owner is a fixed role name from the restore caller (no user input). +runuser -u postgres -- psql -d "$db" -v ON_ERROR_STOP=1 < >(tee -a "$LOG") 2>&1 + +echo "==== VANILLA RESTORE START $(date -u +%Y-%m-%dT%H:%M:%SZ) ====" +echo "CTR=$CTR DUMP_ROOT=$DUMP_ROOT SECMOD_SRC=$SECMOD_SRC LOG=$LOG" + +need() { [ -f "$1" ] || { echo "FAIL missing $1"; exit 1; }; } +need "$DUMP_ROOT/bank/libeufin.dump" +need "$DUMP_ROOT/exchange/taler-exchange.dump" +need "$DUMP_ROOT/merchant/taler-merchant.dump" +need "$MASTER_BAK" +need "$SECMOD_SRC/master.priv" +[ -d "$SECMOD_SRC/secmod-rsa" ] || { echo "FAIL missing secmod-rsa"; exit 1; } + +BAK_SHA="$(sha256sum "$MASTER_BAK" | awk '{print $1}')" +STG_SHA="$(sha256sum "$SECMOD_SRC/master.priv" | awk '{print $1}')" +echo "MASTER_BAK_SHA=$BAK_SHA" +echo "MASTER_STG_SHA=$STG_SHA" +[ "$BAK_SHA" = "$STG_SHA" ] || { echo "FAIL master.priv bak≠staging"; exit 1; } + +podman container exists "$CTR" || { echo "FAIL CTR missing $CTR"; exit 1; } + +wait_pg() { + # EMPTY_IMG may need long crash-recovery after unclean shutdown. + local max="${1:-600}" i=0 st + echo "==== WAIT PG accept (max ${max}s) ====" + podman exec -u root "$CTR" bash -c ' +set +e +systemctl start postgresql 2>/dev/null || true +systemctl start postgresql@17-main 2>/dev/null || true +' + while [ "$i" -lt "$max" ]; do + st=$(podman exec -u root "$CTR" bash -c ' +set +e +systemctl is-active postgresql@17-main 2>/dev/null +runuser -u postgres -- psql -d postgres -v ON_ERROR_STOP=1 -tAc "SELECT 1" 2>/dev/null +' 2>/dev/null | tr '\n' ' ') + if echo "$st" | grep -q 'active' && echo "$st" | grep -qw '1'; then + echo "PG_ACCEPT after ${i}s ($st)" + return 0 + fi + if [ $((i % 30)) -eq 0 ]; then + echo "PG_WAIT ${i}s state=[$st]" + podman exec -u root "$CTR" bash -c 'tail -n 3 /var/log/postgresql/postgresql-17-main.log 2>/dev/null || journalctl -u postgresql@17-main -n 3 --no-pager 2>/dev/null | tail -n 3' || true + fi + sleep 2 + i=$((i + 2)) + done + echo "FAIL PG not accepting after ${max}s" + podman exec -u root "$CTR" bash -c 'systemctl status postgresql@17-main --no-pager -l 2>&1 | head -40; journalctl -u postgresql@17-main -n 40 --no-pager 2>&1 | tail -40' || true + return 1 +} + +wait_unit_active() { + local unit="$1" max="${2:-120}" i=0 st + while [ "$i" -lt "$max" ]; do + st=$(podman exec -u root "$CTR" systemctl is-active "$unit" 2>/dev/null || echo missing) + if [ "$st" = "active" ]; then + echo "UNIT_ACTIVE $unit after ${i}s" + return 0 + fi + if [ "$st" = "failed" ]; then + echo "UNIT_FAILED $unit after ${i}s" + podman exec -u root "$CTR" bash -c "systemctl status $unit --no-pager -l 2>&1 | head -50; journalctl -u $unit -n 80 --no-pager 2>&1 | tail -80" || true + return 1 + fi + sleep 2 + i=$((i + 2)) + done + echo "UNIT_TIMEOUT $unit last=$st after ${max}s" + podman exec -u root "$CTR" bash -c "systemctl status $unit --no-pager -l 2>&1 | head -50; journalctl -u $unit -n 80 --no-pager 2>&1 | tail -80" || true + return 1 +} + +echo "==== STOP app units (keep postgresql) ====" +podman exec -u root "$CTR" bash -c ' +set -e +for u in \ + goa-demo-withdraw-api \ + taler-merchant-httpd \ + taler-exchange-httpd \ + taler-exchange-wirewatch \ + taler-exchange-aggregator \ + taler-exchange-closer \ + taler-exchange-expire \ + taler-exchange-transfer \ + taler-exchange-secmod-rsa \ + taler-exchange-secmod-eddsa \ + taler-exchange-secmod-cs \ + libeufin-bank \ + libeufin-bank-nginx \ + nginx +do + systemctl stop "$u" 2>/dev/null || true +done +systemctl reset-failed 2>/dev/null || true +' + +wait_pg 600 || exit 1 + +echo "==== COPY dumps into CTR ====" +podman cp "$DUMP_ROOT/bank/libeufin.dump" "${CTR}:/tmp/libeufin.dump" +podman cp "$DUMP_ROOT/exchange/taler-exchange.dump" "${CTR}:/tmp/taler-exchange.dump" +podman cp "$DUMP_ROOT/merchant/taler-merchant.dump" "${CTR}:/tmp/taler-merchant.dump" +# ACL helper must be in CTR before restore_one (post pg_restore --no-acl) +if [ ! -f /tmp/goa-pg-acl-fixup.sh ]; then + echo "FAIL missing /tmp/goa-pg-acl-fixup.sh on host" + exit 2 +fi +podman cp /tmp/goa-pg-acl-fixup.sh "${CTR}:/tmp/goa-pg-acl-fixup.sh" + +echo "==== DROP+CREATE+RESTORE DBs ====" +# Live vanilla CONFIG=postgres:///exchange (bak dump dbname was taler-exchange). +podman exec -u root "$CTR" bash -c ' +set -euo pipefail +restore_one() { + local db="$1" dump="$2" owner="$3" + echo "--- restore $db from $dump owner=$owner ---" + runuser -u postgres -- psql -d postgres -c \ + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '"'"'$db'"'"' AND pid <> pg_backend_pid();" \ + || true + runuser -u postgres -- dropdb --if-exists --force "$db" || true + runuser -u postgres -- createdb -O "$owner" "$db" + set +e + runuser -u postgres -- pg_restore --no-owner --no-acl --dbname="$db" "$dump" + RC=$? + set -e + # pg_restore exits 1 on warnings; 0 ok; >=2 hard fail + echo "PG_RESTORE_$db RC=$RC" + if [ "$RC" -ge 2 ]; then + echo "FAIL pg_restore $db" + exit 2 + fi + rm -f "$dump" + # --no-owner --no-acl leaves schemas owned by postgres with no grants → + # app roles hit "permission denied for schema _v". Fix ownership + GRANTs. + # Skip ALTER SEQUENCE OWNER for linked serials; ALTER TABLE OWNER covers them. + bash /tmp/goa-pg-acl-fixup.sh "$db" "$owner" + runuser -u postgres -- psql -d "$db" -v ON_ERROR_STOP=1 -c "SELECT current_database() AS db;" + runuser -u postgres -- psql -d "$db" -tAc "SELECT pg_size_pretty(pg_database_size(current_database()));" + runuser -u postgres -- psql -d "$db" -tAc "SELECT count(*) FROM information_schema.tables WHERE table_schema='"'"'public'"'"';" + # also count exchange schema (bak dumps use schema exchange, not public) + runuser -u postgres -- psql -d "$db" -tAc "SELECT '"'"'schema_exchange_tables='"'"'||count(*)::text FROM information_schema.tables WHERE table_schema='"'"'exchange'"'"';" || true +} + +restore_one libeufin /tmp/libeufin.dump libeufin-bank +restore_one exchange /tmp/taler-exchange.dump taler-exchange-httpd +restore_one taler-merchant /tmp/taler-merchant.dump taler-merchant-httpd + +# drop leftover bak-named DB if any +runuser -u postgres -- dropdb --if-exists --force taler-exchange || true +runuser -u postgres -- psql -d postgres -c "\l" +' + +echo "==== INSTALL master.priv + secmod from staging ====" +HOST_STG="/tmp/goa-restore-host-secmod-$$" +rm -rf "$HOST_STG" +mkdir -p "$HOST_STG" +cp -a "$SECMOD_SRC/." "$HOST_STG/" +cp -a "$MASTER_BAK" "$HOST_STG/master.priv" +# also secm_tofus.pub from bak offline if present +if [ -f "$DUMP_ROOT/exchange/offline/secm_tofus.pub" ]; then + # staging copy is often mode 0400; plain cp -a cannot overwrite it + rm -f "$HOST_STG/secm_tofus.pub" + cp -a "$DUMP_ROOT/exchange/offline/secm_tofus.pub" "$HOST_STG/secm_tofus.pub" +fi + +podman exec -u root "$CTR" bash -c ' +set -euo pipefail +rm -rf /tmp/secmod-in +mkdir -p /tmp/secmod-in +' +podman cp "$HOST_STG/." "${CTR}:/tmp/secmod-in/" + +podman exec -u root "$CTR" bash -c ' +set -euo pipefail +OFF=/var/lib/taler-exchange/offline +mkdir -p "$OFF" +install -o taler-exchange-offline -g taler-exchange-offline -m 400 /tmp/secmod-in/master.priv "$OFF/master.priv" +if [ -f /tmp/secmod-in/secm_tofus.pub ]; then + install -o taler-exchange-offline -g taler-exchange-offline -m 400 /tmp/secmod-in/secm_tofus.pub "$OFF/secm_tofus.pub" +fi +# offline-keys home path (also used live) +HOME_OFF=/home/taler-exchange-offline/.local/share/taler/exchange/offline-keys +mkdir -p "$HOME_OFF" +install -o taler-exchange-offline -g taler-exchange-offline -m 400 /tmp/secmod-in/master.priv "$HOME_OFF/master.priv" + +# secmod trees +for kind in rsa eddsa cs; do + src="/tmp/secmod-in/secmod-$kind" + dst="/var/lib/taler-exchange/secmod-$kind" + [ -d "$src" ] || { echo "FAIL missing $src"; exit 1; } + rm -rf "$dst" + mkdir -p "$dst" + cp -a "$src/." "$dst/" + # ownership: taler-exchange-secmod- : taler-exchange-secmod + u="taler-exchange-secmod-$kind" + chown -R "$u:taler-exchange-secmod" "$dst" + chmod -R u=rwX,g=rX,o= "$dst" + find "$dst" -type f -name "*private*" -exec chmod 400 {} \; || true +done +sha256sum "$OFF/master.priv" +rm -rf /tmp/secmod-in +echo SECMOD_INSTALL_OK +' +rm -rf "$HOST_STG" + +echo "==== START units ====" +podman exec -u root "$CTR" bash -c ' +set -e +systemctl start postgresql +systemctl start postgresql@17-main || true +systemctl start taler-exchange-secmod-rsa taler-exchange-secmod-eddsa taler-exchange-secmod-cs +sleep 2 +systemctl start taler-exchange-httpd +systemctl start libeufin-bank +systemctl start taler-merchant-httpd +systemctl start nginx || true +systemctl start goa-demo-withdraw-api || true +# helpers +for u in taler-exchange-wirewatch taler-exchange-aggregator taler-exchange-closer taler-exchange-expire taler-exchange-transfer; do + systemctl start "$u" 2>/dev/null || true +done +' + +UNIT_FAIL=0 +for u in taler-exchange-secmod-rsa taler-exchange-secmod-eddsa taler-exchange-secmod-cs \ + taler-exchange-httpd libeufin-bank taler-merchant-httpd nginx; do + if ! wait_unit_active "$u" 180; then + UNIT_FAIL=1 + fi +done +# demo-withdraw optional +wait_unit_active goa-demo-withdraw-api 60 || echo "WARN goa-demo-withdraw-api not active (optional)" + +echo "==== UNIT STATUS SNAPSHOT ====" +podman exec -u root "$CTR" bash -c ' +for u in postgresql postgresql@17-main taler-exchange-httpd libeufin-bank taler-merchant-httpd \ + taler-exchange-secmod-rsa taler-exchange-secmod-eddsa taler-exchange-secmod-cs \ + nginx goa-demo-withdraw-api; do + printf "%s=%s\n" "$u" "$(systemctl is-active "$u" 2>/dev/null || echo missing)" +done +' + +if [ "$UNIT_FAIL" -ne 0 ]; then + echo "FAIL required units not active" + exit 1 +fi + +echo "==== WIRE POST-RESTORE (hosts + nginx 9012 + dbconfig + -a drop-ins) ====" +if [ ! -f /tmp/goa-wire-post-restore.sh ]; then + echo "FAIL missing /tmp/goa-wire-post-restore.sh on host" + exit 2 +fi +podman cp /tmp/goa-wire-post-restore.sh "${CTR}:/tmp/goa-wire-post-restore.sh" +podman exec -u root "$CTR" bash /tmp/goa-wire-post-restore.sh + +echo "==== LOCAL SMOKE (scheme-aware) ====" +SMOKE_FAIL=0 +code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 http://127.0.0.1:9012/config 2>/dev/null || echo ERR) +echo "bank_http_9012=$code" +case "$code" in 200|204) ;; *) SMOKE_FAIL=1 ;; esac + +code=$(curl -skS -o /dev/null -w "%{http_code}" --max-time 10 https://127.0.0.1:9010/config 2>/dev/null || echo ERR) +echo "merchant_https_9010=$code" +case "$code" in 200|204) ;; *) SMOKE_FAIL=1 ;; esac + +code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 http://127.0.0.1:9011/config 2>/dev/null || echo ERR) +echo "exchange_http_9011=$code" +case "$code" in 200|204) ;; *) + # fallback https in case nginx wraps it + code2=$(curl -skS -o /dev/null -w "%{http_code}" --max-time 10 https://127.0.0.1:9011/config 2>/dev/null || echo ERR) + echo "exchange_https_9011=$code2" + case "$code2" in 200|204) ;; *) SMOKE_FAIL=1 ;; esac + ;; +esac + +ww=$(podman exec -u root "$CTR" systemctl is-active taler-exchange-wirewatch.service 2>/dev/null || echo missing) +echo "wirewatch=$ww" +[ "$ww" = "active" ] || SMOKE_FAIL=1 + +if [ "$SMOKE_FAIL" -ne 0 ]; then + echo "FAIL local smoke probes" + podman exec -u root "$CTR" bash -c ' +ss -lntp 2>/dev/null | grep -E ":901[0-5]|:80 " || netstat -lntp 2>/dev/null | grep -E ":901[0-5]|:80 " || true +for u in taler-exchange-httpd libeufin-bank taler-merchant-httpd nginx taler-exchange-wirewatch; do + echo "--- journal $u ---" + journalctl -u "$u" -n 40 --no-pager 2>&1 | tail -40 +done +' || true + exit 1 +fi + +echo "==== PUBLIC SMOKE ====" +for u in \ + https://exchange.hacktivism.ch/config \ + https://bank.hacktivism.ch/config \ + https://backend.hacktivism.ch/config +do + code=$(curl -skS -o /dev/null -w "%{http_code}" --max-time 15 "$u" || echo ERR) + echo "PUB $code $u" +done + +echo "==== POST DB SIZES ====" +podman exec -u root "$CTR" runuser -u postgres -- psql -d postgres -tAc \ + "SELECT datname||'='||pg_size_pretty(pg_database_size(datname)) FROM pg_database WHERE datname IN ('exchange','libeufin','taler-merchant') ORDER BY 1;" +podman exec -u root "$CTR" runuser -u postgres -- psql -d exchange -tAc \ + "SELECT 'exchange_schema_tables='||count(*)::text FROM information_schema.tables WHERE table_schema='exchange';" + +echo "RESTORE_SCRIPT_OK $(date -u +%Y-%m-%dT%H:%M:%SZ) LOG=$LOG" diff --git a/scripts/taler-shared/goa-regio-ng/goa-wire-post-restore.sh b/scripts/taler-shared/goa-regio-ng/goa-wire-post-restore.sh new file mode 100755 index 0000000..4eda445 --- /dev/null +++ b/scripts/taler-shared/goa-regio-ng/goa-wire-post-restore.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Inside CTR (as root): wire path after empty→restore. +# - /etc/hosts *.goa.local +# - nginx bank proxy → libeufin :9012 (empty image still has :8080) +# - taler-exchange-dbconfig -p -s (schema + grants for wire role) +# - systemd -a drop-ins (dual exchange-account-1 + exchange-account-default) +# - start wirewatch + transfer; require wirewatch active +set -euo pipefail + +echo "==== WIRE POST-RESTORE ====" + +if ! grep -q 'bank.goa.local' /etc/hosts 2>/dev/null; then + echo '127.0.0.1 bank.goa.local exchange.goa.local backend.goa.local' >> /etc/hosts +fi +grep -E 'goa\.local' /etc/hosts || true + +SNIP=/etc/nginx/snippets/regional-currency-bank-proxy.conf +if [ -f "$SNIP" ] && grep -q '127.0.0.1:8080' "$SNIP"; then + sed -i 's|proxy_pass http://127.0.0.1:8080;|proxy_pass http://127.0.0.1:9012;|' "$SNIP" + echo "NGINX_PROXY_9012=patched" +fi +if command -v nginx >/dev/null 2>&1; then + nginx -t + systemctl reload nginx 2>/dev/null || systemctl restart nginx || true +fi + +taler-exchange-dbconfig -p -s +echo "DBCONFIG_OK" + +ACCT=exchange-account-default +for unit in taler-exchange-wirewatch taler-exchange-transfer; do + bin="/usr/bin/${unit}" + d="/etc/systemd/system/${unit}.service.d" + mkdir -p "$d" + cat > "${d}/account.conf" </dev/null || true +systemctl restart taler-exchange-wirewatch.service +systemctl start taler-exchange-transfer.service 2>/dev/null || true + +# wait wirewatch +ok=0 +for i in $(seq 1 30); do + st=$(systemctl is-active taler-exchange-wirewatch.service 2>/dev/null || echo missing) + if [ "$st" = "active" ]; then + ok=1 + break + fi + sleep 1 +done +echo "WIREWATCH=$(systemctl is-active taler-exchange-wirewatch.service 2>/dev/null || echo missing)" +echo "TRANSFER=$(systemctl is-active taler-exchange-transfer.service 2>/dev/null || echo missing)" +if [ "$ok" -ne 1 ]; then + journalctl -u taler-exchange-wirewatch.service -n 40 --no-pager || true + echo "FAIL wirewatch not active" + exit 1 +fi + +TOKEN=$(awk -F= '/^TOKEN=/{print $2; exit}' \ + /etc/taler-exchange/secrets/exchange-accountcredentials-default.secret.conf 2>/dev/null || true) +if [ -n "${TOKEN:-}" ]; then + code=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 \ + -H "Authorization: Bearer ${TOKEN}" \ + http://bank.goa.local/accounts/exchange/taler-wire-gateway/config 2>/dev/null || echo ERR) + echo "WIRE_GATEWAY_VIA_NGINX=${code}" + case "$code" in 200|204) ;; *) + echo "FAIL wire gateway via bank.goa.local" + exit 1 + ;; + esac +fi + +echo "WIRE_POST_RESTORE_OK"