#!/usr/bin/env bash
# Upgrade hernani podman containers on koopa: per-target dry-run (prod-safe),
# then full backup (compose + mounts/volumes + DB dumps), then apply.
#
# Run on koopa as hernani:
#   ~/upgrade-containers.sh                      # dry-run only (default)
#   ~/upgrade-containers.sh --apply              # dry-run → backup → upgrade per target
#   ~/upgrade-containers.sh --apply --images-only
#   ~/upgrade-containers.sh --apply --os-only
#   ~/upgrade-containers.sh --apply --recreate-only
#   ~/upgrade-containers.sh --apply --only koopa-forgejo,koopa-notes
#   ~/upgrade-containers.sh --apply --skip 'taler-hacktivism*,goa-regio-ng'
#   ~/upgrade-containers.sh --apply --taler-debs
#   ~/upgrade-containers.sh --apply --rebuild-local
#   ~/upgrade-containers.sh --apply --include-stopped
#   ~/upgrade-containers.sh --apply --backup-max-gb 80
#   ~/upgrade-containers.sh --apply --no-backup    # emergency only
#
# Per-target --apply flow:
#   1) dry-run checks that do not recreate/restart or mutate packages
#   2) if dry-run fails → skip that target (others continue)
#   3) full backup under ~/backups/container-upgrades/<ts>/<name>/
#   4) if backup fails → skip apply for that target
#   5) pull / recreate / OS upgrade as selected
set -euo pipefail

APPLY=0
DO_IMAGES=1
DO_RECREATE=1
DO_OS=1
DO_TALER_DEBS=0
REBUILD_LOCAL=0
INCLUDE_STOPPED=0
DO_BACKUP=1
BACKUP_MAX_GB=50
ONLY_RAW=""
SKIP_RAW=""
HOME_DIR="${HOME:-/home/hernani}"
ADMIN_LOG="${ADMIN_LOG:-$HOME_DIR/koopa-admin-log}"
BACKUP_ROOT="${BACKUP_ROOT:-$HOME_DIR/backups/container-upgrades}"
RUN_TS="$(date '+%Y%m%d-%H%M%S')"
BACKUP_RUN_DIR="$BACKUP_ROOT/$RUN_TS"

LOG_TS() { date '+%Y-%m-%dT%H:%M:%S%z'; }
log() { printf '[%s] %s\n' "$(LOG_TS)" "$*"; }
run() {
  if [[ "$APPLY" -eq 1 ]]; then
    log "RUN: $*"
    "$@"
  else
    log "DRY: $*"
  fi
}

usage() {
  sed -n '2,28p' "$0" | sed 's/^# \{0,1\}//'
  exit "${1:-0}"
}

match_glob_list() {
  local name="$1" raw="$2" g
  [[ -z "$raw" ]] && return 1
  raw="${raw//,/ }"
  for g in $raw; do
    # shellcheck disable=SC2254
    case "$name" in
      $g) return 0 ;;
    esac
  done
  return 1
}

while [[ $# -gt 0 ]]; do
  case "$1" in
    -h|--help) usage 0 ;;
    --apply) APPLY=1; shift ;;
    --dry-run) APPLY=0; shift ;;
    --images-only) DO_IMAGES=1; DO_RECREATE=0; DO_OS=0; shift ;;
    --os-only) DO_IMAGES=0; DO_RECREATE=0; DO_OS=1; shift ;;
    --recreate-only) DO_IMAGES=0; DO_RECREATE=1; DO_OS=0; shift ;;
    --taler-debs) DO_TALER_DEBS=1; shift ;;
    --rebuild-local) REBUILD_LOCAL=1; shift ;;
    --include-stopped) INCLUDE_STOPPED=1; shift ;;
    --no-backup) DO_BACKUP=0; shift ;;
    --backup-max-gb) BACKUP_MAX_GB="${2:-50}"; shift 2 ;;
    --backup-max-gb=*) BACKUP_MAX_GB="${1#*=}"; shift ;;
    --only) ONLY_RAW="${2:-}"; shift 2 ;;
    --skip) SKIP_RAW="${2:-}"; shift 2 ;;
    --only=*) ONLY_RAW="${1#*=}"; shift ;;
    --skip=*) SKIP_RAW="${1#*=}"; shift ;;
    *) log "ERROR: unknown arg: $1"; usage 2 ;;
  esac
done

if [[ "$(id -un)" != "hernani" ]] && [[ "${FORCE_USER:-}" != "1" ]]; then
  log "WARN: expected user hernani (got $(id -un)); set FORCE_USER=1 to override"
fi

declare -a NAMES=()
declare -A IMG=()
declare -A CDIR=()
declare -A STATUS=()   # ok|dry_fail|backup_fail|upgraded|skipped
declare -A NOTE=()

list_cmd=(podman ps --format '{{.Names}}|{{.Image}}')
if [[ "$INCLUDE_STOPPED" -eq 1 ]]; then
  list_cmd=(podman ps -a --format '{{.Names}}|{{.Image}}')
fi

while IFS='|' read -r name image; do
  [[ -z "$name" ]] && continue
  if [[ ! "$name" =~ ^(koopa-|taler-|goa-|mytops-) ]]; then
    continue
  fi
  if [[ -n "$ONLY_RAW" ]] && ! match_glob_list "$name" "$ONLY_RAW"; then
    continue
  fi
  if match_glob_list "$name" "$SKIP_RAW"; then
    log "SKIP name=$name (matched --skip)"
    continue
  fi
  NAMES+=("$name")
  IMG["$name"]="$image"
  dir=$(podman inspect -f '{{index .Config.Labels "com.docker.compose.project.working_dir"}}' "$name" 2>/dev/null || true)
  CDIR["$name"]="$dir"
  STATUS["$name"]="pending"
done < <("${list_cmd[@]}")

if [[ ${#NAMES[@]} -eq 0 ]]; then
  log "No matching containers."
  exit 0
fi

log "MODE=$([ "$APPLY" -eq 1 ] && echo apply || echo dry-run) images=$DO_IMAGES recreate=$DO_RECREATE os=$DO_OS backup=$DO_BACKUP backup_max_gb=$BACKUP_MAX_GB"
log "Targets (${#NAMES[@]}): ${NAMES[*]}"
if [[ "$APPLY" -eq 1 && "$DO_BACKUP" -eq 1 ]]; then
  mkdir -p "$BACKUP_RUN_DIR"
  log "BACKUP_DIR=$BACKUP_RUN_DIR"
fi

is_pinned_tag() {
  local ref="$1" tag="${1##*:}"
  [[ "$ref" != *:* ]] && return 1
  case "$tag" in
    latest|stable|stable-*|main|master|landing|live|setup|rootless|*-rootless|alpine|*-alpine|[0-9]|[0-9][0-9]|[0-9].*-alpine|[0-9][0-9]-alpine) return 1 ;;
  esac
  if [[ "$tag" =~ ^[0-9]+\.[0-9]+ ]]; then
    return 0
  fi
  return 1
}

# --- prod-safe dry-run for one container (no recreate, no package mutate) ---
dry_run_one() {
  local name="$1" ref="${IMG[$name]}" dir="${CDIR[$name]}"
  local rc=0
  local tmp
  tmp=$(mktemp)

  # running?
  if ! podman inspect "$name" >/dev/null 2>&1; then
    log "DRYFAIL $name: inspect failed"
    NOTE["$name"]="inspect failed"
    rm -f "$tmp"
    return 1
  fi

  # image phase checks
  if [[ "$DO_IMAGES" -eq 1 ]]; then
    if [[ "$ref" == localhost/* ]]; then
      if ! podman image exists "$ref" 2>/dev/null; then
        log "DRYFAIL $name: local image missing $ref"
        NOTE["$name"]="local image missing"
        rm -f "$tmp"
        return 1
      fi
      log "DRYOK $name: local image present $ref"
    else
      # pull --dry-run is not universal; probe registry without replacing running CT
      if podman pull --help 2>&1 | grep -q -- '--quiet'; then
        :
      fi
      if ! podman image exists "$ref" 2>/dev/null; then
        log "DRYWARN $name: image not local yet ($ref) — apply will pull"
      else
        log "DRYOK $name: image present locally $ref"
      fi
      # non-mutating registry probe when possible
      if command -v curl >/dev/null 2>&1; then
        :
      fi
      # Try a no-restart pull into local store is mutating cache but not prod CT.
      # For dry-run we only validate the ref string / inspect local digest.
      if is_pinned_tag "$ref"; then
        log "DRYOK $name: pinned tag $ref (compose bump needed to move)"
      fi
    fi
  fi

  # compose config validate (no up)
  if [[ "$DO_RECREATE" -eq 1 ]]; then
    if [[ -n "$dir" ]]; then
      if [[ ! -f "$dir/compose.yml" && ! -f "$dir/compose.yaml" && ! -f "$dir/docker-compose.yml" ]]; then
        log "DRYFAIL $name: compose dir without compose file: $dir"
        NOTE["$name"]="no compose file"
        rm -f "$tmp"
        return 1
      fi
      if ! (cd "$dir" && podman-compose config >/dev/null 2>"$tmp"); then
        log "DRYFAIL $name: podman-compose config failed"
        sed -n '1,20p' "$tmp" | while IFS= read -r line; do log "  $line"; done
        NOTE["$name"]="compose config failed"
        rm -f "$tmp"
        return 1
      fi
      log "DRYOK $name: compose config ok ($dir)"
    else
      log "DRYOK $name: no compose label — recreate will be skipped"
    fi
  fi

  # OS simulate (no real upgrade)
  if [[ "$DO_OS" -eq 1 ]]; then
    if podman ps --format '{{.Names}}' | grep -qx "$name"; then
      local os_script
      os_script=$(cat <<'INNER'
set -e
export DEBIAN_FRONTEND=noninteractive
if command -v apt-get >/dev/null 2>&1; then
  apt-get update -qq
  apt-get -s -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold upgrade >/tmp/os-dry.out
  echo OS_DRY=apt ok
elif command -v apk >/dev/null 2>&1; then
  apk update >/dev/null
  apk upgrade --simulate >/tmp/os-dry.out 2>&1 || apk upgrade -s >/tmp/os-dry.out 2>&1 || true
  echo OS_DRY=apk ok
elif command -v zypper >/dev/null 2>&1; then
  zypper -n refresh >/dev/null
  zypper -n update -D >/tmp/os-dry.out 2>&1 || true
  echo OS_DRY=zypper ok
elif command -v microdnf >/dev/null 2>&1; then
  echo OS_DRY=microdnf none-sim
elif command -v dnf >/dev/null 2>&1; then
  dnf -y check-update >/tmp/os-dry.out 2>&1 || true
  echo OS_DRY=dnf ok
else
  echo OS_DRY=none
fi
INNER
)
      if podman exec -u 0 "$name" bash -lc "$os_script" >"$tmp" 2>&1 \
        || podman exec -u 0 "$name" sh -lc "$os_script" >"$tmp" 2>&1; then
        log "DRYOK $name: OS simulate ($(grep -E '^OS_DRY=' "$tmp" | tail -1 || echo ok))"
      else
        # many distroless/app images have no pkg manager — treat as ok/skip
        if grep -q 'OS_DRY=none' "$tmp" 2>/dev/null; then
          log "DRYOK $name: no package manager (OS skip)"
        else
          log "DRYWARN $name: OS simulate soft-fail (will try apply carefully)"
          sed -n '1,8p' "$tmp" | while IFS= read -r line; do log "  $line"; done
        fi
      fi
    else
      log "DRYOK $name: not running — OS phase will skip"
    fi
  fi

  rm -f "$tmp"
  return "$rc"
}

# --- full backup with data ---
backup_one() {
  local name="$1"
  local dest="$BACKUP_RUN_DIR/$name"
  local meta="$dest/META.txt"
  local max_bytes=$((BACKUP_MAX_GB * 1024 * 1024 * 1024))
  mkdir -p "$dest/binds" "$dest/volumes" "$dest/db" "$dest/compose"

  {
    echo "name=$name"
    echo "image=${IMG[$name]}"
    echo "compose_dir=${CDIR[$name]:-}"
    echo "ts=$RUN_TS"
  } >"$meta"

  podman inspect "$name" >"$dest/inspect.json" 2>/dev/null || true

  # compose project files
  local dir="${CDIR[$name]}"
  if [[ -n "$dir" && -d "$dir" ]]; then
    for f in compose.yml compose.yaml docker-compose.yml docker-compose.yaml .env; do
      if [[ -f "$dir/$f" ]]; then
        cp -a "$dir/$f" "$dest/compose/$f"
      fi
    done
    # shallow copy of known config sidecars (not whole tree)
    for f in Caddyfile Containerfile Dockerfile; do
      [[ -f "$dir/$f" ]] && cp -a "$dir/$f" "$dest/compose/$f"
    done
  fi

  # bind mounts + named volumes from inspect
  local mounts
  mounts=$(podman inspect -f '{{range .Mounts}}{{.Type}}|{{.Name}}|{{.Source}}|{{.Destination}}|{{.RW}}{{println}}{{end}}' "$name" 2>/dev/null || true)
  while IFS='|' read -r mtype mname src dst rw; do
    [[ -z "$mtype" ]] && continue
    # skip ephemeral / host tz
    case "$dst" in
      /etc/localtime|/etc/timezone|/run|/run/*|/tmp|/tmp/*|/dev|/dev/*|/proc|/proc/*|/sys|/sys/*) continue ;;
    esac
    case "$src" in
      /usr/share/zoneinfo/*|/etc/localtime) continue ;;
    esac

    if [[ "$mtype" == "bind" ]]; then
      if [[ ! -e "$src" ]]; then
        log "BACKUPWARN $name: bind missing $src"
        continue
      fi
      local sz
      sz=$(du -sb "$src" 2>/dev/null | awk '{print $1}')
      if [[ -n "$sz" && "$sz" -gt "$max_bytes" ]]; then
        log "BACKUPWARN $name: skip huge bind $src (${sz}B > ${BACKUP_MAX_GB}G); raise --backup-max-gb"
        echo "SKIP_HUGE_BIND $src size=$sz" >>"$meta"
        continue
      fi
      local base
      base=$(echo "$dst" | sed 's#^/##; s#/#_#g')
      [[ -z "$base" ]] && base="root"
      log "BACKUP $name: bind $src → binds/${base}.tar.gz"
      if ! tar -C "$(dirname "$src")" -czf "$dest/binds/${base}.tar.gz" "$(basename "$src")" 2>>"$dest/backup.err"; then
        log "BACKUPFAIL $name: tar bind $src"
        NOTE["$name"]="backup bind failed: $src"
        return 1
      fi
      echo "BIND $src -> binds/${base}.tar.gz" >>"$meta"
    elif [[ "$mtype" == "volume" ]]; then
      local vname="${mname:-}"
      if [[ -z "$vname" ]]; then
        # fallback: export via source path
        if [[ -d "$src" ]]; then
          local vbase
          vbase=$(basename "$(dirname "$src")")
          log "BACKUP $name: volume-path $src → volumes/${vbase}.tar.gz"
          if ! tar -C "$src" -czf "$dest/volumes/${vbase}.tar.gz" . 2>>"$dest/backup.err"; then
            log "BACKUPFAIL $name: tar volume path $src"
            NOTE["$name"]="backup volume failed"
            return 1
          fi
          echo "VOLPATH $src -> volumes/${vbase}.tar.gz" >>"$meta"
        fi
        continue
      fi
      local sz
      sz=$(du -sb "$src" 2>/dev/null | awk '{print $1}')
      if [[ -n "$sz" && "$sz" -gt "$max_bytes" ]]; then
        log "BACKUPWARN $name: skip huge volume $vname (${sz}B > ${BACKUP_MAX_GB}G)"
        echo "SKIP_HUGE_VOL $vname size=$sz" >>"$meta"
        continue
      fi
      log "BACKUP $name: volume $vname → volumes/${vname}.tar"
      if podman volume export "$vname" -o "$dest/volumes/${vname}.tar" 2>>"$dest/backup.err"; then
        gzip -f "$dest/volumes/${vname}.tar" 2>/dev/null || true
        echo "VOLUME $vname -> volumes/${vname}.tar.gz" >>"$meta"
      else
        # fallback tar of mount source
        if [[ -d "$src" ]] && tar -C "$src" -czf "$dest/volumes/${vname}.tar.gz" . 2>>"$dest/backup.err"; then
          echo "VOLPATH $src -> volumes/${vname}.tar.gz" >>"$meta"
        else
          log "BACKUPFAIL $name: volume $vname"
          NOTE["$name"]="backup volume failed: $vname"
          return 1
        fi
      fi
    fi
  done <<<"$mounts"

  # DB dumps when obvious
  if podman ps --format '{{.Names}}' | grep -qx "$name"; then
    local img_l
    img_l=$(echo "${IMG[$name]}" | tr '[:upper:]' '[:lower:]')
    case "$img_l" in
      *postgres*|*postgis*)
        local dbuser
        dbuser=$(podman exec "$name" printenv POSTGRES_USER 2>/dev/null || echo postgres)
        local dbname
        dbname=$(podman exec "$name" printenv POSTGRES_DB 2>/dev/null || echo postgres)
        log "BACKUP $name: pg_dumpall"
        if podman exec -u 0 "$name" bash -lc "pg_dumpall -U '$dbuser'" >"$dest/db/pg_dumpall.sql" 2>>"$dest/backup.err"; then
          gzip -f "$dest/db/pg_dumpall.sql"
          echo "PG_DUMPALL ok db=$dbname user=$dbuser" >>"$meta"
        else
          # single-db fallback
          if podman exec -u 0 "$name" bash -lc "pg_dump -U '$dbuser' '$dbname'" >"$dest/db/pg_dump.sql" 2>>"$dest/backup.err"; then
            gzip -f "$dest/db/pg_dump.sql"
            echo "PG_DUMP ok db=$dbname" >>"$meta"
          else
            log "BACKUPWARN $name: postgres dump failed (mounts still saved)"
          fi
        fi
        ;;
      *mariadb*|*mysql*)
        local muser mpass mdb
        muser=$(podman exec "$name" printenv MYSQL_USER 2>/dev/null || podman exec "$name" printenv MARIADB_USER 2>/dev/null || echo root)
        mpass=$(podman exec "$name" printenv MYSQL_ROOT_PASSWORD 2>/dev/null || podman exec "$name" printenv MARIADB_ROOT_PASSWORD 2>/dev/null || podman exec "$name" printenv MYSQL_PASSWORD 2>/dev/null || true)
        mdb=$(podman exec "$name" printenv MYSQL_DATABASE 2>/dev/null || podman exec "$name" printenv MARIADB_DATABASE 2>/dev/null || echo "")
        log "BACKUP $name: mariadb/mysql dump"
        if [[ -n "$mpass" ]]; then
          if podman exec -u 0 "$name" bash -lc "MYSQL_PWD='$mpass' mysqldump -u root --all-databases" >"$dest/db/mysql_all.sql" 2>>"$dest/backup.err" \
            || podman exec -u 0 "$name" bash -lc "MYSQL_PWD='$mpass' mariadb-dump -u root --all-databases" >"$dest/db/mysql_all.sql" 2>>"$dest/backup.err"; then
            gzip -f "$dest/db/mysql_all.sql"
            echo "MYSQL_DUMPALL ok" >>"$meta"
          else
            log "BACKUPWARN $name: mysql dump failed (mounts still saved)"
          fi
        else
          log "BACKUPWARN $name: no mysql password in env — skip dump"
        fi
        ;;
    esac
  fi

  echo "BACKUP_OK" >>"$meta"
  log "BACKUPOK $name → $dest"
  return 0
}

# --- apply phases for one container ---
apply_images_one() {
  local name="$1" ref="${IMG[$name]}"
  [[ "$DO_IMAGES" -eq 1 ]] || return 0
  if [[ "$ref" == localhost/* ]]; then
    log "LOCAL image=$ref (container=$name) — pull skipped"
    return 0
  fi
  if is_pinned_tag "$ref"; then
    log "PINNED image=$ref (container=$name) — pull same tag only"
  fi
  run podman pull "$ref"
}

rebuild_one_local() {
  local name="$1" ref="${IMG[$name]}" dir="${CDIR[$name]}"
  [[ "$ref" == localhost/* ]] || return 0
  local cf=""
  if [[ -n "$dir" && -f "$dir/Containerfile" ]]; then
    cf="$dir/Containerfile"
  elif [[ -n "$dir" && -f "$dir/Dockerfile" ]]; then
    cf="$dir/Dockerfile"
  fi
  case "$name" in
    koopa-nym)
      [[ -x "$ADMIN_LOG/scripts/nym/build.sh" ]] && { run bash "$ADMIN_LOG/scripts/nym/build.sh"; return; }
      ;;
    koopa-tor-relay)
      if [[ -f "$HOME_DIR/koopa-tor-relay/build/Containerfile" ]]; then
        run podman build -t localhost/koopa-tor-relay:latest -f "$HOME_DIR/koopa-tor-relay/build/Containerfile" "$HOME_DIR/koopa-tor-relay/build"
        return
      fi
      ;;
    koopa-paivana)
      if [[ -f "$dir/Containerfile" ]]; then
        run podman build -t localhost/koopa-paivana:latest -f "$dir/Containerfile" "$dir"
        return
      fi
      ;;
    koopa-bbb-front)
      [[ -x "$ADMIN_LOG/scripts/bbb/build.sh" ]] && { run bash "$ADMIN_LOG/scripts/bbb/build.sh"; return; }
      ;;
    koopa-meet-web)
      [[ -x "$ADMIN_LOG/scripts/meet/build.sh" ]] && { run bash "$ADMIN_LOG/scripts/meet/build.sh"; return; }
      ;;
    koopa-galene|koopa-paivana-galene)
      [[ -x "$ADMIN_LOG/scripts/galene/build.sh" ]] && { run bash "$ADMIN_LOG/scripts/galene/build.sh"; return; }
      ;;
    koopa-cryptpad|koopa-cryptpad-edge|koopa-paivana-cryptpad)
      [[ -x "$ADMIN_LOG/scripts/cryptpad/build.sh" ]] && { run bash "$ADMIN_LOG/scripts/cryptpad/build.sh"; return; }
      ;;
    koopa-notes|koopa-silverbullet)
      [[ -x "$ADMIN_LOG/scripts/notes/build.sh" ]] && { run bash "$ADMIN_LOG/scripts/notes/build.sh"; return; }
      ;;
  esac
  if [[ -n "$cf" ]]; then
    run podman build -t "$ref" -f "$cf" "$(dirname "$cf")"
  else
    log "NO-REBUILD path for $name ($ref)"
  fi
}

apply_recreate_one() {
  local name="$1" dir="${CDIR[$name]}"
  [[ "$DO_RECREATE" -eq 1 ]] || return 0
  if [[ -z "$dir" ]]; then
    log "NO-COMPOSE container=$name — recreate skipped"
    return 0
  fi
  if [[ ! -f "$dir/compose.yml" && ! -f "$dir/compose.yaml" && ! -f "$dir/docker-compose.yml" ]]; then
    log "WARN: compose dir without compose file: $dir"
    return 0
  fi
  if [[ "$APPLY" -eq 1 ]]; then
    log "RUN: (cd $dir && podman-compose pull && podman-compose up -d)"
    (cd "$dir" && podman-compose pull && podman-compose up -d)
  else
    log "DRY: (cd $dir && podman-compose pull && podman-compose up -d)"
  fi
}

os_upgrade_one() {
  local name="$1"
  [[ "$DO_OS" -eq 1 ]] || return 0
  if ! podman ps --format '{{.Names}}' | grep -qx "$name"; then
    log "SKIP os name=$name (not running)"
    return 0
  fi
  local script
  script=$(cat <<'INNER'
set -e
export DEBIAN_FRONTEND=noninteractive
if command -v apt-get >/dev/null 2>&1; then
  apt-get update -qq
  apt-get -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold upgrade
  echo OS=apt ok
elif command -v apk >/dev/null 2>&1; then
  apk update
  apk upgrade
  echo OS=apk ok
elif command -v zypper >/dev/null 2>&1; then
  zypper -n refresh
  zypper -n update
  echo OS=zypper ok
elif command -v microdnf >/dev/null 2>&1; then
  microdnf -y upgrade
  echo OS=microdnf ok
elif command -v dnf >/dev/null 2>&1; then
  dnf -y upgrade
  echo OS=dnf ok
else
  echo OS=none
  exit 0
fi
INNER
)
  if [[ "$APPLY" -eq 1 ]]; then
    log "RUN: podman exec -u 0 $name OS-upgrade"
    if ! podman exec -u 0 "$name" bash -lc "$script"; then
      podman exec -u 0 "$name" sh -lc "$script" || log "WARN: OS upgrade failed for $name"
    fi
  else
    log "DRY: podman exec -u 0 $name OS-upgrade"
  fi
}

# Deduplicate compose recreate by dir
declare -A COMPOSE_DONE=()
declare -A IMAGE_DONE=()
declare -A BACKUP_DONE=()

process_one() {
  local name="$1"
  log "==== target: $name (${IMG[$name]}) ===="

  # 1) always dry-run first (prod-safe)
  if ! dry_run_one "$name"; then
    STATUS["$name"]="dry_fail"
    log "SKIP apply $name (dry-run failed)"
    return 0
  fi
  STATUS["$name"]="dry_ok"

  # global dry-run mode: stop after checks
  if [[ "$APPLY" -eq 0 ]]; then
    log "DRY-RUN only for $name (no backup/apply)"
    return 0
  fi

  # 2) full backup with data BEFORE any mutate (even after dry-run ok)
  #    every container — do not skip siblings in a compose stack (DB/data)
  if [[ "$DO_BACKUP" -eq 1 ]]; then
    if [[ -z "${BACKUP_DONE[$name]:-}" ]]; then
      if ! backup_one "$name"; then
        STATUS["$name"]="backup_fail"
        log "SKIP apply $name (backup failed)"
        return 0
      fi
      BACKUP_DONE["$name"]=1
    else
      log "BACKUP skip $name (already backed this container)"
    fi
  else
    log "WARN: --no-backup set; proceeding without backup for $name"
  fi

  # 3) apply
  if [[ -z "${IMAGE_DONE[${IMG[$name]}]:-}" ]]; then
    apply_images_one "$name" || { STATUS["$name"]="apply_fail"; NOTE["$name"]="image pull failed"; return 0; }
    IMAGE_DONE["${IMG[$name]}"]=1
  fi
  if [[ "$REBUILD_LOCAL" -eq 1 ]]; then
    rebuild_one_local "$name" || true
  fi
  local dir="${CDIR[$name]}"
  if [[ -n "$dir" ]]; then
    if [[ -z "${COMPOSE_DONE[$dir]:-}" ]]; then
      apply_recreate_one "$name" || { STATUS["$name"]="apply_fail"; NOTE["$name"]="recreate failed"; return 0; }
      COMPOSE_DONE["$dir"]=1
    else
      log "recreate skip $name (compose dir already done)"
    fi
  else
    apply_recreate_one "$name" || true
  fi
  os_upgrade_one "$name" || { STATUS["$name"]="apply_fail"; NOTE["$name"]="os upgrade failed"; return 0; }
  STATUS["$name"]="upgraded"
  log "DONE $name"
}

# --- main loop: per container ---
for name in "${NAMES[@]}"; do
  process_one "$name"
done

# optional Taler GOA debs (only on --apply, after per-CT work)
if [[ "$DO_TALER_DEBS" -eq 1 ]]; then
  log "==== phase: taler-debs ===="
  tscript="$ADMIN_LOG/scripts/taler-shared/upgrade-goa-debs.sh"
  if [[ "$APPLY" -eq 0 ]]; then
    log "DRY: bash $tscript"
  elif [[ -x "$tscript" ]]; then
    run bash "$tscript"
  else
    log "ERROR: missing $tscript"
    exit 1
  fi
fi

log "==== summary ===="
fail=0
for name in "${NAMES[@]}"; do
  st="${STATUS[$name]:-?}"
  printf '  %-42s %-12s %s  %s\n' "$name" "$st" "${IMG[$name]}" "${NOTE[$name]:-}"
  case "$st" in
    dry_fail|backup_fail|apply_fail) fail=1 ;;
  esac
done
if [[ "$APPLY" -eq 0 ]]; then
  log "Dry-run only (default). Re-run with --apply for backup+upgrade per target that passed dry-run."
fi
if [[ "$fail" -eq 1 ]]; then
  log "STATUS=partial_fail"
  exit 1
fi
log "STATUS=ok"