#!/usr/bin/env bash
# check_apt_deploy.sh — podman apt-src merchant deploy tests on koopa.
#
# Containers (default):
#   Fresh (rebuild when repo versions change):
#     koopa-taler-deploy-test-apt-src-trixie
#     koopa-taler-deploy-test-apt-src-trixie-testing
#   Upgrade track (initial install, then mytops-style apt upgrade):
#     koopa-taler-deploy-test-apt-src-trixie-upgrade
#     koopa-taler-deploy-test-apt-src-trixie-testing-upgrade
#
# Emits a compact container board (board · key=value) for the HTML sticky/top
# overview: which pods are active and which fail (httpd / ldd / …).
#
# Env:
#   APT_DEPLOY_CONTAINERS  "name:suite:mode …"  mode=fresh|upgrade (optional, default fresh)
#   APT_DEPLOY_SKIP=1
#   APT_DEPLOY_PODMAN=podman
#
set -euo pipefail
ROOT=$(cd "$(dirname "$0")" && pwd)
# shellcheck source=lib.sh
source "$ROOT/lib.sh"

if [ "${APT_DEPLOY_SKIP:-0}" = "1" ]; then
  echo "[INFO]  skip phase aptdeploy (APT_DEPLOY_SKIP=1)"
  exit 0
fi

set_area aptdeploy
section "aptdeploy · podman apt-src merchant (fresh + upgrade tracks)"

PODMAN_BIN="${APT_DEPLOY_PODMAN:-podman}"
DEFAULT_LIST="\
koopa-taler-deploy-test-apt-src-trixie:trixie:fresh \
koopa-taler-deploy-test-apt-src-trixie-testing:trixie-testing:fresh \
koopa-taler-deploy-test-apt-src-trixie-upgrade:trixie:upgrade \
koopa-taler-deploy-test-apt-src-trixie-testing-upgrade:trixie-testing:upgrade"
LIST="${APT_DEPLOY_CONTAINERS:-$DEFAULT_LIST}"

if ! command -v "$PODMAN_BIN" >/dev/null 2>&1; then
  err "podman" "podman binary missing ($PODMAN_BIN)"
  summary
  exit 1
fi
ok "podman" "$("$PODMAN_BIN" --version 2>/dev/null | head -1)"
info "host" "$(hostname 2>/dev/null || echo unknown) · whoami=$(whoami)"

fail_any=0
# board rows for end overview (human table + machine board · lines)
BOARD_ROWS=()

_short_name() {
  # koopa-taler-deploy-test-apt-src-trixie-testing-upgrade → trixie-testing-upgrade
  local n="$1"
  n="${n#koopa-taler-deploy-test-apt-src-}"
  printf '%s' "$n"
}

# Record one container for the top-of-page board (HTML parses "board · key=value").
board_record() {
  local short="$1" name="$2" suite="$3" mode="$4" state="$5"
  local merchant="$6" httpd="$7" ldd="$8" verdict="$9" note="${10:-}"
  local kv
  kv="short=${short} name=${name} suite=${suite} mode=${mode} state=${state}"
  kv+=" merchant=${merchant} httpd=${httpd} ldd=${ldd} verdict=${verdict}"
  # note: no spaces (HTML/parser splits on whitespace)
  if [ -n "$note" ]; then
    note_safe=$(printf '%s' "$note" | tr ' ' '_' | tr -d '|')
    kv+=" note=${note_safe}"
  fi
  info "board" "$kv"
  BOARD_ROWS+=("$short|$suite|$mode|$state|$merchant|$httpd|$ldd|$verdict|$note")
}

print_board_table() {
  section "aptdeploy · container overview (active / errors)"
  info "board-table" "| short | suite | mode | podman | merchant | httpd | ldd | verdict | note |"
  info "board-table" "|-------|-------|------|--------|----------|-------|-----|---------|------|"
  local r short suite mode state merchant httpd ldd verdict note
  local n_ok=0 n_err=0 n_miss=0
  for r in "${BOARD_ROWS[@]+"${BOARD_ROWS[@]}"}"; do
    IFS='|' read -r short suite mode state merchant httpd ldd verdict note <<<"$r"
    info "board-table" "| ${short} | ${suite} | ${mode} | ${state} | ${merchant} | ${httpd} | ${ldd} | ${verdict} | ${note} |"
    case "$verdict" in
      OK|ok) n_ok=$((n_ok + 1)) ;;
      MISSING|missing) n_miss=$((n_miss + 1)) ;;
      *) n_err=$((n_err + 1)) ;;
    esac
  done
  info "board-summary" "containers=${#BOARD_ROWS[@]} ok=${n_ok} error=${n_err} missing=${n_miss}"
  # one-line human skim (INFO/OK only — do not re-emit ERROR, details already counted)
  for r in "${BOARD_ROWS[@]+"${BOARD_ROWS[@]}"}"; do
    IFS='|' read -r short suite mode state merchant httpd ldd verdict note <<<"$r"
    if [ "$verdict" = "OK" ]; then
      ok "overview" "${short}: ${state} · merchant=${merchant} · httpd=${httpd} · ldd=${ldd}"
    else
      info "overview" "${short}: ${verdict} · state=${state} merchant=${merchant} httpd=${httpd} ldd=${ldd}${note:+ · ${note}}"
    fi
  done
}

print_pkg_table() {
  local name=$1
  local rows
  rows=$("$PODMAN_BIN" exec "$name" bash -lc '
    echo "| package | version |"
    echo "|---------|---------|"
    for p in taler-merchant libtalermerchant libtalerexchange libdonau libgnunet \
             taler-merchant-webui taler-merchant-typst taler-terms-generator; do
      v=$(dpkg-query -W -f="\${Version}" "$p" 2>/dev/null || echo missing)
      echo "| $p | $v |"
    done
  ' 2>/dev/null || echo "| (query failed) | |")
  info "pkg-table" "$name"
  while IFS= read -r line; do
    [ -n "$line" ] || continue
    info "pkg" "$line"
  done <<<"$rows"
}

# merchant package version only (for board)
_merchant_ver() {
  local name=$1
  "$PODMAN_BIN" exec "$name" dpkg-query -W -f='${Version}' taler-merchant 2>/dev/null || echo "?"
}

check_ldd_deep() {
  local name=$1
  local out missing
  out=$("$PODMAN_BIN" exec "$name" bash -lc '
    set +e
    bin=$(command -v taler-merchant-httpd)
    echo "=== ldd taler-merchant-httpd ==="
    ldd "$bin" 2>&1 | grep -E "libtalerutil|libdonau|libgnunet|not found" || true
    echo "=== ldd libdonau (if present) ==="
    for f in /usr/lib/*/libdonau.so* /usr/lib/*/libdonauutil.so*; do
      [ -e "$f" ] || continue
      echo "-- $f --"
      ldd "$f" 2>&1 | grep -E "libtalerutil|not found" || true
    done
    echo "=== libtalerutil files ==="
    ls -la /usr/lib/*/libtalerutil* 2>/dev/null || echo "(none)"
  ' 2>/dev/null || echo "ldd probe failed")
  while IFS= read -r line; do
    [ -n "$line" ] || continue
    info "ldd" "$line"
  done <<<"$out"
  if echo "$out" | grep -q 'not found'; then
    missing=$(echo "$out" | grep 'not found' | tr '\n' ' ' | head -c 280)
    err "ldd" "shared library missing in $name" "${missing:-not found (see ldd INFO above)}"
    return 1
  fi
  return 0
}

# Build a single-line diagnosis for err() so jump lists are self-contained
# (operators should not need to hunt nearby INFO lines).
_apt_systemd_diagnose() {
  local out="$1"
  local before after failed journal status_line result
  before=$(echo "$out" | sed -n 's/^httpd_active: //p' | tail -1)
  after=$(echo "$out" | sed -n 's/^after_start_httpd: //p' | tail -1)
  failed=$(echo "$out" | sed -n 's/^failed_taler: //p' | grep -v '^none$' | tr '\n' ';' | sed 's/;$//')
  [ -z "$failed" ] && failed=$(echo "$out" | grep -E '^\s*●\s+taler-' | head -3 | tr '\n' ';' | sed 's/;$//')
  result=$(echo "$out" | sed -n 's/^httpd_Result: //p' | tail -1)
  [ -z "$result" ] && result=$(echo "$out" | sed -n 's/^ *Result: //p' | head -1)
  status_line=$(echo "$out" | sed -n 's/^httpd_status_line: //p' | tail -1)
  journal=$(echo "$out" | sed -n 's/^httpd_journal: //p' | tr '\n' ' ' | head -c 220)
  # fallback: first non-meta journal-ish line
  if [ -z "$journal" ]; then
    journal=$(echo "$out" | grep -E 'taler-merchant-httpd\[[0-9]+\]|Failed|error|Error|NRestarts|Main PID' | head -3 | tr '\n' ' ' | head -c 220)
  fi
  local parts=()
  [ -n "$before" ] && parts+=("before=${before}")
  [ -n "$after" ] && parts+=("after_start=${after}")
  [ -n "$result" ] && parts+=("Result=${result}")
  [ -n "$status_line" ] && parts+=("status=${status_line}")
  [ -n "$failed" ] && [ "$failed" != "none" ] && parts+=("failed_units=${failed}")
  [ -n "$journal" ] && parts+=("journal=${journal}")
  if [ "${#parts[@]}" -eq 0 ]; then
    printf '%s' "no status/journal captured (podman exec empty?)"
    return
  fi
  local IFS=' · '
  printf '%s' "${parts[*]}"
}

check_systemd() {
  local name=$1
  local out active_httpd before_httpd detail
  out=$("$PODMAN_BIN" exec "$name" bash -lc '
    set +e
    echo "system: $(systemctl is-system-running 2>&1)"
    echo "target_enabled: $(systemctl is-enabled taler-merchant.target 2>&1)"
    echo "target_active: $(systemctl is-active taler-merchant.target 2>&1)"
    echo "httpd_active: $(systemctl is-active taler-merchant-httpd.service 2>&1)"
    echo "httpd_enabled: $(systemctl is-enabled taler-merchant-httpd.service 2>&1)"
    # unit properties help when is-active is only "activating"/"failed"
    echo "httpd_Result: $(systemctl show -p Result --value taler-merchant-httpd.service 2>&1)"
    echo "httpd_SubState: $(systemctl show -p SubState --value taler-merchant-httpd.service 2>&1)"
    echo "httpd_NRestarts: $(systemctl show -p NRestarts --value taler-merchant-httpd.service 2>&1)"
    echo "httpd_ExecMainStatus: $(systemctl show -p ExecMainStatus --value taler-merchant-httpd.service 2>&1)"
    echo "httpd_ExecMainCode: $(systemctl show -p ExecMainCode --value taler-merchant-httpd.service 2>&1)"
    # one-line status (Active: failed / activating (start) …)
    systemctl status taler-merchant-httpd.service --no-pager -l 2>&1 | head -12 | while IFS= read -r sl; do
      echo "httpd_status: $sl"
    done
    # compact single-line for err detail
    st1=$(systemctl status taler-merchant-httpd.service --no-pager -l 2>&1 | sed -n "s/^ *Active: //p" | head -1)
    echo "httpd_status_line: ${st1:-?}"
    systemctl start taler-merchant.target 2>&1 | tail -8 | while IFS= read -r sl; do
      echo "start_target: $sl"
    done
    # wait longer: "activating" often clears only after dbinit
    sleep 5
    echo "after_start_target: $(systemctl is-active taler-merchant.target 2>&1)"
    echo "after_start_httpd: $(systemctl is-active taler-merchant-httpd.service 2>&1)"
    echo "after_httpd_Result: $(systemctl show -p Result --value taler-merchant-httpd.service 2>&1)"
    echo "after_httpd_SubState: $(systemctl show -p SubState --value taler-merchant-httpd.service 2>&1)"
    # journal: last failure reasons (self-contained for ERROR jump list)
    journalctl -u taler-merchant-httpd.service -n 12 --no-pager -o cat 2>/dev/null \
      | sed "/^$/d" | tail -8 | while IFS= read -r jl; do
          echo "httpd_journal: $jl"
        done
    journalctl -u taler-merchant-dbinit.service -n 6 --no-pager -o cat 2>/dev/null \
      | sed "/^$/d" | tail -4 | while IFS= read -r jl; do
          echo "dbinit_journal: $jl"
        done
    ft=$(systemctl --failed --no-legend 2>/dev/null | grep -i taler || true)
    if [ -n "$ft" ]; then
      echo "$ft" | while IFS= read -r fl; do echo "failed_taler: $fl"; done
    else
      echo "failed_taler: none"
    fi
  ' 2>/dev/null || echo "systemd probe failed")
  while IFS= read -r line; do
    [ -n "$line" ] || continue
    info "systemd" "$line"
  done <<<"$out"

  before_httpd=$(echo "$out" | sed -n 's/^httpd_active: //p' | tail -1)
  active_httpd=$(echo "$out" | sed -n 's/^after_start_httpd: //p' | tail -1)
  # export for caller board
  APT_LAST_HTTPD="${active_httpd:-?}"
  if [ "$active_httpd" = "active" ]; then
    ok "systemd httpd" "taler-merchant-httpd active after start taler-merchant.target"
    return 0
  fi
  detail=$(_apt_systemd_diagnose "$out")
  # Prefer after_* Result if present
  local after_result after_sub
  after_result=$(echo "$out" | sed -n 's/^after_httpd_Result: //p' | tail -1)
  after_sub=$(echo "$out" | sed -n 's/^after_httpd_SubState: //p' | tail -1)
  [ -n "$after_result" ] && [ "$after_result" != "success" ] && detail="${detail} · after_Result=${after_result}"
  [ -n "$after_sub" ] && detail="${detail} · after_SubState=${after_sub}"
  # Human one-liner for jump list (no "see …" without content)
  err "systemd httpd" \
    "taler-merchant-httpd not active (container=${name})" \
    "${detail}"
  return 1
}

check_merchant_basics() {
  local name=$1
  local out
  out=$("$PODMAN_BIN" exec "$name" bash -lc '
    set +e
    echo "=== binaries ==="
    command -v taler-merchant-httpd taler-merchant-dbinit taler-config 2>/dev/null
    echo "=== --version ==="
    taler-merchant-httpd --version 2>&1
    echo "=== config probe ==="
    for u in \
      http://127.0.0.1:9966/config \
      http://127.0.0.1:8081/config \
      http://127.0.0.1/config
    do
      code=$(curl -sS -m 2 -o /tmp/mcfg -w "%{http_code}" "$u" 2>/dev/null || echo 000)
      echo "curl $u -> $code"
      [ "$code" = "200" ] && head -c 120 /tmp/mcfg && echo
    done
    sock=$(ls /run/taler-merchant/httpd/*.sock 2>/dev/null | head -1)
    if [ -n "$sock" ]; then
      echo "socket: $sock"
      code=$(curl -sS -m 2 --unix-socket "$sock" -o /tmp/mcfg -w "%{http_code}" http://localhost/config 2>/dev/null || echo 000)
      echo "curl --unix-socket -> $code"
      [ "$code" = "200" ] && head -c 160 /tmp/mcfg && echo
    else
      echo "socket: (none under /run/taler-merchant/httpd/)"
    fi
  ' 2>/dev/null || echo "basics probe failed")

  while IFS= read -r line; do
    [ -n "$line" ] || continue
    info "basics" "$line"
  done <<<"$out"

  if echo "$out" | grep -qi 'error while loading shared libraries'; then
    local lib_err
    lib_err=$(echo "$out" | grep -i 'error while loading shared libraries' | head -1 | tr '\n' ' ' | head -c 240)
    err "httpd" "shared library error on --version" "${lib_err:-see basics INFO lines}"
    return 1
  fi

  set +e
  local ver_out ec
  ver_out=$("$PODMAN_BIN" exec "$name" taler-merchant-httpd --version 2>&1)
  ec=$?
  set -e
  if [ "$ec" -eq 0 ]; then
    ok "httpd --version" "exit 0"
  else
    err "httpd --version" "exit $ec" "$(printf '%s' "$ver_out" | tr '\n' ' ' | head -c 240)"
    return 1
  fi

  if echo "$out" | grep -qE 'curl .* -> 200'; then
    ok "merchant /config" "HTTP 200 (local)"
  else
    warn "merchant /config" "no local HTTP 200 (unit may need BASE_URL/db — still report version/ldd)"
  fi
  return 0
}

check_one() {
  local name="$1" expect_suite="$2" mode="${3:-fresh}"
  local st short merchant="-" httpd="-" ldd="-" verdict="OK" note="" c_fail=0

  short="$(_short_name "$name")"
  APT_LAST_HTTPD="-"

  # group ids: aptdeploy.trixie-01 / aptdeploy.trixie-testing-upgrade-01
  set_group "${expect_suite}${mode:+-$mode}"
  section "aptdeploy · $name (suite=$expect_suite mode=$mode)"

  if ! "$PODMAN_BIN" container exists "$name" 2>/dev/null; then
    err "container" "$name missing" "run ensure-apt-deploy-test-containers.sh"
    board_record "$short" "$name" "$expect_suite" "$mode" "missing" "-" "-" "-" "MISSING" "ensure containers"
    fail_any=1
    return
  fi
  st=$("$PODMAN_BIN" inspect -f '{{.State.Status}}' "$name" 2>/dev/null || echo unknown)
  if [ "$st" != "running" ]; then
    err "container" "$name not running" "status=$st"
    board_record "$short" "$name" "$expect_suite" "$mode" "$st" "-" "-" "-" "ERROR" "not running"
    fail_any=1
    return
  fi
  ok "container" "$name running (mode=$mode)"

  suite_line=$("$PODMAN_BIN" exec "$name" bash -lc \
    'grep -h "^Suites:" /etc/apt/sources.list.d/*taler* 2>/dev/null | head -1' 2>/dev/null || true)
  if echo "$suite_line" | grep -qE "Suites:[[:space:]]*${expect_suite}([[:space:]]|$)"; then
    ok "apt suite" "$expect_suite"
  else
    err "apt suite" "expected $expect_suite" "got: ${suite_line:-empty}"
    c_fail=1
    note="suite mismatch"
  fi

  merchant="$(_merchant_ver "$name")"
  print_pkg_table "$name"

  if ! check_merchant_basics "$name"; then
    c_fail=1
    if ! check_ldd_deep "$name"; then
      ldd="FAIL"
      c_fail=1
    else
      ldd="OK"
    fi
  else
    ldd_out=$("$PODMAN_BIN" exec "$name" bash -lc \
      'ldd "$(command -v taler-merchant-httpd)" 2>&1 | grep -E "libtalerutil|not found" || true' 2>/dev/null || true)
    if echo "$ldd_out" | grep -q 'not found'; then
      err "libtalerutil" "not found" "$ldd_out"
      check_ldd_deep "$name" || true
      ldd="FAIL"
      c_fail=1
    else
      ldd="OK"
      info "libtalerutil" "$(echo "$ldd_out" | tr '\n' ' ')"
    fi
  fi

  if ! check_systemd "$name"; then
    c_fail=1
    httpd="${APT_LAST_HTTPD:-FAIL}"
    note="${note:+$note; }httpd=${httpd}"
    # deeper ldd if httpd failed
    if [ "$ldd" != "FAIL" ]; then
      check_ldd_deep "$name" || { ldd="FAIL"; c_fail=1; note="${note:+$note; }ldd"; }
    fi
  else
    httpd="active"
  fi
  # normalize httpd cell
  [ -z "$httpd" ] || [ "$httpd" = "-" ] && httpd="${APT_LAST_HTTPD:-?}"

  # Disk free inside this deploy-test container (+ host once, first container only)
  if [ "${_APT_DEPLOY_HOST_DISK_DONE:-0}" != "1" ]; then
    set_group disk
    mon_disk_check_host "host" || fail_any=1
    _APT_DEPLOY_HOST_DISK_DONE=1
  fi
  set_group disk
  if ! mon_disk_check_podman "$name" "$PODMAN_BIN"; then
    c_fail=1
    note="${note:+$note; }disk"
  fi

  if [ "$c_fail" -ne 0 ]; then
    verdict="ERROR"
    fail_any=1
  else
    verdict="OK"
  fi
  board_record "$short" "$name" "$expect_suite" "$mode" "running" "$merchant" "$httpd" "$ldd" "$verdict" "$note"
}

# --- inventory first (what exists before checks) ---
section "aptdeploy · inventory (podman)"
info "inventory" "expected tracks: trixie/trixie-testing × fresh/upgrade (4 containers)"
for entry in $LIST; do
  cname=${entry%%:*}
  rest=${entry#*:}
  case "$rest" in
    *:*) suite=${rest%%:*}; mode=${rest#*:} ;;
    *) suite=$rest; mode=fresh ;;
  esac
  short="$(_short_name "$cname")"
  if "$PODMAN_BIN" container exists "$cname" 2>/dev/null; then
    st=$("$PODMAN_BIN" inspect -f '{{.State.Status}}' "$cname" 2>/dev/null || echo unknown)
    ok "inventory" "${short}: ${st} (${suite}/${mode})"
  else
    warn "inventory" "${short}: missing (${suite}/${mode})" "will ERROR in check"
  fi
done

for entry in $LIST; do
  cname=${entry%%:*}
  rest=${entry#*:}
  case "$rest" in
    *:*)
      suite=${rest%%:*}
      mode=${rest#*:}
      ;;
    *)
      suite=$rest
      mode=fresh
      ;;
  esac
  check_one "$cname" "$suite" "$mode"
done

# Compact matrix + overview ERROR lines (jump targets for HTML)
print_board_table

summary
if [ "$fail_any" -ne 0 ]; then
  exit 1
fi
exit 0