taler-monitoring/check_apt_deploy.sh

#!/usr/bin/env bash
# check_apt_deploy.sh — podman apt-src merchant deploy tests on koopa.
#
# Containers (default):
#   Fresh (rebuild when repo versions change):
#     koopa-taler-deploy-test-apt-src-trixie
#     koopa-taler-deploy-test-apt-src-trixie-testing
#   Upgrade track (initial install, then mytops-style apt upgrade):
#     koopa-taler-deploy-test-apt-src-trixie-upgrade
#     koopa-taler-deploy-test-apt-src-trixie-testing-upgrade
#
# Emits a compact container board (board · key=value) for the HTML sticky/top
# overview: which pods are active and which fail (httpd / ldd / …).
#
# Env:
#   APT_DEPLOY_CONTAINERS  "name:suite:mode …"  mode=fresh|upgrade (optional, default fresh)
#   APT_DEPLOY_SKIP=1
#   APT_DEPLOY_PODMAN=podman
#
set -euo pipefail
ROOT=$(cd "$(dirname "$0")" && pwd)
# shellcheck source=lib.sh
source "$ROOT/lib.sh"

if [ "${APT_DEPLOY_SKIP:-0}" = "1" ]; then
  echo "[INFO]  skip phase aptdeploy (APT_DEPLOY_SKIP=1)"
  exit 0
fi

set_area aptdeploy
section "aptdeploy · podman apt-src merchant (fresh + upgrade tracks)"

PODMAN_BIN="${APT_DEPLOY_PODMAN:-podman}"
DEFAULT_LIST="\
koopa-taler-deploy-test-apt-src-trixie:trixie:fresh \
koopa-taler-deploy-test-apt-src-trixie-testing:trixie-testing:fresh \
koopa-taler-deploy-test-apt-src-trixie-upgrade:trixie:upgrade \
koopa-taler-deploy-test-apt-src-trixie-testing-upgrade:trixie-testing:upgrade"
LIST="${APT_DEPLOY_CONTAINERS:-$DEFAULT_LIST}"

if ! command -v "$PODMAN_BIN" >/dev/null 2>&1; then
  err "podman" "podman binary missing ($PODMAN_BIN)"
  summary
  exit 1
fi
ok "podman" "$("$PODMAN_BIN" --version 2>/dev/null | head -1)"
info "host" "$(hostname 2>/dev/null || echo unknown) · whoami=$(whoami)"

fail_any=0
# board rows for end overview (human table + machine board · lines)
BOARD_ROWS=()

_short_name() {
  # koopa-taler-deploy-test-apt-src-trixie-testing-upgrade → trixie-testing-upgrade
  local n="$1"
  n="${n#koopa-taler-deploy-test-apt-src-}"
  printf '%s' "$n"
}

# Record one container for the top-of-page board (HTML parses "board · key=value").
board_record() {
  local short="$1" name="$2" suite="$3" mode="$4" state="$5"
  local merchant="$6" httpd="$7" ldd="$8" verdict="$9" note="${10:-}"
  local kv
  kv="short=${short} name=${name} suite=${suite} mode=${mode} state=${state}"
  kv+=" merchant=${merchant} httpd=${httpd} ldd=${ldd} verdict=${verdict}"
  # note: no spaces (HTML/parser splits on whitespace)
  if [ -n "$note" ]; then
    note_safe=$(printf '%s' "$note" | tr ' ' '_' | tr -d '|')
    kv+=" note=${note_safe}"
  fi
  info "board" "$kv"
  BOARD_ROWS+=("$short|$suite|$mode|$state|$merchant|$httpd|$ldd|$verdict|$note")
}

print_board_table() {
  section "aptdeploy · container overview (active / errors)"
  info "board-table" "| short | suite | mode | podman | merchant | httpd | ldd | verdict | note |"
  info "board-table" "|-------|-------|------|--------|----------|-------|-----|---------|------|"
  local r short suite mode state merchant httpd ldd verdict note
  local n_ok=0 n_err=0 n_miss=0
  for r in "${BOARD_ROWS[@]+"${BOARD_ROWS[@]}"}"; do
    IFS='|' read -r short suite mode state merchant httpd ldd verdict note <<<"$r"
    info "board-table" "| ${short} | ${suite} | ${mode} | ${state} | ${merchant} | ${httpd} | ${ldd} | ${verdict} | ${note} |"
    case "$verdict" in
      OK|ok) n_ok=$((n_ok + 1)) ;;
      MISSING|missing) n_miss=$((n_miss + 1)) ;;
      *) n_err=$((n_err + 1)) ;;
    esac
  done
  info "board-summary" "containers=${#BOARD_ROWS[@]} ok=${n_ok} error=${n_err} missing=${n_miss}"
  # one-line human skim (INFO/OK only — do not re-emit ERROR, details already counted)
  for r in "${BOARD_ROWS[@]+"${BOARD_ROWS[@]}"}"; do
    IFS='|' read -r short suite mode state merchant httpd ldd verdict note <<<"$r"
    if [ "$verdict" = "OK" ]; then
      ok "overview" "${short}: ${state} · merchant=${merchant} · httpd=${httpd} · ldd=${ldd}"
    else
      info "overview" "${short}: ${verdict} · state=${state} merchant=${merchant} httpd=${httpd} ldd=${ldd}${note:+ · ${note}}"
    fi
  done
}

print_pkg_table() {
  local name=$1
  local rows
  rows=$("$PODMAN_BIN" exec "$name" bash -lc '
    echo "| package | version |"
    echo "|---------|---------|"
    for p in taler-merchant libtalermerchant libtalerexchange libdonau libgnunet \
             taler-merchant-webui taler-merchant-typst taler-terms-generator; do
      v=$(dpkg-query -W -f="\${Version}" "$p" 2>/dev/null || echo missing)
      echo "| $p | $v |"
    done
  ' 2>/dev/null || echo "| (query failed) | |")
  info "pkg-table" "$name"
  while IFS= read -r line; do
    [ -n "$line" ] || continue
    info "pkg" "$line"
  done <<<"$rows"
}

# merchant package version only (for board)
_merchant_ver() {
  local name=$1
  "$PODMAN_BIN" exec "$name" dpkg-query -W -f='${Version}' taler-merchant 2>/dev/null || echo "?"
}

check_ldd_deep() {
  local name=$1
  local out
  out=$("$PODMAN_BIN" exec "$name" bash -lc '
    set +e
    bin=$(command -v taler-merchant-httpd)
    echo "=== ldd taler-merchant-httpd ==="
    ldd "$bin" 2>&1 | grep -E "libtalerutil|libdonau|libgnunet|not found" || true
    echo "=== ldd libdonau (if present) ==="
    for f in /usr/lib/*/libdonau.so* /usr/lib/*/libdonauutil.so*; do
      [ -e "$f" ] || continue
      echo "-- $f --"
      ldd "$f" 2>&1 | grep -E "libtalerutil|not found" || true
    done
    echo "=== libtalerutil files ==="
    ls -la /usr/lib/*/libtalerutil* 2>/dev/null || echo "(none)"
  ' 2>/dev/null || echo "ldd probe failed")
  while IFS= read -r line; do
    [ -n "$line" ] || continue
    info "ldd" "$line"
  done <<<"$out"
  if echo "$out" | grep -q 'not found'; then
    err "ldd" "shared library missing in $name" "see ldd lines above"
    return 1
  fi
  return 0
}

check_systemd() {
  local name=$1
  local out active_httpd
  out=$("$PODMAN_BIN" exec "$name" bash -lc '
    set +e
    echo "system: $(systemctl is-system-running 2>&1)"
    echo "target_enabled: $(systemctl is-enabled taler-merchant.target 2>&1)"
    echo "target_active: $(systemctl is-active taler-merchant.target 2>&1)"
    echo "httpd_active: $(systemctl is-active taler-merchant-httpd.service 2>&1)"
    echo "httpd_enabled: $(systemctl is-enabled taler-merchant-httpd.service 2>&1)"
    systemctl start taler-merchant.target 2>&1 | tail -5
    sleep 2
    echo "after_start_target: $(systemctl is-active taler-merchant.target 2>&1)"
    echo "after_start_httpd: $(systemctl is-active taler-merchant-httpd.service 2>&1)"
    systemctl --failed --no-legend 2>/dev/null | grep -i taler || echo "failed_taler: none"
  ' 2>/dev/null || echo "systemd probe failed")
  while IFS= read -r line; do
    [ -n "$line" ] || continue
    info "systemd" "$line"
  done <<<"$out"

  active_httpd=$(echo "$out" | sed -n 's/^after_start_httpd: //p' | tail -1)
  # export for caller board
  APT_LAST_HTTPD="${active_httpd:-?}"
  if [ "$active_httpd" = "active" ]; then
    ok "systemd httpd" "taler-merchant-httpd active after start taler-merchant.target"
    return 0
  fi
  err "systemd httpd" "taler-merchant-httpd not active" "after_start_httpd=${active_httpd:-?} · see systemd lines"
  return 1
}

check_merchant_basics() {
  local name=$1
  local out
  out=$("$PODMAN_BIN" exec "$name" bash -lc '
    set +e
    echo "=== binaries ==="
    command -v taler-merchant-httpd taler-merchant-dbinit taler-config 2>/dev/null
    echo "=== --version ==="
    taler-merchant-httpd --version 2>&1
    echo "=== config probe ==="
    for u in \
      http://127.0.0.1:9966/config \
      http://127.0.0.1:8081/config \
      http://127.0.0.1/config
    do
      code=$(curl -sS -m 2 -o /tmp/mcfg -w "%{http_code}" "$u" 2>/dev/null || echo 000)
      echo "curl $u -> $code"
      [ "$code" = "200" ] && head -c 120 /tmp/mcfg && echo
    done
    sock=$(ls /run/taler-merchant/httpd/*.sock 2>/dev/null | head -1)
    if [ -n "$sock" ]; then
      echo "socket: $sock"
      code=$(curl -sS -m 2 --unix-socket "$sock" -o /tmp/mcfg -w "%{http_code}" http://localhost/config 2>/dev/null || echo 000)
      echo "curl --unix-socket -> $code"
      [ "$code" = "200" ] && head -c 160 /tmp/mcfg && echo
    else
      echo "socket: (none under /run/taler-merchant/httpd/)"
    fi
  ' 2>/dev/null || echo "basics probe failed")

  while IFS= read -r line; do
    [ -n "$line" ] || continue
    info "basics" "$line"
  done <<<"$out"

  if echo "$out" | grep -qi 'error while loading shared libraries'; then
    err "httpd" "shared library error on --version" "see basics/ldd"
    return 1
  fi

  set +e
  "$PODMAN_BIN" exec "$name" taler-merchant-httpd --version >/dev/null 2>&1
  local ec=$?
  set -e
  if [ "$ec" -eq 0 ]; then
    ok "httpd --version" "exit 0"
  else
    err "httpd --version" "exit $ec"
    return 1
  fi

  if echo "$out" | grep -qE 'curl .* -> 200'; then
    ok "merchant /config" "HTTP 200 (local)"
  else
    warn "merchant /config" "no local HTTP 200 (unit may need BASE_URL/db — still report version/ldd)"
  fi
  return 0
}

check_one() {
  local name="$1" expect_suite="$2" mode="${3:-fresh}"
  local st short merchant="-" httpd="-" ldd="-" verdict="OK" note="" c_fail=0

  short="$(_short_name "$name")"
  APT_LAST_HTTPD="-"

  # group ids: aptdeploy.trixie-01 / aptdeploy.trixie-testing-upgrade-01
  set_group "${expect_suite}${mode:+-$mode}"
  section "aptdeploy · $name (suite=$expect_suite mode=$mode)"

  if ! "$PODMAN_BIN" container exists "$name" 2>/dev/null; then
    err "container" "$name missing" "run ensure-apt-deploy-test-containers.sh"
    board_record "$short" "$name" "$expect_suite" "$mode" "missing" "-" "-" "-" "MISSING" "ensure containers"
    fail_any=1
    return
  fi
  st=$("$PODMAN_BIN" inspect -f '{{.State.Status}}' "$name" 2>/dev/null || echo unknown)
  if [ "$st" != "running" ]; then
    err "container" "$name not running" "status=$st"
    board_record "$short" "$name" "$expect_suite" "$mode" "$st" "-" "-" "-" "ERROR" "not running"
    fail_any=1
    return
  fi
  ok "container" "$name running (mode=$mode)"

  suite_line=$("$PODMAN_BIN" exec "$name" bash -lc \
    'grep -h "^Suites:" /etc/apt/sources.list.d/*taler* 2>/dev/null | head -1' 2>/dev/null || true)
  if echo "$suite_line" | grep -qE "Suites:[[:space:]]*${expect_suite}([[:space:]]|$)"; then
    ok "apt suite" "$expect_suite"
  else
    err "apt suite" "expected $expect_suite" "got: ${suite_line:-empty}"
    c_fail=1
    note="suite mismatch"
  fi

  merchant="$(_merchant_ver "$name")"
  print_pkg_table "$name"

  if ! check_merchant_basics "$name"; then
    c_fail=1
    if ! check_ldd_deep "$name"; then
      ldd="FAIL"
      c_fail=1
    else
      ldd="OK"
    fi
  else
    ldd_out=$("$PODMAN_BIN" exec "$name" bash -lc \
      'ldd "$(command -v taler-merchant-httpd)" 2>&1 | grep -E "libtalerutil|not found" || true' 2>/dev/null || true)
    if echo "$ldd_out" | grep -q 'not found'; then
      err "libtalerutil" "not found" "$ldd_out"
      check_ldd_deep "$name" || true
      ldd="FAIL"
      c_fail=1
    else
      ldd="OK"
      info "libtalerutil" "$(echo "$ldd_out" | tr '\n' ' ')"
    fi
  fi

  if ! check_systemd "$name"; then
    c_fail=1
    httpd="${APT_LAST_HTTPD:-FAIL}"
    # deeper ldd if httpd failed
    if [ "$ldd" != "FAIL" ]; then
      check_ldd_deep "$name" || { ldd="FAIL"; c_fail=1; }
    fi
  else
    httpd="active"
  fi
  # normalize httpd cell
  [ -z "$httpd" ] || [ "$httpd" = "-" ] && httpd="${APT_LAST_HTTPD:-?}"

  # Disk free inside this deploy-test container (+ host once, first container only)
  if [ "${_APT_DEPLOY_HOST_DISK_DONE:-0}" != "1" ]; then
    set_group disk
    mon_disk_check_host "host" || fail_any=1
    _APT_DEPLOY_HOST_DISK_DONE=1
  fi
  set_group disk
  if ! mon_disk_check_podman "$name" "$PODMAN_BIN"; then
    c_fail=1
    note="${note:+$note; }disk"
  fi

  if [ "$c_fail" -ne 0 ]; then
    verdict="ERROR"
    fail_any=1
  else
    verdict="OK"
  fi
  board_record "$short" "$name" "$expect_suite" "$mode" "running" "$merchant" "$httpd" "$ldd" "$verdict" "$note"
}

# --- inventory first (what exists before checks) ---
section "aptdeploy · inventory (podman)"
info "inventory" "expected tracks: trixie/trixie-testing × fresh/upgrade (4 containers)"
for entry in $LIST; do
  cname=${entry%%:*}
  rest=${entry#*:}
  case "$rest" in
    *:*) suite=${rest%%:*}; mode=${rest#*:} ;;
    *) suite=$rest; mode=fresh ;;
  esac
  short="$(_short_name "$cname")"
  if "$PODMAN_BIN" container exists "$cname" 2>/dev/null; then
    st=$("$PODMAN_BIN" inspect -f '{{.State.Status}}' "$cname" 2>/dev/null || echo unknown)
    ok "inventory" "${short}: ${st} (${suite}/${mode})"
  else
    warn "inventory" "${short}: missing (${suite}/${mode})" "will ERROR in check"
  fi
done

for entry in $LIST; do
  cname=${entry%%:*}
  rest=${entry#*:}
  case "$rest" in
    *:*)
      suite=${rest%%:*}
      mode=${rest#*:}
      ;;
    *)
      suite=$rest
      mode=fresh
      ;;
  esac
  check_one "$cname" "$suite" "$mode"
done

# Compact matrix + overview ERROR lines (jump targets for HTML)
print_board_table

summary
if [ "$fail_any" -ne 0 ]; then
  exit 1
fi
exit 0