monitoring: surface inventory and apt-deploy check phases

scripts/taler-monitoring/lib.sh

@@ -23,6 +23,9 @@ MERCHANT_INSTANCE=$(printf '%s' "${MERCHANT_INSTANCE}" | tr '[:upper:]' '[:lower
 : "${TIMEOUT:=12}"
 : "${E2E_TIMEOUT:=55}"       # whole e2e budget; skip rest when exceeded (e2e raises as needed)
 : "${E2E_PAY_SECS:=22}"      # dedicated seconds for pay handle-uri (avoid Alarm clock)
+# Whole-run wall clock for taler-monitoring.sh (seconds). 0 = unlimited.
+# Host-agent default phases (urls/inside/versions) should finish under this.
+: "${RUN_TIMEOUT:=600}"
 # Local GOA: public paivana paywall (https://paivana.hacktivism.ch · template GOA:4200)
 : "${PAIVANA_PUBLIC:=https://paivana.hacktivism.ch}"
 : "${E2E_PAIVANA:=1}"        # 0 = skip paivana section in e2e
@@ -855,6 +858,113 @@ summary() {
   [ "$FAIL_N" -eq 0 ]
 }
 
+# ---------------------------------------------------------------------------
+# Disk space on hosts / containers under test
+# WARN when tight, ERROR when full / critically low
+#   DISK_WARN_USED_PCT=85   # free below 15% → WARN
+#   DISK_ERR_USED_PCT=95    # free below 5%  → ERROR
+#   DISK_ERR_FREE_BYTES=0   # free == 0 always ERROR
+# ---------------------------------------------------------------------------
+: "${DISK_WARN_USED_PCT:=85}"
+: "${DISK_ERR_USED_PCT:=95}"
+
+# Parse one POSIX `df -P` body line (no header): fs size used avail capacity mount
+# size/used/avail in 1K-blocks when using df -Pk
+_mon_disk_eval_line() {
+  local where="$1" fs="$2" size="$3" used="$4" avail="$5" cap="$6" mnt="$7"
+  local pct label detail
+  pct=${cap%%%}
+  # non-numeric capacity → skip
+  case "$pct" in
+    ''|*[!0-9]*) return 0 ;;
+  esac
+  label="disk ${where} ${mnt}"
+  detail="fs=$fs size_1k=$size used_1k=$used avail_1k=$avail use=${pct}%"
+  if [ "$avail" = "0" ] || [ "$pct" -ge 100 ]; then
+    err "disk" "${where} ${mnt} FULL / overflow" "$detail"
+    return 1
+  fi
+  if [ "$pct" -ge "${DISK_ERR_USED_PCT}" ]; then
+    err "disk" "${where} ${mnt} critically low free space (≥${DISK_ERR_USED_PCT}% used)" "$detail"
+    return 1
+  fi
+  if [ "$pct" -ge "${DISK_WARN_USED_PCT}" ]; then
+    warn "disk" "${where} ${mnt} free space tight (≥${DISK_WARN_USED_PCT}% used)" "$detail"
+    return 0
+  fi
+  ok "$label" "$detail"
+  return 0
+}
+
+# Report all filesystems from `df -P` / `df -Pk` output (with header).
+# $1 = where label (host, container:name, ssh:host)
+# $2 = full df text
+mon_disk_report_df() {
+  local where="$1" text="$2" line fs size used avail cap mnt rest ec=0
+  [ -n "$text" ] || {
+    warn "disk" "${where}: no df output"
+    return 0
+  }
+  while IFS= read -r line || [ -n "$line" ]; do
+    [ -n "$line" ] || continue
+    case "$line" in
+      Filesystem*|Filesystem*) continue ;;
+    esac
+    # df -P: Filesystem 1024-blocks Used Available Capacity Mounted on
+    # shellcheck disable=SC2086
+    set -- $line
+    [ "$#" -ge 6 ] || continue
+    fs=$1
+    size=$2
+    used=$3
+    avail=$4
+    cap=$5
+    shift 5
+    mnt=$*
+    # skip special/pseudo if tiny and not root-ish
+    case "$fs" in
+      tmpfs|devtmpfs|overlay|shm|nsfs|proc|sysfs|cgroup*) 
+        # still check root-like mounts that fill (overlay / in containers)
+        case "$mnt" in
+          /|/var|/var/*|/home|/home/*|/data|/mnt/*) ;;
+          *) continue ;;
+        esac
+        ;;
+    esac
+    if ! _mon_disk_eval_line "$where" "$fs" "$size" "$used" "$avail" "$cap" "$mnt"; then
+      ec=1
+    fi
+  done <<<"$text"
+  return "$ec"
+}
+
+# Local host: important mounts
+mon_disk_check_host() {
+  local where text
+  where="${1:-host}"
+  text=""
+  text=$(df -Pk / /var /home /tmp 2>/dev/null | awk 'NR==1 || !seen[$1,$6]++' || true)
+  # fallback whole table
+  if [ -z "$text" ]; then
+    text=$(df -Pk 2>/dev/null || true)
+  fi
+  mon_disk_report_df "$where" "$text"
+}
+
+# Run df inside podman container
+mon_disk_check_podman() {
+  local cname="$1" text
+  local bin="${2:-podman}"
+  text=$("$bin" exec "$cname" df -Pk / /var /tmp 2>/dev/null || "$bin" exec "$cname" df -Pk 2>/dev/null || true)
+  mon_disk_report_df "ctr:${cname}" "$text"
+}
+
+# df text already collected remotely
+mon_disk_check_remote_text() {
+  local where="$1" text="$2"
+  mon_disk_report_df "$where" "$text"
+}
+
 http_code() {
   local url="$1"; shift
   curl -skS --max-redirs 0 -m "${TIMEOUT}" -o /dev/null -w '%{http_code}' "$@" "$url" 2>/dev/null || echo 000