taler-monitoring v1.0: standalone suite
This commit is contained in:
Hernâni Marques
commit
a2afe690b7
73 changed files with 18268 additions and 0 deletions
358
host-agent/run-host-report.sh
Executable file
358
host-agent/run-host-report.sh
Executable file
|
|
@ -0,0 +1,358 @@
|
|||
#!/usr/bin/env bash
|
||||
# run-host-report.sh — global reporting generation for all stacks
|
||||
# (GOA/hacktivism, FP stage, FP prod, …).
|
||||
#
|
||||
# Shared behaviour (always):
|
||||
# 1) update-suite → latest git.hacktivism.ch (local ~/.config never overwritten)
|
||||
# 2) line-buffered run log (written continuously)
|
||||
# 3) RUN_TIMEOUT wall clock for taler-monitoring.sh (default 600s)
|
||||
# 4) always write HTML (first run / empty staging too) + Forgejo commit link
|
||||
# 5) optional rsync to DEPLOY_WWW_ROOT if writable
|
||||
#
|
||||
# Configure via env file (not in git):
|
||||
# ${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring/env
|
||||
# or TALER_MONITORING_ENV=/path/to/env
|
||||
#
|
||||
# Thin wrappers only set stack defaults then exec this script:
|
||||
# run-hacktivism-monitoring.sh
|
||||
# run-fp-stage-monitoring.sh
|
||||
# run-fp-prod-monitoring.sh
|
||||
#
|
||||
set -uo pipefail
|
||||
|
||||
export PATH="${HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin:${PATH:-}"
|
||||
export PYTHONUNBUFFERED=1
|
||||
|
||||
AGENT_DIR=$(cd "$(dirname "$0")" && pwd)
|
||||
AGENT_MON=$(cd "$AGENT_DIR/.." && pwd)
|
||||
CFG_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring"
|
||||
ENV_FILE="${TALER_MONITORING_ENV:-$CFG_DIR/env}"
|
||||
mkdir -p "$CFG_DIR"
|
||||
|
||||
# Thin wrappers (surface, aptdeploy-only, …) export stack knobs before exec.
|
||||
# Snapshot them so ~/.config/taler-monitoring/env cannot clobber dedicated jobs
|
||||
# (env often has PHASES="urls inside versions aptdeploy" for the main timer).
|
||||
_wrap_keys=(
|
||||
PHASES HTML_OUT HTML_OK_DIR HTML_ERR_DIR HTML_URL_OK HTML_URL_ERR PAGE_LABEL
|
||||
RUN_TIMEOUT STATE_NAME AGENT_LABEL MON_HOSTS TALER_DOMAIN DEPLOY_WWW_ROOT
|
||||
INSIDE_PODMAN INSIDE_MODE LOCAL_STACK SKIP_SSH CONTINUE_ON_ERROR AUTH401_CONTINUE
|
||||
APT_DEPLOY_ENSURE SURFACE_SCOPE TALER_DOMAIN_FROM_CLI STRICT_EXIT SOURCE_SUITE_PATH
|
||||
)
|
||||
for _k in "${_wrap_keys[@]}"; do
|
||||
if [ -n "${!_k+x}" ]; then
|
||||
printf -v "_WRAP_SET_${_k}" '%s' 1
|
||||
printf -v "_WRAP_VAL_${_k}" '%s' "${!_k}"
|
||||
fi
|
||||
done
|
||||
unset _k
|
||||
|
||||
if [ -f "$ENV_FILE" ]; then
|
||||
# shellcheck disable=SC1090
|
||||
set -a
|
||||
# shellcheck source=/dev/null
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
# Tell update-suite.sh not to re-source env (would clobber wrapper PHASES etc.)
|
||||
export _TALER_MON_ENV_LOADED=1
|
||||
|
||||
# Restore wrapper overrides (dedicated jobs win over shared env file)
|
||||
for _k in "${_wrap_keys[@]}"; do
|
||||
_set_var="_WRAP_SET_${_k}"
|
||||
_val_var="_WRAP_VAL_${_k}"
|
||||
if [ "${!_set_var:-}" = "1" ]; then
|
||||
printf -v "$_k" '%s' "${!_val_var}"
|
||||
export "$_k"
|
||||
fi
|
||||
done
|
||||
unset _k _set_var _val_var
|
||||
|
||||
# --- global report defaults (env file / wrapper wins if already set) ---
|
||||
export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}"
|
||||
export AUTH401_CONTINUE="${AUTH401_CONTINUE:-${CONTINUE_ON_ERROR}}"
|
||||
export RUN_TIMEOUT="${RUN_TIMEOUT:-600}"
|
||||
export TALER_DOMAIN="${TALER_DOMAIN:-hacktivism.ch}"
|
||||
PHASES="${PHASES:-urls inside versions}"
|
||||
HTML_BASE="${HTML_OUT:-$HOME/monitoring-sites-staging}"
|
||||
MON_HOSTS="${MON_HOSTS:-}"
|
||||
SUITE_PATH="${SOURCE_SUITE_PATH:-.}"
|
||||
AGENT_LABEL="${AGENT_LABEL:-host-report}"
|
||||
STATE_NAME="${STATE_NAME:-taler-monitoring-${TALER_DOMAIN}}"
|
||||
STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/${STATE_NAME}"
|
||||
LOG_DIR="$STATE_DIR/logs"
|
||||
DEPLOY_WWW="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}"
|
||||
# HTML path layout under each host (monitoring / surface / …)
|
||||
HTML_OK_DIR="${HTML_OK_DIR:-monitoring}"
|
||||
HTML_ERR_DIR="${HTML_ERR_DIR:-monitoring_err}"
|
||||
HTML_URL_OK="${HTML_URL_OK:-/monitoring/}"
|
||||
HTML_URL_ERR="${HTML_URL_ERR:-/monitoring_err/}"
|
||||
PAGE_LABEL="${PAGE_LABEL:-monitoring}"
|
||||
|
||||
# Sensible MON_HOSTS from domain if not set
|
||||
if [ -z "$MON_HOSTS" ]; then
|
||||
case "$TALER_DOMAIN" in
|
||||
hacktivism.ch|koopa)
|
||||
MON_HOSTS="bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch"
|
||||
;;
|
||||
stage.lefrancpaysan.ch|stage.*lefrancpaysan*)
|
||||
MON_HOSTS="stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch"
|
||||
;;
|
||||
lefrancpaysan.ch)
|
||||
MON_HOSTS="bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch"
|
||||
;;
|
||||
*)
|
||||
MON_HOSTS="${TALER_DOMAIN}"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
mkdir -p "$LOG_DIR" "$HTML_BASE"
|
||||
|
||||
STAMP=$(date +%Y%m%d-%H%M%S)
|
||||
LOG="$LOG_DIR/run-${STAMP}.log"
|
||||
: >"$LOG"
|
||||
if command -v stdbuf >/dev/null 2>&1; then
|
||||
exec > >(stdbuf -oL -eL tee -a "$LOG") 2>&1
|
||||
else
|
||||
exec > >(tee -a "$LOG") 2>&1
|
||||
fi
|
||||
|
||||
echo "======== $(date -Iseconds 2>/dev/null || date) ${AGENT_LABEL} ========"
|
||||
echo "env_file=$ENV_FILE"
|
||||
echo "domain=$TALER_DOMAIN · phases=$PHASES · RUN_TIMEOUT=${RUN_TIMEOUT}s"
|
||||
echo "hosts=$MON_HOSTS"
|
||||
echo "flags INSIDE_PODMAN=${INSIDE_PODMAN:-} INSIDE_MODE=${INSIDE_MODE:-} LOCAL_STACK=${LOCAL_STACK:-} SKIP_SSH=${SKIP_SSH:-} INSIDE_PROFILE=${INSIDE_PROFILE:-}"
|
||||
|
||||
# --- refresh from Forgejo; re-apply suite-overlay if Forgejo lacks needed phases ---
|
||||
OVERLAY_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring/suite-overlay"
|
||||
# preserve capable mon before reset wipes tracked files
|
||||
if [ -f "$AGENT_MON/taler-monitoring.sh" ]; then
|
||||
mkdir -p "$OVERLAY_DIR"
|
||||
rsync -a \
|
||||
--exclude secrets.env --exclude 'android-test/artifacts' --exclude 'android-test/apks' \
|
||||
--exclude 'android-test/out*' --exclude '__pycache__' \
|
||||
"$AGENT_MON/" "$OVERLAY_DIR/" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# shellcheck source=update-suite.sh
|
||||
source "$AGENT_DIR/update-suite.sh"
|
||||
|
||||
SUITE_MON="${SUITE_DIR}"
|
||||
ROOT="$SUITE_MON"
|
||||
_need_overlay=0
|
||||
# Detect missing features in Forgejo tree (untracked helper files alone are not enough —
|
||||
# taler-monitoring.sh must accept the phase name).
|
||||
case " $PHASES " in
|
||||
*" aptdeploy "*|*" apt-deploy "*|*" apt_src "*)
|
||||
if ! grep -qE 'aptdeploy\|apt-deploy' "$SUITE_MON/taler-monitoring.sh" 2>/dev/null; then
|
||||
_need_overlay=1
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
case " $PHASES " in
|
||||
*" surface "*|*" ecosystem "*)
|
||||
if ! grep -qE 'surface\|ecosystem' "$SUITE_MON/taler-monitoring.sh" 2>/dev/null \
|
||||
|| [ ! -f "$SUITE_MON/check_surface.sh" ]; then
|
||||
_need_overlay=1
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
# Wrapper-preserve + HTML path knobs may live only in local overlay until pushed
|
||||
if [ -f "$OVERLAY_DIR/host-agent/run-host-report.sh" ] \
|
||||
&& ! grep -q '_wrap_keys\|_WRAP_SET_' "$SUITE_MON/host-agent/run-host-report.sh" 2>/dev/null \
|
||||
&& grep -q '_wrap_keys\|_WRAP_SET_' "$OVERLAY_DIR/host-agent/run-host-report.sh" 2>/dev/null; then
|
||||
_need_overlay=1
|
||||
fi
|
||||
if [ "$_need_overlay" = "1" ] && [ -f "$OVERLAY_DIR/taler-monitoring.sh" ]; then
|
||||
echo "WARN: Forgejo @ ${COMMIT_SHORT:-?} lagging local suite-overlay (phases=[$PHASES])" >&2
|
||||
echo " re-applying suite-overlay → $SUITE_MON" >&2
|
||||
mkdir -p "$SUITE_MON"
|
||||
rsync -a \
|
||||
--exclude secrets.env --exclude 'android-test/artifacts' --exclude 'android-test/apks' \
|
||||
--exclude 'android-test/out*' --exclude '__pycache__' \
|
||||
"$OVERLAY_DIR/" "$SUITE_MON/"
|
||||
ROOT="$SUITE_MON"
|
||||
elif [ "$_need_overlay" = "1" ]; then
|
||||
echo "WARN: Forgejo missing features and no suite-overlay — using agent mon $AGENT_MON" >&2
|
||||
ROOT="$AGENT_MON"
|
||||
else
|
||||
echo "using suite mon @ ${COMMIT_SHORT:-?} · $ROOT"
|
||||
fi
|
||||
# Always restore host-agent + site-gen from pre-reset snapshot when present
|
||||
# (systemd ExecStart points at koopa-admin-log which may be the reset suite tree).
|
||||
if [ -d "$OVERLAY_DIR/host-agent" ]; then
|
||||
mkdir -p "$SUITE_MON/host-agent" 2>/dev/null || true
|
||||
rsync -a "$OVERLAY_DIR/host-agent/" "$SUITE_MON/host-agent/" 2>/dev/null || true
|
||||
fi
|
||||
if [ -d "$OVERLAY_DIR/site-gen" ]; then
|
||||
mkdir -p "$SUITE_MON/site-gen"
|
||||
rsync -a "$OVERLAY_DIR/site-gen/" "$SUITE_MON/site-gen/" 2>/dev/null || true
|
||||
fi
|
||||
unset _need_overlay
|
||||
|
||||
if [ -f "$ROOT/site-gen/console_to_html.py" ]; then
|
||||
SITE_GEN="$ROOT/site-gen"
|
||||
elif [ -f "$AGENT_MON/site-gen/console_to_html.py" ]; then
|
||||
SITE_GEN="$AGENT_MON/site-gen"
|
||||
echo "note: using agent site-gen at $SITE_GEN"
|
||||
elif [ -f "$SUITE_MON/site-gen/console_to_html.py" ]; then
|
||||
SITE_GEN="$SUITE_MON/site-gen"
|
||||
else
|
||||
SITE_GEN=""
|
||||
fi
|
||||
|
||||
if [ ! -x "$ROOT/taler-monitoring.sh" ]; then
|
||||
if [ -x "$AGENT_MON/taler-monitoring.sh" ]; then
|
||||
ROOT="$AGENT_MON"
|
||||
echo "note: using agent mon tree $ROOT"
|
||||
else
|
||||
echo "ERROR: suite missing at $ROOT" >&2
|
||||
COMMIT="${COMMIT:-unknown}"
|
||||
COMMIT_SHORT="${COMMIT_SHORT:-unknown}"
|
||||
COMMIT_URL="${COMMIT_URL:-}"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "commit=${COMMIT_SHORT:-?} · ${COMMIT_URL:-}"
|
||||
|
||||
ec=1
|
||||
if [ -x "${ROOT:-}/taler-monitoring.sh" ]; then
|
||||
cd "$ROOT" || exit 1
|
||||
chmod +x taler-monitoring.sh check_*.sh host-agent/*.sh site-gen/*.sh site-gen/*.py 2>/dev/null || true
|
||||
|
||||
# Overlay newer inside/versions host-podman bits if suite clone is stale
|
||||
for f in check_inside.sh check_versions.sh taler-monitoring.sh lib.sh; do
|
||||
if [ -f "$AGENT_MON/$f" ] && [ -f "$ROOT/$f" ]; then
|
||||
if ! grep -q 'host-podman\|INSIDE_ACCESS\|RUN_TIMEOUT' "$ROOT/$f" 2>/dev/null \
|
||||
&& grep -q 'host-podman\|INSIDE_ACCESS\|RUN_TIMEOUT' "$AGENT_MON/$f" 2>/dev/null; then
|
||||
echo "note: overlay $f from agent install"
|
||||
cp -f "$AGENT_MON/$f" "$ROOT/$f"
|
||||
chmod +x "$ROOT/$f" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
./taler-monitoring.sh -d "$TALER_DOMAIN" $PHASES
|
||||
ec=$?
|
||||
fi
|
||||
|
||||
# --- always generate HTML (first run / missing pages / every run) ---
|
||||
write_bootstrap_html() {
|
||||
local out="$1" host="$2" mode="$3"
|
||||
mkdir -p "$(dirname "$out")"
|
||||
local cshort="${COMMIT_SHORT:-unknown}"
|
||||
local curl="${COMMIT_URL:-}"
|
||||
local clink="$cshort"
|
||||
local gen iso
|
||||
gen=$(date -u +"%Y-%m-%d %H:%M:%SZ" 2>/dev/null || date)
|
||||
iso=$(date -u +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -Iseconds)
|
||||
if [ -n "$curl" ]; then
|
||||
clink="<a href=\"$(printf '%s' "$curl" | sed 's/&/\&/g')\">$cshort</a>"
|
||||
fi
|
||||
cat >"$out" <<EOF
|
||||
<!DOCTYPE html>
|
||||
<html lang="en"><head><meta charset="utf-8"/>
|
||||
<meta name="generated" content="${iso}"/>
|
||||
<title>monitoring ${mode} · ${host}</title>
|
||||
<style>
|
||||
body{margin:0;background:#0c0c0c;color:#c8c8c8;font-family:ui-monospace,monospace}
|
||||
a{color:#61afef}.err{color:#ff5c5c}
|
||||
.sticky-bar{position:sticky;top:0;z-index:100;padding:10px 14px;background:#2a1010f2;border-bottom:2px solid #a33}
|
||||
.sticky-bar .pill{font-weight:800;color:#ff5c5c;margin-right:8px}
|
||||
main{padding:1.5rem}
|
||||
</style></head><body>
|
||||
<div class="sticky-bar" id="status-bar">
|
||||
<span class="pill">BOOTSTRAP</span>
|
||||
${PAGE_LABEL:-monitoring} · ${host} ·
|
||||
<span data-generated-iso="${iso}">generated ${gen}</span>
|
||||
· source ${clink}
|
||||
</div>
|
||||
<main>
|
||||
<p class="err">Bootstrap page (converter missing or first install). See host-agent log.</p>
|
||||
<pre style="white-space:pre-wrap;font-size:12px">$(tail -n 80 "$LOG" 2>/dev/null | sed 's/&/\&/g;s/</\</g' || true)</pre>
|
||||
</main>
|
||||
</body></html>
|
||||
EOF
|
||||
echo "bootstrap html → $out"
|
||||
}
|
||||
|
||||
htmlify_host() {
|
||||
local host="$1"
|
||||
local mon="$HTML_BASE/$host/${HTML_OK_DIR}/index.html"
|
||||
local mon_err="$HTML_BASE/$host/${HTML_ERR_DIR}/index.html"
|
||||
mkdir -p "$HTML_BASE/$host/${HTML_OK_DIR}" "$HTML_BASE/$host/${HTML_ERR_DIR}"
|
||||
|
||||
if [ -n "$SITE_GEN" ] && [ -f "$SITE_GEN/console_to_html.py" ]; then
|
||||
python3 "$SITE_GEN/console_to_html.py" \
|
||||
--log "$LOG" \
|
||||
--out "$mon_err" \
|
||||
--hostname "$host" \
|
||||
--mode err \
|
||||
--commit "${COMMIT:-}" \
|
||||
--commit-url "${COMMIT_URL:-}" \
|
||||
--suite-path "$SUITE_PATH" \
|
||||
--page-label "$PAGE_LABEL" \
|
||||
--path-err "$HTML_URL_ERR" \
|
||||
--link-other "$HTML_URL_OK"
|
||||
if [ "$ec" -eq 0 ]; then
|
||||
python3 "$SITE_GEN/console_to_html.py" \
|
||||
--log "$LOG" \
|
||||
--out "$mon" \
|
||||
--hostname "$host" \
|
||||
--mode ok \
|
||||
--commit "${COMMIT:-}" \
|
||||
--commit-url "${COMMIT_URL:-}" \
|
||||
--suite-path "$SUITE_PATH" \
|
||||
--page-label "$PAGE_LABEL" \
|
||||
--path-err "$HTML_URL_ERR" \
|
||||
--link-other ""
|
||||
rm -rf "$HTML_BASE/$host/${HTML_ERR_DIR}"
|
||||
echo "html $host → ${HTML_URL_OK} only (clean · ${COMMIT_SHORT:-?})"
|
||||
else
|
||||
python3 "$SITE_GEN/console_to_html.py" \
|
||||
--log "$LOG" \
|
||||
--out "$mon" \
|
||||
--hostname "$host" \
|
||||
--mode redirect \
|
||||
--commit "${COMMIT:-}" \
|
||||
--commit-url "${COMMIT_URL:-}" \
|
||||
--suite-path "$SUITE_PATH" \
|
||||
--page-label "$PAGE_LABEL" \
|
||||
--path-err "$HTML_URL_ERR"
|
||||
echo "html $host → ${HTML_URL_OK} stub + ${HTML_URL_ERR} (ec=$ec · ${COMMIT_SHORT:-?})"
|
||||
fi
|
||||
else
|
||||
echo "WARN: console_to_html.py missing — bootstrap pages"
|
||||
write_bootstrap_html "$mon_err" "$host" "err"
|
||||
if [ "$ec" -eq 0 ]; then
|
||||
write_bootstrap_html "$mon" "$host" "ok"
|
||||
rm -rf "$HTML_BASE/$host/${HTML_ERR_DIR}"
|
||||
else
|
||||
write_bootstrap_html "$mon" "$host" "redirect"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ! -f "$mon" ]; then
|
||||
write_bootstrap_html "$mon" "$host" "err"
|
||||
fi
|
||||
|
||||
printf '%s\n' "${COMMIT:-unknown}" >"$HTML_BASE/$host/COMMIT"
|
||||
printf '%s\n' "${COMMIT_URL:-}" >"$HTML_BASE/$host/COMMIT_URL"
|
||||
}
|
||||
for host in $MON_HOSTS; do
|
||||
htmlify_host "$host"
|
||||
done
|
||||
|
||||
if [ -n "$DEPLOY_WWW" ] && [ -w "$DEPLOY_WWW" ] 2>/dev/null; then
|
||||
rsync -a --delete "$HTML_BASE/" "$DEPLOY_WWW/" && \
|
||||
echo "synced → $DEPLOY_WWW/"
|
||||
fi
|
||||
|
||||
ls -1t "$LOG_DIR"/run-*.log 2>/dev/null | tail -n +30 | xargs rm -f 2>/dev/null || true
|
||||
|
||||
echo "======== done ec=$ec · log=$LOG · ${COMMIT_URL:-} ========"
|
||||
if [ "${STRICT_EXIT:-0}" = "1" ]; then
|
||||
exit "$ec"
|
||||
fi
|
||||
exit 0
|
||||
Loading…
Add table
Add a link
Reference in a new issue