5a5431bfa9
Multi-phase global SUMMARY, warn-filter dimmed context, SUMMARY chrome as meta; FP stage host-agent via francpaysan-stage-user (stagepaysan); monpages GOA + FP stage.
277 lines
9.6 KiB
Bash
Executable file
277 lines
9.6 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# generate-monitoring-sites.sh — run taler-monitoring, build console HTML for
|
|
# /monitoring and /monitoring_err (static files for host Caddy later).
|
|
#
|
|
# Usage:
|
|
# ./generate-monitoring-sites.sh # all SITES (or ONLY_HOSTS)
|
|
# ONLY_HOSTS="bank.hacktivism.ch taler.hacktivism.ch" ./generate-monitoring-sites.sh
|
|
# ./generate-monitoring-sites.sh --dry-run # HTML from empty/missing logs only
|
|
# ./generate-monitoring-sites.sh --skip-run # only convert existing logs
|
|
#
|
|
# Settings: settings.conf (from settings.conf.example). No secrets in settings.
|
|
# Secrets for e2e: ../secrets.env (optional; default phases are urls versions).
|
|
#
|
|
set -uo pipefail
|
|
|
|
export PYTHONUNBUFFERED=1
|
|
# Global reporting defaults (same as host-agent run-host-report.sh)
|
|
: "${RUN_TIMEOUT:=600}"
|
|
export RUN_TIMEOUT
|
|
|
|
HERE=$(cd "$(dirname "$0")" && pwd)
|
|
SUITE_ROOT=$(cd "$HERE/../.." && pwd)
|
|
SITE_GEN="${SITE_GEN_DIR:-$SUITE_ROOT/site-gen}"
|
|
ROOT="$HERE"
|
|
SETTINGS="${SITE_GEN_SETTINGS:-$SITE_GEN/settings.conf}"
|
|
EXAMPLE="${SITE_GEN}/settings.conf.example"
|
|
[ -f "$EXAMPLE" ] || EXAMPLE="$HERE/settings.conf.example"
|
|
|
|
if [ ! -f "$SETTINGS" ]; then
|
|
if [ -f "$EXAMPLE" ]; then
|
|
echo "note: no $SETTINGS — using example defaults (copy to settings.conf to customize)"
|
|
SETTINGS="$EXAMPLE"
|
|
else
|
|
echo "missing settings: $EXAMPLE" >&2
|
|
exit 2
|
|
fi
|
|
fi
|
|
|
|
# Load KEY=value lines; allow SITES already set in environment to win.
|
|
# Multiline SITES in the conf file: use SITES_FILE= or env SITES= instead.
|
|
while IFS= read -r _line || [ -n "$_line" ]; do
|
|
_line=${_line%%#*}
|
|
_line=$(echo "$_line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
|
|
[ -z "$_line" ] && continue
|
|
case "$_line" in
|
|
SITES=*|SITES_FILE=*) continue ;; # handled below / env
|
|
*=*)
|
|
_k=${_line%%=*}
|
|
_v=${_line#*=}
|
|
# do not override pre-set env
|
|
if [ -z "${!_k+x}" ] 2>/dev/null || [ -z "${!_k}" ]; then
|
|
export "$_k=$_v"
|
|
fi
|
|
;;
|
|
esac
|
|
done < "$SETTINGS"
|
|
|
|
# SITES from dedicated file or keep env; else parse heredoc-style from example list
|
|
if [ -n "${SITES_FILE:-}" ] && [ -f "$SITES_FILE" ]; then
|
|
SITES=$(grep -v '^\s*#' "$SITES_FILE" | grep -v '^\s*$')
|
|
export SITES
|
|
elif [ -z "${SITES:-}" ]; then
|
|
# default 9 fronts
|
|
SITES="bank.hacktivism.ch|hacktivism.ch
|
|
exchange.hacktivism.ch|hacktivism.ch
|
|
taler.hacktivism.ch|hacktivism.ch
|
|
stage.bank.lefrancpaysan.ch|stage.lefrancpaysan.ch
|
|
stage.exchange.lefrancpaysan.ch|stage.lefrancpaysan.ch
|
|
stage.monnaie.lefrancpaysan.ch|stage.lefrancpaysan.ch
|
|
bank.lefrancpaysan.ch|lefrancpaysan.ch
|
|
exchange.lefrancpaysan.ch|lefrancpaysan.ch
|
|
monnaie.lefrancpaysan.ch|lefrancpaysan.ch"
|
|
export SITES
|
|
fi
|
|
|
|
SKIP_RUN=0
|
|
DRY=0
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--skip-run) SKIP_RUN=1; shift ;;
|
|
--dry-run) DRY=1; SKIP_RUN=1; shift ;;
|
|
-h|--help) sed -n '1,20p' "$0"; exit 0 ;;
|
|
*) echo "unknown arg: $1" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
MONITORING_ROOT=$(cd "${MONITORING_ROOT:-$SUITE_ROOT}" && pwd)
|
|
WORK_ROOT="${WORK_ROOT:-/tmp/taler-monitoring-sites-work}"
|
|
LOG_DIR="$WORK_ROOT/logs"
|
|
HTML_DIR="$WORK_ROOT/html"
|
|
mkdir -p "$LOG_DIR" "$HTML_DIR"
|
|
|
|
# commit of monitoring suite used for this run
|
|
if git -C "$MONITORING_ROOT/../.." rev-parse HEAD >/dev/null 2>&1; then
|
|
REPO_ROOT=$(cd "$MONITORING_ROOT/../.." && pwd)
|
|
elif git -C "$MONITORING_ROOT" rev-parse HEAD >/dev/null 2>&1; then
|
|
REPO_ROOT=$(cd "$MONITORING_ROOT" && pwd)
|
|
else
|
|
REPO_ROOT=$(cd "$ROOT/../../.." && pwd)
|
|
fi
|
|
COMMIT=$(git -C "$REPO_ROOT" rev-parse HEAD 2>/dev/null || echo unknown)
|
|
COMMIT_SHORT=$(git -C "$REPO_ROOT" rev-parse --short=12 HEAD 2>/dev/null || echo unknown)
|
|
SOURCE_REPO_WEB="${SOURCE_REPO_WEB:-https://git.hacktivism.ch/hernani/taler-monitoring}"
|
|
TMPL="${SOURCE_COMMIT_URL_TMPL:-\{repo\}/src/commit/\{commit\}}"
|
|
COMMIT_URL=$(printf '%s' "$TMPL" | sed "s|{repo}|$SOURCE_REPO_WEB|g; s|{commit}|$COMMIT|g")
|
|
SUITE_PATH="${SOURCE_SUITE_PATH:-.}"
|
|
|
|
# Outside-only public checks (no SSH into koopa/stage containers)
|
|
PHASES_ERR="${PHASES_ERR:-urls versions}"
|
|
PHASES_OK="${PHASES_OK:-urls versions}"
|
|
CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}"
|
|
SKIP_SSH="${SKIP_SSH:-1}"
|
|
# Drop SSH phases if someone passes "all" / inside / server / e2e by mistake
|
|
filter_phases_outside() {
|
|
local out=() p
|
|
for p in "$@"; do
|
|
case "$p" in
|
|
inside|server|e2e|ladder|goa-ladder|auth401) continue ;;
|
|
all) out+=(urls versions);;
|
|
*) out+=("$p") ;;
|
|
esac
|
|
done
|
|
printf '%s\n' "${out[@]}"
|
|
}
|
|
|
|
run_mon() {
|
|
local domain="$1" phases="$2" logfile="$3" cont="$4"
|
|
# shellcheck disable=SC2206
|
|
local ph=( $phases )
|
|
mapfile -t ph < <(filter_phases_outside "${ph[@]}")
|
|
[ "${#ph[@]}" -gt 0 ] || ph=(urls versions)
|
|
|
|
: >"$logfile"
|
|
echo "+ -d $domain · phases=${ph[*]} · CONTINUE=$cont · SKIP_SSH=$SKIP_SSH · RUN_TIMEOUT=$RUN_TIMEOUT"
|
|
|
|
if [ -n "${RUNNER_SSH:-}" ]; then
|
|
echo "+ via ssh $RUNNER_SSH"
|
|
rsync -az --delete \
|
|
--exclude secrets.env \
|
|
--exclude 'android-test/artifacts' \
|
|
--exclude '.git' \
|
|
"$MONITORING_ROOT/" \
|
|
"${RUNNER_SSH}:${RUNNER_REMOTE_WORKDIR}/" \
|
|
|| return 1
|
|
# shellcheck disable=SC2029
|
|
if command -v stdbuf >/dev/null 2>&1; then
|
|
ssh -o BatchMode=yes -o ConnectTimeout=30 "$RUNNER_SSH" \
|
|
"cd '${RUNNER_REMOTE_WORKDIR}' && chmod +x taler-monitoring.sh check_*.sh 2>/dev/null; \
|
|
export CONTINUE_ON_ERROR='$cont' AUTH401_CONTINUE='$cont' SKIP_SSH='$SKIP_SSH' RUN_TIMEOUT='$RUN_TIMEOUT'; \
|
|
stdbuf -oL -eL ./taler-monitoring.sh -d '$domain' ${ph[*]}" \
|
|
2>&1 | stdbuf -oL -eL tee -a "$logfile"
|
|
else
|
|
ssh -o BatchMode=yes -o ConnectTimeout=30 "$RUNNER_SSH" \
|
|
"cd '${RUNNER_REMOTE_WORKDIR}' && chmod +x taler-monitoring.sh check_*.sh 2>/dev/null; \
|
|
export CONTINUE_ON_ERROR='$cont' AUTH401_CONTINUE='$cont' SKIP_SSH='$SKIP_SSH' RUN_TIMEOUT='$RUN_TIMEOUT'; \
|
|
./taler-monitoring.sh -d '$domain' ${ph[*]}" \
|
|
2>&1 | tee -a "$logfile"
|
|
fi
|
|
return "${PIPESTATUS[0]}"
|
|
fi
|
|
|
|
if command -v stdbuf >/dev/null 2>&1; then
|
|
( cd "$MONITORING_ROOT" && \
|
|
CONTINUE_ON_ERROR="$cont" AUTH401_CONTINUE="$cont" SKIP_SSH="$SKIP_SSH" \
|
|
RUN_TIMEOUT="$RUN_TIMEOUT" \
|
|
stdbuf -oL -eL ./taler-monitoring.sh -d "$domain" "${ph[@]}" ) \
|
|
2>&1 | stdbuf -oL -eL tee -a "$logfile"
|
|
else
|
|
( cd "$MONITORING_ROOT" && \
|
|
CONTINUE_ON_ERROR="$cont" AUTH401_CONTINUE="$cont" SKIP_SSH="$SKIP_SSH" \
|
|
RUN_TIMEOUT="$RUN_TIMEOUT" \
|
|
./taler-monitoring.sh -d "$domain" "${ph[@]}" ) \
|
|
2>&1 | tee -a "$logfile"
|
|
fi
|
|
return "${PIPESTATUS[0]}"
|
|
}
|
|
|
|
htmlify() {
|
|
local host="$1" mode="$2" log="$3" out="$4" other="$5"
|
|
mkdir -p "$(dirname "$out")"
|
|
if [ ! -f "$SITE_GEN/console_to_html.py" ]; then
|
|
echo "WARN: console_to_html.py missing — bootstrap $out"
|
|
{
|
|
echo "<!DOCTYPE html><html><head><meta charset=utf-8><title>$host $mode</title></head>"
|
|
echo "<body><h1>$host · $mode</h1><p>commit <a href=\"$COMMIT_URL\">$COMMIT_SHORT</a></p>"
|
|
echo "<pre>"
|
|
tail -n 100 "$log" 2>/dev/null | sed 's/&/\&/g;s/</\</g' || true
|
|
echo "</pre></body></html>"
|
|
} >"$out"
|
|
return 0
|
|
fi
|
|
python3 "$SITE_GEN/console_to_html.py" \
|
|
--log "$log" \
|
|
--out "$out" \
|
|
--hostname "$host" \
|
|
--mode "$mode" \
|
|
--commit "$COMMIT" \
|
|
--commit-url "$COMMIT_URL" \
|
|
--suite-path "$SUITE_PATH" \
|
|
--link-other "$other"
|
|
}
|
|
|
|
# Parse SITES
|
|
mapfile -t SITE_LINES < <(printf '%s\n' "$SITES" | sed '/^\s*$/d' | sed 's/#.*//')
|
|
ONLY="${ONLY_HOSTS:-}"
|
|
|
|
for line in "${SITE_LINES[@]}"; do
|
|
line=$(echo "$line" | tr -d ' \t')
|
|
[ -z "$line" ] && continue
|
|
host=${line%%|*}
|
|
domain=${line#*|}
|
|
if [ -n "$ONLY" ]; then
|
|
echo " $ONLY " | grep -q " $host " || continue
|
|
fi
|
|
|
|
echo "======== $host (domain=$domain) commit=$COMMIT_SHORT ========"
|
|
base_html="$HTML_DIR/$host"
|
|
mkdir -p "$base_html/monitoring" "$base_html/monitoring_err"
|
|
log_err="$LOG_DIR/${host}.err.log"
|
|
log_ok="$LOG_DIR/${host}.ok.log"
|
|
ec_err=1
|
|
ec_ok=1
|
|
|
|
if [ "$SKIP_RUN" != "1" ]; then
|
|
run_mon "$domain" "$PHASES_ERR" "$log_err" "$CONTINUE_ON_ERROR"
|
|
ec_err=$?
|
|
# ok-run without continue (strict)
|
|
run_mon "$domain" "$PHASES_OK" "$log_ok" "0"
|
|
ec_ok=$?
|
|
else
|
|
[ -f "$log_err" ] || : >"$log_err"
|
|
[ -f "$log_ok" ] || : >"$log_ok"
|
|
# infer exit from log if present
|
|
if grep -q '\[ ERROR \]' "$log_err" 2>/dev/null || grep -q 'ERROR' "$log_err" 2>/dev/null; then
|
|
ec_err=1
|
|
else
|
|
ec_err=0
|
|
fi
|
|
ec_ok=$ec_err
|
|
fi
|
|
|
|
# Always write pages (first run / empty logs too)
|
|
[ -f "$log_err" ] || : >"$log_err"
|
|
[ -f "$log_ok" ] || : >"$log_ok"
|
|
|
|
htmlify "$host" "err" "$log_err" \
|
|
"$base_html/monitoring_err/index.html" "/monitoring/"
|
|
|
|
if [ "$ec_ok" -eq 0 ] && [ "$ec_err" -eq 0 ]; then
|
|
htmlify "$host" "ok" "$log_ok" \
|
|
"$base_html/monitoring/index.html" "/monitoring_err/"
|
|
echo " → /monitoring (clean) + /monitoring_err"
|
|
else
|
|
htmlify "$host" "redirect" "$log_err" \
|
|
"$base_html/monitoring/index.html" "/monitoring_err/"
|
|
echo " → /monitoring (redirect stub) + /monitoring_err (errors) ec_err=$ec_err ec_ok=$ec_ok"
|
|
fi
|
|
|
|
# first-run safety
|
|
if [ ! -f "$base_html/monitoring/index.html" ]; then
|
|
htmlify "$host" "err" "$log_err" \
|
|
"$base_html/monitoring/index.html" "/monitoring_err/"
|
|
fi
|
|
|
|
printf '%s\n' "$COMMIT" >"$base_html/COMMIT"
|
|
printf '%s\n' "$COMMIT_URL" >"$base_html/COMMIT_URL"
|
|
printf 'host=%s domain=%s ec_err=%s ec_ok=%s commit=%s run_timeout=%s\n' \
|
|
"$host" "$domain" "$ec_err" "$ec_ok" "$COMMIT" "$RUN_TIMEOUT" | tee "$base_html/STATUS.txt"
|
|
done
|
|
|
|
echo
|
|
echo "HTML under $HTML_DIR"
|
|
echo "commit $COMMIT_SHORT $COMMIT_URL"
|
|
echo "RUN_TIMEOUT=${RUN_TIMEOUT}s (global; 0=unlimited)"
|
|
echo "Deploy (as hernani, may need sudo for /var/www):"
|
|
echo " $HERE/deploy-monitoring-sites.sh"
|
|
echo "Root on koopa: see ROOT-ON-KOOPA.md"
|