#!/usr/bin/env bash # run-host-report.sh — global reporting generation for all stacks # (GOA/hacktivism, FP stage, FP prod, …). # # Shared behaviour (always): # 1) update-suite → latest git.hacktivism.ch (local ~/.config never overwritten) # 2) line-buffered run log (written continuously) # 3) RUN_TIMEOUT wall clock for taler-monitoring.sh (default 600s) # 4) always write HTML (first run / empty staging too) + Forgejo commit link # 5) rsync to DEPLOY_WWW_ROOT (ERROR if not writable — v1.3.1) # 6) monpages post-check: public FQDN must serve monitoring HTML # 7) exit code: STRICT_EXIT default 1 (v1.7.1) — unit fails if publish/public pages fail # # Configure via env file (not in git): # ${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring/env # or TALER_MONITORING_ENV=/path/to/env # # Thin wrappers only set stack defaults then exec this script: # run-hacktivism-monitoring.sh # run-fp-stage-monitoring.sh # run-fp-prod-monitoring.sh # set -uo pipefail export PATH="${HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin:${PATH:-}" export PYTHONUNBUFFERED=1 AGENT_DIR=$(cd "$(dirname "$0")" && pwd) AGENT_MON=$(cd "$AGENT_DIR/.." && pwd) CFG_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring" ENV_FILE="${TALER_MONITORING_ENV:-$CFG_DIR/env}" mkdir -p "$CFG_DIR" # Thin wrappers (surface, aptdeploy-only, …) export stack knobs before exec. # Snapshot them so ~/.config/taler-monitoring/env cannot clobber dedicated jobs # (env often has PHASES="urls inside versions aptdeploy" for the main timer). _wrap_keys=( PHASES HTML_OUT HTML_OK_DIR HTML_ERR_DIR HTML_URL_OK HTML_URL_ERR PAGE_LABEL RUN_TIMEOUT STATE_NAME AGENT_LABEL MON_HOSTS TALER_DOMAIN DEPLOY_WWW_ROOT INSIDE_PODMAN INSIDE_MODE LOCAL_STACK SKIP_SSH CONTINUE_ON_ERROR AUTH401_CONTINUE APT_DEPLOY_ENSURE SURFACE_SCOPE TALER_DOMAIN_FROM_CLI STRICT_EXIT SOURCE_SUITE_PATH MONPAGES_INVENTORY MONPAGES_CHECK_BARE MONPAGES_BARE_STRICT MONPAGES_REQUIRE_PUBLIC CHECK_BANK MERCHANT_REQUIRED CHECK_LANDING ) for _k in "${_wrap_keys[@]}"; do if [ -n "${!_k+x}" ]; then printf -v "_WRAP_SET_${_k}" '%s' 1 printf -v "_WRAP_VAL_${_k}" '%s' "${!_k}" fi done unset _k if [ -f "$ENV_FILE" ]; then # shellcheck disable=SC1090 set -a # shellcheck source=/dev/null source "$ENV_FILE" set +a fi # Tell update-suite.sh not to re-source env (would clobber wrapper PHASES etc.) export _TALER_MON_ENV_LOADED=1 # Restore wrapper overrides (dedicated jobs win over shared env file) for _k in "${_wrap_keys[@]}"; do _set_var="_WRAP_SET_${_k}" _val_var="_WRAP_VAL_${_k}" if [ "${!_set_var:-}" = "1" ]; then printf -v "$_k" '%s' "${!_val_var}" export "$_k" fi done unset _k _set_var _val_var # --- global report defaults (env file / wrapper wins if already set) --- export CONTINUE_ON_ERROR="${CONTINUE_ON_ERROR:-1}" export AUTH401_CONTINUE="${AUTH401_CONTINUE:-${CONTINUE_ON_ERROR}}" export RUN_TIMEOUT="${RUN_TIMEOUT:-600}" export TALER_DOMAIN="${TALER_DOMAIN:-hacktivism.ch}" # UI language: selectable via TALER_MON_LANG / --lang (en|fr). # If unset, auto: FrancPaysan domains → fr, else en. if [ "${TALER_MON_LANG_SET:-0}" != "1" ] && [ -z "${TALER_MON_LANG:-}" ]; then case "$TALER_DOMAIN" in *lefrancpaysan*|*francpaysan*) export TALER_MON_LANG=fr ;; *) export TALER_MON_LANG=en ;; esac elif [ -n "${TALER_MON_LANG:-}" ]; then case "$TALER_MON_LANG" in fr|FR|fra|french) export TALER_MON_LANG=fr ;; *) export TALER_MON_LANG=en ;; esac export TALER_MON_LANG_SET=1 fi export TALER_MON_LANG PHASES="${PHASES:-urls inside versions monpages}" HTML_BASE="${HTML_OUT:-$HOME/monitoring-sites-staging}" MON_HOSTS="${MON_HOSTS:-}" SUITE_PATH="${SOURCE_SUITE_PATH:-.}" AGENT_LABEL="${AGENT_LABEL:-host-report}" STATE_NAME="${STATE_NAME:-taler-monitoring-${TALER_DOMAIN}}" STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/${STATE_NAME}" LOG_DIR="$STATE_DIR/logs" DEPLOY_WWW="${DEPLOY_WWW_ROOT:-/var/www/monitoring-sites}" # HTML path layout under each host (monitoring / surface / …) HTML_OK_DIR="${HTML_OK_DIR:-monitoring}" HTML_ERR_DIR="${HTML_ERR_DIR:-monitoring_err}" HTML_URL_OK="${HTML_URL_OK:-/monitoring/}" HTML_URL_ERR="${HTML_URL_ERR:-/monitoring_err/}" PAGE_LABEL="${PAGE_LABEL:-monitoring}" SUITE_VERSION="${SUITE_VERSION:-}" SUITE_VERSION_URL="${SUITE_VERSION_URL:-}" if [ -z "$SUITE_VERSION" ] && [ -d "${ROOT:-$AGENT_MON}/.git" ]; then _vd="${ROOT:-$AGENT_MON}" SUITE_VERSION=$(git -C "$_vd" describe --tags --exact-match 2>/dev/null || git -C "$_vd" describe --tags --abbrev=0 2>/dev/null || true) fi if [ -z "$SUITE_VERSION" ] && [ -f "${ROOT:-$AGENT_MON}/VERSION" ]; then SUITE_VERSION="v$(tr -d '[:space:]' <"${ROOT:-$AGENT_MON}/VERSION" | sed 's/^v//')" fi [ -z "$SUITE_VERSION" ] && SUITE_VERSION="unknown" case "$SUITE_VERSION" in unknown|v*) ;; *) SUITE_VERSION="v$SUITE_VERSION" ;; esac if [ -z "$SUITE_VERSION_URL" ] && [ "$SUITE_VERSION" != "unknown" ]; then _web="${SOURCE_REPO_WEB:-https://git.hacktivism.ch/hernani/taler-monitoring}" SUITE_VERSION_URL="${_web}/src/tag/${SUITE_VERSION}" fi export SUITE_VERSION SUITE_VERSION_URL # Sensible MON_HOSTS from domain if not set if [ -z "$MON_HOSTS" ]; then case "$TALER_DOMAIN" in hacktivism.ch|koopa) MON_HOSTS="bank.hacktivism.ch exchange.hacktivism.ch taler.hacktivism.ch" ;; stage.lefrancpaysan.ch|stage.*lefrancpaysan*) MON_HOSTS="stage.bank.lefrancpaysan.ch stage.exchange.lefrancpaysan.ch stage.monnaie.lefrancpaysan.ch" ;; lefrancpaysan.ch) MON_HOSTS="bank.lefrancpaysan.ch exchange.lefrancpaysan.ch monnaie.lefrancpaysan.ch" ;; *) MON_HOSTS="${TALER_DOMAIN}" ;; esac fi mkdir -p "$LOG_DIR" "$HTML_BASE" STAMP=$(date +%Y%m%d-%H%M%S) LOG="$LOG_DIR/run-${STAMP}.log" UPD_LOG="$LOG_DIR/suite-update-${STAMP}.log" : >"$LOG" # --- suite auto-upgrade BEFORE the HTML log (git noise must not pollute mon pages) --- OVERLAY_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring/suite-overlay" if [ -f "$AGENT_MON/taler-monitoring.sh" ]; then mkdir -p "$OVERLAY_DIR" rsync -a \ --exclude secrets.env --exclude 'android-test/artifacts' --exclude 'android-test/apks' \ --exclude 'android-test/out*' --exclude '__pycache__' \ "$AGENT_MON/" "$OVERLAY_DIR/" 2>/dev/null || true fi # shellcheck source=update-suite.sh if ! source "$AGENT_DIR/update-suite.sh" >"$UPD_LOG" 2>&1; then echo "ERROR: suite auto-update failed — refusing to run with stale tree" >&2 echo " detail: $UPD_LOG" >&2 tail -n 30 "$UPD_LOG" >&2 || true echo " fix git/network or set SUITE_UPDATE_STRICT=0 only as emergency" >&2 exit 1 fi if [ -d "${SUITE_DIR:-}/host-agent" ]; then AGENT_DIR="${SUITE_DIR}/host-agent" fi # Now open the public HTML log (clean agent header + suite summary only) if command -v stdbuf >/dev/null 2>&1; then exec > >(stdbuf -oL -eL tee -a "$LOG") 2>&1 else exec > >(tee -a "$LOG") 2>&1 fi echo "======== $(date -Iseconds 2>/dev/null || date) ${AGENT_LABEL} ========" echo "env_file=$ENV_FILE" echo "domain=$TALER_DOMAIN · phases=$PHASES · RUN_TIMEOUT=${RUN_TIMEOUT}s" echo "hosts=$MON_HOSTS" echo "flags INSIDE_PODMAN=${INSIDE_PODMAN:-} INSIDE_MODE=${INSIDE_MODE:-} LOCAL_STACK=${LOCAL_STACK:-} SKIP_SSH=${SKIP_SSH:-} INSIDE_PROFILE=${INSIDE_PROFILE:-}" # One-line suite pin (no git M/checkout noise) echo "suite ${SUITE_VERSION:-?} @ ${COMMIT_SHORT:-?} ${COMMIT_URL:-}" echo " dir=${SUITE_DIR:-}" SUITE_MON="${SUITE_DIR}" ROOT="$SUITE_MON" _need_overlay=0 # Detect missing features in Forgejo tree (untracked helper files alone are not enough — # taler-monitoring.sh must accept the phase name). case " $PHASES " in *" aptdeploy "*|*" apt-deploy "*|*" apt_src "*) if ! grep -qE 'aptdeploy\|apt-deploy' "$SUITE_MON/taler-monitoring.sh" 2>/dev/null; then _need_overlay=1 fi ;; esac case " $PHASES " in *" surface "*|*" ecosystem "*) if ! grep -qE 'surface\|ecosystem' "$SUITE_MON/taler-monitoring.sh" 2>/dev/null \ || [ ! -f "$SUITE_MON/check_surface.sh" ]; then _need_overlay=1 fi ;; esac # Wrapper-preserve + HTML path knobs may live only in local overlay until pushed if [ -f "$OVERLAY_DIR/host-agent/run-host-report.sh" ] \ && ! grep -q '_wrap_keys\|_WRAP_SET_' "$SUITE_MON/host-agent/run-host-report.sh" 2>/dev/null \ && grep -q '_wrap_keys\|_WRAP_SET_' "$OVERLAY_DIR/host-agent/run-host-report.sh" 2>/dev/null; then _need_overlay=1 fi if [ "$_need_overlay" = "1" ] && [ -f "$OVERLAY_DIR/taler-monitoring.sh" ]; then echo "WARN: Forgejo @ ${COMMIT_SHORT:-?} lagging local suite-overlay (phases=[$PHASES])" >&2 echo " re-applying suite-overlay → $SUITE_MON" >&2 mkdir -p "$SUITE_MON" rsync -a \ --exclude secrets.env --exclude 'android-test/artifacts' --exclude 'android-test/apks' \ --exclude 'android-test/out*' --exclude '__pycache__' \ "$OVERLAY_DIR/" "$SUITE_MON/" ROOT="$SUITE_MON" elif [ "$_need_overlay" = "1" ]; then echo "WARN: Forgejo missing features and no suite-overlay — using agent mon $AGENT_MON" >&2 ROOT="$AGENT_MON" else echo "using suite mon @ ${COMMIT_SHORT:-?} · $ROOT" # Do NOT rsync suite-overlay over a fresh Forgejo tree — that reintroduced # stale site-gen/host-agent (bootstrap sticky, old monpages) on stage/betel. fi unset _need_overlay if [ -f "$ROOT/site-gen/console_to_html.py" ]; then SITE_GEN="$ROOT/site-gen" elif [ -f "$AGENT_MON/site-gen/console_to_html.py" ]; then SITE_GEN="$AGENT_MON/site-gen" echo "note: using agent site-gen at $SITE_GEN" elif [ -f "$SUITE_MON/site-gen/console_to_html.py" ]; then SITE_GEN="$SUITE_MON/site-gen" else SITE_GEN="" fi if [ ! -x "$ROOT/taler-monitoring.sh" ]; then if [ -x "$AGENT_MON/taler-monitoring.sh" ]; then ROOT="$AGENT_MON" echo "note: using agent mon tree $ROOT" else echo "ERROR: suite missing at $ROOT" >&2 COMMIT="${COMMIT:-unknown}" COMMIT_SHORT="${COMMIT_SHORT:-unknown}" COMMIT_URL="${COMMIT_URL:-}" fi fi echo "commit=${COMMIT_SHORT:-?} · ${COMMIT_URL:-}" ec=1 if [ -x "${ROOT:-}/taler-monitoring.sh" ]; then cd "$ROOT" || exit 1 chmod +x taler-monitoring.sh check_*.sh host-agent/*.sh site-gen/*.sh site-gen/*.py 2>/dev/null || true # Overlay newer inside/versions host-podman bits if suite clone is stale for f in check_inside.sh check_versions.sh taler-monitoring.sh lib.sh; do if [ -f "$AGENT_MON/$f" ] && [ -f "$ROOT/$f" ]; then if ! grep -q 'host-podman\|INSIDE_ACCESS\|RUN_TIMEOUT' "$ROOT/$f" 2>/dev/null \ && grep -q 'host-podman\|INSIDE_ACCESS\|RUN_TIMEOUT' "$AGENT_MON/$f" 2>/dev/null; then echo "note: overlay $f from agent install" cp -f "$AGENT_MON/$f" "$ROOT/$f" chmod +x "$ROOT/$f" 2>/dev/null || true fi fi done # shellcheck disable=SC2086 ./taler-monitoring.sh -d "$TALER_DOMAIN" $PHASES ec=$? fi # --- always generate HTML (first run / missing pages / every run) --- write_bootstrap_html() { local out="$1" host="$2" mode="$3" mkdir -p "$(dirname "$out")" local cshort="${COMMIT_SHORT:-unknown}" local curl="${COMMIT_URL:-}" local clink="$cshort" local gen iso gen=$(date -u +"%Y-%m-%d %H:%M:%SZ" 2>/dev/null || date) iso=$(date -u +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -Iseconds) if [ -n "$curl" ]; then clink="$cshort" fi cat >"$out" < monitoring ${mode} · ${host}

Bootstrap page (converter missing or first install). See host-agent log.

$(tail -n 80 "$LOG" 2>/dev/null | sed 's/&/\&/g;s/
EOF echo "bootstrap html → $out" } htmlify_host() { local host="$1" local mon="$HTML_BASE/$host/${HTML_OK_DIR}/index.html" local mon_err="$HTML_BASE/$host/${HTML_ERR_DIR}/index.html" mkdir -p "$HTML_BASE/$host/${HTML_OK_DIR}" "$HTML_BASE/$host/${HTML_ERR_DIR}" if [ -n "$SITE_GEN" ] && [ -f "$SITE_GEN/console_to_html.py" ]; then python3 "$SITE_GEN/console_to_html.py" \ --lang "${TALER_MON_LANG:-en}" \ --log "$LOG" \ --out "$mon_err" \ --hostname "$host" \ --mode err \ --commit "${COMMIT:-}" \ --commit-url "${COMMIT_URL:-}" \ --suite-version "${SUITE_VERSION:-unknown}" \ --suite-version-url "${SUITE_VERSION_URL:-}" \ --suite-path "$SUITE_PATH" \ --page-label "$PAGE_LABEL" \ --path-err "$HTML_URL_ERR" \ --link-other "$HTML_URL_OK" if [ "$ec" -eq 0 ]; then python3 "$SITE_GEN/console_to_html.py" \ --lang "${TALER_MON_LANG:-en}" \ --log "$LOG" \ --out "$mon" \ --hostname "$host" \ --mode ok \ --commit "${COMMIT:-}" \ --commit-url "${COMMIT_URL:-}" \ --suite-version "${SUITE_VERSION:-unknown}" \ --suite-version-url "${SUITE_VERSION_URL:-}" \ --suite-path "$SUITE_PATH" \ --page-label "$PAGE_LABEL" \ --path-err "$HTML_URL_ERR" \ --link-other "" rm -rf "$HTML_BASE/$host/${HTML_ERR_DIR}" echo "html $host → ${HTML_URL_OK} only (clean · ${COMMIT_SHORT:-?})" else python3 "$SITE_GEN/console_to_html.py" \ --lang "${TALER_MON_LANG:-en}" \ --log "$LOG" \ --out "$mon" \ --hostname "$host" \ --mode redirect \ --commit "${COMMIT:-}" \ --commit-url "${COMMIT_URL:-}" \ --suite-version "${SUITE_VERSION:-unknown}" \ --suite-version-url "${SUITE_VERSION_URL:-}" \ --suite-path "$SUITE_PATH" \ --page-label "$PAGE_LABEL" \ --path-err "$HTML_URL_ERR" echo "html $host → ${HTML_URL_OK} stub + ${HTML_URL_ERR} (ec=$ec · ${COMMIT_SHORT:-?})" fi else echo "WARN: console_to_html.py missing — bootstrap pages" write_bootstrap_html "$mon_err" "$host" "err" if [ "$ec" -eq 0 ]; then write_bootstrap_html "$mon" "$host" "ok" rm -rf "$HTML_BASE/$host/${HTML_ERR_DIR}" else write_bootstrap_html "$mon" "$host" "redirect" fi fi if [ ! -f "$mon" ]; then write_bootstrap_html "$mon" "$host" "err" fi printf '%s\n' "${COMMIT:-unknown}" >"$HTML_BASE/$host/COMMIT" printf '%s\n' "${COMMIT_URL:-}" >"$HTML_BASE/$host/COMMIT_URL" } for host in $MON_HOSTS; do htmlify_host "$host" echo "public URL (expected): https://${host}${HTML_URL_OK}" done # Publish to live web root when possible (v1.3.1+: never silent-skip) if [ -n "$DEPLOY_WWW" ]; then if [ -w "$DEPLOY_WWW" ] 2>/dev/null; then if rsync -a --delete "$HTML_BASE/" "$DEPLOY_WWW/"; then echo "synced → $DEPLOY_WWW/" else echo "ERROR: rsync to DEPLOY_WWW_ROOT=$DEPLOY_WWW failed" >&2 ec=1 fi elif [ -e "$DEPLOY_WWW" ]; then echo "ERROR: DEPLOY_WWW_ROOT=$DEPLOY_WWW exists but is not writable by $(id -un)" >&2 echo " public FQDN pages will stay missing/stale — fix ownership or run:" >&2 echo " sudo /home/hernani/koopa-caddy/apply-monitoring-live.sh" >&2 ec=1 else echo "ERROR: DEPLOY_WWW_ROOT=$DEPLOY_WWW missing — public pages not published" >&2 echo " create tree + Caddy handles: sudo /home/hernani/koopa-caddy/apply-monitoring-live.sh" >&2 ec=1 fi else echo "WARN: DEPLOY_WWW_ROOT empty — skipping live publish (staging only: $HTML_BASE)" >&2 fi # Post-publish: monpages obligatory (v1.7.6+) — GOA full inventory / FP only FP hosts _monpages_failed=0 if [ "${MONPAGES_POSTCHECK:-1}" = "1" ] && [ -x "$ROOT/check_monitoring_pages.sh" ]; then echo "--- monpages post-check (public FQDN, obligatory) ---" export MON_HOSTS HTML_URL_OK HTML_URL_ERR TALER_DOMAIN HTML_OUT HTML_BASE export MONPAGES_STAGING_BASE="${HTML_BASE}" export DEPLOY_WWW_ROOT="${DEPLOY_WWW:-${DEPLOY_WWW_ROOT:-}}" export MONPAGES_REQUIRE_PUBLIC="${MONPAGES_REQUIRE_PUBLIC:-1}" # auto: GOA = all suite mon sites; FP = only FP mon hosts; job paths always included export MONPAGES_INVENTORY="${MONPAGES_INVENTORY:-auto}" if ! bash "$ROOT/check_monitoring_pages.sh"; then echo "ERROR: public monitoring page(s) missing or not suite HTML via FQDN" >&2 echo " staging HTML_OUT=${HTML_BASE} · DEPLOY_WWW=${DEPLOY_WWW:-}" >&2 ec=1 _monpages_failed=1 fi fi # If public check failed after first HTML write, rewrite pages so sticky bar shows monpages ERRORs if [ "$_monpages_failed" = "1" ]; then echo "--- re-htmlify after monpages failure (include public outage in report) ---" # append marker to log so converter sees ERROR lines if monpages was post-only { echo "" echo "ERROR monpages: public FQDN monitoring HTML missing or merchant JSON code 21" echo "ERROR monpages: fix with sudo /home/hernani/koopa-caddy/apply-monitoring-live.sh (www + Caddy handles)" } >>"$LOG" for host in $MON_HOSTS; do htmlify_host "$host" done fi ls -1t "$LOG_DIR"/run-*.log 2>/dev/null | tail -n +30 | xargs rm -f 2>/dev/null || true echo "======== done ec=$ec · log=$LOG · ${COMMIT_URL:-} ========" # v1.7.1: default STRICT_EXIT=1 so systemd/timer reflects publish/public failures if [ "${STRICT_EXIT:-1}" = "1" ]; then exit "$ec" fi exit 0