f5abc50e55
Multi-phase runs no longer print intermediate progress bars, numbered checks, or SUMMARY boxes after each phase (e.g. 34/52 after urls). Only the last phase (or a solo check_*.sh) may full-summary; global progress_finish still ends the run.
1350 lines
46 KiB
Bash
Executable file
1350 lines
46 KiB
Bash
Executable file
# shellcheck shell=bash
|
||
# Shared helpers for taler-monitoring (laptop or koopa).
|
||
#
|
||
# Machine paths/SSH hosts: only from env (~/.config/taler-monitoring/env or
|
||
# TALER_MONITORING_ENV=…), installed via taler-monitoring-env — never hardcode.
|
||
|
||
# Load host env once (before any := defaults)
|
||
if [ "${_TALER_MON_ENV_LOADED:-0}" != "1" ]; then
|
||
_mon_cfg="${TALER_MONITORING_ENV:-${XDG_CONFIG_HOME:-$HOME/.config}/taler-monitoring/env}"
|
||
if [ -f "$_mon_cfg" ]; then
|
||
# shellcheck disable=SC1090
|
||
set -a
|
||
# shellcheck disable=SC1091
|
||
. "$_mon_cfg"
|
||
set +a
|
||
fi
|
||
unset _mon_cfg
|
||
export _TALER_MON_ENV_LOADED=1
|
||
fi
|
||
|
||
# Default stack = GOA / hacktivism (overridden by TALER_DOMAIN / --domain)
|
||
# i18n (en default; fr for FrancPaysan — see i18n.sh / TALER_MON_LANG)
|
||
_I18N_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||
if [ -f "$_I18N_DIR/i18n.sh" ]; then
|
||
# shellcheck source=i18n.sh
|
||
source "$_I18N_DIR/i18n.sh"
|
||
i18n_init
|
||
else
|
||
i18n_tag() { printf '%s' "$1"; }
|
||
i18n_text() { printf '%s' "$*"; }
|
||
i18n_init() { :; }
|
||
fi
|
||
|
||
: "${TALER_DOMAIN:=hacktivism.ch}"
|
||
: "${BANK_PUBLIC:=https://bank.hacktivism.ch}"
|
||
: "${EXCHANGE_PUBLIC:=https://exchange.hacktivism.ch}"
|
||
: "${MERCHANT_PUBLIC:=https://taler.hacktivism.ch}"
|
||
: "${BANK_LOCAL:=http://127.0.0.1:9012}"
|
||
: "${EXCHANGE_LOCAL:=http://127.0.0.1:9011}"
|
||
: "${MERCHANT_LOCAL:=https://127.0.0.1:9010}"
|
||
: "${LANDING_LOCAL:=http://127.0.0.1:9013}"
|
||
# Hostnames/paths: set via ~/.config/taler-monitoring/env (no hardcoded machine values).
|
||
: "${KOOPA_SSH:=}"
|
||
: "${KOOPA_SSH_FALLBACKS:=}"
|
||
: "${INSIDE_SSH:=}"
|
||
: "${FRANCPAYSAN_SSH:=}"
|
||
: "${DEVTESTING_SSH:=}"
|
||
: "${DEPLOY_SSH:=}"
|
||
: "${FIRECUDA_SSH:=}"
|
||
: "${HTML_OUT:=${HOME}/monitoring-sites-staging}"
|
||
: "${DEPLOY_STAGING:=${HTML_OUT}}"
|
||
: "${DEPLOY_WWW_ROOT:=}"
|
||
: "${APPLY_MONITORING_LIVE:=}"
|
||
: "${KOOPA_CADDY_DIR:=}"
|
||
: "${FRANCPAYSAN_SECRETS:=}"
|
||
: "${FRANCPAYSAN_REMOTE_SECRETS_ROOT:=}"
|
||
: "${FRANCPAYSAN_REMOTE_BANK_ADMIN_SECRET:=}"
|
||
: "${FRANCPAYSAN_REMOTE_BANK_EXPLORER_SECRET:=}"
|
||
: "${FRANCPAYSAN_REMOTE_MERCHANT_TOKEN:=}"
|
||
: "${WALLET_CLI:=}"
|
||
: "${MERCHANT_INSTANCE:=goa-demo-cp4zqk}"
|
||
# Merchant portal /private API: instance ids must be lowercase or you get odd 401s.
|
||
# (Display names may use capitals; path segment /instances/{id}/ must not.)
|
||
MERCHANT_INSTANCE=$(printf '%s' "${MERCHANT_INSTANCE}" | tr '[:upper:]' '[:lower:]')
|
||
: "${WITHDRAW_AMT:=GOA:20}" # single-shot fallback; e2e ladder uses ATM notes
|
||
: "${PAY_AMT:=GOA:0.01}"
|
||
: "${CREDIT_AMT:=GOA:4700}" # covers ATM ladder 20+50+100+200+4200 (paivana)
|
||
: "${TIMEOUT:=12}"
|
||
: "${E2E_TIMEOUT:=55}" # whole e2e budget; skip rest when exceeded (e2e raises as needed)
|
||
: "${E2E_PAY_SECS:=22}" # dedicated seconds for pay handle-uri (avoid Alarm clock)
|
||
# Whole-run wall clock for taler-monitoring.sh (seconds). 0 = unlimited.
|
||
# Host-agent default phases (urls/inside/versions) should finish under this.
|
||
: "${RUN_TIMEOUT:=600}"
|
||
# Local GOA: public paivana paywall (https://paivana.hacktivism.ch · template GOA:4200)
|
||
: "${PAIVANA_PUBLIC:=https://paivana.hacktivism.ch}"
|
||
: "${E2E_PAIVANA:=1}" # 0 = skip paivana section in e2e
|
||
# Devtest: inject reserve credit via wire-gateway admin/add-incoming (optional).
|
||
# Default off once wirewatch DNS works; set E2E_FAKE_INCOMING=1 to force.
|
||
: "${E2E_FAKE_INCOMING:=0}"
|
||
# SSH must never hang the monitoring run
|
||
: "${SSH_CONNECT_TIMEOUT:=3}"
|
||
: "${SSH_CMD_TIMEOUT:=12}" # hard cap for whole remote script (seconds)
|
||
: "${SKIP_SSH:=0}"
|
||
# Expected currency for public /config checks (empty = report only, don't fail)
|
||
# Only if unset (not if intentionally empty = pre-launch / report-only)
|
||
: "${EXPECT_CURRENCY=GOA}"
|
||
# 1 = this is the local koopa/hacktivism stack (inside/e2e/SSH make sense)
|
||
: "${LOCAL_STACK:=1}"
|
||
# Probe merchant host candidates when applying a generic domain (0=off)
|
||
: "${TALER_DOMAIN_PROBE:=1}"
|
||
|
||
BANK_PUBLIC=${BANK_PUBLIC%/}
|
||
EXCHANGE_PUBLIC=${EXCHANGE_PUBLIC%/}
|
||
MERCHANT_PUBLIC=${MERCHANT_PUBLIC%/}
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Domain profiles → bank / exchange / merchant base URLs
|
||
#
|
||
# Single place to declare a stack: domains.conf (suite root)
|
||
# (or TALER_DOMAINS_CONF). Each profile names the three public endpoints.
|
||
#
|
||
# CLI still wins after profile load:
|
||
# --bank URL --exchange URL --merchant URL --currency CODE
|
||
# Env: BANK_PUBLIC EXCHANGE_PUBLIC MERCHANT_PUBLIC EXPECT_CURRENCY
|
||
#
|
||
# Unknown domains fall back to heuristics (see apply_taler_domain).
|
||
# ---------------------------------------------------------------------------
|
||
|
||
_MONITOR_LIB_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||
: "${TALER_DOMAINS_CONF:=${_MONITOR_LIB_DIR}/domains.conf}"
|
||
|
||
# host_or_url → https://… (no trailing slash)
|
||
_to_https_base() {
|
||
local x="${1%/}"
|
||
case "$x" in
|
||
https://*|http://*) printf '%s' "$x" ;;
|
||
"") printf '' ;;
|
||
*) printf 'https://%s' "$x" ;;
|
||
esac
|
||
}
|
||
|
||
# 1 = check GOA-style /intro landing pages (assets, link crawl, demo-withdraw).
|
||
# 0 = skip (e.g. taler-ops.ch / mytops — no public landings).
|
||
: "${CHECK_LANDING:=1}"
|
||
|
||
# Declare bank + exchange + merchant (+ currency, local stack, landing, domain id).
|
||
# Usage:
|
||
# set_taler_stack BANK EXCHANGE MERCHANT [CURRENCY] [LOCAL 0|1] [LANDING 0|1] [TALER_DOMAIN]
|
||
# BANK/EXCHANGE/MERCHANT: hostname or full URL
|
||
set_taler_stack() {
|
||
local bank="${1:-}" exchange="${2:-}" merchant="${3:-}"
|
||
local currency="${4:-}" local_stack="${5:-0}" landing="${6:-}" domain_id="${7:-}"
|
||
|
||
BANK_PUBLIC=$(_to_https_base "$bank")
|
||
EXCHANGE_PUBLIC=$(_to_https_base "$exchange")
|
||
MERCHANT_PUBLIC=$(_to_https_base "$merchant")
|
||
# currency "-" or empty → report-only (do not force GOA)
|
||
if [ -n "$currency" ] && [ "$currency" != "-" ]; then
|
||
EXPECT_CURRENCY="$currency"
|
||
elif [ "$currency" = "-" ]; then
|
||
EXPECT_CURRENCY=""
|
||
fi
|
||
LOCAL_STACK="$local_stack"
|
||
if [ "$LOCAL_STACK" = "1" ]; then
|
||
SKIP_SSH=0
|
||
else
|
||
SKIP_SSH=1
|
||
fi
|
||
# default: landings only matter on local GOA stack
|
||
if [ -n "$landing" ]; then
|
||
CHECK_LANDING="$landing"
|
||
elif [ "$LOCAL_STACK" = "1" ]; then
|
||
CHECK_LANDING=1
|
||
else
|
||
CHECK_LANDING=0
|
||
fi
|
||
[ -n "$domain_id" ] && TALER_DOMAIN="$domain_id"
|
||
|
||
BANK_PUBLIC=${BANK_PUBLIC%/}
|
||
EXCHANGE_PUBLIC=${EXCHANGE_PUBLIC%/}
|
||
MERCHANT_PUBLIC=${MERCHANT_PUBLIC%/}
|
||
# re-evaluate language after domain profile (FrancPaysan → fr)
|
||
if [ "${TALER_MON_LANG_SET:-0}" != "1" ]; then
|
||
TALER_MON_LANG=
|
||
i18n_init
|
||
fi
|
||
}
|
||
|
||
# Load first matching profile from domains.conf.
|
||
# Fields (whitespace-separated; # comments; blank lines ignored):
|
||
# name bank exchange merchant currency local[0|1] landing[0|1] [canonical_domain]
|
||
# Returns 0 if found, 1 if not.
|
||
load_domain_profile() {
|
||
local want="$1" conf="${TALER_DOMAINS_CONF:-}"
|
||
local line name bank exchange merchant currency local_stack landing canon
|
||
[ -n "$want" ] || return 1
|
||
[ -n "$conf" ] && [ -f "$conf" ] || return 1
|
||
|
||
while IFS= read -r line || [ -n "$line" ]; do
|
||
line="${line%%#*}"
|
||
# shellcheck disable=SC2086
|
||
set -- $line
|
||
[ $# -ge 5 ] || continue
|
||
name="$1"
|
||
[ "$name" = "$want" ] || continue
|
||
bank="$2"
|
||
exchange="$3"
|
||
merchant="$4"
|
||
currency="$5"
|
||
local_stack="${6:-0}"
|
||
landing="${7:-}"
|
||
canon="${8:-$name}"
|
||
# Backward compat: old 7th field was canonical domain (contains a dot)
|
||
if [ -n "$landing" ] && [[ "$landing" == *.* && "$landing" != "0" && "$landing" != "1" ]]; then
|
||
canon="$landing"
|
||
landing=""
|
||
fi
|
||
set_taler_stack "$bank" "$exchange" "$merchant" "$currency" "$local_stack" "$landing" "$canon"
|
||
return 0
|
||
done <"$conf"
|
||
return 1
|
||
}
|
||
|
||
_normalize_domain() {
|
||
local d="$1"
|
||
d="${d#https://}"
|
||
d="${d#http://}"
|
||
d="${d%%/*}"
|
||
d="${d%%:*}"
|
||
# Strip service host prefix only if a real domain remains (has a dot).
|
||
# e.g. bank.demo.taler.net → demo.taler.net, taler.hacktivism.ch → hacktivism.ch
|
||
# but NOT taler.net → net
|
||
case "$d" in
|
||
bank.*|exchange.*|taler.*|backend.*|merchant.*|shop.*|libeufin.*|my.*|map.*|monnaie.*)
|
||
rest="${d#*.}"
|
||
if [[ "$rest" == *.* ]]; then
|
||
d="$rest"
|
||
fi
|
||
;;
|
||
esac
|
||
printf '%s' "$d"
|
||
}
|
||
|
||
_probe_https_config() {
|
||
# 0 if https://$1/config returns 200
|
||
local host="$1" code
|
||
code=$(curl -skS --max-redirs 0 -m 4 -o /dev/null -w '%{http_code}' "https://${host}/config" 2>/dev/null || echo 000)
|
||
[ "$code" = "200" ]
|
||
}
|
||
|
||
apply_taler_domain() {
|
||
local raw="${1:-}"
|
||
local d
|
||
[ -n "$raw" ] || return 0
|
||
d=$(_normalize_domain "$raw")
|
||
TALER_DOMAIN="$d"
|
||
|
||
# 1) Explicit profile (domains.conf) — preferred way to add stacks
|
||
if load_domain_profile "$d" || load_domain_profile "$raw"; then
|
||
# Optional tiny defaults for e2e ladders by currency
|
||
case "${EXPECT_CURRENCY:-}" in
|
||
GOA)
|
||
: "${WITHDRAW_AMT:=GOA:20}"
|
||
: "${PAY_AMT:=GOA:0.01}"
|
||
: "${CREDIT_AMT:=GOA:400}"
|
||
;;
|
||
KUDOS|TESTKUDOS)
|
||
MERCHANT_INSTANCE="${MERCHANT_INSTANCE:-sandbox}"
|
||
WITHDRAW_AMT="${WITHDRAW_AMT:-${EXPECT_CURRENCY}:20}"
|
||
PAY_AMT="${PAY_AMT:-${EXPECT_CURRENCY}:0.01}"
|
||
CREDIT_AMT="${CREDIT_AMT:-${EXPECT_CURRENCY}:100}"
|
||
;;
|
||
TESTPAYSAN)
|
||
# Stage FrancPaysan: farmer shops (override GOA default goa-demo-cp4zqk)
|
||
case "${MERCHANT_INSTANCE:-}" in
|
||
""|goa-demo*|goa-shop)
|
||
MERCHANT_INSTANCE=fermes-des-collines
|
||
;;
|
||
esac
|
||
MERCHANT_INSTANCE=$(printf '%s' "${MERCHANT_INSTANCE}" | tr '[:upper:]' '[:lower:]')
|
||
WITHDRAW_AMT="${WITHDRAW_AMT:-TESTPAYSAN:50}"
|
||
PAY_AMT="${PAY_AMT:-TESTPAYSAN:5}"
|
||
# Cover oeufs(5)+fromage(8.5)+jus(6)+shop pick(~5–12) + fees
|
||
CREDIT_AMT="${CREDIT_AMT:-TESTPAYSAN:200}"
|
||
# Public template pays on by default (shops landings)
|
||
: "${E2E_USE_TEMPLATES:=1}"
|
||
# Map pay-ladder amounts → public shop templates (instance set in e2e)
|
||
: "${E2E_TEMPLATE_MAP:=5=oeufs-6 4.5=pain-seigle 6=jus-pomme 8.5=fromage-chevre 12=miel-printemps 25=panier-legumes}"
|
||
: "${E2E_PAY_VALUES:=5 8.5 6}"
|
||
: "${E2E_WITHDRAW_VALUES:=20 50}"
|
||
# Inside checks as low-priv stagepaysan (podman, no sudo) — not koopa.
|
||
# INSIDE_SSH / container names: set in host env (no hardcoded SSH host).
|
||
INSIDE_PROFILE="${INSIDE_PROFILE:-stage-lfp}"
|
||
INSIDE_BANK_CTR="${INSIDE_BANK_CTR:-stage-lfp-bank}"
|
||
INSIDE_EXCHANGE_CTR="${INSIDE_EXCHANGE_CTR:-stage-lfp-exchange-ansible}"
|
||
INSIDE_MERCHANT_CTR="${INSIDE_MERCHANT_CTR:-stage-lfp-merchant}"
|
||
# Host pasta ports (127.0.0.1 on the FP host (INSIDE_SSH))
|
||
INSIDE_BANK_PORT="${INSIDE_BANK_PORT:-9032}"
|
||
INSIDE_EXCHANGE_PORT="${INSIDE_EXCHANGE_PORT:-9031}"
|
||
INSIDE_MERCHANT_PORT="${INSIDE_MERCHANT_PORT:-9030}"
|
||
INSIDE_DNS_BANK="${INSIDE_DNS_BANK:-stage.bank.lefrancpaysan.ch}"
|
||
INSIDE_DNS_EXCHANGE="${INSIDE_DNS_EXCHANGE:-stage.exchange.lefrancpaysan.ch}"
|
||
INSIDE_DNS_MERCHANT="${INSIDE_DNS_MERCHANT:-stage.monnaie.lefrancpaysan.ch}"
|
||
;;
|
||
CHF)
|
||
WITHDRAW_AMT="${WITHDRAW_AMT:-CHF:20}"
|
||
PAY_AMT="${PAY_AMT:-CHF:0.01}"
|
||
CREDIT_AMT="${CREDIT_AMT:-CHF:100}"
|
||
;;
|
||
esac
|
||
else
|
||
# 2) Unknown domain — heuristics only (prefer profile in domains.conf)
|
||
# Override: --bank / --exchange / --merchant / --currency
|
||
# Do not keep default GOA: empty currency = report only, no hard fail.
|
||
BANK_PUBLIC="https://bank.${d}"
|
||
EXCHANGE_PUBLIC="https://exchange.${d}"
|
||
# TOPS-style multi-tenant merchant first, then legacy names
|
||
MERCHANT_PUBLIC="https://my.${d}"
|
||
EXPECT_CURRENCY=""
|
||
LOCAL_STACK=0
|
||
SKIP_SSH=1
|
||
CHECK_LANDING=0
|
||
if [ "${TALER_DOMAIN_PROBE}" = "1" ]; then
|
||
local h
|
||
for h in "monnaie.${d}" "backend.${d}" "my.${d}" "taler.${d}" "merchant.${d}" "shop.${d}"; do
|
||
_probe_https_config "$h" && { MERCHANT_PUBLIC="https://$h"; break; }
|
||
done
|
||
for h in "bank.${d}" "libeufin.${d}"; do
|
||
_probe_https_config "$h" && { BANK_PUBLIC="https://$h"; break; }
|
||
done
|
||
_probe_https_config "exchange.${d}" || true
|
||
fi
|
||
fi
|
||
|
||
BANK_PUBLIC=${BANK_PUBLIC%/}
|
||
EXCHANGE_PUBLIC=${EXCHANGE_PUBLIC%/}
|
||
MERCHANT_PUBLIC=${MERCHANT_PUBLIC%/}
|
||
# re-evaluate language after domain profile (FrancPaysan → fr)
|
||
if [ "${TALER_MON_LANG_SET:-0}" != "1" ]; then
|
||
TALER_MON_LANG=
|
||
i18n_init
|
||
fi
|
||
|
||
# Koopa SSH only for LOCAL_STACK=1. Stage LFP uses INSIDE_SSH (stagepaysan) separately.
|
||
if [ "${LOCAL_STACK}" != "1" ]; then
|
||
SKIP_SSH=1
|
||
fi
|
||
# Default inside profile for local GOA
|
||
if [ "${LOCAL_STACK}" = "1" ]; then
|
||
INSIDE_PROFILE="${INSIDE_PROFILE:-koopa}"
|
||
fi
|
||
}
|
||
|
||
# Apply TALER_DOMAIN from env once (CLI exports TALER_DOMAIN_APPLIED=1 after overrides).
|
||
if [ "${TALER_DOMAIN_APPLIED:-0}" != "1" ] \
|
||
&& [ -n "${TALER_DOMAIN:-}" ] && [ "${TALER_DOMAIN}" != "hacktivism.ch" ]; then
|
||
apply_taler_domain "$TALER_DOMAIN"
|
||
TALER_DOMAIN_APPLIED=1
|
||
fi
|
||
|
||
# Safe SSH: publickey only, short connect, overall alarm so we never block forever.
|
||
SSH_BASE_OPTS=(
|
||
-o BatchMode=yes
|
||
-o ConnectTimeout="${SSH_CONNECT_TIMEOUT}"
|
||
-o ConnectionAttempts=1
|
||
-o ServerAliveInterval=3
|
||
-o ServerAliveCountMax=2
|
||
-o StrictHostKeyChecking=accept-new
|
||
-o PreferredAuthentications=publickey
|
||
-o PasswordAuthentication=no
|
||
-o KbdInteractiveAuthentication=no
|
||
-o GSSAPIAuthentication=no
|
||
-o NumberOfPasswordPrompts=0
|
||
)
|
||
|
||
# Hard wall-clock timeout so ssh/curl never block the monitoring run forever.
|
||
with_timeout() {
|
||
local secs="$1"; shift
|
||
local rc
|
||
if command -v gtimeout >/dev/null 2>&1; then
|
||
gtimeout --kill-after=2 "$secs" "$@"
|
||
return $?
|
||
fi
|
||
if command -v timeout >/dev/null 2>&1; then
|
||
# Prefer GNU -k; if that flag form is unsupported, fall back once.
|
||
# Do NOT use `cmd || cmd` on the child — a phase exit 1 would re-run it.
|
||
set +e
|
||
timeout -k 2 "$secs" "$@" 2>/dev/null
|
||
rc=$?
|
||
set -e
|
||
# 125 = timeout(1) itself failed (bad options); 126/127 = not executable
|
||
if [ "$rc" -eq 125 ] || [ "$rc" -eq 126 ] || [ "$rc" -eq 127 ]; then
|
||
timeout --kill-after=2 "$secs" "$@"
|
||
return $?
|
||
fi
|
||
return "$rc"
|
||
fi
|
||
# Portable: perl alarm + process group kill
|
||
perl -e '
|
||
use strict; use warnings;
|
||
my $secs = shift @ARGV;
|
||
my $pid = fork();
|
||
die "fork: $!" unless defined $pid;
|
||
if ($pid == 0) {
|
||
setpgrp(0, 0);
|
||
exec @ARGV;
|
||
exit 127;
|
||
}
|
||
$SIG{ALRM} = sub {
|
||
kill "TERM", -$pid;
|
||
select(undef, undef, undef, 1.0);
|
||
kill "KILL", -$pid;
|
||
exit 124;
|
||
};
|
||
alarm $secs;
|
||
waitpid($pid, 0);
|
||
my $code = $? >> 8;
|
||
alarm 0;
|
||
exit $code;
|
||
' "$secs" "$@"
|
||
}
|
||
|
||
# Generic short SSH to a named host (low-priv stagepaysan or koopa).
|
||
# usage: mon_ssh_bash HOST [timeout] <<'EOF' ... EOF
|
||
mon_ssh_bash() {
|
||
local host="$1" t="${2:-$SSH_CMD_TIMEOUT}"
|
||
[ -n "$host" ] || return 1
|
||
with_timeout "$t" ssh "${SSH_BASE_OPTS[@]}" "$host" bash -s
|
||
}
|
||
mon_ssh_ok() {
|
||
local host="$1"
|
||
[ -n "$host" ] || return 1
|
||
with_timeout $((SSH_CONNECT_TIMEOUT + 5)) \
|
||
ssh "${SSH_BASE_OPTS[@]}" "$host" true >/dev/null 2>&1
|
||
}
|
||
|
||
# Pick a working SSH host: KOOPA_SSH first, then KOOPA_SSH_FALLBACKS (from env).
|
||
# Sets KOOPA_SSH to the first host that answers. 0 = ok, 1 = none.
|
||
KOOPA_SSH_RESOLVED=0
|
||
resolve_koopa_ssh() {
|
||
[ "${SKIP_SSH:-0}" = "1" ] && return 1
|
||
if [ "${KOOPA_SSH_RESOLVED}" = "1" ]; then
|
||
return 0
|
||
fi
|
||
local cands=() c f seen=" "
|
||
cands+=("${KOOPA_SSH}")
|
||
# shellcheck disable=SC2086
|
||
for f in ${KOOPA_SSH_FALLBACKS}; do
|
||
case "$seen" in *" $f "*) continue ;; esac
|
||
cands+=("$f")
|
||
seen="$seen$f "
|
||
done
|
||
for c in "${cands[@]}"; do
|
||
if with_timeout $((SSH_CONNECT_TIMEOUT + 5)) \
|
||
ssh "${SSH_BASE_OPTS[@]}" "$c" 'echo ok' >/dev/null 2>&1; then
|
||
if [ "$c" != "${KOOPA_SSH}" ]; then
|
||
# surface once so operators know we used WAN jump
|
||
printf '[INFO] SSH host %s unreachable — using %s\n' "${KOOPA_SSH}" "$c" >&2 || true
|
||
fi
|
||
KOOPA_SSH="$c"
|
||
KOOPA_SSH_RESOLVED=1
|
||
export KOOPA_SSH
|
||
return 0
|
||
fi
|
||
done
|
||
return 1
|
||
}
|
||
|
||
# Probe: 0 if any koopa SSH host works quickly
|
||
koopa_ssh_ok() {
|
||
[ "${SKIP_SSH}" = "1" ] && return 1
|
||
resolve_koopa_ssh
|
||
}
|
||
|
||
# Run remote bash -s with optional stdin script; hard-capped
|
||
# usage: koopa_ssh_bash [timeout_secs] <<'EOF' ... EOF
|
||
# or: koopa_ssh_run timeout_secs 'remote command'
|
||
koopa_ssh_run() {
|
||
local t="${1:-$SSH_CMD_TIMEOUT}"
|
||
shift
|
||
resolve_koopa_ssh || return 1
|
||
with_timeout "$t" ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" "$@"
|
||
}
|
||
|
||
koopa_ssh_bash() {
|
||
local t="${1:-$SSH_CMD_TIMEOUT}"
|
||
resolve_koopa_ssh || return 1
|
||
with_timeout "$t" ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" 'bash -s'
|
||
}
|
||
|
||
# stdin → remote python3 - (for metrics load probe)
|
||
koopa_ssh_python() {
|
||
local t="${1:-60}"
|
||
resolve_koopa_ssh || return 1
|
||
with_timeout "$t" ssh "${SSH_BASE_OPTS[@]}" "${KOOPA_SSH}" python3 -
|
||
}
|
||
|
||
# ANSI colours — left badge (clear severity) + body label colour + dim detail.
|
||
# Off: NO_COLOR=1 or CLICOLOR=0.
|
||
# OK green badge · green label
|
||
# INFO blue badge · cyan label
|
||
# WARN yellow badge · yellow label (never red)
|
||
# ERROR red badge · red label
|
||
# BLOCKER magenta badge· magenta label
|
||
# progress cyan badge = " 42%" + bar fills left→right
|
||
if [ "${NO_COLOR:-0}" = "1" ] || [ "${CLICOLOR:-1}" = "0" ]; then
|
||
G= R= Y= C= M= D= W= B= N=
|
||
BG_OK= BG_INFO= BG_WARN= BG_ERR= BG_BLK= BG_SEC= BG_PROG=
|
||
BOX=0
|
||
else
|
||
G=$'\e[1;32m' # green
|
||
R=$'\e[1;31m' # red
|
||
Y=$'\e[1;33m' # yellow
|
||
C=$'\e[1;36m' # cyan
|
||
M=$'\e[1;35m' # magenta
|
||
D=$'\e[2m' # dim tid / detail
|
||
W=$'\e[1;37m' # bold white
|
||
B=$'\e[1m' # bold
|
||
N=$'\e[0m' # reset
|
||
# High-contrast left badges (bright white / black on strong bg)
|
||
BG_OK=$'\e[1;97;42m' # white on green
|
||
BG_INFO=$'\e[1;97;44m' # white on blue
|
||
BG_WARN=$'\e[1;30;103m' # black on bright yellow — not red
|
||
BG_ERR=$'\e[1;97;41m' # white on red — errors only
|
||
BG_BLK=$'\e[1;97;45m' # white on magenta — blockers
|
||
BG_SEC=$'\e[1;97;44m' # white on blue (section)
|
||
BG_PROG=$'\e[1;30;106m' # black on bright cyan (progress)
|
||
BOX=1
|
||
fi
|
||
|
||
PASS_N=0
|
||
FAIL_N=0
|
||
WARN_N=0
|
||
INFO_N=0
|
||
BLOCKERS=() # human-readable payment/withdraw blockers
|
||
ERRORS=() # all ERROR lines (component scope)
|
||
|
||
# Grouped test IDs: area.group-NN (e.g. www.exchange-01, e2e.pay-03)
|
||
# plus global run number: #042 (monotonic for the whole ./taler-monitoring.sh run)
|
||
# set_area www # phase: www | e2e | inside | versions | …
|
||
# set_group exchange # → www.exchange-01
|
||
# set_group bank # → www.bank-01
|
||
# Without set_group: area-01, area-02, … (flat within area)
|
||
# Line shape: ┌ OK ┐ #003 www.exchange-02 label · detail
|
||
TEST_AREA=""
|
||
TEST_GROUP=""
|
||
TEST_N=0
|
||
LAST_TID=""
|
||
# Global / progress: may continue across check_*.sh processes via TALER_MON_STATE
|
||
GLOBAL_N="${GLOBAL_N:-0}"
|
||
LAST_GLOBAL=""
|
||
PROGRESS_DONE="${PROGRESS_DONE:-0}"
|
||
PROGRESS_TOTAL="${PROGRESS_TOTAL:-0}"
|
||
PROGRESS_SHOW_EVERY="${PROGRESS_SHOW_EVERY:-8}"
|
||
PROGRESS_LAST_SHOWN="${PROGRESS_LAST_SHOWN:-0}"
|
||
_mon_state_load() {
|
||
[ -n "${TALER_MON_STATE:-}" ] && [ -f "${TALER_MON_STATE}" ] || return 0
|
||
# shellcheck disable=SC1090
|
||
. "${TALER_MON_STATE}"
|
||
}
|
||
_mon_state_save() {
|
||
[ -n "${TALER_MON_STATE:-}" ] || return 0
|
||
{
|
||
printf 'GLOBAL_N=%s\n' "$GLOBAL_N"
|
||
printf 'PROGRESS_DONE=%s\n' "$PROGRESS_DONE"
|
||
printf 'PROGRESS_TOTAL=%s\n' "$PROGRESS_TOTAL"
|
||
printf 'PROGRESS_LAST_SHOWN=%s\n' "$PROGRESS_LAST_SHOWN"
|
||
} >"${TALER_MON_STATE}"
|
||
}
|
||
_mon_state_load
|
||
# Per-group counters so re-entering www.exchange after perf continues NN
|
||
# (shell vars TEST_CNT_<area>_<group>).
|
||
_tid_key() {
|
||
# $1=area $2=group → safe identifier
|
||
printf 'TEST_CNT_%s_%s' "$1" "$2" | tr -c 'A-Za-z0-9_' '_'
|
||
}
|
||
_tid_save() {
|
||
[ -z "${TEST_AREA:-}" ] || [ -z "${TEST_GROUP:-}" ] && return 0
|
||
local k
|
||
k=$(_tid_key "$TEST_AREA" "$TEST_GROUP")
|
||
eval "$k=\$TEST_N"
|
||
}
|
||
_tid_load() {
|
||
local k
|
||
k=$(_tid_key "$TEST_AREA" "$1")
|
||
eval "TEST_N=\${$k:-0}"
|
||
}
|
||
set_area() {
|
||
# leave previous group counter saved
|
||
_tid_save
|
||
TEST_AREA="$1"
|
||
TEST_GROUP=""
|
||
TEST_N=0
|
||
LAST_TID=""
|
||
}
|
||
# Start a logical sub-group. Short stable names:
|
||
# exchange bank merchant perf stats landing paivana
|
||
# prereq wallet atm settle pay shop dig load report
|
||
# ssh caddy outside inside compare withdraw
|
||
# Counter continues if the same group is re-entered later in the area.
|
||
# Prints a compact group chip so log + issue text map cleanly (www.bank-03).
|
||
set_group() {
|
||
local g="$1"
|
||
# no-op if same group already active (e.g. inside emit loop)
|
||
if [ "$g" = "${TEST_GROUP:-}" ] && [ -n "$g" ]; then
|
||
return 0
|
||
fi
|
||
_tid_save
|
||
TEST_GROUP="$g"
|
||
LAST_TID=""
|
||
if [ -z "${TEST_AREA:-}" ] || [ -z "$g" ]; then
|
||
TEST_N=0
|
||
return 0
|
||
fi
|
||
_tid_load "$g"
|
||
local tag="${TEST_AREA}.${g}"
|
||
if [ "${BOX:-0}" = "1" ]; then
|
||
# cyan-outline group chip: ┌ www.exchange ┐
|
||
printf '%s%s┌ %s ┐%s\n' "$D" "$C" "$tag" "$N"
|
||
else
|
||
printf -- '-- %s --\n' "$tag"
|
||
fi
|
||
# show progress when entering a new group (if totals known)
|
||
_progress_maybe_show 1
|
||
}
|
||
set_progress_total() {
|
||
# Optional: expected number of numbered checks in this run.
|
||
# PROGRESS_TOTAL=0 → no percent, only "done=N".
|
||
# Does not reset GLOBAL_N / PROGRESS_DONE if already mid-run (state file).
|
||
# May shrink when parent re-fits after a short phase (total = done + remaining est),
|
||
# but never below PROGRESS_DONE.
|
||
local n="${1:-0}"
|
||
if [ "${PROGRESS_DONE:-0}" -gt 0 ] && [ "$n" -gt 0 ] && [ "$n" -lt "$PROGRESS_DONE" ]; then
|
||
n=$PROGRESS_DONE
|
||
fi
|
||
PROGRESS_TOTAL="$n"
|
||
if [ "${PROGRESS_DONE:-0}" -eq 0 ]; then
|
||
PROGRESS_LAST_SHOWN=0
|
||
fi
|
||
_mon_state_save
|
||
}
|
||
add_progress_total() {
|
||
local n="${1:-0}"
|
||
PROGRESS_TOTAL=$((PROGRESS_TOTAL + n))
|
||
_mon_state_save
|
||
}
|
||
# When the global estimate was short, grow total just enough so done≤total.
|
||
# Never shrink. Mid-run headroom is small and monotonic (stable N in done/N).
|
||
_progress_rebalance() {
|
||
[ "${PROGRESS_TOTAL:-0}" -gt 0 ] || return 0
|
||
[ "${PROGRESS_DONE:-0}" -gt 0 ] || return 0
|
||
if [ "$PROGRESS_DONE" -gt "$PROGRESS_TOTAL" ]; then
|
||
# + max(8, 5% of done) so we do not snap to 100% then jump again next check
|
||
local head=$((PROGRESS_DONE / 20))
|
||
[ "$head" -lt 8 ] && head=8
|
||
PROGRESS_TOTAL=$((PROGRESS_DONE + head))
|
||
_mon_state_save
|
||
fi
|
||
}
|
||
# Snap total to actual done — only at whole-run end (not after each phase).
|
||
progress_finish() {
|
||
[ "${PROGRESS_OFF:-0}" = "1" ] && return 0
|
||
[ "${PROGRESS_DONE:-0}" -gt 0 ] || return 0
|
||
if [ "${PROGRESS_TOTAL:-0}" -ne "$PROGRESS_DONE" ]; then
|
||
PROGRESS_TOTAL=$PROGRESS_DONE
|
||
_mon_state_save
|
||
fi
|
||
_progress_bar_line "$PROGRESS_DONE" "$PROGRESS_TOTAL"
|
||
if [ "${GLOBAL_N:-0}" -gt 0 ]; then
|
||
printf -- '%s %s %d/%d (#001…#%03d)%s\n' \
|
||
"$D" "$(i18n_text 'global checks:')" \
|
||
"$PROGRESS_DONE" "$PROGRESS_TOTAL" "$GLOBAL_N" "$N"
|
||
fi
|
||
}
|
||
_progress_bar_line() {
|
||
# Format (grows left → right; badge = percent):
|
||
# ┌ 42% ┐ ████████░░░░░░░░░░░░░░░░ 31/74
|
||
# done/total = global numbered checks so far / global expected total
|
||
local done="$1" total="$2" width=28 pct=0 filled empty i bar pct_lab
|
||
# Defensive: never display done > total
|
||
if [ "$total" -gt 0 ] && [ "$done" -gt "$total" ]; then
|
||
total=$done
|
||
fi
|
||
if [ "$total" -gt 0 ]; then
|
||
# integer percent; while mid-run past a short estimate, cap display < 100%
|
||
pct=$((done * 100 / total))
|
||
[ "$pct" -gt 100 ] && pct=100
|
||
if [ "$done" -lt "$total" ] && [ "$pct" -ge 100 ]; then
|
||
pct=99
|
||
fi
|
||
# fill cells from the left (progress moves rightward as filled grows)
|
||
filled=$((done * width / total))
|
||
[ "$filled" -gt "$width" ] && filled=$width
|
||
# show at least 1 block once any work done (unless 0%)
|
||
if [ "$done" -gt 0 ] && [ "$filled" -eq 0 ]; then
|
||
filled=1
|
||
fi
|
||
# at 100% fill complete bar
|
||
if [ "$done" -ge "$total" ]; then
|
||
filled=$width
|
||
pct=100
|
||
fi
|
||
empty=$((width - filled))
|
||
bar=""
|
||
i=0
|
||
while [ "$i" -lt "$filled" ]; do bar="${bar}█"; i=$((i + 1)); done
|
||
i=0
|
||
while [ "$i" -lt "$empty" ]; do bar="${bar}░"; i=$((i + 1)); done
|
||
pct_lab=$(printf '%3d%%' "$pct")
|
||
if [ "${BOX:-0}" = "1" ]; then
|
||
# badge = percent; bar grows ░→█ left to right; N/M = global check count
|
||
printf '%s┌%s┐%s %s%s%s %s%d/%d%s\n' \
|
||
"$BG_PROG" "$pct_lab" "$N" \
|
||
"$C" "$bar" "$N" \
|
||
"$D" "$done" "$total" "$N"
|
||
else
|
||
printf -- '%s%4s%s %s %d/%d\n' \
|
||
"$C" "$pct_lab" "$N" "$bar" "$done" "$total"
|
||
fi
|
||
else
|
||
if [ "${BOX:-0}" = "1" ]; then
|
||
printf '%s┌ … ┐%s %s%d/?%s (set PROGRESS_TOTAL= for global %%)\n' \
|
||
"$BG_PROG" "$N" "$D" "$done" "$N"
|
||
else
|
||
printf -- '%s%d/?%s (set PROGRESS_TOTAL= for global %%)\n' "$D" "$done" "$N"
|
||
fi
|
||
fi
|
||
}
|
||
_progress_maybe_show() {
|
||
local force="${1:-0}"
|
||
[ "${PROGRESS_OFF:-0}" = "1" ] && return 0
|
||
[ "$PROGRESS_DONE" -le 0 ] && [ "$force" != "1" ] && return 0
|
||
_progress_rebalance
|
||
if [ "$force" = "1" ] || \
|
||
[ $((PROGRESS_DONE - PROGRESS_LAST_SHOWN)) -ge "$PROGRESS_SHOW_EVERY" ] || \
|
||
{ [ "$PROGRESS_TOTAL" -gt 0 ] && [ "$PROGRESS_DONE" -ge "$PROGRESS_TOTAL" ]; }; then
|
||
_progress_bar_line "$PROGRESS_DONE" "$PROGRESS_TOTAL"
|
||
PROGRESS_LAST_SHOWN=$PROGRESS_DONE
|
||
fi
|
||
}
|
||
# Assign next id into LAST_TID (must not run in a subshell).
|
||
_take_tid() {
|
||
LAST_TID=""
|
||
LAST_GLOBAL=""
|
||
if [ -z "${TEST_AREA:-}" ]; then
|
||
return
|
||
fi
|
||
# Global monotonic number for the whole monitoring run (#001 …)
|
||
GLOBAL_N=$((GLOBAL_N + 1))
|
||
LAST_GLOBAL=$(printf '#%03d' "$GLOBAL_N")
|
||
TEST_N=$((TEST_N + 1))
|
||
if [ -n "${TEST_GROUP:-}" ]; then
|
||
LAST_TID=$(printf '%s.%s-%02d' "$TEST_AREA" "$TEST_GROUP" "$TEST_N")
|
||
_tid_save
|
||
else
|
||
LAST_TID=$(printf '%s-%02d' "$TEST_AREA" "$TEST_N")
|
||
fi
|
||
PROGRESS_DONE=$((PROGRESS_DONE + 1))
|
||
_progress_rebalance
|
||
_mon_state_save
|
||
_progress_maybe_show 0
|
||
}
|
||
# Dim ids: "#003 www.exchange-01 " or empty
|
||
_fmt_tid() {
|
||
if [ -n "${LAST_GLOBAL:-}" ]; then
|
||
printf '%s%s%s ' "$D" "$LAST_GLOBAL" "$N"
|
||
fi
|
||
if [ -n "${LAST_TID:-}" ]; then
|
||
printf '%s%s%s ' "$D" "$LAST_TID" "$N"
|
||
fi
|
||
}
|
||
# Boxed badge cell: ┌ OK ┐ with filled bg (tag left-padded, width 7 for BLOCKER)
|
||
# $1=badge style $2=tag text
|
||
_fmt_badge() {
|
||
local badge="$1" tag="$2" inner
|
||
inner=$(printf -- '%-7s' "$tag")
|
||
if [ "${BOX:-0}" = "1" ]; then
|
||
printf '%s┌ %s┐%s' "$badge" "$inner" "$N"
|
||
else
|
||
printf '%s[ %s]%s' "$badge" "$inner" "$N"
|
||
fi
|
||
}
|
||
# One concrete line: ┌ OK ┐ tid label · detail
|
||
# $1=badge style $2=tag text $3=label colour $4=label $5=detail
|
||
_msg_line() {
|
||
local badge="$1" tag="$2" lcol="$3" label="$4" detail="${5:-}"
|
||
# i18n: tags + free text (en default; fr for FrancPaysan)
|
||
tag=$(i18n_tag "$tag")
|
||
label=$(i18n_text "$label")
|
||
[ -n "$detail" ] && detail=$(i18n_text "$detail")
|
||
if [ -n "$detail" ]; then
|
||
printf -- '%s %s%s%s%s %s·%s %s%s%s\n' \
|
||
"$(_fmt_badge "$badge" "$tag")" "$(_fmt_tid)" "$lcol" "$label" "$N" "$D" "$N" "$D" "$detail" "$N"
|
||
else
|
||
printf -- '%s %s%s%s%s\n' \
|
||
"$(_fmt_badge "$badge" "$tag")" "$(_fmt_tid)" "$lcol" "$label" "$N"
|
||
fi
|
||
}
|
||
|
||
ok() {
|
||
# ok "what is good" ["concrete evidence: HTTP 200 · 12ms · …"]
|
||
local label="$1" detail="${2:-}"
|
||
_take_tid
|
||
_msg_line "$BG_OK" "OK" "$G" "$label" "$detail"
|
||
PASS_N=$((PASS_N + 1))
|
||
}
|
||
# component-scoped error: err bank "what failed" "why / HTTP / path"
|
||
err() {
|
||
local comp="$1" msg="$2" detail="${3:-}"
|
||
_take_tid
|
||
_msg_line "$BG_ERR" "ERROR" "$R" "${comp}: ${msg}" "$detail"
|
||
FAIL_N=$((FAIL_N + 1))
|
||
ERRORS+=("${LAST_TID:+$LAST_TID }[$comp] $msg${detail:+ · $detail}")
|
||
}
|
||
# fail "what failed" ["why / HTTP code / path"]
|
||
fail() {
|
||
local label="$1" detail="${2:-}"
|
||
_take_tid
|
||
_msg_line "$BG_ERR" "ERROR" "$R" "$label" "$detail"
|
||
FAIL_N=$((FAIL_N + 1))
|
||
ERRORS+=("${LAST_TID:+$LAST_TID }$label${detail:+ · $detail}")
|
||
}
|
||
warn() {
|
||
# warn "what is soft-bad" ["why still ok to continue"]
|
||
# warn component "what" "why"
|
||
# Yellow only — never red (red = ERROR / BLOCKER path)
|
||
local a1="${1:-}" a2="${2:-}" a3="${3:-}"
|
||
local head detail
|
||
_take_tid
|
||
if [ -n "$a3" ]; then
|
||
head="${a1}: ${a2}"
|
||
detail="$a3"
|
||
elif [ -n "$a2" ]; then
|
||
head="$a1"
|
||
detail="$a2"
|
||
else
|
||
head="$a1"
|
||
detail=""
|
||
fi
|
||
_msg_line "$BG_WARN" "WARN" "$Y" "$head" "$detail"
|
||
WARN_N=$((WARN_N + 1))
|
||
}
|
||
info() {
|
||
# info "topic" ["concrete fact / value / next step"]
|
||
local label="$1" detail="${2:-}"
|
||
_take_tid
|
||
_msg_line "$BG_INFO" "INFO" "$C" "$label" "$detail"
|
||
INFO_N=$((INFO_N + 1))
|
||
}
|
||
blocker() {
|
||
# Hard stop on pay/withdraw path: blocker "step" "why it cannot continue"
|
||
local step="$1" msg="$2"
|
||
_take_tid
|
||
_msg_line "$BG_BLK" "BLOCKER" "$M" "${step}" "$msg"
|
||
BLOCKERS+=("${LAST_TID:+$LAST_TID }[$step] $msg")
|
||
FAIL_N=$((FAIL_N + 1))
|
||
ERRORS+=("BLOCKER ${LAST_TID:+$LAST_TID }[$step] $msg")
|
||
}
|
||
section() {
|
||
# Boxed section header (3 lines when colour on)
|
||
local title
|
||
title=$(i18n_text "$*")
|
||
local w=${#title}
|
||
[ "$w" -lt 24 ] && w=24
|
||
[ "$w" -gt 56 ] && w=56
|
||
local pad line i
|
||
pad=$(printf -- "%-${w}s" "$title")
|
||
if [ "${BOX:-0}" = "1" ]; then
|
||
line=""
|
||
i=0
|
||
while [ "$i" -lt $((w + 2)) ]; do
|
||
line="${line}═"
|
||
i=$((i + 1))
|
||
done
|
||
printf -- '\n%s╔%s╗%s\n' "$BG_SEC" "$line" "$N"
|
||
printf -- '%s║%s %s%s%s %s║%s\n' "$BG_SEC" "$N" "$B" "$pad" "$N" "$BG_SEC" "$N"
|
||
printf -- '%s╚%s╝%s\n' "$BG_SEC" "$line" "$N"
|
||
else
|
||
printf -- '\n== %s ==\n' "$title"
|
||
fi
|
||
}
|
||
|
||
summary() {
|
||
local blk_n=${#BLOCKERS[@]}
|
||
# Multi-phase runs: no intermediate progress bar / numbered checks / SUMMARY box
|
||
# (v1.13.9+). Solo check_*.sh still full summary. Parent sets TALER_MON_MULTI_PHASE=1
|
||
# and only the last phase gets PROGRESS_FINAL=1. Mid-run still lists ERRORS/BLOCKERS.
|
||
local _full=1
|
||
if [ "${TALER_MON_MULTI_PHASE:-0}" = "1" ] && [ "${PROGRESS_FINAL:-0}" != "1" ]; then
|
||
_full=0
|
||
fi
|
||
|
||
echo ""
|
||
if [ "$_full" = "1" ]; then
|
||
# Progress snapshot — whole-run snap also in progress_finish.
|
||
if [ "${PROGRESS_OFF:-0}" != "1" ] && [ "${PROGRESS_DONE:-0}" -gt 0 ]; then
|
||
if [ "${PROGRESS_FINAL:-0}" = "1" ]; then
|
||
if [ "${PROGRESS_TOTAL:-0}" -gt 0 ] && [ "$PROGRESS_DONE" -ne "$PROGRESS_TOTAL" ]; then
|
||
PROGRESS_TOTAL=$PROGRESS_DONE
|
||
_mon_state_save
|
||
fi
|
||
fi
|
||
_progress_rebalance
|
||
_progress_bar_line "$PROGRESS_DONE" "$PROGRESS_TOTAL"
|
||
fi
|
||
if [ "$GLOBAL_N" -gt 0 ]; then
|
||
if [ "${PROGRESS_TOTAL:-0}" -gt 0 ]; then
|
||
printf -- '%s %s #001…#%03d · %d/%d %s%s\n' \
|
||
"$D" "$(i18n_text 'numbered checks:')" "$GLOBAL_N" \
|
||
"$PROGRESS_DONE" "$PROGRESS_TOTAL" \
|
||
"$(i18n_text '(global done/total)')" "$N"
|
||
else
|
||
printf -- '%s %s #001…#%03d%s\n' "$D" "$(i18n_text 'numbered checks this run:')" "$GLOBAL_N" "$N"
|
||
fi
|
||
fi
|
||
fi
|
||
if [ "$blk_n" -gt 0 ]; then
|
||
if [ "${BOX:-0}" = "1" ]; then
|
||
printf -- '%s┌ %s ┐%s\n' "$BG_BLK" "$(i18n_text 'BLOCKERS · pay/withdraw cannot finish')" "$N"
|
||
else
|
||
printf -- '%s--- %s ---%s\n' "$M" "$(i18n_text 'BLOCKERS (pay/withdraw cannot finish)')" "$N"
|
||
fi
|
||
local b
|
||
for b in "${BLOCKERS[@]}"; do
|
||
printf -- ' %s•%s %s%s%s\n' "$M" "$N" "$W" "$b" "$N"
|
||
done
|
||
fi
|
||
if [ "${#ERRORS[@]}" -gt 0 ] && [ "$blk_n" -lt "${#ERRORS[@]}" ]; then
|
||
if [ "${BOX:-0}" = "1" ]; then
|
||
printf -- '%s┌ %s ┐%s\n' "$BG_ERR" "$(i18n_text 'ERRORS · failed checks')" "$N"
|
||
else
|
||
printf -- '%s--- %s ---%s\n' "$R" "$(i18n_text 'ERRORS (failed checks)')" "$N"
|
||
fi
|
||
local e
|
||
for e in "${ERRORS[@]}"; do
|
||
case "$e" in BLOCKER*) continue ;; esac
|
||
printf -- ' %s•%s %s%s%s\n' "$R" "$N" "$W" "$e" "$N"
|
||
done
|
||
fi
|
||
|
||
# Coloured totals — only at full/final summary (not mid multi-phase)
|
||
if [ "$_full" != "1" ]; then
|
||
return 0
|
||
fi
|
||
if [ "${BOX:-0}" = "1" ]; then
|
||
printf -- '\n%s┌ %s ┐%s\n' "$BG_SEC" "$(i18n_tag SUMMARY)" "$N"
|
||
_sum_badge() { # $1=bg $2=tag $3=fg $4=n
|
||
local _tg
|
||
_tg=$(i18n_tag "$2")
|
||
printf -- ' %s┌ %-5s┐%s %s%4d%s\n' "$1" "$_tg" "$N" "$3" "$4" "$N"
|
||
}
|
||
_sum_badge "$BG_OK" "OK" "$G" "$PASS_N"
|
||
[ "$FAIL_N" -gt 0 ] && _sum_badge "$BG_ERR" "ERROR" "$R" "$FAIL_N"
|
||
[ "$WARN_N" -gt 0 ] && _sum_badge "$BG_WARN" "WARN" "$Y" "$WARN_N"
|
||
[ "$INFO_N" -gt 0 ] && _sum_badge "$BG_INFO" "INFO" "$C" "$INFO_N"
|
||
[ "$blk_n" -gt 0 ] && _sum_badge "$BG_BLK" "BLOCK" "$M" "$blk_n"
|
||
else
|
||
printf -- '\n[ %s ]\n' "$(i18n_tag SUMMARY)"
|
||
printf -- ' [ %-5s ] %s%4d%s\n' "$(i18n_tag OK)" "$G" "$PASS_N" "$N"
|
||
[ "$FAIL_N" -gt 0 ] && printf -- ' [ %-5s ] %s%4d%s\n' "$(i18n_tag ERROR)" "$R" "$FAIL_N" "$N"
|
||
[ "$WARN_N" -gt 0 ] && printf -- ' [ %-5s ] %s%4d%s\n' "$(i18n_tag WARN)" "$Y" "$WARN_N" "$N"
|
||
[ "$INFO_N" -gt 0 ] && printf -- ' [ %-5s ] %s%4d%s\n' "$(i18n_tag INFO)" "$C" "$INFO_N" "$N"
|
||
[ "$blk_n" -gt 0 ] && printf -- ' [ %-5s ] %s%4d%s\n' "$(i18n_tag BLOCK)" "$M" "$blk_n" "$N"
|
||
fi
|
||
# one-line rollup (bold label, severity-coloured counts)
|
||
printf -- ' %stotals:%s %s%d OK%s' "$B" "$N" "$G" "$PASS_N" "$N"
|
||
[ "$FAIL_N" -gt 0 ] && printf -- ' · %s%d ERROR%s' "$R" "$FAIL_N" "$N"
|
||
[ "$WARN_N" -gt 0 ] && printf -- ' · %s%d WARN%s' "$Y" "$WARN_N" "$N"
|
||
[ "$INFO_N" -gt 0 ] && printf -- ' · %s%d INFO%s' "$C" "$INFO_N" "$N"
|
||
[ "$blk_n" -gt 0 ] && printf -- ' · %s%d BLOCKER%s' "$M" "$blk_n" "$N"
|
||
printf '\n'
|
||
# overall verdict
|
||
if [ "$FAIL_N" -eq 0 ] && [ "$blk_n" -eq 0 ] && [ "$WARN_N" -eq 0 ]; then
|
||
if [ "${BOX:-0}" = "1" ]; then
|
||
printf -- ' %s┌ OK ┐%s %sall clear%s\n' "$BG_OK" "$N" "$G" "$N"
|
||
else
|
||
printf -- ' %s[ OK ] all clear%s\n' "$G" "$N"
|
||
fi
|
||
elif [ "$FAIL_N" -eq 0 ] && [ "$blk_n" -eq 0 ]; then
|
||
if [ "${BOX:-0}" = "1" ]; then
|
||
printf -- ' %s┌ WARN ┐%s %swarnings only (no hard fail)%s\n' "$BG_WARN" "$N" "$Y" "$N"
|
||
else
|
||
printf -- ' %s[ WARN ] warnings only (no hard fail)%s\n' "$Y" "$N"
|
||
fi
|
||
else
|
||
_fail_msg=$(i18n_text 'failed — see list above')
|
||
if [ "${BOX:-0}" = "1" ]; then
|
||
printf -- ' %s┌ ERROR ┐%s %s%s%s\n' "$BG_ERR" "$N" "$R" "$_fail_msg" "$N"
|
||
else
|
||
printf -- ' %s[ ERROR ] %s%s\n' "$R" "$_fail_msg" "$N"
|
||
fi
|
||
fi
|
||
[ "$FAIL_N" -eq 0 ]
|
||
}
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Disk space on hosts / containers under test
|
||
# WARN when tight, ERROR when full / critically low
|
||
# DISK_WARN_USED_PCT=85 # free below 15% → WARN
|
||
# DISK_ERR_USED_PCT=95 # free below 5% → ERROR
|
||
# DISK_ERR_FREE_BYTES=0 # free == 0 always ERROR
|
||
# ---------------------------------------------------------------------------
|
||
: "${DISK_WARN_USED_PCT:=85}"
|
||
: "${DISK_ERR_USED_PCT:=95}"
|
||
|
||
# Parse one POSIX `df -P` body line (no header): fs size used avail capacity mount
|
||
# size/used/avail in 1K-blocks when using df -Pk
|
||
_mon_disk_eval_line() {
|
||
local where="$1" fs="$2" size="$3" used="$4" avail="$5" cap="$6" mnt="$7"
|
||
local pct label detail
|
||
pct=${cap%%%}
|
||
# non-numeric capacity → skip
|
||
case "$pct" in
|
||
''|*[!0-9]*) return 0 ;;
|
||
esac
|
||
label="disk ${where} ${mnt}"
|
||
detail="fs=$fs size_1k=$size used_1k=$used avail_1k=$avail use=${pct}%"
|
||
if [ "$avail" = "0" ] || [ "$pct" -ge 100 ]; then
|
||
err "disk" "${where} ${mnt} FULL / overflow" "$detail"
|
||
return 1
|
||
fi
|
||
if [ "$pct" -ge "${DISK_ERR_USED_PCT}" ]; then
|
||
err "disk" "${where} ${mnt} critically low free space (≥${DISK_ERR_USED_PCT}% used)" "$detail"
|
||
return 1
|
||
fi
|
||
if [ "$pct" -ge "${DISK_WARN_USED_PCT}" ]; then
|
||
warn "disk" "${where} ${mnt} free space tight (≥${DISK_WARN_USED_PCT}% used)" "$detail"
|
||
return 0
|
||
fi
|
||
ok "$label" "$detail"
|
||
return 0
|
||
}
|
||
|
||
# Report all filesystems from `df -P` / `df -Pk` output (with header).
|
||
# $1 = where label (host, container:name, ssh:host)
|
||
# $2 = full df text
|
||
mon_disk_report_df() {
|
||
local where="$1" text="$2" line fs size used avail cap mnt rest ec=0
|
||
[ -n "$text" ] || {
|
||
warn "disk" "${where}: no df output"
|
||
return 0
|
||
}
|
||
while IFS= read -r line || [ -n "$line" ]; do
|
||
[ -n "$line" ] || continue
|
||
case "$line" in
|
||
Filesystem*|Filesystem*) continue ;;
|
||
esac
|
||
# df -P: Filesystem 1024-blocks Used Available Capacity Mounted on
|
||
# shellcheck disable=SC2086
|
||
set -- $line
|
||
[ "$#" -ge 6 ] || continue
|
||
fs=$1
|
||
size=$2
|
||
used=$3
|
||
avail=$4
|
||
cap=$5
|
||
shift 5
|
||
mnt=$*
|
||
# skip special/pseudo if tiny and not root-ish
|
||
case "$fs" in
|
||
tmpfs|devtmpfs|overlay|shm|nsfs|proc|sysfs|cgroup*)
|
||
# still check root-like mounts that fill (overlay / in containers)
|
||
case "$mnt" in
|
||
/|/var|/var/*|/home|/home/*|/data|/mnt/*) ;;
|
||
*) continue ;;
|
||
esac
|
||
;;
|
||
esac
|
||
if ! _mon_disk_eval_line "$where" "$fs" "$size" "$used" "$avail" "$cap" "$mnt"; then
|
||
ec=1
|
||
fi
|
||
done <<<"$text"
|
||
return "$ec"
|
||
}
|
||
|
||
# Local host: important mounts
|
||
mon_disk_check_host() {
|
||
local where text
|
||
where="${1:-host}"
|
||
text=""
|
||
text=$(df -Pk / /var /home /tmp 2>/dev/null | awk 'NR==1 || !seen[$1,$6]++' || true)
|
||
# fallback whole table
|
||
if [ -z "$text" ]; then
|
||
text=$(df -Pk 2>/dev/null || true)
|
||
fi
|
||
mon_disk_report_df "$where" "$text"
|
||
}
|
||
|
||
# Run df inside podman container
|
||
mon_disk_check_podman() {
|
||
local cname="$1" text
|
||
local bin="${2:-podman}"
|
||
text=$("$bin" exec "$cname" df -Pk / /var /tmp 2>/dev/null || "$bin" exec "$cname" df -Pk 2>/dev/null || true)
|
||
mon_disk_report_df "ctr:${cname}" "$text"
|
||
}
|
||
|
||
# df text already collected remotely
|
||
mon_disk_check_remote_text() {
|
||
local where="$1" text="$2"
|
||
mon_disk_report_df "$where" "$text"
|
||
}
|
||
|
||
http_code() {
|
||
local url="$1"; shift
|
||
curl -skS --max-redirs 0 -m "${TIMEOUT}" -o /dev/null -w '%{http_code}' "$@" "$url" 2>/dev/null || echo 000
|
||
}
|
||
|
||
http_body() {
|
||
local url="$1" out="$2"; shift 2
|
||
curl -skS --max-redirs 0 -m "${TIMEOUT}" -o "$out" -w '%{http_code}' "$@" "$url" 2>/dev/null || echo 000
|
||
}
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Currency unit map checks (wallet codec: alt_unit_names must include "0")
|
||
# ---------------------------------------------------------------------------
|
||
# Returns 0 if JSON body at $1 has usable alt_unit_names.
|
||
# Supports:
|
||
# - exchange/bank: currency_specification.alt_unit_names
|
||
# - merchant: currencies.<CODE>.alt_unit_names for each code
|
||
# Optional $2 = required currency code for currency_specification.currency
|
||
json_has_alt_unit_names() {
|
||
local file="$1" want_cur="${2:-}"
|
||
python3 - "$file" "$want_cur" <<'PY'
|
||
import json, sys
|
||
path, want = sys.argv[1], sys.argv[2]
|
||
try:
|
||
d = json.load(open(path))
|
||
except Exception as e:
|
||
print(f"json-error: {e}")
|
||
sys.exit(2)
|
||
|
||
def check_au(au, label):
|
||
if not isinstance(au, dict) or not au:
|
||
print(f"{label}: missing/empty alt_unit_names")
|
||
return False
|
||
if "0" not in au or not str(au.get("0") or "").strip():
|
||
print(f"{label}: alt_unit_names missing non-empty key \"0\" (have {sorted(au.keys())})")
|
||
return False
|
||
print(f"{label}: alt_unit_names ok (0={au.get('0')!r}, n={len(au)})")
|
||
return True
|
||
|
||
ok = True
|
||
cs = d.get("currency_specification")
|
||
if isinstance(cs, dict):
|
||
if want and cs.get("currency") and cs.get("currency") != want:
|
||
print(f"currency_specification.currency={cs.get('currency')!r} want {want!r}")
|
||
ok = False
|
||
if not check_au(cs.get("alt_unit_names"), "currency_specification"):
|
||
ok = False
|
||
elif "currency_specification" in d:
|
||
print("currency_specification: not an object")
|
||
ok = False
|
||
|
||
curs = d.get("currencies")
|
||
if isinstance(curs, dict) and curs:
|
||
for code, spec in curs.items():
|
||
if not isinstance(spec, dict):
|
||
print(f"currencies.{code}: not an object")
|
||
ok = False
|
||
continue
|
||
if not check_au(spec.get("alt_unit_names"), f"currencies.{code}"):
|
||
ok = False
|
||
|
||
if not isinstance(cs, dict) and not (isinstance(curs, dict) and curs):
|
||
# neither shape — fail
|
||
print("no currency_specification or currencies map")
|
||
ok = False
|
||
|
||
sys.exit(0 if ok else 1)
|
||
PY
|
||
}
|
||
|
||
# Check one exchange base URL's /config for alt_unit_names.
|
||
# $1=label $2=base_url $3=expected currency (optional) $4=strict(1) or soft(0)
|
||
check_exchange_alt_units() {
|
||
local label="$1" base="$2" want_cur="${3:-}" strict="${4:-1}"
|
||
local f code
|
||
base="${base%/}"
|
||
f=$(mktemp)
|
||
code=$(http_body "${base}/config" "$f")
|
||
if [ "$code" != "200" ]; then
|
||
rm -f "$f"
|
||
if [ "$strict" = "1" ]; then
|
||
fail "$label /config" "HTTP $code ($base)"
|
||
else
|
||
warn "$label /config" "HTTP $code ($base)"
|
||
fi
|
||
return
|
||
fi
|
||
local out ec
|
||
set +e
|
||
out=$(json_has_alt_unit_names "$f" "$want_cur" 2>&1)
|
||
ec=$?
|
||
set -e
|
||
rm -f "$f"
|
||
if [ "$ec" -eq 0 ]; then
|
||
ok "$label alt_unit_names" "$(echo "$out" | tr '\n' '; ' | sed 's/; $//')"
|
||
else
|
||
if [ "$strict" = "1" ]; then
|
||
fail "$label alt_unit_names" "$(echo "$out" | tr '\n' '; ' | sed 's/; $//')"
|
||
else
|
||
warn "$label alt_unit_names" "$(echo "$out" | tr '\n' '; ' | sed 's/; $//')"
|
||
fi
|
||
fi
|
||
}
|
||
|
||
# From merchant /config JSON file: walk exchanges[] and check each base_url/config.
|
||
# Local stack hosts (hacktivism.ch or $EXCHANGE_PUBLIC host) are strict; others soft.
|
||
check_merchant_listed_exchanges_alt_units() {
|
||
local mer_json="$1"
|
||
local list
|
||
list=$(python3 - "$mer_json" <<'PY'
|
||
import json, sys
|
||
d = json.load(open(sys.argv[1]))
|
||
for e in d.get("exchanges") or []:
|
||
if not isinstance(e, dict):
|
||
continue
|
||
u = (e.get("base_url") or e.get("url") or "").rstrip("/")
|
||
c = e.get("currency") or ""
|
||
if u:
|
||
print(f"{c}\t{u}")
|
||
PY
|
||
)
|
||
if [ -z "$list" ]; then
|
||
fail "merchant exchanges[]" "empty — no exchanges to check for alt_unit_names"
|
||
return
|
||
fi
|
||
local line cur url strict host
|
||
while IFS=$'\t' read -r cur url; do
|
||
[ -n "$url" ] || continue
|
||
host="${url#https://}"; host="${host#http://}"; host="${host%%/*}"
|
||
strict=1
|
||
case "$host" in
|
||
*hacktivism.ch) strict=1 ;;
|
||
*)
|
||
# foreign exchange (e.g. taler-ops) — soft unless it is our configured EXCHANGE_PUBLIC
|
||
if [ "$url" = "${EXCHANGE_PUBLIC}" ] || [ "$url" = "${EXCHANGE_PUBLIC}/" ]; then
|
||
strict=1
|
||
else
|
||
strict=0
|
||
fi
|
||
;;
|
||
esac
|
||
check_exchange_alt_units "exchange ${cur:-?} ${host}" "$url" "$cur" "$strict"
|
||
done <<<"$list"
|
||
}
|
||
|
||
# Live bank/merchant passwords: sibling **koopa-admin-secrets** (never in admin-log).
|
||
# Override: SECRETS_ROOT=/path/to/koopa-admin-secrets/koopa/host-root
|
||
# or KOOPA_ADMIN_SECRETS=/path/to/koopa-admin-secrets
|
||
SECRETS_ROOT="${SECRETS_ROOT:-}"
|
||
_secrets_search_log() { :; } # placeholder if we later want debug
|
||
if [ -z "$SECRETS_ROOT" ]; then
|
||
_mon_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
_admin_log="$(cd "${_mon_dir}/../.." && pwd)"
|
||
# Prefer explicit env pointing at the secrets *repo* root
|
||
if [ -n "${KOOPA_ADMIN_SECRETS:-}" ] && [ -d "${KOOPA_ADMIN_SECRETS}/koopa/host-root/taler-bank" ]; then
|
||
SECRETS_ROOT="${KOOPA_ADMIN_SECRETS}/koopa/host-root"
|
||
fi
|
||
fi
|
||
# No further home-path probes — set SECRETS_ROOT or KOOPA_ADMIN_SECRETS in env.
|
||
|
||
|
||
read_secret() {
|
||
local rel="$1" base val=""
|
||
base=$(basename "$rel")
|
||
if [ -n "${SECRETS_ROOT:-}" ]; then
|
||
if [ -f "${SECRETS_ROOT}/${rel}" ]; then
|
||
tr -d '\n\r' <"${SECRETS_ROOT}/${rel}"
|
||
return 0
|
||
fi
|
||
# also accept flat layout under host-root/
|
||
if [ -f "${SECRETS_ROOT}/${base}" ]; then
|
||
tr -d '\n\r' <"${SECRETS_ROOT}/${base}"
|
||
return 0
|
||
fi
|
||
fi
|
||
# Optional: copy under ~/.config/taler-landing/ (landing-stats style)
|
||
if [ -f "${HOME}/.config/taler-landing/${base}" ]; then
|
||
tr -d '\n\r' <"${HOME}/.config/taler-landing/${base}"
|
||
return 0
|
||
fi
|
||
# Live on koopa host /root (same basenames as deploy)
|
||
if koopa_ssh_ok; then
|
||
val=$(koopa_ssh_run 12 "tr -d '\\n\\r' </root/${base} 2>/dev/null || true" 2>/dev/null || true)
|
||
if [ -n "$val" ]; then
|
||
printf '%s' "$val"
|
||
return 0
|
||
fi
|
||
# container path used by some installs
|
||
val=$(koopa_ssh_run 12 \
|
||
"podman exec taler-hacktivism-bank tr -d '\\n\\r' </root/${base} 2>/dev/null || true" 2>/dev/null || true)
|
||
if [ -n "$val" ]; then
|
||
printf '%s' "$val"
|
||
return 0
|
||
fi
|
||
fi
|
||
return 1
|
||
}
|
||
|
||
# Human hint when secrets missing (e2e prereq)
|
||
secrets_hint() {
|
||
cat <<EOF
|
||
Prefer a local secrets file (not committed):
|
||
cp secrets.env.example secrets.env
|
||
\$EDITOR secrets.env
|
||
Or set SECRETS_ROOT / KOOPA_ADMIN_SECRETS in ~/.config/taler-monitoring/env
|
||
Or export E2E_BANK_ADMIN_PASS / E2E_MERCHANT_TOKEN for one shot.
|
||
EOF
|
||
}
|
||
|
||
# Load optional secrets.env next to this suite (gitignored). Safe KEY=value only.
|
||
# Call once from taler-monitoring.sh after source lib.sh (or any check_*.sh entry).
|
||
load_monitoring_secrets_env() {
|
||
local mon_dir f
|
||
mon_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
f="${MONITORING_SECRETS_ENV:-${mon_dir}/secrets.env}"
|
||
if [ ! -f "$f" ]; then
|
||
return 0
|
||
fi
|
||
# shellcheck disable=SC1090
|
||
set -a
|
||
# Only simple KEY=VALUE lines; ignore comments / blanks
|
||
# shellcheck disable=SC1091
|
||
. "$f"
|
||
set +a
|
||
# Map common aliases → e2e vars if not already set
|
||
: "${E2E_BANK_ADMIN_PASS:=${BANK_ADMIN_PASS:-${BANK_ADMIN_PASSWORD:-}}}"
|
||
: "${E2E_MERCHANT_TOKEN:=${MERCHANT_TOKEN:-${MERCHANT_INSTANCE_PASSWORD:-}}}"
|
||
: "${SECRETS_ROOT:=${KOOPA_SECRETS_HOST_ROOT:-}}"
|
||
if [ -z "${SECRETS_ROOT:-}" ] && [ -n "${KOOPA_ADMIN_SECRETS:-}" ]; then
|
||
if [ -d "${KOOPA_ADMIN_SECRETS}/koopa/host-root/taler-bank" ]; then
|
||
SECRETS_ROOT="${KOOPA_ADMIN_SECRETS}/koopa/host-root"
|
||
fi
|
||
fi
|
||
export E2E_BANK_ADMIN_PASS E2E_MERCHANT_TOKEN SECRETS_ROOT KOOPA_ADMIN_SECRETS
|
||
}
|
||
|
||
find_wallet_cli() {
|
||
# Must return a path suitable for: node "$WALLET_CLI" …
|
||
# Prefer *.mjs / bundled entry; plain /usr/bin/taler-wallet-cli is often a
|
||
# shell wrapper and cannot be passed to node.
|
||
if [ -n "${WALLET_CLI:-}" ] && [ -f "$WALLET_CLI" ]; then
|
||
echo "$WALLET_CLI"; return 0
|
||
fi
|
||
local c cand
|
||
# WALLET_CLI / WALLET_CLI_CANDIDATES from env; then portable package paths only.
|
||
# shellcheck disable=SC2086
|
||
for c in ${WALLET_CLI_CANDIDATES:-} \
|
||
"${HOME}/.local/bin/taler-wallet-cli.mjs" \
|
||
/usr/lib/taler-wallet-cli/node_modules/taler-wallet-cli/bin/taler-wallet-cli.mjs \
|
||
/usr/share/taler-wallet-cli/bin/taler-wallet-cli.mjs
|
||
do
|
||
[ -n "$c" ] && [ -f "$c" ] && { echo "$c"; return 0; }
|
||
done
|
||
# Debian package wrapper → resolve real .mjs
|
||
cand=$(command -v taler-wallet-cli 2>/dev/null || true)
|
||
if [ -n "$cand" ] && [ -f "$cand" ]; then
|
||
if head -1 "$cand" 2>/dev/null | grep -q 'node\|mjs'; then
|
||
# shebang node script or .mjs
|
||
echo "$cand"; return 0
|
||
fi
|
||
# follow symlink into package tree
|
||
if [ -L "$cand" ]; then
|
||
c=$(readlink -f "$cand" 2>/dev/null || true)
|
||
[ -n "$c" ] && [ -f "$c" ] && { echo "$c"; return 0; }
|
||
fi
|
||
fi
|
||
return 1
|
||
}
|