taler-monitoring/check_devtesting.sh

#!/usr/bin/env bash
# check_devtesting.sh — fake-franken (CHF) via rusty.taler-ops.ch taler-devtesting
#
# Requires SSH to Host DEVTESTING_SSH (default: taler-rusty-devtesting →
# rusty.taler-ops.ch user devtesting, forced command taler-devtesting).
#
# Probes:
#   1) CLI reachable (fake-incoming --help)
#   2) geniban → Swiss IBAN (debit side — “enough money” is synthetic per rung)
#   3) baseline fake-incoming CHF:1.00 (plumbing)
#   4) optional amount ladder: DEVTESTING_LADDER=1 (default) — 23 rungs up to 50k CHF
#
# Env:
#   DEVTESTING_SSH              default taler-rusty-devtesting (or user@host)
#   DEVTESTING_AMOUNT           baseline amount (default CHF:1.00)
#   DEVTESTING_CREDIT_PAYTO     exchange IBAN payto
#   DEVTESTING_EXCHANGE_IBAN    fallback CH6808573105529100001 (TOPS CHF exchange)
#   DEVTESTING_LADDER           1 (default) run amount ladder; 0 = baseline only
#   DEVTESTING_LADDER_STEPS     default 23
#   DEVTESTING_LADDER_MAX       default 50000 (CHF major units)
#   DEVTESTING_LADDER_MIN       default 0.01
#   DEVTESTING_LADDER_STOP_ON_FAIL  1 (default) stop ladder on hard fail; 0 = continue
#   DEVTESTING_SKIP=1           skip phase
#
set -euo pipefail
ROOT=$(cd "$(dirname "$0")" && pwd)
# shellcheck source=lib.sh
source "$ROOT/lib.sh"

set_area devtesting
section "devtesting · fake-franken CHF (rusty taler-devtesting)"

# Do NOT honor SKIP_SSH here: stage/tops profiles set SKIP_SSH=1 for container
# inside checks, but rusty.taler-ops.ch is a separate SSH hop (devtesting key).
if [ "${DEVTESTING_SKIP:-0}" = "1" ]; then
  info "skip" "DEVTESTING_SKIP=1"
  exit 0
fi

: "${DEVTESTING_SSH:=}"
: "${DEVTESTING_AMOUNT:=CHF:1.00}"
: "${DEVTESTING_EXCHANGE_IBAN:=CH6808573105529100001}"
: "${DEVTESTING_LADDER:=1}"
: "${DEVTESTING_LADDER_STEPS:=23}"
: "${DEVTESTING_LADDER_MAX:=50000}"
: "${DEVTESTING_LADDER_MIN:=0.01}"
: "${DEVTESTING_LADDER_STOP_ON_FAIL:=1}"
: "${SSH_CONNECT_TIMEOUT:=8}"
: "${SSH_CMD_TIMEOUT:=60}"

ssh_dt() {
  # Prefer Host alias; fall back to explicit user@host if config missing
  local target="$DEVTESTING_SSH"
  if ! ssh -G "$target" >/dev/null 2>&1; then
    target="devtesting@rusty.taler-ops.ch"
  fi
  if command -v timeout >/dev/null 2>&1; then
    timeout "${SSH_CMD_TIMEOUT}" ssh \
      -o BatchMode=yes \
      -o ConnectTimeout="${SSH_CONNECT_TIMEOUT}" \
      -o StrictHostKeyChecking=accept-new \
      "$target" "$@"
  else
    ssh \
      -o BatchMode=yes \
      -o ConnectTimeout="${SSH_CONNECT_TIMEOUT}" \
      -o StrictHostKeyChecking=accept-new \
      "$target" "$@"
  fi
}

# Run one fake-incoming. Sets globals: FI_RC FI_FLAT FI_STATUS
# FI_STATUS: ok_bounce | ok_in | ok_cli | fail_ssh | fail_creditor | fail_nexus | fail_other
run_fake_incoming() {
  local amount="$1" subject="$2" credit="$3" debit="$4"
  local fi_out
  FI_RC=0
  FI_FLAT=""
  FI_STATUS="fail_other"
  fi_out=$(ssh_dt fake-incoming \
    --amount "${amount}" \
    --subject "${subject}" \
    --credit-payto "${credit}" \
    --debit-payto "${debit}" 2>&1) || FI_RC=$?
  FI_FLAT=$(printf '%s' "$fi_out" | tr '\n' ' ')

  if printf '%s' "$FI_FLAT" | grep -qiE 'Permission denied|Connection refused|Could not resolve|No such command'; then
    FI_STATUS=fail_ssh
    return 1
  fi
  if printf '%s' "$FI_FLAT" | grep -qiE 'Creditor must be the exchange'; then
    FI_STATUS=fail_creditor
    return 1
  fi
  if printf '%s' "$FI_FLAT" | grep -qiE 'Missing amount|unable to read secrets|CalledProcessError'; then
    if ! printf '%s' "$FI_FLAT" | grep -qE 'libeufin-nexus - IN .* CHF'; then
      FI_STATUS=fail_nexus
      return 1
    fi
  fi
  if printf '%s' "$FI_FLAT" | grep -qE 'libeufin-nexus - IN .* CHF:[0-9]'; then
    if printf '%s' "$FI_FLAT" | grep -qiE 'bounced.*missing reserve public key'; then
      FI_STATUS=ok_bounce
      return 0
    fi
    if printf '%s' "$FI_FLAT" | grep -qiE 'bounced'; then
      FI_STATUS=ok_bounce
      return 0
    fi
    FI_STATUS=ok_in
    return 0
  fi
  if [ "$FI_RC" -eq 0 ] && printf '%s' "$FI_FLAT" | grep -qi 'Faking incoming CHF'; then
    FI_STATUS=ok_cli
    return 0
  fi
  FI_STATUS=fail_other
  return 1
}

set_group cli
info "ssh" "DEVTESTING_SSH=${DEVTESTING_SSH} baseline=${DEVTESTING_AMOUNT} ladder=${DEVTESTING_LADDER} steps=${DEVTESTING_LADDER_STEPS} max=CHF:${DEVTESTING_LADDER_MAX}"

help_out=$(ssh_dt fake-incoming --help 2>&1) || true
if printf '%s' "$help_out" | grep -q 'fake-incoming'; then
  ok "cli" "taler-devtesting fake-incoming reachable via ${DEVTESTING_SSH}"
else
  fail "cli" "cannot run taler-devtesting on ${DEVTESTING_SSH}" \
    "$(printf '%s' "$help_out" | tr '\n' ' ' | head -c 200)"
  exit 1
fi

set_group geniban
iban_out=$(ssh_dt geniban 2>&1) || true
iban=$(printf '%s\n' "$iban_out" | grep -E '^CH[0-9A-Z]+$' | tail -1 || true)
if [ -n "$iban" ]; then
  ok "geniban" "synthetic IBAN $iban (debit / “withdraw” side for each rung)"
else
  fail "geniban" "expected CH… IBAN" "$(printf '%s' "$iban_out" | tr '\n' ' ' | head -c 160)"
  exit 1
fi

# Credit payto: rusty libeufin-nexus only accepts the *production* exchange IBAN
# configured there (DEVTESTING_EXCHANGE_IBAN). Do not use stage /keys first-account.
set_group credit-payto
credit_payto="${DEVTESTING_CREDIT_PAYTO:-}"
if [ -z "$credit_payto" ]; then
  credit_payto="payto://iban/${DEVTESTING_EXCHANGE_IBAN}?receiver-name=Taler+Operations+AG"
fi
ok "credit-payto" "$credit_payto"

debit_payto="payto://iban/${iban}?receiver-name=MonDevTest"
ts_base=$(date -u +%Y%m%d%H%M%S)

# --- baseline plumbing probe ---
set_group baseline
subject="mon-fake-franken-${ts_base}-base-$$"
info "fake-incoming" "amount=${DEVTESTING_AMOUNT} subject=${subject}"
if run_fake_incoming "$DEVTESTING_AMOUNT" "$subject" "$credit_payto" "$debit_payto"; then
  case "$FI_STATUS" in
    ok_bounce)
      ok "fake-incoming" "CHF IN accepted by nexus · bounce/missing reserve pub — plumbing OK"
      info "note" "use real reserve pub as --subject for wirewatch credit (not required for mon probe)"
      ;;
    ok_in) ok "fake-incoming" "CHF IN recorded by nexus (no bounce line)" ;;
    ok_cli) ok "fake-incoming" "CLI completed CHF fake-incoming" ;;
    *) ok "fake-incoming" "status=${FI_STATUS}" ;;
  esac
else
  case "$FI_STATUS" in
    fail_ssh)
      fail "fake-incoming" "SSH/CLI failure" "$(printf '%s' "$FI_FLAT" | head -c 200)"
      exit 1
      ;;
    fail_creditor)
      fail "fake-incoming" "wrong credit IBAN (not exchange)" "$(printf '%s' "$FI_FLAT" | head -c 220)"
      exit 1
      ;;
    fail_nexus)
      fail "fake-incoming" "nexus rejected request" "$(printf '%s' "$FI_FLAT" | head -c 220)"
      exit 1
      ;;
    *)
      fail "fake-incoming" "no CHF IN line from nexus" "rc=${FI_RC} $(printf '%s' "$FI_FLAT" | head -c 200)"
      exit 1
      ;;
  esac
fi

# --- CHF amount ladder (fake-incoming rungs) ---
if [ "${DEVTESTING_LADDER}" = "1" ]; then
  set_group ladder
  section "devtesting · CHF fake-incoming ladder (→ ${DEVTESTING_LADDER_MAX})"

  LADDER_FILE=$(mktemp "${TMPDIR:-/tmp}/devtesting-ladder.XXXXXX")
  python3 - <<'PY' "${DEVTESTING_LADDER_MAX}" "${DEVTESTING_LADDER_STEPS}" "${DEVTESTING_LADDER_MIN}" "$LADDER_FILE"
import math, sys
from decimal import Decimal, ROUND_HALF_UP
from pathlib import Path

max_amt = Decimal(sys.argv[1])
steps = max(1, int(sys.argv[2]))
min_amt = Decimal(sys.argv[3])
out = Path(sys.argv[4])
if min_amt <= 0:
    min_amt = Decimal("0.01")
if min_amt >= max_amt:
    min_amt = max_amt / Decimal(1000)
# CHF: 2 decimal places
q2 = Decimal("0.01")

def quant(v: Decimal) -> Decimal:
    if v < min_amt:
        v = min_amt
    return v.quantize(q2, rounding=ROUND_HALF_UP)

def fmt(v: Decimal) -> str:
    q = quant(v)
    if q == q.to_integral():
        return str(int(q))
    return format(q, "f")

if steps == 1:
    amts = [quant(max_amt)]
elif steps == 2:
    amts = [quant(min_amt), quant(max_amt)]
else:
    # Exactly `steps` log-spaced rungs from min → max (includes both ends).
    # Guarantees max rung = DEVTESTING_LADDER_MAX (“enough” volume at the top).
    amts = []
    lo, hi = float(min_amt), float(max_amt)
    if hi <= lo:
        hi = lo * 10
    for i in range(steps):
        if steps == 1:
            v = max_amt
        else:
            t = i / (steps - 1)
            v = Decimal(str(math.exp(math.log(lo) + t * (math.log(hi) - math.log(lo)))))
        amts.append(quant(v))
    amts[0] = quant(min_amt)
    amts[-1] = quant(max_amt)
    # enforce strictly increasing
    for i in range(1, len(amts)):
        if amts[i] <= amts[i - 1]:
            amts[i] = quant(amts[i - 1] + min_amt)
    amts[-1] = quant(max_amt)
    if amts[-1] <= amts[-2]:
        # last-but-one squeezed; leave max exact and drop collision by nudging previous
        amts[-2] = quant(max(amts[-3] + min_amt if len(amts) > 2 else min_amt, max_amt - Decimal(1)))
        if amts[-2] >= amts[-1]:
            amts[-2] = quant(amts[-1] - min_amt)

lines = ["CHF:%s" % fmt(a) for a in amts]
out.write_text("\n".join(lines) + "\n")
PY
  n_rungs=$(grep -cE '^CHF:' "$LADDER_FILE" || echo 0)
  ladder_sum=$(python3 -c '
from decimal import Decimal
s=Decimal(0)
for ln in open("'"$LADDER_FILE"'"):
  ln=ln.strip()
  if not ln: continue
  s += Decimal(ln.split(":",1)[-1])
print(s)
' 2>/dev/null || echo "?")

  info "ladder plan" "steps=${n_rungs} max=CHF:${DEVTESTING_LADDER_MAX} min=CHF:${DEVTESTING_LADDER_MIN} sum≈CHF:${ladder_sum} (synthetic debit geniban each rung)"
  info "ladder funding" "each rung uses geniban debit payto — no real bank withdraw; volume covers max rung CHF:${DEVTESTING_LADDER_MAX}"

  LADDER_OK=0
  LADDER_FAIL=0
  LADDER_WARN=0
  rung=0
  while IFS= read -r amt || [ -n "$amt" ]; do
    [ -n "$amt" ] || continue
    rung=$((rung + 1))
    subject="mon-fake-ladder-${ts_base}-r${rung}-$$"
    info "rung ${rung}/${n_rungs}" "amount=${amt} subject=${subject}"
    if run_fake_incoming "$amt" "$subject" "$credit_payto" "$debit_payto"; then
      case "$FI_STATUS" in
        ok_bounce)
          ok "rung ${rung}" "${amt} · nexus IN + bounce (plumbing OK)"
          LADDER_OK=$((LADDER_OK + 1))
          ;;
        ok_in)
          ok "rung ${rung}" "${amt} · nexus IN recorded"
          LADDER_OK=$((LADDER_OK + 1))
          ;;
        ok_cli)
          ok "rung ${rung}" "${amt} · CLI ok"
          LADDER_OK=$((LADDER_OK + 1))
          ;;
        *)
          warn "rung ${rung}" "${amt} · status=${FI_STATUS}"
          LADDER_WARN=$((LADDER_WARN + 1))
          ;;
      esac
    else
      fail "rung ${rung}" "${amt} · ${FI_STATUS}" "$(printf '%s' "$FI_FLAT" | head -c 180)"
      LADDER_FAIL=$((LADDER_FAIL + 1))
      if [ "${DEVTESTING_LADDER_STOP_ON_FAIL}" = "1" ]; then
        warn "ladder stop" "DEVTESTING_LADDER_STOP_ON_FAIL=1 after rung ${rung}"
        break
      fi
    fi
  done <"$LADDER_FILE"
  rm -f "$LADDER_FILE"

  info "ladder summary" "ok=${LADDER_OK} warn=${LADDER_WARN} fail=${LADDER_FAIL} / planned=${n_rungs} · max=CHF:${DEVTESTING_LADDER_MAX}"
  if [ "$LADDER_FAIL" -gt 0 ] && [ "$LADDER_OK" -eq 0 ]; then
    fail "ladder" "all rungs failed"
  elif [ "$LADDER_FAIL" -gt 0 ]; then
    warn "ladder" "partial failures fail=${LADDER_FAIL} ok=${LADDER_OK}"
  else
    ok "ladder complete" "ok=${LADDER_OK} through max CHF:${DEVTESTING_LADDER_MAX}"
  fi
else
  info "ladder" "skipped (DEVTESTING_LADDER=0)"
fi

info "done" "devtesting fake-franken probe finished"
exit 0