monitoring: stats.json freshness + display fields outside-in

scripts/taler-exchange/landing-stats-exchange.sh

@@ -238,6 +238,7 @@ fi
 
 GEN=$(now_iso)
 HUMAN=$(now_human)
+GEN_UNIX=$(date +%s)
 num_or_null() { case "${1:-}" in ''|null) echo null ;; *) echo "$1" ;; esac; }
 
 MEM_JSON='"container_rss_human": "—"'
@@ -257,6 +258,7 @@ cat >"$TMP" <<EOF
   "timezone": $(json_str "$TZ"),
   "generated_at": $(json_str "$GEN"),
   "generated_at_human": $(json_str "$HUMAN"),
+  "generated_at_unix": ${GEN_UNIX},
   "reserves": ${RESERVES:-0},
   "wire_in_count": ${RESERVES_IN:-0},
   "wire_in_amount": $(json_str "$WIRE_IN_AMT"),

scripts/taler-merchant/landing-stats-merchant.sh

@@ -363,6 +363,7 @@ CUR_JSON="${CUR_JSON}
 
 GEN=$(now_iso)
 HUMAN=$(now_human)
+GEN_UNIX=$(date +%s)
 
 # Live performance probes (exchange-style)
 measure_ms() {
@@ -406,6 +407,7 @@ cat >"$TMP" <<EOF
   "timezone": $(json_str "$TZ"),
   "generated_at": $(json_str "$GEN"),
   "generated_at_human": $(json_str "$HUMAN"),
+  "generated_at_unix": ${GEN_UNIX},
   "instances": ${INSTANCES:-0},
   "orders": ${TOTAL_CONTRACTS:-0},
   "paid": ${TOTAL_PAID:-0},

scripts/taler-monitoring/README.md

@@ -43,8 +43,27 @@ End of each phase: totals + list of **BLOCKERS** and **ERRORS** (ids included).
 ./taler-monitoring.sh --bank https://bank.demo.taler.net \
   --exchange https://exchange.demo.taler.net \
   --merchant https://backend.demo.taler.net --currency KUDOS urls
+
+# Landing stats.json freshness (outside-in; age via generated_at_unix)
+STATS_STALE_SECS=900 STATS_FAIL_SECS=3600 ./taler-monitoring.sh -d stage.lefrancpaysan.ch urls
 ```
 
+### Landing `stats.json` (www.stats)
+
+Public `GET …/intro/stats.json` must expose timestamps so monitoring can tell
+whether the collector is still updating:
+
+| Field | Role |
+|-------|------|
+| `generated_at_unix` | epoch seconds — **preferred** for age = now − unix |
+| `generated_at` | ISO-8601 (fallback if unix missing) |
+| `generated_at_human` | UI “updated” / footer string |
+
+Also checks display payloads (bank-shape `withdraws` / accounts, or exchange-db /
+merchant-db counters). Soft WARN if age > `STATS_STALE_SECS` (default 900);
+ERROR if age > `STATS_FAIL_SECS` (default 3600; set `0` to disable).
+Stage shared feed: same `gen_unix` on bank + exchange + monnaies.
+
 ## Domain profiles (`domains.conf`)
 
 Each monitored stack must say **what is bank, exchange, and merchant-backend**.

scripts/taler-monitoring/TESTS.md

@@ -44,7 +44,7 @@ Numbering follows **executed** checks (early skip may shift later NN inside the
 |-------|------------------|
 | **www.exchange-** | `/config`, currency, **alt_unit_names**; `/keys` (+ alt soft); `/intro/`, `/`; **`/terms`**, **`/privacy`**, `/terms/` |
 | **www.perf-** | outside-in RTT: bank/exchange/merchant `/config`, keys, webui, intro — ms; WARN ≥ `PERF_WARN_MS` (8000); ERROR ≥ `PERF_FAIL_MS` (20000) |
-| **www.stats-** | `/intro/stats.json` loadavg / container RSS / probe ms (public; SSH fallback) |
+| **www.stats-** | `/intro/stats.json` **reachable**; **timestamps** (`generated_at_unix` + human/ISO); **freshness** vs wall clock (`STATS_STALE_SECS` WARN, `STATS_FAIL_SECS` ERROR); **display fields** (bank-shape withdraws/accounts, or exchange-db / merchant-db keys); optional performance block; shared-feed equal `gen_unix` when bank collector is published to all three |
 | **www.bank-** | `/config`, currency, alt_unit_names; integration/webui/intro; **auto-account.json**; `/terms`, `/privacy` |
 | **www.merchant-** | `/config` currency + currencies alt_unit_names; listed exchanges alt; webui/intro; **`/terms`**, **`/privacy`** |
 | **www.paivana-** | local GOA paywall front (redirect to template) |

scripts/taler-monitoring/check_urls.sh

@@ -294,11 +294,18 @@ fi
 info "perf note" "RTT measured from this host (outside-in)"
 
 # ---------------------------------------------------------------------------
-# Landing load stats (stats.json written *inside* containers, usually public).
+# Landing stats.json (public outside-in) — freshness + display fields.
-# Prefer HTTPS /intro/stats.json (no SSH). If missing and SSH works, cat from
+# Timestamps (required for external measurement):
-# the container. Report loadavg + RSS + in-container probe ms — one line each.
+#   generated_at_unix  — epoch seconds (preferred)
+#   generated_at       — ISO-8601
+#   generated_at_human — footer / “updated” UI string
+# Bank collector always sets all three; exchange/merchant scripts set unix too.
+# Env:
+#   STATS_STALE_SECS   warn if age > this (default 900 = 15m; timer often 5–15m)
+#   STATS_FAIL_SECS    hard fail if age > this (default 3600); 0 = never hard-fail on age
 # ---------------------------------------------------------------------------
-: "${STATS_STALE_SECS:=900}"   # warn if generated_at older than 15m
+: "${STATS_STALE_SECS:=900}"
+: "${STATS_FAIL_SECS:=3600}"
 
 _fetch_landing_stats_json() {
   # $1=name bank|exchange|merchant  $2=base URL  → writes $tmp/stats-$1.json, exit 0 if ok
@@ -326,82 +333,265 @@ _fetch_landing_stats_json() {
   return 1
 }
 
-report_landing_load_stats() {
+# Validate one stats.json for monitoring display + outside age measurement.
+# Prints lines: OK|WARN|ERR|STALE|FAIL <detail>
+# Sets $tmp/stats-meta-$name.txt with gen_unix|source for shared-feed compare.
+check_landing_stats_json() {
   local name="$1" base="$2"
   local f="$tmp/stats-${name}.json"
-  local line age_s
+  local line hard_missing=0
 
   if ! _fetch_landing_stats_json "$name" "$base"; then
     if [ "${CHECK_LANDING:-1}" = "1" ] || [ "${LOCAL_STACK:-0}" = "1" ]; then
-      warn "perf ${name} landing-stats" "stats.json not available (public or SSH)"
+      fail "stats ${name}" "HTTP/body missing · ${base}/intro/stats.json"
+    else
+      warn "stats ${name}" "stats.json not available"
     fi
-    return
+    return 1
   fi
+  ok "stats ${name} reachable" "${base}/intro/stats.json"
+
+  # Multi-line verdict from python (STATUS detail)
+  while IFS= read -r line; do
+    case "$line" in
+      META\ *)
+        printf '%s\n' "${line#META }" >"$tmp/stats-meta-${name}.txt"
+        ;;
+      OK\ *)
+        ok "stats ${name}" "${line#OK }"
+        ;;
+      WARN\ *)
+        warn "stats ${name}" "${line#WARN }"
+        ;;
+      STALE\ *)
+        warn "stats ${name} freshness" "stale · ${line#STALE }"
+        ;;
+      FAIL\ *)
+        fail "stats ${name}" "${line#FAIL }"
+        hard_missing=1
+        ;;
+      ERR\ *)
+        if [ "${LOCAL_STACK:-0}" = "1" ] || [ "${CHECK_LANDING:-1}" = "1" ]; then
+          fail "stats ${name}" "${line#ERR }"
+        else
+          warn "stats ${name}" "${line#ERR }"
+        fi
+        hard_missing=1
+        ;;
+      *)
+        [ -n "$line" ] && warn "stats ${name}" "unparsed · $line"
+        ;;
+    esac
+  done < <(python3 - "$f" "$name" "${STATS_STALE_SECS}" "${STATS_FAIL_SECS}" <<'PY' 2>/dev/null || echo "ERR python check failed"
+import json, sys, time, re
+from datetime import datetime, timezone
+
+path, role, stale_s, fail_s = sys.argv[1], sys.argv[2], int(sys.argv[3]), int(sys.argv[4])
+now = int(time.time())
+
+def out(kind, msg):
+    print("%s %s" % (kind, msg))
 
-  line=$(python3 - "$f" "${STATS_STALE_SECS}" <<'PY' 2>/dev/null || true
-import json, sys, time
-path, stale = sys.argv[1], int(sys.argv[2])
 try:
     d = json.load(open(path))
 except Exception as e:
-    print("ERR parse:" + str(e)[:80])
+    out("ERR", "JSON parse: %s" % str(e)[:100])
     sys.exit(0)
-if not d.get("ok", True):
+
-    print("ERR ok=false " + str(d.get("error") or "")[:80])
+if not isinstance(d, dict):
+    out("ERR", "body is not a JSON object")
     sys.exit(0)
-p = d.get("performance") or {}
+
-if not isinstance(p, dict) or not p:
+if d.get("ok") is False:
-    print("ERR no performance block")
+    out("ERR", "ok=false · %s" % str(d.get("error") or d.get("hint") or "")[:80])
     sys.exit(0)
-mem = p.get("memory") if isinstance(p.get("memory"), dict) else {}
+
-rss = mem.get("container_rss_human") or mem.get("proc_sum_rss_human") or "?"
+# --- timestamps (outside-in age measurement) ---
-load = p.get("loadavg") or "?"
+gu = d.get("generated_at_unix")
-# latency fields differ by site
+giso = d.get("generated_at") or d.get("generated_at_iso")
-bits = []
+ghum = d.get("generated_at_human")
-for k in ("config_ms", "integration_ms", "webui_ms", "keys_ms", "terms_ms"):
+age = None
-    if k in p and p[k] is not None:
+src_u = None
-        bits.append("%s=%sms" % (k.replace("_ms", ""), p[k]))
+
-lat = " ".join(bits) if bits else "latency=?"
+if gu is not None and str(gu).strip() != "":
-gen = d.get("generated_at_human") or d.get("generated_at") or "?"
+    try:
-# staleness
+        src_u = int(float(gu))
-age = ""
+        # tolerate ms accidental timestamps
-try:
+        if src_u > 10_000_000_000:
-    gu = d.get("generated_at_unix")
+            src_u //= 1000
-    if gu is not None:
+        age = now - src_u
-        age_s = int(time.time()) - int(gu)
+    except Exception:
-        age = " age=%ss" % age_s
+        out("ERR", "generated_at_unix not numeric: %r" % gu)
-        if age_s > stale:
+        src_u = None
-            print("STALE loadavg=%s RSS=%s %s gen=%s%s" % (load, rss, lat, gen, age))
+elif giso:
-            sys.exit(0)
+    # parse ISO when unix missing (legacy exchange/merchant before unix field)
-except Exception:
+    try:
-    pass
+        s = str(giso).strip()
-print("OK loadavg=%s RSS=%s %s gen=%s%s" % (load, rss, lat, gen, age))
+        if s.endswith("Z"):
+            s = s[:-1] + "+00:00"
+        # allow +0200 without colon
+        s = re.sub(r"([+-]\d{2})(\d{2})$", r"\1:\2", s)
+        dt = datetime.fromisoformat(s)
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=timezone.utc)
+        src_u = int(dt.timestamp())
+        age = now - src_u
+        out("WARN", "no generated_at_unix — derived age from generated_at ISO (deploy collector with unix)")
+    except Exception as e:
+        out("ERR", "cannot parse generated_at=%r (%s)" % (giso, e))
+else:
+    out("ERR", "missing generated_at_unix and generated_at — cannot measure freshness outside")
+
+if not ghum and not giso:
+    out("ERR", "missing generated_at_human/generated_at — UI cannot show “updated” time")
+elif ghum:
+    out("OK", "timestamp human=%s" % ghum)
+else:
+    out("OK", "timestamp iso=%s" % giso)
+
+if src_u is not None:
+    out("META", "%s|%s|%s" % (src_u, d.get("source") or "", d.get("currency") or ""))
+    if age is not None and age < -120:
+        out("WARN", "clock skew · generated_at_unix in the future by %ss" % (-age))
+    if age is not None:
+        out("OK", "generated_at_unix=%s age=%ss (now=%s)" % (src_u, age, now))
+        if fail_s > 0 and age > fail_s:
+            out("FAIL", "age=%ss > STATS_FAIL_SECS=%s · collector not updating" % (age, fail_s))
+        elif age > stale_s:
+            out("STALE", "age=%ss > STATS_STALE_SECS=%s · timer lag or stuck publish" % (age, stale_s))
+
+# --- display fields for landings / monitoring ---
+src = str(d.get("source") or "")
+# Bank-shaped (incl. stage shared feed on all three landings)
+bankish = (
+    role == "bank"
+    or "collect_bank_stats" in src
+    or "shared" in src
+    or isinstance(d.get("withdraws"), dict)
+    or isinstance(d.get("bank_accounts"), dict)
+)
+if bankish:
+    missing = []
+    if not isinstance(d.get("withdraws"), dict):
+        missing.append("withdraws")
+    ba = d.get("bank_accounts")
+    wl = d.get("wallets")
+    if not isinstance(ba, dict) and not isinstance(wl, dict):
+        missing.append("bank_accounts|wallets")
+    if d.get("currency") in (None, ""):
+        missing.append("currency")
+    if missing:
+        out("ERR", "bank-shaped display missing: %s" % ",".join(missing))
+    else:
+        w = d.get("withdraws") or {}
+        ba = ba if isinstance(ba, dict) else {}
+        n_acc = ba.get("total") if ba.get("total") is not None else ba.get("users")
+        out(
+            "OK",
+            "display bank-shape currency=%s accounts=%s withdraws.count=%s source=%s"
+            % (d.get("currency"), n_acc, w.get("count"), (src or "?")[:48]),
+        )
+elif src == "exchange-db" or (role == "exchange" and "reserves" in d):
+    need = []
+    for k in ("reserves", "wire_in_count", "known_coins"):
+        if k not in d and k.replace("known_coins", "coins_live") not in d:
+            if k == "known_coins" and ("coins_live" in d or "coins_remaining_amount" in d):
+                continue
+            need.append(k)
+    if need:
+        out("WARN", "exchange display keys missing: %s" % ",".join(need))
+    else:
+        out(
+            "OK",
+            "display exchange-db reserves=%s wire_in=%s coins=%s"
+            % (d.get("reserves"), d.get("wire_in_count"), d.get("known_coins") or d.get("coins_live")),
+        )
+elif src == "merchant-db" or (role == "merchant" and "instances" in d):
+    if "instances" not in d and "orders" not in d:
+        out("ERR", "merchant display missing instances/orders")
+    else:
+        out(
+            "OK",
+            "display merchant-db instances=%s orders=%s paid=%s"
+            % (d.get("instances"), d.get("orders"), d.get("paid")),
+        )
+else:
+    out("WARN", "unknown stats schema role=%s source=%s keys=%s" % (role, src[:40], ",".join(list(d.keys())[:8])))
+
+# performance block optional (stage shared bank feed may omit heavy probes)
+p = d.get("performance")
+if isinstance(p, dict) and p:
+    mem = p.get("memory") if isinstance(p.get("memory"), dict) else {}
+    rss = mem.get("container_rss_human") or mem.get("proc_sum_rss_human") or "—"
+    load = p.get("loadavg") or "—"
+    bits = []
+    for k in ("config_ms", "integration_ms", "webui_ms", "keys_ms", "terms_ms"):
+        if k in p and p[k] is not None:
+            bits.append("%s=%s" % (k.replace("_ms", ""), p[k]))
+    lat = " ".join(bits) if bits else "latency=—"
+    out("OK", "performance loadavg=%s RSS=%s %s" % (load, rss, lat))
+else:
+    out("OK", "performance block optional/absent")
 PY
 )
 
-  case "$line" in
+  return 0
-    OK\ *)
-      ok "perf ${name} landing-stats" "${line#OK }"
-      ;;
-    STALE\ *)
-      warn "perf ${name} landing-stats" "stale · ${line#STALE }"
-      ;;
-    ERR\ *)
-      warn "perf ${name} landing-stats" "${line#ERR }"
-      ;;
-    *)
-      warn "perf ${name} landing-stats" "unreadable stats.json"
-      ;;
-  esac
 }
 
 if [ "${CHECK_LANDING:-1}" = "1" ] || [ "${LOCAL_STACK:-0}" = "1" ]; then
   set_group stats
-  section "www · performance · landing load stats (stats.json · in-container probes)"
+  section "www · landing stats.json (freshness + display fields · outside-in)"
-  report_landing_load_stats "bank" "$BANK_PUBLIC"
+  info "stats policy" "STALE≥${STATS_STALE_SECS}s WARN · FAIL≥${STATS_FAIL_SECS}s ERROR (0=disable fail) · need generated_at_unix"
-  report_landing_load_stats "exchange" "$EXCHANGE_PUBLIC"
+  check_landing_stats_json "bank" "$BANK_PUBLIC" || true
-  report_landing_load_stats "merchant" "$MERCHANT_PUBLIC"
+  check_landing_stats_json "exchange" "$EXCHANGE_PUBLIC" || true
-  info "perf landing-stats note" "from /intro/stats.json (public); SSH container fallback if needed"
+  check_landing_stats_json "merchant" "$MERCHANT_PUBLIC" || true
+
+  # Shared feed: stage-style publish copies same bank stats to all three URLs
+  if [ -f "$tmp/stats-meta-bank.txt" ] && [ -f "$tmp/stats-meta-exchange.txt" ] && [ -f "$tmp/stats-meta-merchant.txt" ]; then
+    shared_line=$(python3 - "$tmp/stats-meta-bank.txt" "$tmp/stats-meta-exchange.txt" "$tmp/stats-meta-merchant.txt" <<'PY' 2>/dev/null || true
+import sys
+def parse(p):
+    try:
+        t = open(p).read().strip().split("|", 2)
+        return t[0], (t[1] if len(t) > 1 else ""), (t[2] if len(t) > 2 else "")
+    except Exception:
+        return "", "", ""
+b, bs, bc = parse(sys.argv[1])
+e, es, ec = parse(sys.argv[2])
+m, ms, mc = parse(sys.argv[3])
+if not b or not e or not m:
+    print("SKIP incomplete meta")
+elif b == e == m:
+    print("SHARED gen_unix=%s bank_src=%s" % (b, (bs or "?")[:50]))
+else:
+    # Shared feed only when all three claim the same bank-collector source
+    # (e.g. stage TESTPAYSAN "… shared"). GOA uses independent exchange-db /
+    # merchant-db collectors — different gen_unix is expected.
+    def bankish(s):
+        s = (s or "").lower()
+        return "shared" in s or "collect_bank_stats" in s
+    if bankish(bs) and bankish(es) and bankish(ms):
+        print("DRIFT bank=%s exchange=%s merchant=%s (expected equal for shared feed)" % (b, e, m))
+    else:
+        print("INDEPENDENT bank=%s exchange=%s merchant=%s" % (b, e, m))
+PY
+)
+    case "$shared_line" in
+      SHARED\ *)
+        ok "stats shared feed" "${shared_line#SHARED }"
+        ;;
+      DRIFT\ *)
+        warn "stats shared feed" "timestamps differ · ${shared_line#DRIFT }"
+        ;;
+      INDEPENDENT\ *)
+        info "stats feeds" "independent collectors · ${shared_line#INDEPENDENT }"
+        ;;
+      *)
+        info "stats feeds" "${shared_line:-n/a}"
+        ;;
+    esac
+  fi
+  info "stats note" "public GET /intro/stats.json; measure age via generated_at_unix vs wall clock"
 fi
 # Rollup + optional JSON for metrics_print_overall
 if [ -s "$PERF_TSV" ]; then