From 8a49e51cbdb9b5713e5e71e84c35ce6140dab35e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Fri, 17 Jul 2026 21:53:53 +0200 Subject: [PATCH] android-test: stable-self failures are WARN only, not blockers Published F-Droid remains the hard gate (FAIL_FAST). Self-build of the release tag continues the matrix and does not alone fail overall exit. --- .../android-test/GUI-AUTOMATION-NOTES.md | 14 +-- .../taler-monitoring/android-test/README.md | 14 +-- .../run-android-variant-matrix.sh | 107 +++++++++++++----- 3 files changed, 91 insertions(+), 44 deletions(-) diff --git a/scripts/taler-monitoring/android-test/GUI-AUTOMATION-NOTES.md b/scripts/taler-monitoring/android-test/GUI-AUTOMATION-NOTES.md index 16023eb..836526d 100644 --- a/scripts/taler-monitoring/android-test/GUI-AUTOMATION-NOTES.md +++ b/scripts/taler-monitoring/android-test/GUI-AUTOMATION-NOTES.md @@ -243,14 +243,14 @@ Scripts **today** implement Android only; they should honour: ### Build variants (anti–“weird local build”) -| Variant | Source | Package | Purpose | -|---------|--------|---------|---------| -| **published** | F-Droid APK URL | `net.taler.wallet.fdroid` | What users install | -| **stable-self** | tag `wallet-1.6.1` (override `STABLE_TAG`) rebuilt here | `…fdroid.debug` | Same release tree as stable, **our** Gradle — excludes odd publisher-only or dirty-tree builds | -| **master** | `origin/master` rebuilt here | `…fdroid.debug` | Tip of mainline | -| **fix** (optional) | inference fix branch | `…fdroid.debug` | GOA automation fixes | +| Variant | Source | Severity | Purpose | +|---------|--------|----------|---------| +| **stable-self** | tag `wallet-1.6.1` rebuilt here | **WARN only** | Same release tree, our Gradle — never hard-blocks the matrix | +| **published** | F-Droid APK URL | **BLOCKER** | What users install; FAIL_FAST stops here | +| **master** | `origin/master` | soft fail | Tip of mainline | +| **fix** (optional) | inference fix branch | soft fail | GOA automation fixes | -Default matrix: `published,stable-self,master`. +Default matrix order: `stable-self,published,master`. `./run-android-variant-matrix.sh` — see README. --- diff --git a/scripts/taler-monitoring/android-test/README.md b/scripts/taler-monitoring/android-test/README.md index fc02faf..3ab8d6f 100644 --- a/scripts/taler-monitoring/android-test/README.md +++ b/scripts/taler-monitoring/android-test/README.md @@ -36,18 +36,18 @@ Runs several smokes so a single odd developer build cannot be the only sample: | Variant | What | |---------|------| -| Order | Variant | What | -|------|---------|------| -| **1 (gate)** | **stable-self** | Rebuild tag `wallet-1.6.1` on this host | -| **2 (gate)** | **published** | F-Droid stable APK (`net.taler.wallet.fdroid` 1.6.1 / 854) | -| **3** | **master** | Rebuild `origin/master` (only if gates pass when `FAIL_FAST=1`) | +| Order | Variant | Severity | What | +|------|---------|----------|------| +| **1** | **stable-self** | **WARN only** (not a blocker) | Rebuild tag `wallet-1.6.1` on this host | +| **2** | **published** | **BLOCKER** | F-Droid stable APK (`net.taler.wallet.fdroid` 1.6.1 / 854) | +| **3** | **master** | fail counts / FAIL_FAST | Rebuild `origin/master` | -**Default:** `FAIL_FAST=1` — first non-zero gate stops the matrix (no master if stable fails). +**Default:** `FAIL_FAST=1` only stops on **blocker** failures (`published`). +`stable-self` failure → warning in `SUMMARY.txt`, matrix continues, exit code unaffected by that warn alone. ```bash ./run-android-variant-matrix.sh STACK=goa ./run-android-variant-matrix.sh -# only the hard gates: VARIANTS=stable-self,published FAIL_FAST=1 ./run-android-variant-matrix.sh ``` diff --git a/scripts/taler-monitoring/android-test/run-android-variant-matrix.sh b/scripts/taler-monitoring/android-test/run-android-variant-matrix.sh index 3bb3e07..9f80700 100755 --- a/scripts/taler-monitoring/android-test/run-android-variant-matrix.sh +++ b/scripts/taler-monitoring/android-test/run-android-variant-matrix.sh @@ -2,22 +2,23 @@ # Run Android wallet smoke against **several build variants** so a single # "weird" developer tree or a bad local build cannot be the only sample. # -# Default matrix (gate order — fail → hard stop when FAIL_FAST=1): +# Default matrix order: # 1) **stable-self** — rebuild release **tag** (same source as published) +# Failure → **WARN only** (not a hard blocker; continue matrix) # 2) **published** — F-Droid stable APK (what users install) -# 3) **master** — rebuild origin/master (only if 1–2 pass with FAIL_FAST) +# Failure → **BLOCKER** (matrix exit ≠ 0; FAIL_FAST stops here) +# 3) **master** — rebuild origin/master +# Failure → counts toward overall fail; does not skip earlier gates # # Why stable-self in addition to published? -# Rare, but a published APK *or* a one-off developer build can be odd -# (wrong flags, dirty tree, wrong ABI). Rebuilding the release tag on *this* -# machine with the same script path as master isolates "our toolchain" vs -# "F-Droid binary" vs "current master source". +# Isolates "our toolchain rebuild of the release tag" vs "F-Droid binary". +# Rebuild issues (signing flavor, dirty host, low-RAM ANR on debug) must not +# block the published-stable gate — hence warn-only for stable-self. # # Usage: # ./run-android-variant-matrix.sh # STACK=goa ./run-android-variant-matrix.sh # VARIANTS=master,stable-self ./run-android-variant-matrix.sh -# VARIANTS=published,stable-self,master,fix GUI=1 ./run-android-variant-matrix.sh # STABLE_TAG=wallet-1.6.1 MASTER_BRANCH=master ./run-android-variant-matrix.sh # # Env: @@ -27,7 +28,7 @@ # FIX_BRANCH inference fix branch (variant "fix") # VARIANTS comma list: published,stable-self,master,fix # GUI=1 use gui-smoke instead of pay-smoke after each build -# FAIL_FAST=1 stop on first non-zero smoke (default 0 = run all) +# FAIL_FAST=1 stop on first **blocker** failure (default 1) # MATRIX_OUT summary dir (default out-matrix-) # set -euo pipefail @@ -40,7 +41,7 @@ ROOT=$(cd "$(dirname "$0")" && pwd) : "${STABLE_APK_NAME:=net.taler.wallet.fdroid_854.apk}" : "${MASTER_BRANCH:=master}" : "${FIX_BRANCH:=dev/hernani-inference/fix-bank-withdraw-auto-exchange}" -# Gate: stable self-build → published F-Droid; then master. FAIL_FAST default on. +# Order: stable-self (warn) → published (blocker) → master. : "${VARIANTS:=stable-self,published,master}" : "${FAIL_FAST:=1}" : "${GUI:=0}" @@ -55,6 +56,7 @@ SUMMARY="$MATRIX_OUT/SUMMARY.txt" echo "=== Android variant matrix ===" | tee -a "$SUMMARY" echo "STACK=$STACK GUI=$GUI headless=${EMULATOR_HEADLESS} variants=$VARIANTS" | tee -a "$SUMMARY" echo "STABLE_TAG=$STABLE_TAG MASTER_BRANCH=$MASTER_BRANCH" | tee -a "$SUMMARY" +echo "severity: stable-self=WARN published=BLOCKER master=fail-count" | tee -a "$SUMMARY" echo "out: $MATRIX_OUT" | tee -a "$SUMMARY" # Ensure device once (headless AVD by default) @@ -67,13 +69,26 @@ echo "device: $SERIAL" | tee -a "$SUMMARY" IFS=',' read -r -a VLIST <<<"$VARIANTS" declare -a RESULTS=() +declare -a WARNINGS=() overall=0 +# Severity: warn | blocker | soft +# warn — log WARN, never FAIL_FAST, never overall≠0 +# blocker — FAIL_FAST eligible; overall≠0 +# soft — overall≠0 on fail, but FAIL_FAST only if FAIL_FAST=1 and not warn +variant_severity() { + case "$1" in + stable-self|stable|self-stable|release-self) echo warn ;; + published|fdroid|stable-published) echo blocker ;; + *) echo soft ;; + esac +} + run_published() { local out="$MATRIX_OUT/published" mkdir -p "$out" echo "" | tee -a "$SUMMARY" - echo "-------- variant: published (F-Droid stable binary) --------" | tee -a "$SUMMARY" + echo "-------- variant: published (F-Droid stable binary) [BLOCKER] --------" | tee -a "$SUMMARY" local smoke="$ROOT/run-android-pay-smoke.sh" [ "$GUI" = "1" ] && smoke="$ROOT/run-android-gui-smoke.sh" set +e @@ -90,17 +105,23 @@ run_published() { local rc=$? set -e echo "published exit=$rc" | tee -a "$SUMMARY" - echo "published $rc" >>"$MATRIX_OUT/results.tsv" + echo "published $rc blocker" >>"$MATRIX_OUT/results.tsv" return "$rc" } run_selfbuild() { local label="$1" local ref="$2" + local sev="${3:-soft}" local out="$MATRIX_OUT/$label" mkdir -p "$out" echo "" | tee -a "$SUMMARY" - echo "-------- variant: $label (self-build ref=$ref) --------" | tee -a "$SUMMARY" + echo "-------- variant: $label (self-build ref=$ref) [$(echo "$sev" | tr 'a-z' 'A-Z')] --------" | tee -a "$SUMMARY" + # Warn-only self-build: still collect evidence; SMOKE_STRICT optional (default off for warn) + local strict="${SMOKE_STRICT:-1}" + if [ "$sev" = "warn" ]; then + strict="${SMOKE_STRICT_SELF:-0}" + fi set +e env STACK="$STACK" \ BRANCH="$ref" \ @@ -111,12 +132,12 @@ run_selfbuild() { ANDROID_SERIAL="$SERIAL" \ AUTO_START_EMULATOR=0 \ PKG=net.taler.wallet.fdroid.debug \ - SMOKE_STRICT="${SMOKE_STRICT:-1}" \ + SMOKE_STRICT="$strict" \ "$ROOT/run-android-build-and-smoke.sh" local rc=$? set -e - echo "$label (ref=$ref) exit=$rc" | tee -a "$SUMMARY" - echo "$label $rc $ref" >>"$MATRIX_OUT/results.tsv" + echo "$label (ref=$ref) exit=$rc severity=$sev" | tee -a "$SUMMARY" + echo "$label $rc $ref $sev" >>"$MATRIX_OUT/results.tsv" if [ -f "$out/build-meta.txt" ]; then echo "build-meta:" | tee -a "$SUMMARY" cat "$out/build-meta.txt" | tee -a "$SUMMARY" @@ -128,31 +149,54 @@ for raw in "${VLIST[@]}"; do v=$(echo "$raw" | tr -d '[:space:]') [ -n "$v" ] || continue rc=0 + sev=$(variant_severity "$v") case "$v" in published|fdroid|stable-published) run_published || rc=$? ;; stable-self|stable|self-stable|release-self) - run_selfbuild "stable-self" "$STABLE_TAG" || rc=$? + run_selfbuild "stable-self" "$STABLE_TAG" warn || rc=$? ;; master|main) - run_selfbuild "master" "$MASTER_BRANCH" || rc=$? + run_selfbuild "master" "$MASTER_BRANCH" soft || rc=$? ;; fix|fix-branch|inference) - run_selfbuild "fix" "$FIX_BRANCH" || rc=$? + run_selfbuild "fix" "$FIX_BRANCH" soft || rc=$? ;; *) - # treat as raw git ref / label - run_selfbuild "ref-${v//\//-}" "$v" || rc=$? + run_selfbuild "ref-${v//\//-}" "$v" soft || rc=$? ;; esac - RESULTS+=("$v:$rc") + if [ "$rc" -ne 0 ]; then - overall=1 - if [ "$FAIL_FAST" = "1" ]; then - echo "FAIL_FAST: stopping after $v rc=$rc" | tee -a "$SUMMARY" - break - fi + case "$sev" in + warn) + echo "WARN: variant $v failed rc=$rc — self-build only, NOT a blocker; continuing" | tee -a "$SUMMARY" + WARNINGS+=("$v:$rc") + RESULTS+=("$v:WARN($rc)") + # do not set overall; do not FAIL_FAST + ;; + blocker) + echo "BLOCKER: variant $v failed rc=$rc" | tee -a "$SUMMARY" + RESULTS+=("$v:BLOCKER($rc)") + overall=1 + if [ "$FAIL_FAST" = "1" ]; then + echo "FAIL_FAST: stopping after blocker $v" | tee -a "$SUMMARY" + break + fi + ;; + soft) + echo "FAIL: variant $v rc=$rc (counts toward overall; not stable-self)" | tee -a "$SUMMARY" + RESULTS+=("$v:FAIL($rc)") + overall=1 + if [ "$FAIL_FAST" = "1" ]; then + echo "FAIL_FAST: stopping after $v rc=$rc" | tee -a "$SUMMARY" + break + fi + ;; + esac + else + RESULTS+=("$v:OK") fi done @@ -161,10 +205,13 @@ echo "=== matrix done (overall=$overall) ===" | tee -a "$SUMMARY" for r in "${RESULTS[@]}"; do echo " $r" | tee -a "$SUMMARY" done +if [ "${#WARNINGS[@]}" -gt 0 ]; then + echo "warnings (non-blocking):" | tee -a "$SUMMARY" + for w in "${WARNINGS[@]}"; do + echo " WARN $w" | tee -a "$SUMMARY" + done +fi echo "" echo "Compare OUT_DIR trees under $MATRIX_OUT" -echo "Intent: published vs stable-self should behave similarly;" -echo " master may diverge (new features / fixes)." -echo "A odd stable-self vs published suggests toolchain or unsigned-debug differences;" -echo "a odd master-only failure suggests tip breakage, not F-Droid stable." +echo "stable-self failure = WARN only; published failure = BLOCKER." exit "$overall"