From 261c25c87fe157d691b88a073c81d2c1db43fa65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hern=C3=A2ni=20Marques?= Date: Thu, 1 Oct 2026 11:39:45 +0200 Subject: [PATCH] fix(dossiers-ngi): update README and scripts for branch status --- .../user/dossiers-ngi-branch-status.service | 2 +- scripts/dossiers-ngi/README.md | 6 +- scripts/dossiers-ngi/branch-status-hourly.py | 77 +++++++++++++++++-- scripts/dossiers-ngi/smoke-branch-tree.sh | 22 ++++++ .../_cascade/bar.css | 15 +++- .../_cascade/bar.js | 33 ++++++++ .../cascade/_cascade/bar.css | 15 +++- .../cascade/_cascade/bar.js | 33 ++++++++ 8 files changed, 188 insertions(+), 15 deletions(-) diff --git a/configs/systemd/user/dossiers-ngi-branch-status.service b/configs/systemd/user/dossiers-ngi-branch-status.service index 76ef361..971c93b 100644 --- a/configs/systemd/user/dossiers-ngi-branch-status.service +++ b/configs/systemd/user/dossiers-ngi-branch-status.service @@ -14,7 +14,7 @@ Environment=ADMIN_LOG=%h/src/koopa/koopa-admin-log Environment=TDPD_ROOT=%h/git/src/ngi/tech-dossier-project-data Environment=OUT=/var/www/monitoring-sites/dossiers.ngi.hacktivism.ch/branch-status.json Environment=EDITIONS_ROOT=/var/www/monitoring-sites/dossiers.ngi.hacktivism.ch/editions -# Rebuild only when .edition-tip differs / missing (never force from timer) +# Rebuild only when current tip freeze_ok is not already true (never force from timer) Environment=REBUILD=1 EnvironmentFile=-%h/.config/dossiers-ngi/branch-status.env # flock -n: second fire while busy exits 0 without stacking garbage diff --git a/scripts/dossiers-ngi/README.md b/scripts/dossiers-ngi/README.md index dd42487..a288176 100644 --- a/scripts/dossiers-ngi/README.md +++ b/scripts/dossiers-ngi/README.md @@ -58,10 +58,12 @@ Per branch in JSON / sticky bar: | `tip_time` | committer time of tip (`%cI`) | | `built_at` | when this edition freeze was written | | `build_seconds` | wall time of that freeze | +| `freeze_ok` | `true` when the current origin tip freeze succeeded (sticky bar green ✓); `false` when it failed, is a placeholder, or the recorded tip is stale (red ✗) | +| `freeze_error` | last freeze error text (only when not ok) | | `vs_parent_plus` / `vs_parent_minus` | `git diff --numstat` tip vs **parent** tip (sticky bar +grün/−rot; omitted when `parent` is empty or not in the tree, e.g. `main` and `feature/nl-net-injestion-injestor-update`) | | `loc` | text-file line count of the tip vs the empty tree (gray `N LOC` in the plus/minus columns; only when there is no parent) | -Rebuild runs **only when** `.edition-tip` ≠ current tip (timer never forces). +Rebuild runs **only when** the current origin tip is not already a successful freeze (`freeze_ok`; timer never forces). Each run also: `git fetch --prune`, `worktree prune`, drop stale `editions//` (gone remotes), purge leftover `tdpd-*` tmp dirs. @@ -113,7 +115,7 @@ main … N LOC └── project_template … +N −M # vs main ``` -Visible cells: name (tree prefix + branch) · tip `monDD HH:MM` · `Ns monDD HH:MM` (build seconds + freeze time) · plus column · minus column. `+N −M` vs the **parent** tip, in two right-aligned columns so `+898` and `+1200` share an edge. Roots with no parent show gray `N` in the plus column and `LOC` in the minus column. No `@sha`, no `default` label (`data-default` stays for flask/cookie). Inner tree `max-width: 45rem` (same as site `#navigation` / `#content` 45em at root 16px), so names line up with About and the minus column with the page edge; below 45rem the tree is full width with a small pad. Rear cells are right-aligned `tabular-nums`. Poll every 30s **updates when/built/plus/minus in place** when the branch structure is unchanged (no full tree remount). +Visible cells: freeze mark (green ✓ / red ✗) · name (tree prefix + branch) · tip `monDD HH:MM` · `Ns monDD HH:MM` (build seconds + freeze time) · plus column · minus column. `+N −M` vs the **parent** tip, in two right-aligned columns so `+898` and `+1200` share an edge. Roots with no parent show gray `N` in the plus column and `LOC` in the minus column. No `@sha`, no `default` label (`data-default` stays for flask/cookie). Inner tree `max-width: 45rem` (same as site `#navigation` / `#content` 45em at root 16px), so names line up with About and the minus column with the page edge; below 45rem the tree is full width with a small pad. Rear cells are right-aligned `tabular-nums`. Poll every 30s **updates mark/when/built/plus/minus in place** when the branch structure is unchanged (no full tree remount). Each name links to `/_cascade/use/` (cookie + 302 `/`). Caddy map: diff --git a/scripts/dossiers-ngi/branch-status-hourly.py b/scripts/dossiers-ngi/branch-status-hourly.py index 910eb82..7b32e78 100755 --- a/scripts/dossiers-ngi/branch-status-hourly.py +++ b/scripts/dossiers-ngi/branch-status-hourly.py @@ -301,6 +301,42 @@ def write_build_meta(editions: Path, slug: str, seconds: float) -> None: (dest / ".edition-build-seconds").write_text(f"{seconds:.3f}\n", encoding="utf-8") +def write_freeze_status(editions: Path, slug: str, ok: bool, error: str = "") -> None: + dest = editions / slug + dest.mkdir(parents=True, exist_ok=True) + (dest / ".edition-freeze-ok").write_text("1\n" if ok else "0\n", encoding="utf-8") + errp = dest / ".edition-freeze-error" + if ok or not error: + if errp.exists(): + errp.unlink() + return + errp.write_text(error[:500] + "\n", encoding="utf-8") + + +def freeze_ok_for(editions: Path | None, slug: str, tip_full: str) -> tuple[bool, str]: + """Current origin tip freeze succeeded (green ✓) or not (red ✗).""" + if editions is None or not tip_full: + return False, "" + dest = editions / slug + err = "" + errp = dest / ".edition-freeze-error" + if errp.is_file(): + err = errp.read_text(encoding="utf-8").strip() + recorded = tip_recorded(editions, slug) + okp = dest / ".edition-freeze-ok" + if okp.is_file(): + file_ok = okp.read_text(encoding="utf-8").strip().lower() in ( + "1", + "true", + "ok", + "yes", + ) + else: + file_ok = (dest / ".edition-built-at").is_file() + ok = bool(file_ok and recorded == tip_full and (dest / "index.html").is_file()) + return ok, err + + def cleanup_stale_editions(editions: Path, tips: dict[str, str]) -> list[str]: """Drop edition dirs whose branch tip is gone (keep flask alias).""" keep = {edition_slug(b) for b in tips} | {"flask"} @@ -424,12 +460,10 @@ def rebuild_edition(repo: Path, branch: str, tip: str, editions: Path) -> dict: dest = editions / slug t0 = time.perf_counter() force = os.environ.get("FORCE_REBUILD", "").strip() in ("1", "true", "yes") - if ( - not force - and tip_recorded(editions, slug) == tip - and (dest / "index.html").is_file() - ): + if not force and freeze_ok_for(editions, slug, tip)[0]: inject_bar(dest) + if not (dest / ".edition-freeze-ok").is_file(): + write_freeze_status(editions, slug, True) return {"status": "skipped", "seconds": round(time.perf_counter() - t0, 3)} wt: Path | None = None @@ -441,10 +475,12 @@ def rebuild_edition(repo: Path, branch: str, tip: str, editions: Path) -> dict: git(repo, "worktree", "prune", check=False) cp = git(repo, "worktree", "add", "--detach", str(wt), tip, check=False) if cp.returncode != 0: + st = f"error:worktree:{cp.stderr.strip()[:200]}" ensure_placeholder(editions, slug, tip) + write_freeze_status(editions, slug, False, st) inject_bar(dest) return { - "status": f"error:worktree:{cp.stderr.strip()[:200]}", + "status": st, "seconds": round(time.perf_counter() - t0, 3), } @@ -452,6 +488,7 @@ def rebuild_edition(repo: Path, branch: str, tip: str, editions: Path) -> dict: build = wt / "website" / "build" if not build.is_dir(): ensure_placeholder(editions, slug, tip) + write_freeze_status(editions, slug, False, "error:no-build") inject_bar(dest) return {"status": "error:no-build", "seconds": round(time.perf_counter() - t0, 3)} @@ -460,6 +497,8 @@ def rebuild_edition(repo: Path, branch: str, tip: str, editions: Path) -> dict: ".edition-tip", ".edition-built-at", ".edition-build-seconds", + ".edition-freeze-ok", + ".edition-freeze-error", } for child in list(dest.iterdir()): if child.name in keep_meta: @@ -477,13 +516,16 @@ def rebuild_edition(repo: Path, branch: str, tip: str, editions: Path) -> dict: secs = round(time.perf_counter() - t0, 3) (dest / ".edition-tip").write_text(tip + "\n", encoding="utf-8") write_build_meta(editions, slug, secs) + write_freeze_status(editions, slug, True) inject_bar(dest) return {"status": "built", "seconds": secs} except Exception as exc: # noqa: BLE001 — hourly must continue + st = f"error:{type(exc).__name__}:{exc}" ensure_placeholder(editions, slug, tip) + write_freeze_status(editions, slug, False, st) inject_bar(dest) return { - "status": f"error:{type(exc).__name__}:{exc}", + "status": st, "seconds": round(time.perf_counter() - t0, 3), } finally: @@ -510,10 +552,19 @@ def sync_flask_alias(editions: Path, tips: dict[str, str]) -> None: shutil.copytree(src, dst) (dst / ".edition-tip").write_text(tip + "\n", encoding="utf-8") # reuse src build meta if present - for name in (".edition-built-at", ".edition-build-seconds"): + for name in ( + ".edition-built-at", + ".edition-build-seconds", + ".edition-freeze-ok", + ".edition-freeze-error", + ): s = src / name if s.is_file(): shutil.copy2(s, dst / name) + else: + p = dst / name + if p.exists(): + p.unlink() inject_bar(dst) @@ -570,6 +621,7 @@ def build_payload(repo: Path, tips: dict[str, str], rebuild: bool) -> dict: slug = edition_slug(name) if not (ed_root / slug / "index.html").is_file(): ensure_placeholder(ed_root, slug, tip) + write_freeze_status(ed_root, slug, False, "placeholder") rebuild_status[name] = {"status": "placeholder", "seconds": 0.0} if not (ed_root / "flask" / "index.html").is_file(): sync_flask_alias(ed_root, tips) @@ -601,6 +653,15 @@ def build_payload(repo: Path, tips: dict[str, str], rebuild: bool) -> dict: entry["built_at"] = meta["built_at"] if "build_seconds" in meta: entry["build_seconds"] = meta["build_seconds"] + freeze_ok, freeze_err = freeze_ok_for(ed_root, slug, tips[name]) + st = str(rb.get("status") or "") + if st == "built": + freeze_ok, freeze_err = True, "" + elif st.startswith("error") or st == "placeholder": + freeze_ok, freeze_err = False, st + entry["freeze_ok"] = freeze_ok + if freeze_err: + entry["freeze_error"] = freeze_err # sticky bar: +grün/−rot vs parent tip; gray loc when no parent in tree parent = row["parent"] if parent and parent in tips: diff --git a/scripts/dossiers-ngi/smoke-branch-tree.sh b/scripts/dossiers-ngi/smoke-branch-tree.sh index 6c6340b..7703693 100755 --- a/scripts/dossiers-ngi/smoke-branch-tree.sh +++ b/scripts/dossiers-ngi/smoke-branch-tree.sh @@ -62,6 +62,8 @@ assert "smoke/temp-sibling" in branches assert branches["feat/a"]["parent"] == "main", branches["feat/a"] assert branches["feat/b"]["parent"] == "feat/a", branches["feat/b"] assert branches["smoke/temp-sibling"]["parent"] == "main", branches["smoke/temp-sibling"] +for b in data["branches"]: + assert b.get("freeze_ok") is True, b # prefixes semi-graphic prefs = {b["branch"]: b["prefix"] for b in data["branches"]} assert any("├──" in p or "└──" in p for p in prefs.values()), prefs @@ -77,6 +79,26 @@ test -f "$EDITIONS/feat-a/index.html" test -f "$EDITIONS/smoke-temp-sibling/index.html" grep -q "feat/a" "$EDITIONS/feat-a/index.html" grep -q "smoke/temp-sibling" "$EDITIONS/smoke-temp-sibling/index.html" +test "$(cat "$EDITIONS/feat-a/.edition-freeze-ok")" = "1" + +# red ✗ when last freeze of this tip failed +printf '0\n' > "$EDITIONS/feat-a/.edition-freeze-ok" +printf 'error:smoke\n' > "$EDITIONS/feat-a/.edition-freeze-error" +FETCH=0 TDPD_ROOT="$CLONE" OUT="$OUT" EDITIONS_ROOT="$EDITIONS" \ + FREEZE_CMD="$FREEZE_CMD" \ + python3 "$SCRIPT" --no-fetch +python3 - "$OUT" <<'PY' +import json, sys +from pathlib import Path +data = json.loads(Path(sys.argv[1]).read_text()) +rows = {b["branch"]: b for b in data["branches"]} +assert rows["feat/a"]["freeze_ok"] is False, rows["feat/a"] +assert rows["feat/a"].get("freeze_error") == "error:smoke", rows["feat/a"] +assert rows["main"]["freeze_ok"] is True, rows["main"] +print("OK freeze_ok false after failed mark") +PY +printf '1\n' > "$EDITIONS/feat-a/.edition-freeze-ok" +rm -f "$EDITIONS/feat-a/.edition-freeze-error" # delete smoke branch on origin; fetch --prune drops remote-tracking ref cd "$CLONE" diff --git a/sites/dossiers.ngi.hacktivism.ch/_cascade/bar.css b/sites/dossiers.ngi.hacktivism.ch/_cascade/bar.css index 94d3b58..d0a638e 100644 --- a/sites/dossiers.ngi.hacktivism.ch/_cascade/bar.css +++ b/sites/dossiers.ngi.hacktivism.ch/_cascade/bar.css @@ -36,7 +36,7 @@ html { } a.cascade-tree-row { display: grid; - grid-template-columns: minmax(6rem, 1fr) 7.2rem 10.5rem 4.4rem 3.6rem; + grid-template-columns: 1.05rem minmax(6rem, 1fr) 7.2rem 10.5rem 4.4rem 3.6rem; column-gap: 0.55rem; align-items: baseline; color: #7ec8ff; @@ -89,11 +89,22 @@ a.cascade-tree-row.is-active:hover { max-width: 100%; } a.cascade-tree-row { - grid-template-columns: minmax(5rem, 1fr) 5.6rem 8.4rem 3.6rem 3rem; + grid-template-columns: 0.9rem minmax(5rem, 1fr) 5.6rem 8.4rem 3.6rem 3rem; column-gap: 0.35rem; } } +.cascade-mark { + text-align: center; + font-weight: 700; + line-height: 1; +} +.cascade-bar .cascade-ok { + color: #3fb950; +} +.cascade-bar .cascade-fail { + color: #f85149; +} .cascade-bar .diff-plus { color: #3fb950; font-weight: 600; diff --git a/sites/dossiers.ngi.hacktivism.ch/_cascade/bar.js b/sites/dossiers.ngi.hacktivism.ch/_cascade/bar.js index 720e070..8fe4765 100644 --- a/sites/dossiers.ngi.hacktivism.ch/_cascade/bar.js +++ b/sites/dossiers.ngi.hacktivism.ch/_cascade/bar.js @@ -59,6 +59,31 @@ return sec || at; } + function freezeOk(row) { + return row && row.freeze_ok === true; + } + + function freezeLabel(row) { + if (freezeOk(row)) return "freeze ok"; + var err = row && row.freeze_error ? String(row.freeze_error) : ""; + return err ? "freeze failed: " + err : "freeze failed"; + } + + function freezeMarkHtml(row) { + var ok = freezeOk(row); + return ( + '' + + (ok ? "✓" : "✗") + + "" + ); + } + function plusHtml(row) { if (row.vs_parent_plus != null) { var p = Number(row.vs_parent_plus); @@ -111,6 +136,8 @@ (tip ? " @ " + tip : "") + (tipTime ? " tip=" + shortTime(tipTime) : "") + (built ? " build=" + built : "") + + " " + + freezeLabel(row) + vs ); } @@ -191,6 +218,11 @@ else a.removeAttribute("data-default"); setText(a.querySelector(".cascade-when"), shortTime(row.tip_time || "")); setText(a.querySelector(".cascade-built"), builtText(row)); + var markEl = a.querySelector(".cascade-mark"); + if (markEl) { + var markHtml = freezeMarkHtml(row); + if (markEl.outerHTML !== markHtml) markEl.outerHTML = markHtml; + } var plusEl = a.querySelector(".cascade-plus"); if (plusEl) { var ph = plusHtml(row); @@ -256,6 +288,7 @@ ' title="' + esc(rowTitle(row)) + '">' + + freezeMarkHtml(row) + '' + esc(prefix) + "" + diff --git a/sites/dossiers.ngi.hacktivism.ch/cascade/_cascade/bar.css b/sites/dossiers.ngi.hacktivism.ch/cascade/_cascade/bar.css index 94d3b58..d0a638e 100644 --- a/sites/dossiers.ngi.hacktivism.ch/cascade/_cascade/bar.css +++ b/sites/dossiers.ngi.hacktivism.ch/cascade/_cascade/bar.css @@ -36,7 +36,7 @@ html { } a.cascade-tree-row { display: grid; - grid-template-columns: minmax(6rem, 1fr) 7.2rem 10.5rem 4.4rem 3.6rem; + grid-template-columns: 1.05rem minmax(6rem, 1fr) 7.2rem 10.5rem 4.4rem 3.6rem; column-gap: 0.55rem; align-items: baseline; color: #7ec8ff; @@ -89,11 +89,22 @@ a.cascade-tree-row.is-active:hover { max-width: 100%; } a.cascade-tree-row { - grid-template-columns: minmax(5rem, 1fr) 5.6rem 8.4rem 3.6rem 3rem; + grid-template-columns: 0.9rem minmax(5rem, 1fr) 5.6rem 8.4rem 3.6rem 3rem; column-gap: 0.35rem; } } +.cascade-mark { + text-align: center; + font-weight: 700; + line-height: 1; +} +.cascade-bar .cascade-ok { + color: #3fb950; +} +.cascade-bar .cascade-fail { + color: #f85149; +} .cascade-bar .diff-plus { color: #3fb950; font-weight: 600; diff --git a/sites/dossiers.ngi.hacktivism.ch/cascade/_cascade/bar.js b/sites/dossiers.ngi.hacktivism.ch/cascade/_cascade/bar.js index 720e070..8fe4765 100644 --- a/sites/dossiers.ngi.hacktivism.ch/cascade/_cascade/bar.js +++ b/sites/dossiers.ngi.hacktivism.ch/cascade/_cascade/bar.js @@ -59,6 +59,31 @@ return sec || at; } + function freezeOk(row) { + return row && row.freeze_ok === true; + } + + function freezeLabel(row) { + if (freezeOk(row)) return "freeze ok"; + var err = row && row.freeze_error ? String(row.freeze_error) : ""; + return err ? "freeze failed: " + err : "freeze failed"; + } + + function freezeMarkHtml(row) { + var ok = freezeOk(row); + return ( + '' + + (ok ? "✓" : "✗") + + "" + ); + } + function plusHtml(row) { if (row.vs_parent_plus != null) { var p = Number(row.vs_parent_plus); @@ -111,6 +136,8 @@ (tip ? " @ " + tip : "") + (tipTime ? " tip=" + shortTime(tipTime) : "") + (built ? " build=" + built : "") + + " " + + freezeLabel(row) + vs ); } @@ -191,6 +218,11 @@ else a.removeAttribute("data-default"); setText(a.querySelector(".cascade-when"), shortTime(row.tip_time || "")); setText(a.querySelector(".cascade-built"), builtText(row)); + var markEl = a.querySelector(".cascade-mark"); + if (markEl) { + var markHtml = freezeMarkHtml(row); + if (markEl.outerHTML !== markHtml) markEl.outerHTML = markHtml; + } var plusEl = a.querySelector(".cascade-plus"); if (plusEl) { var ph = plusHtml(row); @@ -256,6 +288,7 @@ ' title="' + esc(rowTitle(row)) + '">' + + freezeMarkHtml(row) + '' + esc(prefix) + "" +