docs: new root as prior history lost (orphan + GC); ~215 commits not recoverable
This commit is contained in:
Hernâni Marques
commit
96961f23f5
268 changed files with 24161 additions and 0 deletions
240
scripts/bonfire/gitbot-mirror.py
Executable file
240
scripts/bonfire/gitbot-mirror.py
Executable file
|
|
@ -0,0 +1,240 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Poll git.ngi-0.eu and post new commits to Bonfire as gitbot."""
|
||||
from __future__ import annotations
|
||||
import json, os, pathlib, re, subprocess, sys, time, urllib.error, urllib.request
|
||||
|
||||
GQL = os.environ.get("BONFIRE_GQL", "http://127.0.0.1:9021/api/graphql")
|
||||
GIT = os.environ.get("GIT_BASE", "https://git.ngi-0.eu")
|
||||
STATE = pathlib.Path(os.environ.get("GITBOT_STATE", "/home/hernani/koopa-bonfire/gitbot-state.json"))
|
||||
USERS = pathlib.Path(os.environ.get("BONFIRE_USERS_ENV", "/home/hernani/koopa-bonfire/users.env"))
|
||||
INTERVAL = int(os.environ.get("GITBOT_INTERVAL", "300"))
|
||||
MAX_FAILS = int(os.environ.get("GITBOT_MAX_FAILS", "3"))
|
||||
MAX_TIME = 45
|
||||
OUTBOX_FEED = os.environ.get(
|
||||
"GITBOT_OUTBOX_FEED", "019f487a-df20-4ed0-a334-40ec3eac23e7"
|
||||
)
|
||||
FEED_INTERNET = os.environ.get(
|
||||
"GITBOT_FEED_INTERNET", "0aab414c-eb0a-ac1d-8c81-ef0d74ec55da"
|
||||
)
|
||||
FEED_LOCAL = os.environ.get(
|
||||
"GITBOT_FEED_LOCAL", "797632fc-029e-06f0-1031-410d73a5558e"
|
||||
)
|
||||
PUBLISH_CMD = os.environ.get(
|
||||
"GITBOT_PUBLISH_CMD",
|
||||
"podman exec -i koopa-bonfire-db psql -U postgres -d bonfire_db",
|
||||
)
|
||||
|
||||
_BARE_HOST = re.compile(
|
||||
r"(?<!https://)(?<!http://)"
|
||||
r"(?<![\w./@-])"
|
||||
r"((?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+(?:[a-zA-Z]{2,}))"
|
||||
r"(?![\w./:])"
|
||||
)
|
||||
|
||||
MUT = """
|
||||
mutation CreatePost($pc: PostContentInput!) {
|
||||
createPost(postContent: $pc) { id }
|
||||
}
|
||||
"""
|
||||
|
||||
def sanitize_text(text: str) -> str:
|
||||
"""Add https:// to bare hostnames (Bonfire URI.parse breaks without it)."""
|
||||
return _BARE_HOST.sub(r"https://\1", text)
|
||||
|
||||
def load_users():
|
||||
d = {}
|
||||
for line in USERS.read_text().splitlines():
|
||||
if "=" in line and not line.strip().startswith("#"):
|
||||
k, v = line.split("=", 1)
|
||||
d[k.strip()] = v.strip()
|
||||
return d
|
||||
|
||||
def http_json(url, data=None, headers=None, timeout=MAX_TIME):
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
data=None if data is None else json.dumps(data).encode(),
|
||||
headers=headers or {},
|
||||
method="GET" if data is None else "POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return json.load(r)
|
||||
|
||||
def gql(token, query, variables=None):
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
body = {"query": query}
|
||||
if variables is not None:
|
||||
body["variables"] = variables
|
||||
return http_json(GQL, body, headers)
|
||||
|
||||
def bonfire_ready() -> bool:
|
||||
base = GQL.rsplit("/api/", 1)[0] + "/"
|
||||
try:
|
||||
with urllib.request.urlopen(base, timeout=10) as r:
|
||||
return 200 <= r.status < 500
|
||||
except (urllib.error.URLError, TimeoutError, OSError):
|
||||
return False
|
||||
|
||||
def login(email_or_user, password):
|
||||
res = gql(
|
||||
None,
|
||||
"mutation($e:String!,$p:String!){ login(emailOrUsername:$e, password:$p){ token currentUsername } }",
|
||||
{"e": email_or_user, "p": password},
|
||||
)
|
||||
tok = (res.get("data") or {}).get("login") or {}
|
||||
if not tok.get("token"):
|
||||
raise RuntimeError(f"login failed: {res}")
|
||||
return tok["token"]
|
||||
|
||||
def select_user(token, username):
|
||||
res = gql(
|
||||
token,
|
||||
"mutation($u:String!){ selectUser(username:$u){ token currentUsername } }",
|
||||
{"u": username},
|
||||
)
|
||||
t = (res.get("data") or {}).get("selectUser") or {}
|
||||
return t.get("token") or token
|
||||
|
||||
def create_post(token, body, summary, name):
|
||||
res = gql(
|
||||
token,
|
||||
MUT,
|
||||
{"pc": {"htmlBody": body, "summary": summary[:140], "name": name}},
|
||||
)
|
||||
post = (res.get("data") or {}).get("createPost")
|
||||
if not post:
|
||||
raise RuntimeError(f"createPost failed: {res}")
|
||||
return post["id"]
|
||||
|
||||
def post_commit(token, full, c):
|
||||
sha = c["sha"]
|
||||
short = sha[:8]
|
||||
raw_msg = c["commit"]["message"].split("\n")[0][:140]
|
||||
msg = sanitize_text(raw_msg)
|
||||
html = c.get("html_url") or f"{GIT}/{full}/commit/{sha}"
|
||||
author = (c.get("commit") or {}).get("author", {}).get("name") or "?"
|
||||
full_body = (
|
||||
f"[gitbot] commit {short} on {full}\n"
|
||||
f"{msg}\nby {author}\n{html}\n#git #ngi0 #foss"
|
||||
)
|
||||
summary = sanitize_text(f"{full}@{short}: {msg}")[:140]
|
||||
try:
|
||||
return create_post(token, full_body, summary, f"commit {short}")
|
||||
except RuntimeError:
|
||||
minimal_body = f"[gitbot] commit {short} on {full}\n{html}\n#git #ngi0 #foss"
|
||||
return create_post(token, minimal_body, f"{full}@{short}", f"commit {short}")
|
||||
|
||||
def load_state():
|
||||
state = {"seen": [], "fails": {}}
|
||||
if STATE.exists():
|
||||
state = json.loads(STATE.read_text())
|
||||
state.setdefault("seen", [])
|
||||
state.setdefault("fails", {})
|
||||
return state
|
||||
|
||||
def save_state(state, seen):
|
||||
state["seen"] = list(seen)[-800:]
|
||||
STATE.write_text(json.dumps(state, indent=2))
|
||||
|
||||
def publish_outbox_to_public() -> bool:
|
||||
"""Copy foss outbox into local/internet feeds (ops workaround; no official republish API)."""
|
||||
sql = f"""
|
||||
INSERT INTO bonfire_data_social_feed_publish (id, feed_id)
|
||||
SELECT fp.id, '{FEED_INTERNET}'::uuid
|
||||
FROM bonfire_data_social_feed_publish fp
|
||||
WHERE fp.feed_id = '{OUTBOX_FEED}'::uuid
|
||||
ON CONFLICT DO NOTHING;
|
||||
INSERT INTO bonfire_data_social_feed_publish (id, feed_id)
|
||||
SELECT fp.id, '{FEED_LOCAL}'::uuid
|
||||
FROM bonfire_data_social_feed_publish fp
|
||||
WHERE fp.feed_id = '{OUTBOX_FEED}'::uuid
|
||||
ON CONFLICT DO NOTHING;
|
||||
"""
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
PUBLISH_CMD.split(),
|
||||
input=sql,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=30,
|
||||
check=False,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
print("publish feeds failed:", proc.stderr.strip(), file=sys.stderr)
|
||||
return False
|
||||
print("published outbox -> local/internet feeds", flush=True)
|
||||
return True
|
||||
except (OSError, subprocess.TimeoutExpired) as e:
|
||||
print("publish feeds error:", e, file=sys.stderr)
|
||||
return False
|
||||
|
||||
def once():
|
||||
if not bonfire_ready():
|
||||
print("bonfire not ready, skipping cycle", flush=True)
|
||||
return 0
|
||||
|
||||
u = load_users()
|
||||
tok = login(u.get("FOSS_USER", "foss"), u["FOSS_PW"])
|
||||
try:
|
||||
tok = select_user(tok, u.get("GITBOT_USER", "gitbot"))
|
||||
except Exception as e:
|
||||
print("select gitbot skipped:", e, file=sys.stderr)
|
||||
|
||||
state = load_state()
|
||||
seen = set(state["seen"])
|
||||
fails = state["fails"]
|
||||
skipped = set(state.get("skipped") or [])
|
||||
|
||||
repos = http_json(f"{GIT}/api/v1/repos/search?limit=30&sort=updated")["data"]
|
||||
new_n = 0
|
||||
for repo in repos:
|
||||
full = repo["full_name"]
|
||||
owner, name = full.split("/", 1)
|
||||
try:
|
||||
commits = http_json(f"{GIT}/api/v1/repos/{owner}/{name}/commits?limit=10")
|
||||
except Exception as e:
|
||||
print("repo fail", full, e, file=sys.stderr)
|
||||
continue
|
||||
for c in reversed(commits):
|
||||
sha = c["sha"]
|
||||
if sha in seen or sha in skipped:
|
||||
continue
|
||||
try:
|
||||
pid = post_commit(tok, full, c)
|
||||
print(f"posted {full}@{sha[:8]} -> {pid}", flush=True)
|
||||
seen.add(sha)
|
||||
fails.pop(sha, None)
|
||||
new_n += 1
|
||||
save_state(state, seen)
|
||||
time.sleep(0.4)
|
||||
except Exception as e:
|
||||
n = fails.get(sha, 0) + 1
|
||||
fails[sha] = n
|
||||
state["fails"] = fails
|
||||
save_state(state, seen)
|
||||
print("post fail", full, sha[:8], f"try={n}", e, file=sys.stderr)
|
||||
if n >= MAX_FAILS:
|
||||
skipped.add(sha)
|
||||
state["skipped"] = list(skipped)[-200:]
|
||||
save_state(state, seen)
|
||||
print("skip", full, sha[:8], "after repeated failures", file=sys.stderr)
|
||||
publish_outbox_to_public()
|
||||
print(f"done new={new_n} tracked={len(seen)} skipped={len(skipped)}", flush=True)
|
||||
return new_n
|
||||
|
||||
def main():
|
||||
loop = "--loop" in sys.argv
|
||||
if loop:
|
||||
print(f"gitbot loop interval={INTERVAL}s gql={GQL}", flush=True)
|
||||
while True:
|
||||
try:
|
||||
once()
|
||||
except Exception as e:
|
||||
print("cycle error:", e, file=sys.stderr, flush=True)
|
||||
time.sleep(INTERVAL)
|
||||
else:
|
||||
once()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue