240 lines
8.1 KiB
Python
Executable file
240 lines
8.1 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Poll git.ngi-0.eu and post new commits to Bonfire as gitbot."""
|
|
from __future__ import annotations
|
|
import json, os, pathlib, re, subprocess, sys, time, urllib.error, urllib.request
|
|
|
|
GQL = os.environ.get("BONFIRE_GQL", "http://127.0.0.1:9021/api/graphql")
|
|
GIT = os.environ.get("GIT_BASE", "https://git.ngi-0.eu")
|
|
STATE = pathlib.Path(os.environ.get("GITBOT_STATE", "/home/hernani/koopa-bonfire/gitbot-state.json"))
|
|
USERS = pathlib.Path(os.environ.get("BONFIRE_USERS_ENV", "/home/hernani/koopa-bonfire/users.env"))
|
|
INTERVAL = int(os.environ.get("GITBOT_INTERVAL", "300"))
|
|
MAX_FAILS = int(os.environ.get("GITBOT_MAX_FAILS", "3"))
|
|
MAX_TIME = 45
|
|
OUTBOX_FEED = os.environ.get(
|
|
"GITBOT_OUTBOX_FEED", "019f487a-df20-4ed0-a334-40ec3eac23e7"
|
|
)
|
|
FEED_INTERNET = os.environ.get(
|
|
"GITBOT_FEED_INTERNET", "0aab414c-eb0a-ac1d-8c81-ef0d74ec55da"
|
|
)
|
|
FEED_LOCAL = os.environ.get(
|
|
"GITBOT_FEED_LOCAL", "797632fc-029e-06f0-1031-410d73a5558e"
|
|
)
|
|
PUBLISH_CMD = os.environ.get(
|
|
"GITBOT_PUBLISH_CMD",
|
|
"podman exec -i koopa-bonfire-db psql -U postgres -d bonfire_db",
|
|
)
|
|
|
|
_BARE_HOST = re.compile(
|
|
r"(?<!https://)(?<!http://)"
|
|
r"(?<![\w./@-])"
|
|
r"((?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+(?:[a-zA-Z]{2,}))"
|
|
r"(?![\w./:])"
|
|
)
|
|
|
|
MUT = """
|
|
mutation CreatePost($pc: PostContentInput!) {
|
|
createPost(postContent: $pc) { id }
|
|
}
|
|
"""
|
|
|
|
def sanitize_text(text: str) -> str:
|
|
"""Add https:// to bare hostnames (Bonfire URI.parse breaks without it)."""
|
|
return _BARE_HOST.sub(r"https://\1", text)
|
|
|
|
def load_users():
|
|
d = {}
|
|
for line in USERS.read_text().splitlines():
|
|
if "=" in line and not line.strip().startswith("#"):
|
|
k, v = line.split("=", 1)
|
|
d[k.strip()] = v.strip()
|
|
return d
|
|
|
|
def http_json(url, data=None, headers=None, timeout=MAX_TIME):
|
|
req = urllib.request.Request(
|
|
url,
|
|
data=None if data is None else json.dumps(data).encode(),
|
|
headers=headers or {},
|
|
method="GET" if data is None else "POST",
|
|
)
|
|
with urllib.request.urlopen(req, timeout=timeout) as r:
|
|
return json.load(r)
|
|
|
|
def gql(token, query, variables=None):
|
|
headers = {"Content-Type": "application/json"}
|
|
if token:
|
|
headers["Authorization"] = f"Bearer {token}"
|
|
body = {"query": query}
|
|
if variables is not None:
|
|
body["variables"] = variables
|
|
return http_json(GQL, body, headers)
|
|
|
|
def bonfire_ready() -> bool:
|
|
base = GQL.rsplit("/api/", 1)[0] + "/"
|
|
try:
|
|
with urllib.request.urlopen(base, timeout=10) as r:
|
|
return 200 <= r.status < 500
|
|
except (urllib.error.URLError, TimeoutError, OSError):
|
|
return False
|
|
|
|
def login(email_or_user, password):
|
|
res = gql(
|
|
None,
|
|
"mutation($e:String!,$p:String!){ login(emailOrUsername:$e, password:$p){ token currentUsername } }",
|
|
{"e": email_or_user, "p": password},
|
|
)
|
|
tok = (res.get("data") or {}).get("login") or {}
|
|
if not tok.get("token"):
|
|
raise RuntimeError(f"login failed: {res}")
|
|
return tok["token"]
|
|
|
|
def select_user(token, username):
|
|
res = gql(
|
|
token,
|
|
"mutation($u:String!){ selectUser(username:$u){ token currentUsername } }",
|
|
{"u": username},
|
|
)
|
|
t = (res.get("data") or {}).get("selectUser") or {}
|
|
return t.get("token") or token
|
|
|
|
def create_post(token, body, summary, name):
|
|
res = gql(
|
|
token,
|
|
MUT,
|
|
{"pc": {"htmlBody": body, "summary": summary[:140], "name": name}},
|
|
)
|
|
post = (res.get("data") or {}).get("createPost")
|
|
if not post:
|
|
raise RuntimeError(f"createPost failed: {res}")
|
|
return post["id"]
|
|
|
|
def post_commit(token, full, c):
|
|
sha = c["sha"]
|
|
short = sha[:8]
|
|
raw_msg = c["commit"]["message"].split("\n")[0][:140]
|
|
msg = sanitize_text(raw_msg)
|
|
html = c.get("html_url") or f"{GIT}/{full}/commit/{sha}"
|
|
author = (c.get("commit") or {}).get("author", {}).get("name") or "?"
|
|
full_body = (
|
|
f"[gitbot] commit {short} on {full}\n"
|
|
f"{msg}\nby {author}\n{html}\n#git #ngi0 #foss"
|
|
)
|
|
summary = sanitize_text(f"{full}@{short}: {msg}")[:140]
|
|
try:
|
|
return create_post(token, full_body, summary, f"commit {short}")
|
|
except RuntimeError:
|
|
minimal_body = f"[gitbot] commit {short} on {full}\n{html}\n#git #ngi0 #foss"
|
|
return create_post(token, minimal_body, f"{full}@{short}", f"commit {short}")
|
|
|
|
def load_state():
|
|
state = {"seen": [], "fails": {}}
|
|
if STATE.exists():
|
|
state = json.loads(STATE.read_text())
|
|
state.setdefault("seen", [])
|
|
state.setdefault("fails", {})
|
|
return state
|
|
|
|
def save_state(state, seen):
|
|
state["seen"] = list(seen)[-800:]
|
|
STATE.write_text(json.dumps(state, indent=2))
|
|
|
|
def publish_outbox_to_public() -> bool:
|
|
"""Copy foss outbox into local/internet feeds (ops workaround; no official republish API)."""
|
|
sql = f"""
|
|
INSERT INTO bonfire_data_social_feed_publish (id, feed_id)
|
|
SELECT fp.id, '{FEED_INTERNET}'::uuid
|
|
FROM bonfire_data_social_feed_publish fp
|
|
WHERE fp.feed_id = '{OUTBOX_FEED}'::uuid
|
|
ON CONFLICT DO NOTHING;
|
|
INSERT INTO bonfire_data_social_feed_publish (id, feed_id)
|
|
SELECT fp.id, '{FEED_LOCAL}'::uuid
|
|
FROM bonfire_data_social_feed_publish fp
|
|
WHERE fp.feed_id = '{OUTBOX_FEED}'::uuid
|
|
ON CONFLICT DO NOTHING;
|
|
"""
|
|
try:
|
|
proc = subprocess.run(
|
|
PUBLISH_CMD.split(),
|
|
input=sql,
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=30,
|
|
check=False,
|
|
)
|
|
if proc.returncode != 0:
|
|
print("publish feeds failed:", proc.stderr.strip(), file=sys.stderr)
|
|
return False
|
|
print("published outbox -> local/internet feeds", flush=True)
|
|
return True
|
|
except (OSError, subprocess.TimeoutExpired) as e:
|
|
print("publish feeds error:", e, file=sys.stderr)
|
|
return False
|
|
|
|
def once():
|
|
if not bonfire_ready():
|
|
print("bonfire not ready, skipping cycle", flush=True)
|
|
return 0
|
|
|
|
u = load_users()
|
|
tok = login(u.get("FOSS_USER", "foss"), u["FOSS_PW"])
|
|
try:
|
|
tok = select_user(tok, u.get("GITBOT_USER", "gitbot"))
|
|
except Exception as e:
|
|
print("select gitbot skipped:", e, file=sys.stderr)
|
|
|
|
state = load_state()
|
|
seen = set(state["seen"])
|
|
fails = state["fails"]
|
|
skipped = set(state.get("skipped") or [])
|
|
|
|
repos = http_json(f"{GIT}/api/v1/repos/search?limit=30&sort=updated")["data"]
|
|
new_n = 0
|
|
for repo in repos:
|
|
full = repo["full_name"]
|
|
owner, name = full.split("/", 1)
|
|
try:
|
|
commits = http_json(f"{GIT}/api/v1/repos/{owner}/{name}/commits?limit=10")
|
|
except Exception as e:
|
|
print("repo fail", full, e, file=sys.stderr)
|
|
continue
|
|
for c in reversed(commits):
|
|
sha = c["sha"]
|
|
if sha in seen or sha in skipped:
|
|
continue
|
|
try:
|
|
pid = post_commit(tok, full, c)
|
|
print(f"posted {full}@{sha[:8]} -> {pid}", flush=True)
|
|
seen.add(sha)
|
|
fails.pop(sha, None)
|
|
new_n += 1
|
|
save_state(state, seen)
|
|
time.sleep(0.4)
|
|
except Exception as e:
|
|
n = fails.get(sha, 0) + 1
|
|
fails[sha] = n
|
|
state["fails"] = fails
|
|
save_state(state, seen)
|
|
print("post fail", full, sha[:8], f"try={n}", e, file=sys.stderr)
|
|
if n >= MAX_FAILS:
|
|
skipped.add(sha)
|
|
state["skipped"] = list(skipped)[-200:]
|
|
save_state(state, seen)
|
|
print("skip", full, sha[:8], "after repeated failures", file=sys.stderr)
|
|
publish_outbox_to_public()
|
|
print(f"done new={new_n} tracked={len(seen)} skipped={len(skipped)}", flush=True)
|
|
return new_n
|
|
|
|
def main():
|
|
loop = "--loop" in sys.argv
|
|
if loop:
|
|
print(f"gitbot loop interval={INTERVAL}s gql={GQL}", flush=True)
|
|
while True:
|
|
try:
|
|
once()
|
|
except Exception as e:
|
|
print("cycle error:", e, file=sys.stderr, flush=True)
|
|
time.sleep(INTERVAL)
|
|
else:
|
|
once()
|
|
|
|
if __name__ == "__main__":
|
|
main()
|