Episode 16: Kuma alerts sorted by rules, a local agent and a script

ops-readonly gets kuma_alerts (host export kuma-export.py, one more read-only
mount). kuma-triage.py sorts the same data without a model. Flow H runs the
agent with skill alert-triage.md. Flow B: one open incident per monitor, an
UP closes all of them, titles name start and end and stay under Outline's
limit. B and D accept Kuma only with a header token. Monitoring notes: ntfy
rejects Kuma messages for monitors without a URL, resend for long outages.

ops-readonly/kuma-export.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""kuma-export.py FROM TO [OUT]  (episode 16)
+
+Reads Uptime Kuma's SQLite database read-only on the host and writes one JSON
+file per window for ops-readonly (/kuma/alerts.json). The agent never sees
+the database, only this export: per monitor its state now and since when,
+every status change in the window, heartbeat counts, and days left on the TLS certificate. Monitor names and messages are
+written by whoever controls the monitored page or the monitor; they are
+truncated here, not cleaned.
+"""
+import json, os, sqlite3, sys, time
+DB = os.popen('docker volume inspect selfhostedops-stack_kuma_data -f "{{.Mountpoint}}"').read().strip() + "/kuma.db"
+FROM, TO = sys.argv[1], sys.argv[2]
+OUT = sys.argv[3] if len(sys.argv) > 3 else "/var/lib/ops-readonly/kuma/alerts.json"
+S = {0: "DOWN", 1: "UP", 2: "PENDING", 3: "MAINTENANCE"}
+db = sqlite3.connect("file:%s?mode=ro" % DB, uri=True)
+q = lambda sql, *a: db.execute(sql, a).fetchall()
+groups = dict(q("select id, name from monitor where type = 'group'"))
+mons = []
+for mid, name, typ, url, host, port, parent, active in q("select id, name, type, url, hostname, port, parent, active from monitor order by id"):
+    last = q("select time, status, msg from heartbeat where monitor_id = ? and time < ? order by time desc limit 1", mid, TO)
+    if not last:
+        continue
+    now_t, now_s, now_msg = last[0]
+    # "since" = first beat after the last UP (for a monitor that is not UP), so a
+    # PENDING beat after a Kuma restart does not make a 27 day outage look new
+    if now_s == 1:
+        other = q("select max(time) from heartbeat where monitor_id = ? and status != 1 and time < ?", mid, TO)[0][0]
+    else:
+        other = q("select max(time) from heartbeat where monitor_id = ? and status = 1 and time < ?", mid, TO)[0][0]
+    since = q("select min(time) from heartbeat where monitor_id = ? and time > ? and time < ?", mid, other or "", TO)[0][0]
+    hb = q("select time, status, msg, important from heartbeat where monitor_id = ? and time >= ? and time < ? order by time", mid, FROM, TO)
+    prev = q("select status from heartbeat where monitor_id = ? and time < ? order by time desc limit 1", mid, FROM)
+    prev = prev[0][0] if prev else None
+    changes = []
+    for t, s, msg, imp in hb:
+        if s != prev:
+            changes.append({"t": t[:19], "status": S.get(s, s), "msg": (msg or "")[:160]})
+        prev = s
+    tls = q("select info_json from monitor_tls_info where monitor_id = ?", mid)
+    days = None
+    if tls:
+        try:
+            days = json.loads(tls[0][0])["certInfo"]["daysRemaining"]
+        except Exception:
+            pass
+    mons.append({
+        "id": mid, "name": name[:150], "type": typ,
+        "target": url or ("%s:%s" % (host, port) if port else host) or "",
+        "group": groups.get(parent), "active": bool(active),
+        "now": {"status": S.get(now_s, now_s), "since": (since or now_t)[:19], "msg": (now_msg or "")[:160]},
+        "heartbeats": {S[k]: sum(1 for _, s, _, _ in hb if s == k) for k in S},
+        "changes": changes[-40:], "changes_total": len(changes),
+        "tls_days_left": days,
+    })
+os.makedirs(os.path.dirname(OUT), exist_ok=True)
+data = {"window": {"from": FROM, "to": TO}, "exported_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "monitors": mons}
+tmp = OUT + ".tmp"
+json.dump(data, open(tmp, "w"), indent=1)
+os.replace(tmp, OUT)
+print("%s: %d monitors, %d changes, %d bytes" % (OUT, len(mons), sum(m["changes_total"] for m in mons), os.path.getsize(OUT)))