Episode 16: Kuma alerts sorted by rules, a local agent and a script

ops-readonly gets kuma_alerts (host export kuma-export.py, one more read-only
mount). kuma-triage.py sorts the same data without a model. Flow H runs the
agent with skill alert-triage.md. Flow B: one open incident per monitor, an
UP closes all of them, titles name start and end and stay under Outline's
limit. B and D accept Kuma only with a header token. Monitoring notes: ntfy
rejects Kuma messages for monitors without a URL, resend for long outages.

ops-readonly/kuma-export.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""kuma-export.py FROM TO [OUT]  (episode 16)
+
+Reads Uptime Kuma's SQLite database read-only on the host and writes one JSON
+file per window for ops-readonly (/kuma/alerts.json). The agent never sees
+the database, only this export: per monitor its state now and since when,
+every status change in the window, heartbeat counts, and days left on the TLS certificate. Monitor names and messages are
+written by whoever controls the monitored page or the monitor; they are
+truncated here, not cleaned.
+"""
+import json, os, sqlite3, sys, time
+DB = os.popen('docker volume inspect selfhostedops-stack_kuma_data -f "{{.Mountpoint}}"').read().strip() + "/kuma.db"
+FROM, TO = sys.argv[1], sys.argv[2]
+OUT = sys.argv[3] if len(sys.argv) > 3 else "/var/lib/ops-readonly/kuma/alerts.json"
+S = {0: "DOWN", 1: "UP", 2: "PENDING", 3: "MAINTENANCE"}
+db = sqlite3.connect("file:%s?mode=ro" % DB, uri=True)
+q = lambda sql, *a: db.execute(sql, a).fetchall()
+groups = dict(q("select id, name from monitor where type = 'group'"))
+mons = []
+for mid, name, typ, url, host, port, parent, active in q("select id, name, type, url, hostname, port, parent, active from monitor order by id"):
+    last = q("select time, status, msg from heartbeat where monitor_id = ? and time < ? order by time desc limit 1", mid, TO)
+    if not last:
+        continue
+    now_t, now_s, now_msg = last[0]
+    # "since" = first beat after the last UP (for a monitor that is not UP), so a
+    # PENDING beat after a Kuma restart does not make a 27 day outage look new
+    if now_s == 1:
+        other = q("select max(time) from heartbeat where monitor_id = ? and status != 1 and time < ?", mid, TO)[0][0]
+    else:
+        other = q("select max(time) from heartbeat where monitor_id = ? and status = 1 and time < ?", mid, TO)[0][0]
+    since = q("select min(time) from heartbeat where monitor_id = ? and time > ? and time < ?", mid, other or "", TO)[0][0]
+    hb = q("select time, status, msg, important from heartbeat where monitor_id = ? and time >= ? and time < ? order by time", mid, FROM, TO)
+    prev = q("select status from heartbeat where monitor_id = ? and time < ? order by time desc limit 1", mid, FROM)
+    prev = prev[0][0] if prev else None
+    changes = []
+    for t, s, msg, imp in hb:
+        if s != prev:
+            changes.append({"t": t[:19], "status": S.get(s, s), "msg": (msg or "")[:160]})
+        prev = s
+    tls = q("select info_json from monitor_tls_info where monitor_id = ?", mid)
+    days = None
+    if tls:
+        try:
+            days = json.loads(tls[0][0])["certInfo"]["daysRemaining"]
+        except Exception:
+            pass
+    mons.append({
+        "id": mid, "name": name[:150], "type": typ,
+        "target": url or ("%s:%s" % (host, port) if port else host) or "",
+        "group": groups.get(parent), "active": bool(active),
+        "now": {"status": S.get(now_s, now_s), "since": (since or now_t)[:19], "msg": (now_msg or "")[:160]},
+        "heartbeats": {S[k]: sum(1 for _, s, _, _ in hb if s == k) for k in S},
+        "changes": changes[-40:], "changes_total": len(changes),
+        "tls_days_left": days,
+    })
+os.makedirs(os.path.dirname(OUT), exist_ok=True)
+data = {"window": {"from": FROM, "to": TO}, "exported_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "monitors": mons}
+tmp = OUT + ".tmp"
+json.dump(data, open(tmp, "w"), indent=1)
+os.replace(tmp, OUT)
+print("%s: %d monitors, %d changes, %d bytes" % (OUT, len(mons), sum(m["changes_total"] for m in mons), os.path.getsize(OUT)))

ops-readonly/kuma-triage.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+# kuma-triage.py < kuma_alerts.json  (episode 16): the same answer as the agent, no model.
+# Down now = alert (still_down if it started before the window). 3+ drops in the window = flapping.
+# Recovered drops are reported once. Cert < 14 days. Same host + same kind + start within 90 s = one alert.
+import json, sys, datetime as dt
+d = json.load(sys.stdin); t0 = d["window"]["from"][:19]; out = []
+ts = lambda s: dt.datetime.fromisoformat(s.replace("T", " ")[:19])
+for m in d["monitors"]:
+    if m["type"] == "group" or not m["active"]:
+        continue
+    downs = [c["t"] for c in m["changes"] if c["status"] == "DOWN"]
+    host = m["target"].split("//")[-1].split("/")[0].split(":")[0]
+    if m["now"]["status"] == "DOWN":
+        out.append(dict(kind="still_down" if m["now"]["since"] < t0 else "outage", start=m["now"]["since"], host=host, monitors=[m["name"]], summary="down since %s: %s" % (m["now"]["since"], m["now"]["msg"][:80])))
+    elif len(downs) >= 3:
+        out.append(dict(kind="flapping", start=downs[0], host=host, monitors=[m["name"]], summary="%d drops in the window" % len(downs)))
+    elif downs:
+        out.append(dict(kind="recovered", start=downs[0], host=host, monitors=[m["name"]], summary="was down from %s, up again" % downs[0]))
+    if m["tls_days_left"] is not None and m["tls_days_left"] < 14:
+        out.append(dict(kind="cert_expiring", start="", host=host, monitors=[m["name"]], summary="certificate expires in %d days" % m["tls_days_left"]))
+merged = []
+for a in sorted(out, key=lambda a: (a["kind"], a["host"], a["start"])):
+    p = merged[-1] if merged else None
+    if p and a["kind"] == p["kind"] != "cert_expiring" and a["host"] == p["host"] and abs((ts(a["start"]) - ts(p["start"])).total_seconds()) <= 90:
+        p["monitors"] += a["monitors"]
+    else:
+        merged.append(a)
+print(json.dumps({"alerts": [{k: a[k] for k in ("monitors", "kind", "summary")} for a in merged]}, indent=1))

ops-readonly/reader.py

@@ -10,6 +10,9 @@ questions, each answered by a few lines of code over three read-only mounts.
   /run/fail2ban            fail2ban's control socket     (status queries only)
   /waf                     WAF alerts as JSON, written by waf-export.py on the
                            host (episode 14); cscli and its credentials stay there
+  /kuma                    Uptime Kuma state per monitor as JSON, written by
+                           kuma-export.py on the host (episode 16); the Kuma
+                           database stays there
 
 No docker socket, no write access, no free-form arguments. An action name that
 is not in ACTIONS is refused, an argument outside its range is refused, and
@@ -280,6 +283,18 @@ def waf_rule_info(rule_id):
     return dict(rule, id=rule_id, known=True)
 
 
+# ---------------------------------------------------------------- Kuma --
+# Episode 16. Monitor names and messages (which quote the monitored page) are
+# written by whoever controls the monitor or the page. Served as exported.
+
+KUMA_DIR = os.environ.get("KUMA_DIR", "/kuma")
+
+
+def kuma_alerts():
+    with open(os.path.join(KUMA_DIR, "alerts.json"), "rb") as fh:
+        return json.load(fh)
+
+
 # ---------------------------------------------------------------- the list --
 
 def arg_n(q):
@@ -313,6 +328,7 @@ ACTIONS = {
     "caddy_top_ips": lambda q: caddy_top_ips(arg_n(q)),
     "waf_blocked": lambda q: waf_blocked(arg_hours(q)),
     "waf_rule_info": lambda q: waf_rule_info(q.get("id", [""])[0]),
+    "kuma_alerts": lambda q: kuma_alerts(),
 }