Episode 14: an agent reads the WAF blocks, a script beats it, a human merges

ops-readonly gets two read-only questions, waf_blocked and waf_rule_info. The
rule ids per blocked request are in CrowdSec's alerts, not in its log, and
alerts need machine credentials that can also write, so waf-export.py runs on
the host and the container gets one more read-only mount.

Workflow G lets a local 3B model draft CRS route exclusions as a Forgejo issue.
The model never writes rule syntax, a code node checks every field against the
tool output, the only write permission is an issue in one repo, trusted sources
live in the workflow and the trigger needs a token.

waf-propose.py does the same without a model: 5 of 5 false positives, no
exclusion for an attack, 0.03 s. The model proposed an exclusion for nearly
every attack it saw. Numbers and the traps we hit are in HARDENING.md.

ops-readonly/reader.py

@@ -8,6 +8,8 @@ questions, each answered by a few lines of code over three read-only mounts.
   /logs/caddy/access.log   Caddy's JSON access log       (episode 10)
   /var/log/journal         the host journal, sshd lines  (read via libsystemd)
   /run/fail2ban            fail2ban's control socket     (status queries only)
+  /waf                     WAF alerts as JSON, written by waf-export.py on the
+                           host (episode 14); cscli and its credentials stay there
 
 No docker socket, no write access, no free-form arguments. An action name that
 is not in ACTIONS is refused, an argument outside its range is refused, and
@@ -18,7 +20,7 @@ The fail2ban socket speaks pickle. Only two verbs are ever sent through it,
 "status" and "banned"; the same protocol could also send "set ... unbanip", so
 the verb list below is the allowlist for that socket.
 """
-import collections, datetime, json, os, pickle, re, socket, time
+import calendar, collections, datetime, json, os, pickle, re, socket, time
 from http.server import BaseHTTPRequestHandler, HTTPServer
 from urllib.parse import urlparse, parse_qs
 
@@ -29,6 +31,8 @@ F2B_SOCK = "/run/fail2ban/fail2ban.sock"
 WINDOW_H = 24
 JAILS = ("sshd", "caddy-probes")
 N_MIN, N_MAX, N_DEFAULT = 1, 20, 5
+WAF_DIR = os.environ.get("WAF_DIR", "/waf")
+H_MIN, H_MAX, H_DEFAULT = 1, 48, 24
 
 
 # ---------------------------------------------------------------- fail2ban --
@@ -203,6 +207,79 @@ def caddy_top_ips(n=N_DEFAULT):
     }
 
 
+# ---------------------------------------------------------------- WAF alerts --
+# Episode 14. Everything in here except rule ids, counts and the source IP was
+# written by whoever sent the request. The reader truncates, it does not clean.
+
+def _waf_json(name):
+    with open(os.path.join(WAF_DIR, name), "rb") as fh:
+        return json.load(fh)
+
+
+def _auth_index():
+    """(client_ip, host, uri) -> times of every 403 that carried an Authorization
+    or Cookie header. Presence only: the WAF blocks before the app checks the
+    credential, so anyone can set this. It is a hint, never proof."""
+    seen = collections.defaultdict(list)
+    with open(CADDY_LOG, "rb") as fh:
+        for line in fh:
+            if b'"status":403' not in line:
+                continue
+            try:
+                e = json.loads(line)
+            except ValueError:
+                continue
+            req = e.get("request", {})
+            h = req.get("headers", {})
+            if "Authorization" in h or "Cookie" in h:
+                key = (str(req.get("client_ip", "")), str(req.get("host", "")), str(req.get("uri", ""))[:300])
+                seen[key].append(float(e.get("ts", 0)))
+    return seen
+
+
+def _had_auth(index, alert):
+    """Same request, not just the same URI: the log line must sit within 3 s of the alert."""
+    t = calendar.timegm(time.strptime(alert["ts"], "%Y-%m-%dT%H:%M:%SZ"))
+    return any(abs(ts - t) <= 3 for ts in index.get((alert["source_ip"], alert["host"], alert["uri"]), ()))
+
+
+def waf_blocked(hours=H_DEFAULT):
+    data = _waf_json("alerts.json")
+    catalog = _waf_json("crs-rules.json")
+    cutoff = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - hours * 3600))
+    auth = _auth_index()
+    groups = collections.OrderedDict()
+    total = 0
+    for a in data.get("alerts", []):
+        if a.get("ts", "") < cutoff:
+            continue
+        total += 1
+        ids = [r["id"] for r in a.get("rules", [])]
+        key = (a["source_ip"], a["host"], a["method"], a["uri"], tuple(ids))
+        g = groups.get(key)
+        if g is None:
+            g = groups[key] = {
+                "source_ip": a["source_ip"], "host": a["host"], "method": a["method"],
+                "uri": a["uri"][:160], "count": 0, "first_seen": a["ts"], "last_seen": a["ts"],
+                "auth_header_present": _had_auth(auth, a),
+                "rule_ids": ids,
+                "attack_classes": sorted({catalog.get(i, {}).get("attack_class", "") for i in ids} - {""}),
+            }
+        g["count"] += 1
+        g["last_seen"] = max(g["last_seen"], a["ts"])
+    return {"window_hours": hours, "exported_at": data.get("exported_at"), "blocked_total": total,
+            "distinct_requests": len(groups), "requests": list(groups.values())}
+
+
+def waf_rule_info(rule_id):
+    if not re.fullmatch(r"\d{6}", rule_id or ""):
+        raise ValueError("id must be a six digit CRS rule id")
+    rule = _waf_json("crs-rules.json").get(rule_id)
+    if rule is None:
+        return {"id": rule_id, "known": False, "note": "no such rule in the installed CRS"}
+    return dict(rule, id=rule_id, known=True)
+
+
 # ---------------------------------------------------------------- the list --
 
 def arg_n(q):
@@ -216,6 +293,17 @@ def arg_n(q):
     return n
 
 
+def arg_hours(q):
+    raw = q.get("hours", [str(H_DEFAULT)])[0]
+    try:
+        h = int(str(raw).strip())
+    except ValueError:
+        raise ValueError("hours must be a whole number")
+    if not H_MIN <= h <= H_MAX:
+        raise ValueError("hours must be between %d and %d" % (H_MIN, H_MAX))
+    return h
+
+
 ACTIONS = {
     "f2b_status": lambda q: f2b_status(q.get("jail", [""])[0]),
     "f2b_banned": lambda q: f2b_banned(),
@@ -223,6 +311,8 @@ ACTIONS = {
     "caddy_status_counts": lambda q: caddy_status_counts(),
     "caddy_top_paths": lambda q: caddy_top_paths(arg_n(q)),
     "caddy_top_ips": lambda q: caddy_top_ips(arg_n(q)),
+    "waf_blocked": lambda q: waf_blocked(arg_hours(q)),
+    "waf_rule_info": lambda q: waf_rule_info(q.get("id", [""])[0]),
 }