Episode 14: an agent reads the WAF blocks, a script beats it, a human merges

ops-readonly gets two read-only questions, waf_blocked and waf_rule_info. The
rule ids per blocked request are in CrowdSec's alerts, not in its log, and
alerts need machine credentials that can also write, so waf-export.py runs on
the host and the container gets one more read-only mount.

Workflow G lets a local 3B model draft CRS route exclusions as a Forgejo issue.
The model never writes rule syntax, a code node checks every field against the
tool output, the only write permission is an issue in one repo, trusted sources
live in the workflow and the trigger needs a token.

waf-propose.py does the same without a model: 5 of 5 false positives, no
exclusion for an attack, 0.03 s. The model proposed an exclusion for nearly
every attack it saw. Numbers and the traps we hit are in HARDENING.md.

ops-readonly/waf-export.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""
+waf-export (episode 14): runs on the HOST, not in a container.
+
+The WAF exception advisor needs to know which requests the WAF blocked and
+which CRS rule ids fired. CrowdSec keeps that in its alerts (kind "waf"), and
+alerts are only readable with machine credentials, which can also WRITE alerts
+and decisions. Handing those to the ops-readonly container would end "read
+only". So cscli stays on the host: this script dumps the WAF alerts to a JSON
+file, and ops-readonly gets one more read-only mount. Run it from the timer
+next to this file.
+
+  /var/lib/ops-readonly/waf/alerts.json     one entry per blocked request
+  /var/lib/ops-readonly/waf/crs-rules.json  rule id -> description, from the
+                                            CRS SecLang files CrowdSec installed
+
+Everything in an alert except the rule ids and the source IP was written by
+whoever sent the request. Treat uri, user_agent and matched data as attacker
+text downstream.
+"""
+import glob, json, os, re, subprocess, sys, tempfile, time
+
+OUT = os.environ.get("WAF_EXPORT_DIR", "/var/lib/ops-readonly/waf")
+CRS_GLOB = "/var/lib/crowdsec/data/*-9[0-9][0-9]-*.conf"
+SINCE = os.environ.get("WAF_EXPORT_SINCE", "48h")
+# CRS bookkeeping rules: they fire on every blocked request and say nothing
+# about why. 949110/959100 are the blocking rules, never a tuning target here.
+BOOKKEEPING = {"901340", "949110", "959100", "980170"}
+
+
+def meta_list(alert, key):
+    for m in alert.get("meta") or []:
+        if m.get("key") == key:
+            try:
+                v = json.loads(m.get("value") or "[]")
+            except ValueError:
+                v = [m.get("value")]
+            return [str(x) for x in (v if isinstance(v, list) else [v])]
+    return []
+
+
+def event_field(alert, key):
+    for ev in alert.get("events") or []:
+        for m in ev.get("meta") or []:
+            if m.get("key") == key and m.get("value"):
+                return str(m["value"])
+    return ""
+
+
+def rules_of(alert):
+    """(id, message) for every rule that fired, bookkeeping rules dropped."""
+    seen, out = set(), []
+    for ev in alert.get("events") or []:
+        m = {x.get("key"): str(x.get("value", "")) for x in ev.get("meta") or []}
+        rid = m.get("rule_name", "").replace("native_rule:", "")
+        if not rid or rid in BOOKKEEPING or rid in seen:
+            continue
+        seen.add(rid)
+        out.append({"id": rid, "msg": m.get("message", "")[:160]})
+    return out
+
+
+def export_alerts():
+    raw = subprocess.run(["cscli", "alerts", "list", "-o", "json", "--since", SINCE, "-l", "5000"],
+                         capture_output=True, text=True, check=True).stdout
+    rows = []
+    for a in json.loads(raw) or []:
+        if a.get("kind") != "waf":
+            continue
+        rows.append({
+            "id": a["id"],
+            "ts": a.get("created_at", "")[:19] + "Z",
+            "source_ip": (a.get("source") or {}).get("ip", ""),
+            "host": event_field(a, "target_fqdn"),
+            "method": (meta_list(a, "method") or [""])[0],
+            "uri": (meta_list(a, "target_uri") or [""])[0][:300],
+            "user_agent": (meta_list(a, "user_agent") or [""])[0][:200],
+            "rules": rules_of(a),
+        })
+    rows.sort(key=lambda r: r["id"])
+    return rows
+
+
+RE_RULE = re.compile(r"""["\s]id:(\d{6}),""")
+RE_MSG = re.compile(r"msg:'([^']*)'")
+RE_PL = re.compile(r"tag:'paranoia-level/(\d)'")
+RE_ATTACK = re.compile(r"tag:'(attack-[a-z-]+)'")
+RE_SEV = re.compile(r"severity:'([A-Z]+)'")
+
+
+def export_catalog():
+    cat = {}
+    for path in sorted(glob.glob(CRS_GLOB)):
+        text = open(path, encoding="utf-8", errors="replace").read()
+        # one logical rule per block; continuation lines were already joined by CrowdSec's copy
+        for block in re.split(r"\n(?=SecRule|SecAction)", text):
+            m = RE_RULE.search(block)
+            msg = RE_MSG.search(block)
+            if not m or not msg:
+                continue
+            pl, atk, sev = RE_PL.search(block), RE_ATTACK.search(block), RE_SEV.search(block)
+            cat[m.group(1)] = {
+                "msg": msg.group(1)[:200],
+                "file": os.path.basename(path),
+                "attack_class": atk.group(1) if atk else "",
+                "paranoia_level": int(pl.group(1)) if pl else None,
+                "severity": sev.group(1) if sev else "",
+            }
+    return cat
+
+
+def write_atomic(name, obj):
+    os.makedirs(OUT, mode=0o750, exist_ok=True)
+    fd, tmp = tempfile.mkstemp(dir=OUT, prefix="." + name)
+    with os.fdopen(fd, "w") as fh:
+        json.dump(obj, fh)
+    os.chmod(tmp, 0o640)
+    os.replace(tmp, os.path.join(OUT, name))
+
+
+if __name__ == "__main__":
+    alerts = export_alerts()
+    write_atomic("alerts.json", {"exported_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+                                 "since": SINCE, "alerts": alerts})
+    if "--catalog" in sys.argv or not os.path.exists(os.path.join(OUT, "crs-rules.json")):
+        write_atomic("crs-rules.json", export_catalog())
+    print("waf-export: %d blocked requests written to %s" % (len(alerts), OUT))