Episode 14: an agent reads the WAF blocks, a script beats it, a human merges

ops-readonly gets two read-only questions, waf_blocked and waf_rule_info. The
rule ids per blocked request are in CrowdSec's alerts, not in its log, and
alerts need machine credentials that can also write, so waf-export.py runs on
the host and the container gets one more read-only mount.

Workflow G lets a local 3B model draft CRS route exclusions as a Forgejo issue.
The model never writes rule syntax, a code node checks every field against the
tool output, the only write permission is an issue in one repo, trusted sources
live in the workflow and the trigger needs a token.

waf-propose.py does the same without a model: 5 of 5 false positives, no
exclusion for an attack, 0.03 s. The model proposed an exclusion for nearly
every attack it saw. Numbers and the traps we hit are in HARDENING.md.

ops-readonly/reader.py

@@ -8,6 +8,8 @@ questions, each answered by a few lines of code over three read-only mounts.
   /logs/caddy/access.log   Caddy's JSON access log       (episode 10)
   /var/log/journal         the host journal, sshd lines  (read via libsystemd)
   /run/fail2ban            fail2ban's control socket     (status queries only)
+  /waf                     WAF alerts as JSON, written by waf-export.py on the
+                           host (episode 14); cscli and its credentials stay there
 
 No docker socket, no write access, no free-form arguments. An action name that
 is not in ACTIONS is refused, an argument outside its range is refused, and
@@ -18,7 +20,7 @@ The fail2ban socket speaks pickle. Only two verbs are ever sent through it,
 "status" and "banned"; the same protocol could also send "set ... unbanip", so
 the verb list below is the allowlist for that socket.
 """
-import collections, datetime, json, os, pickle, re, socket, time
+import calendar, collections, datetime, json, os, pickle, re, socket, time
 from http.server import BaseHTTPRequestHandler, HTTPServer
 from urllib.parse import urlparse, parse_qs
 
@@ -29,6 +31,8 @@ F2B_SOCK = "/run/fail2ban/fail2ban.sock"
 WINDOW_H = 24
 JAILS = ("sshd", "caddy-probes")
 N_MIN, N_MAX, N_DEFAULT = 1, 20, 5
+WAF_DIR = os.environ.get("WAF_DIR", "/waf")
+H_MIN, H_MAX, H_DEFAULT = 1, 48, 24
 
 
 # ---------------------------------------------------------------- fail2ban --
@@ -203,6 +207,79 @@ def caddy_top_ips(n=N_DEFAULT):
     }
 
 
+# ---------------------------------------------------------------- WAF alerts --
+# Episode 14. Everything in here except rule ids, counts and the source IP was
+# written by whoever sent the request. The reader truncates, it does not clean.
+
+def _waf_json(name):
+    with open(os.path.join(WAF_DIR, name), "rb") as fh:
+        return json.load(fh)
+
+
+def _auth_index():
+    """(client_ip, host, uri) -> times of every 403 that carried an Authorization
+    or Cookie header. Presence only: the WAF blocks before the app checks the
+    credential, so anyone can set this. It is a hint, never proof."""
+    seen = collections.defaultdict(list)
+    with open(CADDY_LOG, "rb") as fh:
+        for line in fh:
+            if b'"status":403' not in line:
+                continue
+            try:
+                e = json.loads(line)
+            except ValueError:
+                continue
+            req = e.get("request", {})
+            h = req.get("headers", {})
+            if "Authorization" in h or "Cookie" in h:
+                key = (str(req.get("client_ip", "")), str(req.get("host", "")), str(req.get("uri", ""))[:300])
+                seen[key].append(float(e.get("ts", 0)))
+    return seen
+
+
+def _had_auth(index, alert):
+    """Same request, not just the same URI: the log line must sit within 3 s of the alert."""
+    t = calendar.timegm(time.strptime(alert["ts"], "%Y-%m-%dT%H:%M:%SZ"))
+    return any(abs(ts - t) <= 3 for ts in index.get((alert["source_ip"], alert["host"], alert["uri"]), ()))
+
+
+def waf_blocked(hours=H_DEFAULT):
+    data = _waf_json("alerts.json")
+    catalog = _waf_json("crs-rules.json")
+    cutoff = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - hours * 3600))
+    auth = _auth_index()
+    groups = collections.OrderedDict()
+    total = 0
+    for a in data.get("alerts", []):
+        if a.get("ts", "") < cutoff:
+            continue
+        total += 1
+        ids = [r["id"] for r in a.get("rules", [])]
+        key = (a["source_ip"], a["host"], a["method"], a["uri"], tuple(ids))
+        g = groups.get(key)
+        if g is None:
+            g = groups[key] = {
+                "source_ip": a["source_ip"], "host": a["host"], "method": a["method"],
+                "uri": a["uri"][:160], "count": 0, "first_seen": a["ts"], "last_seen": a["ts"],
+                "auth_header_present": _had_auth(auth, a),
+                "rule_ids": ids,
+                "attack_classes": sorted({catalog.get(i, {}).get("attack_class", "") for i in ids} - {""}),
+            }
+        g["count"] += 1
+        g["last_seen"] = max(g["last_seen"], a["ts"])
+    return {"window_hours": hours, "exported_at": data.get("exported_at"), "blocked_total": total,
+            "distinct_requests": len(groups), "requests": list(groups.values())}
+
+
+def waf_rule_info(rule_id):
+    if not re.fullmatch(r"\d{6}", rule_id or ""):
+        raise ValueError("id must be a six digit CRS rule id")
+    rule = _waf_json("crs-rules.json").get(rule_id)
+    if rule is None:
+        return {"id": rule_id, "known": False, "note": "no such rule in the installed CRS"}
+    return dict(rule, id=rule_id, known=True)
+
+
 # ---------------------------------------------------------------- the list --
 
 def arg_n(q):
@@ -216,6 +293,17 @@ def arg_n(q):
     return n
 
 
+def arg_hours(q):
+    raw = q.get("hours", [str(H_DEFAULT)])[0]
+    try:
+        h = int(str(raw).strip())
+    except ValueError:
+        raise ValueError("hours must be a whole number")
+    if not H_MIN <= h <= H_MAX:
+        raise ValueError("hours must be between %d and %d" % (H_MIN, H_MAX))
+    return h
+
+
 ACTIONS = {
     "f2b_status": lambda q: f2b_status(q.get("jail", [""])[0]),
     "f2b_banned": lambda q: f2b_banned(),
@@ -223,6 +311,8 @@ ACTIONS = {
     "caddy_status_counts": lambda q: caddy_status_counts(),
     "caddy_top_paths": lambda q: caddy_top_paths(arg_n(q)),
     "caddy_top_ips": lambda q: caddy_top_ips(arg_n(q)),
+    "waf_blocked": lambda q: waf_blocked(arg_hours(q)),
+    "waf_rule_info": lambda q: waf_rule_info(q.get("id", [""])[0]),
 }
 
 

ops-readonly/waf-export.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""
+waf-export (episode 14): runs on the HOST, not in a container.
+
+The WAF exception advisor needs to know which requests the WAF blocked and
+which CRS rule ids fired. CrowdSec keeps that in its alerts (kind "waf"), and
+alerts are only readable with machine credentials, which can also WRITE alerts
+and decisions. Handing those to the ops-readonly container would end "read
+only". So cscli stays on the host: this script dumps the WAF alerts to a JSON
+file, and ops-readonly gets one more read-only mount. Run it from the timer
+next to this file.
+
+  /var/lib/ops-readonly/waf/alerts.json     one entry per blocked request
+  /var/lib/ops-readonly/waf/crs-rules.json  rule id -> description, from the
+                                            CRS SecLang files CrowdSec installed
+
+Everything in an alert except the rule ids and the source IP was written by
+whoever sent the request. Treat uri, user_agent and matched data as attacker
+text downstream.
+"""
+import glob, json, os, re, subprocess, sys, tempfile, time
+
+OUT = os.environ.get("WAF_EXPORT_DIR", "/var/lib/ops-readonly/waf")
+CRS_GLOB = "/var/lib/crowdsec/data/*-9[0-9][0-9]-*.conf"
+SINCE = os.environ.get("WAF_EXPORT_SINCE", "48h")
+# CRS bookkeeping rules: they fire on every blocked request and say nothing
+# about why. 949110/959100 are the blocking rules, never a tuning target here.
+BOOKKEEPING = {"901340", "949110", "959100", "980170"}
+
+
+def meta_list(alert, key):
+    for m in alert.get("meta") or []:
+        if m.get("key") == key:
+            try:
+                v = json.loads(m.get("value") or "[]")
+            except ValueError:
+                v = [m.get("value")]
+            return [str(x) for x in (v if isinstance(v, list) else [v])]
+    return []
+
+
+def event_field(alert, key):
+    for ev in alert.get("events") or []:
+        for m in ev.get("meta") or []:
+            if m.get("key") == key and m.get("value"):
+                return str(m["value"])
+    return ""
+
+
+def rules_of(alert):
+    """(id, message) for every rule that fired, bookkeeping rules dropped."""
+    seen, out = set(), []
+    for ev in alert.get("events") or []:
+        m = {x.get("key"): str(x.get("value", "")) for x in ev.get("meta") or []}
+        rid = m.get("rule_name", "").replace("native_rule:", "")
+        if not rid or rid in BOOKKEEPING or rid in seen:
+            continue
+        seen.add(rid)
+        out.append({"id": rid, "msg": m.get("message", "")[:160]})
+    return out
+
+
+def export_alerts():
+    raw = subprocess.run(["cscli", "alerts", "list", "-o", "json", "--since", SINCE, "-l", "5000"],
+                         capture_output=True, text=True, check=True).stdout
+    rows = []
+    for a in json.loads(raw) or []:
+        if a.get("kind") != "waf":
+            continue
+        rows.append({
+            "id": a["id"],
+            "ts": a.get("created_at", "")[:19] + "Z",
+            "source_ip": (a.get("source") or {}).get("ip", ""),
+            "host": event_field(a, "target_fqdn"),
+            "method": (meta_list(a, "method") or [""])[0],
+            "uri": (meta_list(a, "target_uri") or [""])[0][:300],
+            "user_agent": (meta_list(a, "user_agent") or [""])[0][:200],
+            "rules": rules_of(a),
+        })
+    rows.sort(key=lambda r: r["id"])
+    return rows
+
+
+RE_RULE = re.compile(r"""["\s]id:(\d{6}),""")
+RE_MSG = re.compile(r"msg:'([^']*)'")
+RE_PL = re.compile(r"tag:'paranoia-level/(\d)'")
+RE_ATTACK = re.compile(r"tag:'(attack-[a-z-]+)'")
+RE_SEV = re.compile(r"severity:'([A-Z]+)'")
+
+
+def export_catalog():
+    cat = {}
+    for path in sorted(glob.glob(CRS_GLOB)):
+        text = open(path, encoding="utf-8", errors="replace").read()
+        # one logical rule per block; continuation lines were already joined by CrowdSec's copy
+        for block in re.split(r"\n(?=SecRule|SecAction)", text):
+            m = RE_RULE.search(block)
+            msg = RE_MSG.search(block)
+            if not m or not msg:
+                continue
+            pl, atk, sev = RE_PL.search(block), RE_ATTACK.search(block), RE_SEV.search(block)
+            cat[m.group(1)] = {
+                "msg": msg.group(1)[:200],
+                "file": os.path.basename(path),
+                "attack_class": atk.group(1) if atk else "",
+                "paranoia_level": int(pl.group(1)) if pl else None,
+                "severity": sev.group(1) if sev else "",
+            }
+    return cat
+
+
+def write_atomic(name, obj):
+    os.makedirs(OUT, mode=0o750, exist_ok=True)
+    fd, tmp = tempfile.mkstemp(dir=OUT, prefix="." + name)
+    with os.fdopen(fd, "w") as fh:
+        json.dump(obj, fh)
+    os.chmod(tmp, 0o640)
+    os.replace(tmp, os.path.join(OUT, name))
+
+
+if __name__ == "__main__":
+    alerts = export_alerts()
+    write_atomic("alerts.json", {"exported_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+                                 "since": SINCE, "alerts": alerts})
+    if "--catalog" in sys.argv or not os.path.exists(os.path.join(OUT, "crs-rules.json")):
+        write_atomic("crs-rules.json", export_catalog())
+    print("waf-export: %d blocked requests written to %s" % (len(alerts), OUT))

ops-readonly/waf-export.service

@@ -0,0 +1,12 @@
+# Episode 14: dumps CrowdSec's WAF alerts to a JSON file for ops-readonly.
+# Runs on the host, because cscli needs machine credentials that can also write.
+#   cp ops-readonly/waf-export.{service,timer} /etc/systemd/system/
+#   systemctl daemon-reload && systemctl enable --now waf-export.timer
+[Unit]
+Description=Export CrowdSec WAF alerts for the read-only agent tool
+After=crowdsec.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/python3 /opt/selfhostedops-stack/ops-readonly/waf-export.py
+Nice=10

ops-readonly/waf-export.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Export CrowdSec WAF alerts every minute
+
+[Timer]
+OnBootSec=2min
+OnUnitActiveSec=1min
+AccuracySec=5s
+
+[Install]
+WantedBy=timers.target

ops-readonly/waf-propose.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+"""
+waf-propose (episode 14): the same proposal the advisor agent makes, without a model.
+
+  curl -s -H "x-ops-token: $T" http://ops-readonly:8098/read/waf_blocked | ./waf-propose.py 198.51.100.10
+
+Reads the waf_blocked answer on stdin. Arguments: the source IPs you trust
+(office, the box itself). A blocked request counts only if it came from one of
+them, and a route only if it was blocked MIN_COUNT times. No trusted source
+given means no proposal: request text alone proves nothing, the sender wrote it.
+"""
+import collections, json, sys
+
+MIN_COUNT = 2
+trusted = set(sys.argv[1:])
+routes = collections.OrderedDict()
+for r in json.load(sys.stdin)["requests"]:
+    if r["source_ip"] not in trusted:
+        continue
+    path = r["uri"].split("?")[0]
+    seg = path.strip("/").split("/")
+    # ponytail: route = first two path segments. A repo or topic name in there
+    # makes the exclusion per repo; widen by hand when that gets tedious.
+    prefix = "/" + "/".join(seg[:2]) + "/" if len(seg) > 2 else path
+    g = routes.setdefault((r["host"], prefix), {"count": 0, "ids": set()})
+    g["count"] += r["count"]
+    g["ids"].update(i for i in r["rule_ids"] if i.isdigit())
+
+proposals = [{"classification": "false_positive", "host": h, "path": p, "match": "prefix", "rule_ids": sorted(g["ids"]),
+              "reason": "blocked %d times from a trusted source" % g["count"]}
+             for (h, p), g in routes.items() if g["count"] >= MIN_COUNT and g["ids"]]
+json.dump({"proposals": proposals, "rejected": []}, sys.stdout, indent=1)