3edef38ba3
ops-readonly gets two read-only questions, waf_blocked and waf_rule_info. The rule ids per blocked request are in CrowdSec's alerts, not in its log, and alerts need machine credentials that can also write, so waf-export.py runs on the host and the container gets one more read-only mount. Workflow G lets a local 3B model draft CRS route exclusions as a Forgejo issue. The model never writes rule syntax, a code node checks every field against the tool output, the only write permission is an issue in one repo, trusted sources live in the workflow and the trigger needs a token. waf-propose.py does the same without a model: 5 of 5 false positives, no exclusion for an attack, 0.03 s. The model proposed an exclusion for nearly every attack it saw. Numbers and the traps we hit are in HARDENING.md.
127 lines
5 KiB
Python
127 lines
5 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
waf-export (episode 14): runs on the HOST, not in a container.
|
|
|
|
The WAF exception advisor needs to know which requests the WAF blocked and
|
|
which CRS rule ids fired. CrowdSec keeps that in its alerts (kind "waf"), and
|
|
alerts are only readable with machine credentials, which can also WRITE alerts
|
|
and decisions. Handing those to the ops-readonly container would end "read
|
|
only". So cscli stays on the host: this script dumps the WAF alerts to a JSON
|
|
file, and ops-readonly gets one more read-only mount. Run it from the timer
|
|
next to this file.
|
|
|
|
/var/lib/ops-readonly/waf/alerts.json one entry per blocked request
|
|
/var/lib/ops-readonly/waf/crs-rules.json rule id -> description, from the
|
|
CRS SecLang files CrowdSec installed
|
|
|
|
Everything in an alert except the rule ids and the source IP was written by
|
|
whoever sent the request. Treat uri, user_agent and matched data as attacker
|
|
text downstream.
|
|
"""
|
|
import glob, json, os, re, subprocess, sys, tempfile, time
|
|
|
|
OUT = os.environ.get("WAF_EXPORT_DIR", "/var/lib/ops-readonly/waf")
|
|
CRS_GLOB = "/var/lib/crowdsec/data/*-9[0-9][0-9]-*.conf"
|
|
SINCE = os.environ.get("WAF_EXPORT_SINCE", "48h")
|
|
# CRS bookkeeping rules: they fire on every blocked request and say nothing
|
|
# about why. 949110/959100 are the blocking rules, never a tuning target here.
|
|
BOOKKEEPING = {"901340", "949110", "959100", "980170"}
|
|
|
|
|
|
def meta_list(alert, key):
|
|
for m in alert.get("meta") or []:
|
|
if m.get("key") == key:
|
|
try:
|
|
v = json.loads(m.get("value") or "[]")
|
|
except ValueError:
|
|
v = [m.get("value")]
|
|
return [str(x) for x in (v if isinstance(v, list) else [v])]
|
|
return []
|
|
|
|
|
|
def event_field(alert, key):
|
|
for ev in alert.get("events") or []:
|
|
for m in ev.get("meta") or []:
|
|
if m.get("key") == key and m.get("value"):
|
|
return str(m["value"])
|
|
return ""
|
|
|
|
|
|
def rules_of(alert):
|
|
"""(id, message) for every rule that fired, bookkeeping rules dropped."""
|
|
seen, out = set(), []
|
|
for ev in alert.get("events") or []:
|
|
m = {x.get("key"): str(x.get("value", "")) for x in ev.get("meta") or []}
|
|
rid = m.get("rule_name", "").replace("native_rule:", "")
|
|
if not rid or rid in BOOKKEEPING or rid in seen:
|
|
continue
|
|
seen.add(rid)
|
|
out.append({"id": rid, "msg": m.get("message", "")[:160]})
|
|
return out
|
|
|
|
|
|
def export_alerts():
|
|
raw = subprocess.run(["cscli", "alerts", "list", "-o", "json", "--since", SINCE, "-l", "5000"],
|
|
capture_output=True, text=True, check=True).stdout
|
|
rows = []
|
|
for a in json.loads(raw) or []:
|
|
if a.get("kind") != "waf":
|
|
continue
|
|
rows.append({
|
|
"id": a["id"],
|
|
"ts": a.get("created_at", "")[:19] + "Z",
|
|
"source_ip": (a.get("source") or {}).get("ip", ""),
|
|
"host": event_field(a, "target_fqdn"),
|
|
"method": (meta_list(a, "method") or [""])[0],
|
|
"uri": (meta_list(a, "target_uri") or [""])[0][:300],
|
|
"user_agent": (meta_list(a, "user_agent") or [""])[0][:200],
|
|
"rules": rules_of(a),
|
|
})
|
|
rows.sort(key=lambda r: r["id"])
|
|
return rows
|
|
|
|
|
|
RE_RULE = re.compile(r"""["\s]id:(\d{6}),""")
|
|
RE_MSG = re.compile(r"msg:'([^']*)'")
|
|
RE_PL = re.compile(r"tag:'paranoia-level/(\d)'")
|
|
RE_ATTACK = re.compile(r"tag:'(attack-[a-z-]+)'")
|
|
RE_SEV = re.compile(r"severity:'([A-Z]+)'")
|
|
|
|
|
|
def export_catalog():
|
|
cat = {}
|
|
for path in sorted(glob.glob(CRS_GLOB)):
|
|
text = open(path, encoding="utf-8", errors="replace").read()
|
|
# one logical rule per block; continuation lines were already joined by CrowdSec's copy
|
|
for block in re.split(r"\n(?=SecRule|SecAction)", text):
|
|
m = RE_RULE.search(block)
|
|
msg = RE_MSG.search(block)
|
|
if not m or not msg:
|
|
continue
|
|
pl, atk, sev = RE_PL.search(block), RE_ATTACK.search(block), RE_SEV.search(block)
|
|
cat[m.group(1)] = {
|
|
"msg": msg.group(1)[:200],
|
|
"file": os.path.basename(path),
|
|
"attack_class": atk.group(1) if atk else "",
|
|
"paranoia_level": int(pl.group(1)) if pl else None,
|
|
"severity": sev.group(1) if sev else "",
|
|
}
|
|
return cat
|
|
|
|
|
|
def write_atomic(name, obj):
|
|
os.makedirs(OUT, mode=0o750, exist_ok=True)
|
|
fd, tmp = tempfile.mkstemp(dir=OUT, prefix="." + name)
|
|
with os.fdopen(fd, "w") as fh:
|
|
json.dump(obj, fh)
|
|
os.chmod(tmp, 0o640)
|
|
os.replace(tmp, os.path.join(OUT, name))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
alerts = export_alerts()
|
|
write_atomic("alerts.json", {"exported_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
"since": SINCE, "alerts": alerts})
|
|
if "--catalog" in sys.argv or not os.path.exists(os.path.join(OUT, "crs-rules.json")):
|
|
write_atomic("crs-rules.json", export_catalog())
|
|
print("waf-export: %d blocked requests written to %s" % (len(alerts), OUT))
|