From 88670f028f249b640848dc82ec056ab2d9d6a7c0 Mon Sep 17 00:00:00 2001 From: Maximilian Grimm Date: Wed, 16 Sep 2026 20:34:18 +0200 Subject: [PATCH] Episode 12/13: CrowdSec next to fail2ban, and an AppSec WAF in Caddy Measured on the demo box against the same 24h of real traffic. Episode 12 (detection, blocklist, firewall): - crowdsec/ with install notes and the traps: port 8080 clash, empty first blocklist pull, replay forensic-mode dropping events (FIFO workaround), ssh-time-based-bf shipping remediation:false, the AppSec collections that refuse to start without extra rules. - firewall bouncer in nftables mode (input + forward), which blocks Docker-published ports and survives ufw reload; iptables/DOCKER-USER does not. - HARDENING.md episode 12 section with the numbers: fail2ban 4 banned IPs vs CrowdSec 23 detected / 1 blocked, blocklist overlap 50% of IPs but 19% of attempts, and the mode=normal blind spot for key-only root attempts. Episode 13 (AppSec WAF): - caddy/Dockerfile now builds the CrowdSec bouncer and AppSec handler alongside rate-limit, pinned to v0.14.1. - Caddyfile imports a crowdsec snippet (bouncer + appsec) on every site except vpn; compose passes CROWDSEC_BOUNCER_KEY and host.docker.internal. - Measured: virtual patching blocks known CVEs only (7 of 41 probes, near-zero false positives); OWASP CRS raises that to 27 of 41 but breaks git push and wiki editing at default paranoia. Default stays vpatch+generic. fail2ban config unchanged. No secrets or real domains/IPs in tracked files. --- .env.example | 6 ++ Caddyfile | 22 ++++++ HARDENING.md | 73 ++++++++++++++++++++ README.md | 6 ++ caddy/Dockerfile | 10 ++- caddy/README.md | 17 +++-- compose.yml | 5 +- crowdsec/README.md | 121 +++++++++++++++++++++++++++++++++ crowdsec/acquis.d/appsec.yaml | 10 +++ crowdsec/acquis.d/caddy.yaml | 7 ++ crowdsec/firewall-bouncer.yaml | 12 ++++ 11 files changed, 280 insertions(+), 9 deletions(-) create mode 100644 crowdsec/README.md create mode 100644 crowdsec/acquis.d/appsec.yaml create mode 100644 crowdsec/acquis.d/caddy.yaml create mode 100644 crowdsec/firewall-bouncer.yaml diff --git a/.env.example b/.env.example index 25efe7d..01f28e9 100644 --- a/.env.example +++ b/.env.example @@ -78,3 +78,9 @@ RUNNER_UUID= # The runner still refuses any action or any target that is not on its own # hard-coded allowlists, so a leaked token cannot widen the blast radius. OPS_ACTIONS_TOKEN= + +# --- Episode 12: CrowdSec ------------------------------------------------- +# The Caddy bouncer's key for the CrowdSec Local API on the host: +# cscli bouncers add caddy-bouncer -o raw +# Without it Caddy still starts, but blocks nothing and logs auth errors. +CROWDSEC_BOUNCER_KEY= diff --git a/Caddyfile b/Caddyfile index e9f8758..8c90a9d 100644 --- a/Caddyfile +++ b/Caddyfile @@ -5,6 +5,16 @@ email {$ACME_EMAIL} # episode 10: rate_limit has to run before reverse_proxy hands the request on order rate_limit before basic_auth + # episode 12: CrowdSec decides first (banned IP -> 403), then the AppSec WAF + # checks the request itself. Both talk to CrowdSec on the host. + order crowdsec first + order appsec after crowdsec + crowdsec { + api_url http://host.docker.internal:8090 + api_key {$CROWDSEC_BOUNCER_KEY} + ticker_interval 10s + appsec_url http://host.docker.internal:7422 + } } # episode 10: one snippet, imported into every site. JSON access log (fail2ban @@ -22,8 +32,15 @@ header -Server } +# episode 12: blocklist + WAF. Imported everywhere except vpn. (NetBird gRPC streams). +(crowdsec) { + crowdsec + appsec +} + git.{$DOMAIN} { import common + import crowdsec # episode 10: throttle login attempts (10 per minute per source IP) rate_limit { zone git_login { @@ -41,11 +58,13 @@ git.{$DOMAIN} { wiki.{$DOMAIN} { import common + import crowdsec reverse_proxy outline:3000 } vault.{$DOMAIN} { import common + import crowdsec # episode 10: the admin panel is a setup tool, not a public page (was 200) @admin path /admin* respond @admin 404 @@ -65,12 +84,14 @@ vault.{$DOMAIN} { status.{$DOMAIN} { import common + import crowdsec reverse_proxy uptime-kuma:3001 } # Push notifications (episode 6). Websockets and long polling pass through as-is. ntfy.{$DOMAIN} { import common + import crowdsec reverse_proxy ntfy:80 } @@ -80,6 +101,7 @@ ntfy.{$DOMAIN} { # one door that has to stay open - and it is exactly one door wide. n8n.{$DOMAIN} { import common + import crowdsec @webhooks path /webhook/* /webhook-test/* /webhook-waiting/* handle @webhooks { reverse_proxy n8n:5678 diff --git a/HARDENING.md b/HARDENING.md index 4377ee9..d1baf30 100644 --- a/HARDENING.md +++ b/HARDENING.md @@ -86,6 +86,12 @@ advice from episode 1 got wrong: the ban in INPUT; Caddy's traffic is forwarded, so a bot "banned" in INPUT keeps getting 404s (measured). Set `chain = DOCKER-USER` as a plain jail parameter and the same bot runs into a refused connection instead. +- **Once passwords are off, `mode = normal` goes half blind** (episode 12). + Key-only root attempts log as `Connection closed by authenticating user + root [preauth]`, which the normal sshd filter ignores. Measured: 192 attempts + from one IP in 24 h, zero seen. `mode = aggressive` in the jail counts them; + it also counts the second line of each invalid-user attempt, so maxretry 5 + bans after roughly 3 tries. Operational note: `ufw reload` flushes `DOCKER-USER` and takes fail2ban's ban jumps with it, run `systemctl restart fail2ban` afterwards to restore them. @@ -374,3 +380,70 @@ agent doing the whole job against seven tools gets the counts right often but not always; splitting the work into three narrow roles, each with one skill file (`agents/skills/`), trades more tokens for a more reliable report. The numbers are in the lab notes. + +## Episode 12: CrowdSec next to fail2ban, same box, same logs + +A viewer asked for it: does CrowdSec replace fail2ban? We ran both on the demo +box and gave them the same 24 hours of real traffic (fail2ban live, CrowdSec by +replaying the identical sshd log). Setup and traps: `crowdsec/README.md`. + +**What fail2ban caught in 24 h** (password auth already off since episode 10): +1,458 SSH login attempts from 218 IPs. fail2ban banned **4 IPs** (6 bans). +Two blind spots, both measured: + +- *Slow bots.* One IP tried 273 times, one attempt every ~4.5 minutes. Never 5 + in 10 minutes, so never banned. +- *Root without a password path.* `Connection closed by authenticating user + root [preauth]` is not a failure in the default `mode = normal` of the sshd + filter. One IP made 192 of those attempts and fail2ban did not see a single + one. `mode = aggressive` matches 2,192 of 2,345 lines instead of 696. + +**What CrowdSec detected on the same log:** 23 IPs, including the slow one. +**What it would have banned: 1 IP.** 22 of the 23 came from +`ssh-time-based-bf`, and that scenario ships with `remediation: false`: it +raises an alert (and a community signal) but no decision. Out of the box, +CrowdSec *sees* more and *blocks* less than fail2ban on this traffic. If you +want the slow ones banned, add a profile for that scenario instead of editing +it: a modified hub scenario is "tainted", its signals stop counting, and a +non-contributing engine drops to the 3,000-IP Lite blocklist. + +**The community blocklist:** 15,000 IPs on day one (after a restart, see the +README). Of the 218 IPs that tried to log in during the previous 24 h, **109 +were already on it**, but those 109 made only **19 %** of the attempts. The list +knows the one-shot scanners; the persistent bots that hammer a single box were +mostly not on it (1 of the top 10). None of the 4 IPs fail2ban banned were +listed. The two tools overlap less than you would think. + +**The firewall bouncer and Docker, again.** In its default iptables mode the +bouncer only hooks `INPUT`: a banned IP gets a timeout on SSH and still 200 on +every Docker-published port. `DOCKER-USER` fixes it until `ufw reload` flushes +the chain (it takes fail2ban's jump with it; nothing restores either). The +bouncer's **nftables mode** hooks `input` and `forward` in its own table, blocks +the Docker ports, and survived `ufw reload` untouched. + +**Controlled attack, both watching:** + +| | fail2ban | CrowdSec | +|---|---|---| +| invalid user, 1 try/s | banned after 5 tries (~5 s) | detected after 3 tries, firewall drop ~3 s later | +| `root`, key-only, 1 try/s | never | detected after 6 tries, drop ~3 s later | +| 12 web probes on vault (`/.env` answers 422) | no ban (filter counts 401/403/404/429 only) | detected after 3 probes, drop ~3 s later | + +fail2ban acts the moment it matches (REJECT, "connection refused"). CrowdSec +decides fast but the bouncer polls every 10 s, so a few more requests slip +through (DROP, timeout). + +**AppSec (WAF) in Caddy:** every exploit request (`/.env`, `/.git/config`, the +PHPUnit RCE) got a **403 on the first try**, while normal requests in between +stayed 200. Two blocks and the IP is banned at the firewall. Cost: +1 to 2 ms +median time to first byte, one more round-trip to the host per request. + +**What it costs on a 4 GB box:** CrowdSec ~200 MB RSS, firewall bouncer +~30 MB, fail2ban ~50 MB. The engine, the bouncers and the hub are MIT or Apache +licensed and free; the 15k community list is free as long as you share +signals. Bigger lists and the console's premium tier are paid. + +**Verdict for this stack:** run both. fail2ban reacts instantly to the loud +ones on the lines you tell it to watch; CrowdSec adds the slow bots (as +alerts), the blocklist and the WAF, at 4x the memory. Neither replaces the +other on these numbers. diff --git a/README.md b/README.md index 73b2ad4..4980d8a 100644 --- a/README.md +++ b/README.md @@ -247,6 +247,12 @@ security updates, pinned image tags, offsite backups. The ready-to-paste configs from episode 10 (ufw `after.rules`, fail2ban jails and filter, sshd drop-in) live in **[hardening/](hardening/)**. +Episode 12 adds CrowdSec next to fail2ban (not instead of it): community +blocklist, a firewall bouncer in nftables mode that Docker cannot bypass, and an +AppSec WAF inside Caddy. Install notes and the traps we measured are in +**[crowdsec/](crowdsec/)**. Note the Caddy image now needs the key +`CROWDSEC_BOUNCER_KEY` in `.env`. + --- MIT — do whatever. If it saved you money, a YouTube subscription is the price: diff --git a/caddy/Dockerfile b/caddy/Dockerfile index 108cbab..e3f943e 100644 --- a/caddy/Dockerfile +++ b/caddy/Dockerfile @@ -1,9 +1,13 @@ # episode 10: the stock caddy:2-alpine image cannot rate limit. The directive # lives in a plugin (github.com/mholt/caddy-ratelimit) that has to be compiled -# in with xcaddy - there is no runtime "enable" for it. Two stages so the final -# image stays the same small alpine, just with our own caddy binary. +# in with xcaddy - there is no runtime "enable" for it. +# episode 12: same trick for CrowdSec. The bouncer (http) and the AppSec +# handler (appsec) are plugins too, pinned to a release tag instead of @main. FROM caddy:2-builder-alpine AS builder -RUN xcaddy build --with github.com/mholt/caddy-ratelimit +RUN xcaddy build \ + --with github.com/mholt/caddy-ratelimit \ + --with github.com/hslatman/caddy-crowdsec-bouncer/http@v0.14.1 \ + --with github.com/hslatman/caddy-crowdsec-bouncer/appsec@v0.14.1 FROM caddy:2-alpine COPY --from=builder /usr/bin/caddy /usr/bin/caddy diff --git a/caddy/README.md b/caddy/README.md index c3b9916..cc64c9b 100644 --- a/caddy/README.md +++ b/caddy/README.md @@ -1,16 +1,23 @@ -# caddy/ (episode 10) +# caddy/ (episode 10, extended in episode 12) -Custom Caddy image with the rate limiting plugin compiled in. +Custom Caddy image with the rate limiting plugin, the CrowdSec bouncer and the +CrowdSec AppSec handler compiled in (see `crowdsec/README.md`). The stock `caddy:2-alpine` has no `rate_limit` directive; it is not a runtime flag, the module must be built into the binary with `xcaddy`. `compose.yml` carries both `image:` and `build: ./caddy`, so `docker compose build caddy` -produces `selfhostedops/caddy:ratelimit` locally and `docker compose up -d` +produces `selfhostedops/caddy:crowdsec` locally and `docker compose up -d` uses it. Do not build on a small box while the stack is running - a Go build can push a 2 GB machine into OOM and take the live services down with it. Build it elsewhere (a bigger machine, or a throwaway host) and move it over: - docker build -t selfhostedops/caddy:ratelimit ./caddy - docker save selfhostedops/caddy:ratelimit | ssh root@box docker load + docker build -t selfhostedops/caddy:crowdsec ./caddy + docker save selfhostedops/caddy:crowdsec | ssh root@box docker load + +Measured in episode 12: the build with all three modules took 60 s on a 12 GB +build host, the image is 158 MB (the rate-limit-only image was 154 MB). + +The modules are pinned to the release tag (`@v0.14.1`), not `@main` as the +upstream README suggests, so a rebuild next month gives the same binary. diff --git a/compose.yml b/compose.yml index 1ec8724..d8d9a55 100644 --- a/compose.yml +++ b/compose.yml @@ -58,7 +58,7 @@ volumes: services: caddy: - image: selfhostedops/caddy:ratelimit # episode 10: own image with caddy-ratelimit + image: selfhostedops/caddy:crowdsec # episode 12: caddy-ratelimit + CrowdSec bouncer + AppSec build: ./caddy restart: unless-stopped ports: @@ -68,6 +68,9 @@ services: environment: DOMAIN: ${DOMAIN} ACME_EMAIL: ${ACME_EMAIL} + CROWDSEC_BOUNCER_KEY: ${CROWDSEC_BOUNCER_KEY} # episode 12, see crowdsec/README.md + extra_hosts: + - "host.docker.internal:host-gateway" # episode 12: CrowdSec runs on the host volumes: - ./Caddyfile:/etc/caddy/Caddyfile:ro - ./logs/caddy:/var/log/caddy # episode 10: JSON access log on the host for fail2ban diff --git a/crowdsec/README.md b/crowdsec/README.md new file mode 100644 index 0000000..734650f --- /dev/null +++ b/crowdsec/README.md @@ -0,0 +1,121 @@ +# crowdsec/ (episode 12) + +CrowdSec next to fail2ban on the same box, both reading the same logs. This +folder holds the files that differ from a stock install. Everything was +measured on the demo box; the numbers are in `HARDENING.md` (episode 12). + +CrowdSec runs on the **host**, not in the stack: it reads the sshd journal +directly and drives the host firewall. Caddy talks to it over the Docker +bridge. + +## Install (Debian 13) + +``` +curl -fsSL https://install.crowdsec.net | sh +apt install crowdsec +``` + +**Port 8080 trap.** The Local API defaults to `127.0.0.1:8080`. On this stack +the NetBird dashboard already publishes `127.0.0.1:8080`, so the installer +prints one line ("Not attempting to start crowdsec, port 8080 is already +used") and leaves the service **inactive**. Move it: + +``` +sed -i 's#listen_uri: 127.0.0.1:8080#listen_uri: 0.0.0.0:8090#' /etc/crowdsec/config.yaml +sed -i 's#url: http://127.0.0.1:8080#url: http://127.0.0.1:8090#' /etc/crowdsec/local_api_credentials.yaml +``` + +(`0.0.0.0` only because the Caddy container has to reach it, see below. If you +skip the Caddy bouncer, keep `127.0.0.1:8090`.) + +**First start pulls nothing.** The very first community-blocklist pull ran +before the scenario list was registered ("scenario list is empty, will not +pull yet", 0 entries). A `systemctl restart crowdsec` pulled 15,000 IPs a +minute later. Check with `cscli decisions list --origin CAPI | wc -l`. + +## Log sources and collections + +``` +cscli collections install crowdsecurity/caddy +cp acquis.d/caddy.yaml /etc/crowdsec/acquis.d/ +``` + +The installer already adds `crowdsecurity/sshd` and a journald source for +`ssh.service`. + +## Firewall bouncer + +``` +apt install crowdsec-firewall-bouncer-iptables +``` + +The package registers itself with the Local API. Then apply +`firewall-bouncer.yaml` (mode `nftables`) and restart it. Why not the +default iptables mode: it only inserts into `INPUT`. SSH is blocked, but every +Docker-published port (80, 443, 2222) stays reachable for a banned IP, the same +hole as fail2ban in episode 10. Adding `DOCKER-USER` fixes that until the next +`ufw reload`, which flushes the chain and nobody puts the rule back. + +## Caddy bouncer and AppSec (WAF) + +1. Build the Caddy image with the modules (`caddy/Dockerfile`), elsewhere, and + load it on the box. +2. AppSec collections, plus the rules the default config references but the + collections do not install (without them crowdsec **does not start**: + "no appsec-rules found for pattern crowdsecurity/experimental-*", then the + same for `generic-*`): + + ``` + cscli collections install crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules + cscli appsec-rules install crowdsecurity/experimental-no-user-agent \ + crowdsecurity/generic-freemarker-ssti crowdsecurity/generic-wordpress-uploads-listing \ + crowdsecurity/generic-wordpress-uploads-php + cp acquis.d/appsec.yaml /etc/crowdsec/acquis.d/ + crowdsec -t && systemctl restart crowdsec + ``` + + Run `crowdsec -t` first. A failed restart means no Local API; the firewall + keeps its current sets, but no new decisions arrive. +3. Let the Docker network in, and only that network: + + ``` + ufw allow proto tcp from 172.18.0.0/16 to 172.17.0.1 port 8090,7422 + ``` + + The container reaches the host as `host.docker.internal` (`extra_hosts: + host-gateway` in `compose.yml`, which is `172.17.0.1`). From the internet + both ports stay filtered. The mesh interface `wt0` is allowed wholesale by + episode 5, so mesh peers can reach both; each still needs an API key. +4. `cscli bouncers add caddy-bouncer -o raw` and put the key into `.env` as + `CROWDSEC_BOUNCER_KEY`, then `docker compose up -d caddy`. + +The `vpn.` site does not import the `crowdsec` snippet: it carries the NetBird +gRPC streams, and a WAF round-trip on every stream message is not what you want +between your peers. + +## Replaying old logs (and the forensic-mode trap) + +`crowdsec -dsn file:///path/to/log -type syslog` is documented as replay mode. +With CrowdSec 1.8.1 it shut down one second after reading the file and dropped +every event still in the pipeline: 47 or 49 alerts on two identical runs, none +after 02:07 of a 24 h log. Feeding the same file through a FIFO that stays open +60 s after the last line gives the full, repeatable result (133 alerts, twice): + +``` +mkfifo /tmp/replay.fifo +(cat sshd.log; sleep 60) > /tmp/replay.fifo & +crowdsec -c replay.yaml -order-event -dsn file:///tmp/replay.fifo -type syslog +``` + +Use a separate config (`replay.yaml`) with its own database, its own +`listen_uri` and **no `online_client`**, or the replayed alerts land in your +live database and go out as signals. + +## What goes out to CrowdSec + +With the default config, every alert from an unmodified hub scenario is pushed +as a signal: scenario name and hash, timestamp, your machine id, and **the +attacking IP**. Custom or modified ("tainted") scenarios and manual decisions +are not pushed unless you enroll in the console. Your own penetration tests are +real alerts too: switch `sharing: false` under `api.server.online_client` +while you attack your own box, or you report your test machine. diff --git a/crowdsec/acquis.d/appsec.yaml b/crowdsec/acquis.d/appsec.yaml new file mode 100644 index 0000000..1080da0 --- /dev/null +++ b/crowdsec/acquis.d/appsec.yaml @@ -0,0 +1,10 @@ +# episode 12: the AppSec (WAF) endpoint the Caddy bouncer asks on every request. +# Listens on all addresses; ufw only lets the Docker "proxy" network in +# (see crowdsec/README.md). Needs the appsec collections AND the rules listed +# in the README, or crowdsec refuses to start. +appsec_configs: + - crowdsecurity/appsec-default +labels: + type: appsec +listen_addr: 0.0.0.0:7422 +source: appsec diff --git a/crowdsec/acquis.d/caddy.yaml b/crowdsec/acquis.d/caddy.yaml new file mode 100644 index 0000000..185caca --- /dev/null +++ b/crowdsec/acquis.d/caddy.yaml @@ -0,0 +1,7 @@ +# episode 12: CrowdSec reads the same JSON access log fail2ban reads. +# The path is the host side of the ./logs/caddy volume in compose.yml. +filenames: + - /opt/selfhostedops-stack/logs/caddy/access.log +labels: + type: caddy +source: file diff --git a/crowdsec/firewall-bouncer.yaml b/crowdsec/firewall-bouncer.yaml new file mode 100644 index 0000000..270bee5 --- /dev/null +++ b/crowdsec/firewall-bouncer.yaml @@ -0,0 +1,12 @@ +# episode 12: the keys changed in /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml +# (the package writes the rest, including api_url and api_key). +# +# nftables mode instead of the iptables default. The bouncer then owns its +# own tables (ip crowdsec, ip6 crowdsec6) with hooks on input AND forward at +# priority -10, so it drops before Docker's FORWARD rules, and `ufw reload` +# does not touch it. In iptables mode you must add DOCKER-USER to +# iptables_chains, and every `ufw reload` silently removes the rule again. +mode: nftables +nftables_hooks: + - input + - forward