diff --git a/.gitignore b/.gitignore index ba42e3c..0543071 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ backup/.restic-env backup/dumps/ backup/backup.log .playwright-mcp/ +logs/ diff --git a/Caddyfile b/Caddyfile index 91e66dc..e9f8758 100644 --- a/Caddyfile +++ b/Caddyfile @@ -3,26 +3,74 @@ { email {$ACME_EMAIL} + # episode 10: rate_limit has to run before reverse_proxy hands the request on + order rate_limit before basic_auth +} + +# episode 10: one snippet, imported into every site. JSON access log (fail2ban +# on the host reads it) and the Server header stripped so responses no longer +# announce Rocket/Caddy. client_ip in the log is the real remote address, so no +# trusted_proxies needed - the Docker DNAT already carries it through. +(common) { + log { + output file /var/log/caddy/access.log { + roll_size 50mb + roll_keep 5 + } + format json + } + header -Server } git.{$DOMAIN} { + import common + # episode 10: throttle login attempts (10 per minute per source IP) + rate_limit { + zone git_login { + match { + method POST + path /user/login + } + key {remote_host} + events 10 + window 1m + } + } reverse_proxy forgejo:3000 } wiki.{$DOMAIN} { + import common reverse_proxy outline:3000 } vault.{$DOMAIN} { + import common + # episode 10: the admin panel is a setup tool, not a public page (was 200) + @admin path /admin* + respond @admin 404 + # episode 10: throttle the login endpoints (10 per minute per source IP) + rate_limit { + zone vault_login { + match { + path /identity/connect/token /api/accounts/prelogin + } + key {remote_host} + events 10 + window 1m + } + } reverse_proxy vaultwarden:80 } status.{$DOMAIN} { + import common reverse_proxy uptime-kuma:3001 } # Push notifications (episode 6). Websockets and long polling pass through as-is. ntfy.{$DOMAIN} { + import common reverse_proxy ntfy:80 } @@ -31,6 +79,7 @@ ntfy.{$DOMAIN} { # A webhook sender out on the internet cannot join your mesh, so this is the # one door that has to stay open - and it is exactly one door wide. n8n.{$DOMAIN} { + import common @webhooks path /webhook/* /webhook-test/* /webhook-waiting/* handle @webhooks { reverse_proxy n8n:5678 @@ -42,6 +91,7 @@ n8n.{$DOMAIN} { # NetBird control plane (episode 5) - dashboard, API, gRPC, relay, built-in IdP vpn.{$DOMAIN} { + import common # Native gRPC (needs HTTP/2 cleartext to backend) @grpc header Content-Type application/grpc* reverse_proxy @grpc h2c://netbird-server:80 @@ -53,4 +103,3 @@ vpn.{$DOMAIN} { # Dashboard (everything else) reverse_proxy /* netbird-dashboard:80 } - diff --git a/HARDENING.md b/HARDENING.md index f571ab2..5acecaf 100644 --- a/HARDENING.md +++ b/HARDENING.md @@ -24,22 +24,43 @@ systemctl restart ssh # TEST A NEW SSH SESSION BEFORE CLOSING THIS ONE. ``` -## 2. Firewall — four ports, nothing else +## 2. Firewall, and the Docker hole this list used to paper over + +Episodes 1 through 9 shipped the block below. It is wrong, and worth keeping +here as the mistake, because it is the single most common self-host firewall +error. Docker publishes ports by DNAT in the FORWARD path; ufw only filters +INPUT. So `ufw allow 80/tcp` neither opens nor closes port 80: the port was +already open the moment the container published it, and `ufw deny` cannot shut +it. The allow lines are theatre: + +```bash +# WRONG: the 80/443/2222 lines do nothing, the ports are open regardless +ufw allow 80/tcp 443/tcp 443/udp +ufw allow 2222/tcp +``` + +What actually works (full walk-through and files in `hardening/`, episode 10): ```bash apt install -y ufw ufw default deny incoming -ufw allow 80/tcp 443/tcp 443/udp # caddy (http/https/h3) -ufw allow 2222/tcp # git ssh -ufw allow 51820/udp # wireguard (only if using --profile vpn) -ufw allow 22/tcp # your ssh (move it if you want) -ufw enable +ufw default allow outgoing +ufw allow 22/tcp +ufw allow 51820/udp # WireGuard of the NetBird client +ufw allow in on wt0 # the mesh +ufw enable # 80/443/2222 are deliberately NOT here ``` -Note: Docker's published ports bypass ufw INPUT rules — that's exactly why this -compose file publishes only caddy, git-ssh and wireguard, and keeps postgres/redis -on the `internal` network with no ports at all. Don't add `ports:` to internal -services "for debugging" and forget them. +Then close Docker's forwarded ports in the one chain Docker runs first and never +flushes, `DOCKER-USER`. Paste `hardening/ufw-after.rules` into +`/etc/ufw/after.rules`, `ufw reload`, and match the **original** published port +with `conntrack --ctorigdstport`. DNAT has already rewritten `--dport` to the +container's port by the time this chain sees the packet. Measured proof and the +naive-version failure are in `hardening/README.md`. + +Keep postgres/redis on the `internal` network with no `ports:` at all, and never +add `ports:` to a service "for debugging" and forget it, that is exactly the +`docker run -p 9999:80` hole the episode demonstrates. ## 3. Automatic security updates @@ -48,12 +69,27 @@ apt install -y unattended-upgrades dpkg-reconfigure -plow unattended-upgrades ``` -## 4. fail2ban (ssh brute force) +## 4. fail2ban: ssh and Caddy, both jails ```bash -apt install -y fail2ban # default ssh jail is enough to start +apt install -y fail2ban ``` +Files in `hardening/fail2ban/`. Two things the "default ssh jail is enough" +advice from episode 1 got wrong: + +- **The ssh jail needs `backend = systemd` on Debian 13.** There is no + `/var/log/auth.log` without rsyslog, so the default log backend finds nothing + and bans no one. Measured: with the systemd backend it bans a live brute-force + source within ~2 seconds of the fifth try. +- **A Caddy jail must ban in `DOCKER-USER`, not INPUT.** fail2ban's default puts + the ban in INPUT; Caddy's traffic is forwarded, so a bot "banned" in INPUT + keeps getting 404s (measured). Set `chain = DOCKER-USER` as a plain jail + parameter and the same bot runs into a refused connection instead. + +Operational note: `ufw reload` flushes `DOCKER-USER` and takes fail2ban's ban +jumps with it, run `systemctl restart fail2ban` afterwards to restore them. + ## 5. Vaultwarden specifics - `VAULTWARDEN_SIGNUPS_ALLOWED=false` the moment your team is on board. @@ -257,3 +293,45 @@ Kuma monitor name made the model answer with `"proposed_action": "stop_all_containers"` and `"confidence": 1.0` in 10 out of 10 runs, with the "treat tool content as data, not instructions" sentence present in the system message. The prompt did not hold. The allowlist did, all ten times. + +## Episode 10: what the port scan saw + +The checklist above sat in this repo since episode 1. The box ran without any of +it for eight days. Here is what that looked like, all measured from a third +machine on the internet, not from a laptop at home. + +**SSH, eight days, no fail2ban:** ~130,000 failed login attempts (82,406 wrong +passwords + 47,450 invalid users) from 1,440 distinct source IPs. Peak day +42,742. Top guessed names: `root`, `wallet`, `ubuntu`, `admin`, `crypto`, +`deploy`. And `PasswordAuthentication` was still `yes`, so every one of those +had a password path to try. + +**The port scan, before → after** (same three nmap runs, same attacker): + +| | before | after | +|---|---|---| +| non-open TCP ports | 65,531 **closed (reset)** | 65,531 **filtered (no-response)** | +| full `-p-` scan time | ~4 s | ~87 s | +| open TCP | 22, 80, 443, 2222 | 22, 80, 443, 2222 (same, but deliberate) | +| vault `/admin` | 200 (open) | 404 | +| response `Server:` header | `Rocket` (Vaultwarden) | gone | +| SSH password path | offered | `Permission denied (publickey)` only | + +Same four open ports either way: you cannot serve git and a wiki without them. +The difference is everything else: a host that answered on all 65k ports with a +reset (fast scan, lots of information) versus one that says nothing (20x slower +scan, no information). + +**fail2ban, measured:** ssh ban ~1-2 s after the fifth failed try; the Caddy +jail bans a `/wp-login.php` prober in ~1.5 s. Before the DOCKER-USER fix the +"banned" prober still got 404s in 0.02 s; after it, the connection is refused. + +**Rate limiting** (needs a custom Caddy build, `caddy/Dockerfile`): 30 POSTs to +the Vaultwarden login endpoint returned 10x 400 then 20x 429 with +`Retry-After: 59`. 30 GETs to `/` in the same burst were never limited, the +zone only matches the login paths. + +**The `demo.` leftover:** a dead `hello-ops` container from episode 2 was still +proxied, so every bot request to `demo.…/.env` cost Caddy a 502 (608 error-log +lines in 7 days). Removing the site block stops it, bots now get a TLS failure +before any HTTP. diff --git a/README.md b/README.md index 153a834..73b2ad4 100644 --- a/README.md +++ b/README.md @@ -241,8 +241,11 @@ worthless are in **[automation/README.md](automation/README.md)**. ## Hardening Before you put real company data on this box, read **[HARDENING.md](HARDENING.md)**. -The short version: SSH keys only, firewall (80/443/2222/51820 only), unattended -security updates, fail2ban, pinned image tags, offsite backups. +The short version: SSH keys only, a firewall that also closes Docker's forwarded +ports (`DOCKER-USER`, not just ufw), fail2ban on SSH and Caddy, unattended +security updates, pinned image tags, offsite backups. The ready-to-paste configs +from episode 10 (ufw `after.rules`, fail2ban jails and filter, sshd drop-in) live +in **[hardening/](hardening/)**. --- diff --git a/caddy/Dockerfile b/caddy/Dockerfile new file mode 100644 index 0000000..108cbab --- /dev/null +++ b/caddy/Dockerfile @@ -0,0 +1,9 @@ +# episode 10: the stock caddy:2-alpine image cannot rate limit. The directive +# lives in a plugin (github.com/mholt/caddy-ratelimit) that has to be compiled +# in with xcaddy - there is no runtime "enable" for it. Two stages so the final +# image stays the same small alpine, just with our own caddy binary. +FROM caddy:2-builder-alpine AS builder +RUN xcaddy build --with github.com/mholt/caddy-ratelimit + +FROM caddy:2-alpine +COPY --from=builder /usr/bin/caddy /usr/bin/caddy diff --git a/caddy/README.md b/caddy/README.md new file mode 100644 index 0000000..c3b9916 --- /dev/null +++ b/caddy/README.md @@ -0,0 +1,16 @@ +# caddy/ (episode 10) + +Custom Caddy image with the rate limiting plugin compiled in. + +The stock `caddy:2-alpine` has no `rate_limit` directive; it is not a runtime +flag, the module must be built into the binary with `xcaddy`. `compose.yml` +carries both `image:` and `build: ./caddy`, so `docker compose build caddy` +produces `selfhostedops/caddy:ratelimit` locally and `docker compose up -d` +uses it. + +Do not build on a small box while the stack is running - a Go build can push a +2 GB machine into OOM and take the live services down with it. Build it +elsewhere (a bigger machine, or a throwaway host) and move it over: + + docker build -t selfhostedops/caddy:ratelimit ./caddy + docker save selfhostedops/caddy:ratelimit | ssh root@box docker load diff --git a/compose.yml b/compose.yml index c55e2ce..9f39c6a 100644 --- a/compose.yml +++ b/compose.yml @@ -58,7 +58,8 @@ volumes: services: caddy: - image: caddy:2-alpine + image: selfhostedops/caddy:ratelimit # episode 10: own image with caddy-ratelimit + build: ./caddy restart: unless-stopped ports: - "80:80" @@ -69,6 +70,7 @@ services: ACME_EMAIL: ${ACME_EMAIL} volumes: - ./Caddyfile:/etc/caddy/Caddyfile:ro + - ./logs/caddy:/var/log/caddy # episode 10: JSON access log on the host for fail2ban - caddy_data:/data - caddy_config:/config networks: [proxy] diff --git a/hardening/README.md b/hardening/README.md new file mode 100644 index 0000000..a2f8207 --- /dev/null +++ b/hardening/README.md @@ -0,0 +1,65 @@ +# hardening/ (episode 10) + +The four files the episode 1 checklist promised and never delivered: keys-only +SSH, a firewall that actually closes Docker's published ports, and fail2ban with +an SSH jail and a Caddy jail. Everything here was measured on the demo box, not +assumed. + +## Order matters + +1. **SSH first, with a lifeline.** Copy `sshd/hardening.conf` to + `/etc/ssh/sshd_config.d/hardening.conf`, run `sshd -t`, then + `systemctl reload ssh`. Open a second SSH session and confirm it works + **before** you close the first one. Key auth is untouched by this change, but + a typo in the file is how people lock themselves out. + +2. **ufw, then the Docker fix.** Install ufw and allow only what the host itself + listens on: + + ``` + ufw default deny incoming + ufw default allow outgoing + ufw allow 22/tcp + ufw allow 51820/udp # WireGuard of the NetBird client + ufw allow in on wt0 # the mesh + ufw enable + ``` + + Do **not** allow 80/443/2222 here. Those are Docker published ports, and ufw + cannot see them anyway (next point). Set a fallback before `ufw enable` in + case you fat-finger a rule and there is no serial console: + `nohup bash -c 'sleep 300; ufw --force disable' &` - then kill it once a + fresh SSH session confirms you are still in. + +3. **DOCKER-USER.** Paste the block from `ufw-after.rules` at the end of + `/etc/ufw/after.rules` (and `after6.rules`), then `ufw reload`. Without it, + `ufw deny incoming` is a lie: a colleague who runs + `docker run -p 9999:80 ...` to debug something publishes port 9999 straight + to the internet, ufw and all. Confirm with `iptables -S DOCKER-USER`. + + Why `--ctorigdstport` and not `--dport`: DNAT rewrites the destination port + to the container's port before this chain runs, so `--dport` matches the + wrong port. Measured on the box, the naive `--dport 80,443,2222` version let + a `-p 9999:80` container through (its DNAT'd port is 80) and blocked git-ssh + (2222 DNAT'd to 22). `--ctorigdstport` matches the published port. + +4. **fail2ban last.** Copy `fail2ban/jail.local` and + `fail2ban/filter.d/caddy-probes.conf` into `/etc/fail2ban/`, then + `systemctl restart fail2ban`. Two jails: + - `sshd`: `backend = systemd`, because Debian 13 has no `/var/log/auth.log` + without rsyslog. + - `caddy-probes`: reads the JSON access log and bans on repeated + 401/403/404/429. Its ban must land in `DOCKER-USER`, not INPUT - a ban in + INPUT never touches Caddy's forwarded traffic and the bot keeps getting + 404s. Set `chain = DOCKER-USER` as a plain jail parameter, not inside + `banaction[...]`, or the `action_` template's `chain="%(chain)s"` + overrides it back to INPUT. + +## The one operational trap + +`ufw reload` rewrites DOCKER-USER from `after.rules`, which flushes the chain and +throws away fail2ban's ban jumps with it. After any `ufw reload`, run +`systemctl restart fail2ban` to put the jumps (and the active bans, which +fail2ban keeps in its own database) back. A plain `systemctl restart docker` +does not have this problem: DOCKER-USER keeps both our rules and the fail2ban +jumps across a Docker restart (measured). diff --git a/hardening/fail2ban/filter.d/caddy-probes.conf b/hardening/fail2ban/filter.d/caddy-probes.conf new file mode 100644 index 0000000..99b2eac --- /dev/null +++ b/hardening/fail2ban/filter.d/caddy-probes.conf @@ -0,0 +1,8 @@ +# /etc/fail2ban/filter.d/caddy-probes.conf (episode 10) +# Matches the JSON access log: bots that harvest 401/403/404/429 (probing for +# /.env, /wp-login.php, /xmlrpc.php, or hammering a login past the rate limit). +# The whole log entry is one line; client_ip sits in the request object and +# status at the top level, so client_ip always comes before status. +[Definition] +failregex = ^.*"client_ip":"".*"status":(401|403|404|429)\b.*$ +ignoreregex = diff --git a/hardening/fail2ban/jail.local b/hardening/fail2ban/jail.local new file mode 100644 index 0000000..cb16088 --- /dev/null +++ b/hardening/fail2ban/jail.local @@ -0,0 +1,33 @@ +# /etc/fail2ban/jail.local (episode 10) +[DEFAULT] +bantime = 1h +findtime = 10m +maxretry = 5 +# localhost and the NetBird mesh range are never banned. Do NOT put your own +# home/office IP here - if you never guess passwords you never get banned, and +# an ignore entry is one more secret to keep out of a public repo. +ignoreip = 127.0.0.1/8 ::1 100.125.0.0/16 +banaction = iptables-multiport + +[sshd] +enabled = true +# Debian 13 ships no /var/log/auth.log without rsyslog; read the journal instead. +backend = systemd +maxretry = 5 +findtime = 10m +bantime = 1h + +[caddy-probes] +enabled = true +backend = auto +logpath = /opt/selfhostedops-stack/logs/caddy/access.log +filter = caddy-probes +maxretry = 10 +findtime = 1m +bantime = 1h +port = http,https +# The ban has to land in DOCKER-USER, not INPUT - Caddy's traffic is forwarded, +# a ban in INPUT never touches it and the "banned" bot keeps getting answers. +# chain must be a plain parameter here: the default action_ template passes +# chain="%(chain)s", so putting [chain=...] inside banaction gets overridden. +chain = DOCKER-USER diff --git a/hardening/sshd/hardening.conf b/hardening/sshd/hardening.conf new file mode 100644 index 0000000..5cc6c6e --- /dev/null +++ b/hardening/sshd/hardening.conf @@ -0,0 +1,10 @@ +# /etc/ssh/sshd_config.d/hardening.conf (episode 10) +# Keys only. Password guessing is what the 130k login attempts in eight days were. +PasswordAuthentication no +KbdInteractiveAuthentication no +# Root stays reachable by key on this single-admin box. On a box with a real +# ops user, set this to "no" and log in as that user instead. +PermitRootLogin prohibit-password +# +# After editing: sshd -t && systemctl reload ssh +# TEST A SECOND SSH SESSION BEFORE YOU CLOSE THIS ONE. diff --git a/hardening/ufw-after.rules b/hardening/ufw-after.rules new file mode 100644 index 0000000..234d2df --- /dev/null +++ b/hardening/ufw-after.rules @@ -0,0 +1,29 @@ +# episode 10 - paste this block at the END of /etc/ufw/after.rules +# (and the same block in /etc/ufw/after6.rules for IPv6). +# +# Why: Docker publishes ports by DNAT in the FORWARD path, not INPUT. ufw only +# filters INPUT, so `ufw deny incoming` does NOT close a published container +# port - it stays wide open. DOCKER-USER is the one chain Docker runs first in +# FORWARD and never flushes on its own, so the fix goes here. +# +# The catch inside the catch: DNAT rewrites the destination port to the +# container's port BEFORE this chain sees the packet. Matching --dport would +# match the wrong (translated) port. So match the ORIGINAL published port with +# conntrack --ctorigdstport instead. +# +# Only eth0 (the public NIC) is filtered. wt0 (the NetBird mesh) and the docker +# bridges are left alone on purpose - a rule without `-i eth0` here breaks the +# mesh route to n8n (episode 8) and traffic between containers. +# +# Replace eth0 with your public interface, and the port list with what you +# actually publish. +*filter +:DOCKER-USER - [0:0] +-A DOCKER-USER -i eth0 -m conntrack --ctstate RELATED,ESTABLISHED -j RETURN +-A DOCKER-USER -i eth0 -p tcp -m conntrack --ctorigdstport 80 -j RETURN +-A DOCKER-USER -i eth0 -p tcp -m conntrack --ctorigdstport 443 -j RETURN +-A DOCKER-USER -i eth0 -p tcp -m conntrack --ctorigdstport 2222 -j RETURN +-A DOCKER-USER -i eth0 -p udp -m conntrack --ctorigdstport 443 -j RETURN +-A DOCKER-USER -i eth0 -p udp -m conntrack --ctorigdstport 3478 -j RETURN +-A DOCKER-USER -i eth0 -j DROP +COMMIT