From 80d5b00b67ab83f160fa3e520dcf07f301659242 Mon Sep 17 00:00:00 2001 From: Maximilian Grimm Date: Wed, 2 Sep 2026 19:51:35 +0200 Subject: [PATCH] Episode 6: Uptime Kuma monitors that mean something, ntfy push server, status page, second eye Claude-Session: https://claude.ai/code/session_01Y1BTehJjJ4Chr6YscVRXmn --- .env.example | 8 ++- Caddyfile | 5 ++ README.md | 18 +++++ compose.yml | 22 ++++++ monitoring/README.md | 161 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 monitoring/README.md diff --git a/.env.example b/.env.example index 422e345..6a368bd 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,7 @@ # ~$ selfhostedops — copy me: cp .env.example .env # Generate secrets with: openssl rand -hex 32 -# Your base domain. Services live at git.* wiki.* vault.* status.* n8n.* vpn.* +# Your base domain. Services live at git.* wiki.* vault.* status.* ntfy.* vpn.* # DNS: one wildcard A (and AAAA) record *.ops.example.com -> your server IP DOMAIN=ops.example.com @@ -46,3 +46,9 @@ WGEASY_PASSWORD_HASH= RUNNER_SECRET= RUNNER_UUID= + +# --- ntfy push notifications (episode 6) --- +# No secrets here: ntfy keeps its users in its own database. Create them once: +# docker compose exec -e NTFY_PASSWORD= ntfy ntfy user add --role=user kuma +# docker compose exec ntfy ntfy access kuma alerts write-only +# See monitoring/README.md for the full walkthrough. diff --git a/Caddyfile b/Caddyfile index d2cf872..d6aa336 100644 --- a/Caddyfile +++ b/Caddyfile @@ -21,6 +21,11 @@ status.{$DOMAIN} { reverse_proxy uptime-kuma:3001 } +# Push notifications (episode 6). Websockets and long polling pass through as-is. +ntfy.{$DOMAIN} { + reverse_proxy ntfy:80 +} + # n8n has no public route any more - it is reachable over the mesh only (episode 5) # NetBird control plane (episode 5) - dashboard, API, gRPC, relay, built-in IdP diff --git a/README.md b/README.md index 5e38b33..678231b 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,24 @@ people. Note that joining is three steps, not two: you invite, they log in, and then an owner has to **confirm** them. Until that confirmation, a member sees no shared items at all. +## Alerts that reach you (Episode 6) + +A monitor nobody hears is a dashboard. Episode 6 adds a push channel you own +(ntfy, `ntfy.`), keyword monitors for everything a customer sees, +internal monitors for the services that left the internet in episode 5, a +public status page at the root of `status.`, and a second Uptime Kuma +outside the server, because a box cannot report its own death. + +```bash +docker compose up -d ntfy +docker compose exec caddy caddy reload --config /etc/caddy/Caddyfile +docker compose exec -e NTFY_PASSWORD='' ntfy ntfy user add --role=user kuma +docker compose exec ntfy ntfy access kuma alerts write-only +``` + +The full walkthrough, the second-eye compose file and the gotchas are in +**[monitoring/README.md](monitoring/README.md)**. + ## Hardening Before you put real company data on this box, read **[HARDENING.md](HARDENING.md)**. diff --git a/compose.yml b/compose.yml index da57762..580fe88 100644 --- a/compose.yml +++ b/compose.yml @@ -7,6 +7,7 @@ # outline team wiki (logs in via Forgejo SSO) # vaultwarden password manager (Bitwarden-compatible) # uptime-kuma monitoring + alerts +# ntfy push notifications for those alerts (episode 6) # n8n automation hub # wg-easy WireGuard VPN (optional — start with: --profile vpn) # (the full NetBird mesh is in netbird/ — episode 5) @@ -38,6 +39,8 @@ volumes: outline_pg: vaultwarden_data: kuma_data: + ntfy_data: + ntfy_cache: n8n_data: wg_data: docker_certs: @@ -160,6 +163,25 @@ services: - kuma_data:/app/data networks: [proxy] + # Push notifications for the monitoring alerts (episode 6). + # Uptime Kuma publishes to the topic "alerts", your phone subscribes to it. + ntfy: + image: binwiederhier/ntfy:v2.28.0 + restart: unless-stopped + command: serve + environment: + NTFY_BASE_URL: https://ntfy.${DOMAIN} + NTFY_BEHIND_PROXY: "true" + NTFY_AUTH_DEFAULT_ACCESS: deny-all # nobody reads or writes without a user + NTFY_AUTH_FILE: /var/lib/ntfy/user.db + NTFY_CACHE_FILE: /var/cache/ntfy/cache.db + NTFY_ATTACHMENT_CACHE_DIR: /var/cache/ntfy/attachments + NTFY_ENABLE_LOGIN: "true" + volumes: + - ntfy_data:/var/lib/ntfy + - ntfy_cache:/var/cache/ntfy + networks: [proxy] + n8n: image: docker.n8n.io/n8nio/n8n:latest restart: unless-stopped diff --git a/monitoring/README.md b/monitoring/README.md new file mode 100644 index 0000000..859c3ee --- /dev/null +++ b/monitoring/README.md @@ -0,0 +1,161 @@ +# Monitoring and alerting - notice before your customer does (episode 6) + +Uptime Kuma has been in the stack since episode 1, but a monitor nobody hears +is a dashboard, not an alarm. This episode adds three things: a push channel +you own (ntfy), a public status page on its own hostname, and a second Kuma +outside the server, because a box cannot report its own death. + +## ntfy: the push channel + +One container, one topic. Kuma publishes to `alerts`, your phone subscribes to +it with the ntfy app (Android, iOS, or the web UI at `https://ntfy.`). + +```yaml + ntfy: + image: binwiederhier/ntfy:v2.28.0 + restart: unless-stopped + command: serve + environment: + NTFY_BASE_URL: https://ntfy.${DOMAIN} + NTFY_BEHIND_PROXY: "true" + NTFY_AUTH_DEFAULT_ACCESS: deny-all # nobody reads or writes without a user + NTFY_AUTH_FILE: /var/lib/ntfy/user.db + NTFY_CACHE_FILE: /var/cache/ntfy/cache.db + NTFY_ATTACHMENT_CACHE_DIR: /var/cache/ntfy/attachments + NTFY_ENABLE_LOGIN: "true" + volumes: + - ntfy_data:/var/lib/ntfy + - ntfy_cache:/var/cache/ntfy + networks: [proxy] +``` + +Caddy block (websockets and long polling need nothing special): + +``` +ntfy.{$DOMAIN} { + reverse_proxy ntfy:80 +} +``` + +Start it, reload Caddy, then create two users: one that may only write to the +topic (that is Kuma) and one that may read it (that is you). + +```bash +docker compose up -d ntfy +docker compose exec caddy caddy reload --config /etc/caddy/Caddyfile + +docker compose exec -e NTFY_PASSWORD='' ntfy ntfy user add --role=user kuma +docker compose exec -e NTFY_PASSWORD='' ntfy ntfy user add --role=user max +docker compose exec ntfy ntfy access kuma alerts write-only +docker compose exec ntfy ntfy access max alerts read-write +docker compose exec ntfy ntfy access # prints the whole table +``` + +Prove the lock works before you trust it: + +```bash +# anonymous publish: 403 +curl -d "anyone can post here?" https://ntfy./alerts +# authenticated publish: lands +curl -u kuma: -H "Title: Hello from the box" -d "First message" https://ntfy./alerts +# read it back as a reader +curl -u max: "https://ntfy./alerts/json?poll=1" +``` + +## Uptime Kuma: notification, monitors, status page + +All of this is done in the Kuma web UI at `https://status./dashboard`. + +1. Settings -> Notifications -> Setup Notification. Type **ntfy**, server + `https://ntfy.`, topic `alerts`, authentication *username and + password* with the `kuma` user. Tick *Default enabled* and *Apply on all + existing monitors*, press *Test*, watch it arrive on your phone. +2. Monitors. Public services get an **HTTP(s) - Keyword** monitor, because a + 200 from a reverse proxy says nothing about the app behind it. Keywords + that work with the stack: `Forgejo`, `Outline`, `Vaultwarden`, `NetBird`. + Interval 60 s, retries 1, retry interval 20 s, *Certificate Expiry + Notification* on. Settings -> Notifications -> TLS expiry: 7, 14, 21 days. +3. Services that left the public internet in episode 5 are monitored from + inside the Docker network. n8n: `http://n8n:5678/healthz`. Kuma sits on + the same `proxy` network, so the container name resolves. +4. Groups `Public` and `Internal` keep the dashboard readable. A **Push** + monitor `Nightly backup` with a 90000 s interval turns a backup job into + something that alerts when it *does not* run: the job calls the push URL, + Kuma complains when the call is missing for 25 hours. +5. Status page: slug `status`, title *Company Status*, add only the public + monitors. Under *Domain Names* enter `status.`: Kuma then serves + the page at the root of that hostname and keeps the dashboard at + `/dashboard`. No extra Caddy rule needed. + +Verify: + +```bash +curl -s https://status./ | grep -o "[^<]*" # Company Status +curl -s https://status./api/status-page/status | python3 -m json.tool +``` + +## The second eye + +Stop the server and both Kuma and ntfy stop with it: no alert. So a second +Uptime Kuma runs on a machine that is not that server (a laptop at home, a +Raspberry Pi, a colleague's NAS). LAN only, no public route. + +```yaml +name: kuma-home +services: + uptime-kuma: + image: louislam/uptime-kuma:1 + restart: unless-stopped + ports: + - "3001:3001" + volumes: + - kuma_data:/app/data +volumes: + kuma_data: +``` + +Its monitors: keyword on `https://status./` (keyword *Company Status*), +keyword on `https://git./` (keyword *Forgejo*), ping on the server IP. +Its notification must not depend on the server either, so it uses the public +`https://ntfy.sh` with a random, unguessable topic (`sho-` plus twelve random +characters). Subscribe to that topic in the same ntfy app, next to your own. + +## Rehearse the outage + +Do not wait for the real one. `docker stop selfhostedops-stack-outline-1`, +start a stopwatch, `docker start` it when the phone buzzes. With the settings +above the DOWN alert takes between 20 and 80 seconds (interval plus one +retry), the UP alert follows about a minute after the container is healthy. + +## Gotchas + +- `NTFY_AUTH_DEFAULT_ACCESS: deny-all` also blocks the *reader*. A user with + `write-only` cannot poll the topic (403). Give yourself `read-write`, give + Kuma `write-only`, never the other way round. +- The ntfy CLI takes the password from the `NTFY_PASSWORD` environment + variable. Pass it with `docker compose exec -e NTFY_PASSWORD=...`; + `ntfy user add` without it wants a terminal and hangs in scripts. +- A plain HTTP monitor is happy with any 2xx. Caddy answers 2xx for a dead + backend in some setups (a login page, a maintenance page). Use keyword + monitors for anything a customer sees. +- Switching a monitor from a dead public URL to the internal one fires an + *Up* notification the moment it works again. Expected, but it surprises + people the first time. +- Kuma 1.x refuses monitor intervals above 24 days and the push monitor's + interval is also its grace period: 90000 s means "alert after 25 hours + without a ping", not "check every 25 hours". +- The status page is rendered server side for the domain root, the + dashboard stays a single page app. A keyword monitor on the status page + therefore works (the title is in the HTML), a keyword monitor on + `/dashboard` does not. +- The *Domain Names* field only works when the request really arrives with + that `Host` header. Behind Caddy that is the default; behind a proxy that + rewrites the host header, set `trustProxy` in Kuma's settings and forward + the header. +- The second Kuma uses the public ntfy.sh without authentication. Anyone who + guesses the topic can read your alerts and post fake ones. Twelve random + characters make guessing impractical; still, do not put secrets in monitor + names. +- `louislam/uptime-kuma:1` is the 1.x line (1.23.17 at recording time). + 2.x exists (2.5.3) and migrates the database on first start. Pin what you + test, upgrade on purpose, not by `docker compose pull`.