diff --git a/.env.example b/.env.example index 6a368bd..78a0d1f 100644 --- a/.env.example +++ b/.env.example @@ -52,3 +52,21 @@ RUNNER_UUID= # docker compose exec -e NTFY_PASSWORD= ntfy ntfy user add --role=user kuma # docker compose exec ntfy ntfy access kuma alerts write-only # See monitoring/README.md for the full walkthrough. + +# --- n8n automation (episode 8) --- +# No variables needed here either: the workflows keep their credentials inside +# n8n, encrypted with the key in /home/node/.n8n/config. That file is inside the +# n8n_data volume and therefore inside the nightly backup from episode 7. +# +# If you would rather pin the key yourself (recommended once more than one +# person can restore this box), uncomment the line below and put the same value +# in your password manager. It wins over the file, and the file is then never +# written. Changing it later makes every existing credential undecryptable. +# openssl rand -base64 24 +# N8N_ENCRYPTION_KEY= +# +# The ntfy user the workflows publish as: +# docker compose exec -e NTFY_PASSWORD= ntfy ntfy user add --role=user n8n +# docker compose exec ntfy ntfy access n8n alerts write-only +# docker compose exec ntfy ntfy access n8n deploys write-only +# See automation/README.md for the rest. diff --git a/.gitignore b/.gitignore index e3c9429..ba42e3c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ runner-config.yml backup/.restic-env backup/dumps/ backup/backup.log +.playwright-mcp/ diff --git a/Caddyfile b/Caddyfile index d6aa336..91e66dc 100644 --- a/Caddyfile +++ b/Caddyfile @@ -26,7 +26,19 @@ ntfy.{$DOMAIN} { reverse_proxy ntfy:80 } -# n8n has no public route any more - it is reachable over the mesh only (episode 5) +# Automation (episode 8). The editor never becomes public: only the webhook +# paths are served, everything else under this host answers an empty 404. +# A webhook sender out on the internet cannot join your mesh, so this is the +# one door that has to stay open - and it is exactly one door wide. +n8n.{$DOMAIN} { + @webhooks path /webhook/* /webhook-test/* /webhook-waiting/* + handle @webhooks { + reverse_proxy n8n:5678 + } + handle { + respond 404 + } +} # NetBird control plane (episode 5) - dashboard, API, gRPC, relay, built-in IdP vpn.{$DOMAIN} { diff --git a/HARDENING.md b/HARDENING.md index c773553..c845fe7 100644 --- a/HARDENING.md +++ b/HARDENING.md @@ -138,3 +138,58 @@ it — the example in `examples/deploy-hello-ops.sh` strips everything that is n hex character, because it only ever expects an image tag. The deploy user needs Docker access, so the forced command is the thing containing it; without the forced command, membership in the `docker` group is equivalent to root. + +## Automation: the small door and the one file (episode 8) + +**Publish paths, not hosts.** n8n needs a public address for webhooks that come +from outside, and for nothing else. The site block in `Caddyfile` serves +`/webhook/*`, `/webhook-test/*` and `/webhook-waiting/*` and answers an empty +404 for every other path, so the editor, the login page and the REST API are not +part of the public surface. The check is one line: + +```bash +curl -o /dev/null -w '%{http_code}\n' https://n8n./rest/settings # 404 +curl -o /dev/null -w '%{http_code}\n' -X POST https://n8n./webhook/ # 200 +``` + +`/webhook-test/*` only does anything while somebody has "Test workflow" open in +the editor. If you never test against an outside sender, drop it from the +matcher and the surface gets smaller again. + +**Senders inside the box should stay inside the box.** Forgejo and Uptime Kuma +reach `http://n8n:5678/webhook/...` over the docker network. No certificate, no +published port, no trip through the public IP. Forgejo blocks private targets by +default; `FORGEJO__webhook__ALLOWED_HOST_LIST: "external,n8n"` allows exactly +that one name and leaves the rest of the block in place. + +**`/home/node/.n8n/config` is the whole credential store.** It contains +`encryptionKey`, and every credential in `database.sqlite` is encrypted with it. +Lose the file and the database is landfill: n8n starts, writes a fresh key, the +workflows still look fine, and the first run fails with + +``` +Credentials could not be decrypted. The likely reason is that a different +"encryptionKey" was used to encrypt the data. +``` + +The episode 7 backup covers the whole volume, so the key is in the snapshot. +Confirm it instead of assuming: + +```bash +restic ls latest | grep n8n_data/_data/config +``` + +If you set `N8N_ENCRYPTION_KEY` in the environment, that value wins, the file is +never written, and the key then has to live in your password manager and survive +the box. + +**Keep the file nodes on a leash.** `N8N_RESTRICT_FILE_ACCESS_TO: /backup` lets +the digest workflow read the backup log and nothing else. Left unset or widened +to `/`, a workflow, or anyone who can edit one, can read the encryption key from +inside n8n itself. + +**Pin the image, pin the subnets.** `docker.n8n.io/n8nio/n8n:latest` turns every +restore into an unplanned upgrade. The `ipam` blocks under `networks:` do the +same job for addresses: Compose hands subnets out in creation order, so a +restore onto a fresh box can renumber them, and every mesh route that points at +the old range stops working without an error anywhere. diff --git a/README.md b/README.md index 678231b..153a834 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ From the video: **[Self-Hosted Company Stack for €15/Month (Full Setup)](https | [Outline](https://www.getoutline.com) | `wiki.…` | Notion | $10/user/mo | | [Vaultwarden](https://github.com/dani-garcia/vaultwarden) | `vault.…` | 1Password | $8/user/mo | | [Uptime Kuma](https://github.com/louislam/uptime-kuma) | `status.…` | Pingdom & co. | $20+/mo | -| [n8n](https://n8n.io) | `n8n.…` | Zapier | $30+/mo | +| [n8n](https://n8n.io) | `n8n.…` (webhooks only, episode 8) | Zapier | €26+/mo | | [wg-easy](https://github.com/wg-easy/wg-easy) | `vpn.…` | Tailscale | $6/user/mo | All images are multi-arch (amd64 + arm64) — a €14 ARM cloud box (e.g. Hetzner @@ -207,6 +207,37 @@ docker compose exec ntfy ntfy access kuma alerts write-only The full walkthrough, the second-eye compose file and the gotchas are in **[monitoring/README.md](monitoring/README.md)**. +## Automation that connects the rest (Episode 8) + +n8n is the piece that makes the other six talk to each other. A push becomes a +notification, a monitor going red becomes an incident document, and every +morning one document says what the box did overnight. + +The interesting part is the address. Automation needs webhooks, webhooks look +like they need a public host, and the editor still has no business being on the +internet. So the Caddyfile publishes **paths, not a host**: + +``` +n8n.{$DOMAIN} { + @webhooks path /webhook/* /webhook-test/* /webhook-waiting/* + handle @webhooks { + reverse_proxy n8n:5678 + } + handle { + respond 404 + } +} +``` + +Everything else under `n8n.` answers an empty 404: the editor, the login +page, the REST API. You open the editor over the mesh from episode 5. Senders +that live on the box (Forgejo, Uptime Kuma) skip the public address entirely and +call `http://n8n:5678/webhook/...` over the docker network. + +The three workflows, the credentials they need, the two settings that make +Forgejo and n8n cooperate, and the one file whose loss makes every credential +worthless are in **[automation/README.md](automation/README.md)**. + ## Hardening Before you put real company data on this box, read **[HARDENING.md](HARDENING.md)**. diff --git a/automation/README.md b/automation/README.md new file mode 100644 index 0000000..58b4324 --- /dev/null +++ b/automation/README.md @@ -0,0 +1,109 @@ +# Automation (episode 8) + +Three workflows that wire the other services together. They are the reason n8n +sits in this compose file, and they are all built from HTTP requests, so nothing +here depends on a vendor connector that might change next quarter. + +| File | Trigger | What it does | +|---|---|---| +| `a-deploy-notifier.json` | Forgejo `push` webhook | Turns a push into one push notification: repo, branch, author, commit subject, link to the commit. | +| `b-incident-responder.json` | Uptime Kuma webhook notification | On DOWN: opens an incident document in the wiki and sends a high priority alert with a link to it. On UP: finds that document, appends the downtime, renames it to "Resolved", sends the all clear. | +| `c-nightly-ops-digest.json` | Schedule, 07:00 daily | Reads commits of the last 24 hours from Forgejo, availability from the Uptime Kuma status page, and the last restic snapshot from the episode 7 backup log. Writes one wiki document and sends a one line summary. | + +## The point of this episode + +Automation needs webhooks. Webhooks look like they need a public address. They +do not, at least not for everything: + +- **Senders that live on the box** (Forgejo, Uptime Kuma) call + `http://n8n:5678/webhook/...` straight over the docker network. That traffic + never touches the internet, needs no certificate and no port. +- **Senders out on the internet** need one public path, and exactly one. The + Caddyfile publishes `/webhook/*`, `/webhook-test/*` and `/webhook-waiting/*` + under `n8n.` and answers an empty 404 for everything else. The editor, + the REST API and the login page are not reachable from outside at all. +- **You** reach the editor over the mesh from episode 5, on the container + address, on plain HTTP inside the tunnel. + +Two different 404s tell you which layer answered: an empty body is Caddy +refusing the path, a JSON body is n8n saying the webhook is not registered. + +## Importing them + +1. Create the three credentials in n8n first. Their names in these files are + `ntfy (n8n user)`, `Outline API` and `Forgejo API`; keep the names and n8n + will attach them on import. + - `ntfy (n8n user)`: Basic Auth, an ntfy user with write access to the topics. + `docker compose exec -e NTFY_PASSWORD='...' ntfy ntfy user add --role=user n8n` + then `ntfy access n8n alerts write-only` and the same for `deploys`. + - `Outline API`: Header Auth, name `Authorization`, value `Bearer ol_api_...` + (Outline: Settings, API tokens). + - `Forgejo API`: Header Auth, name `Authorization`, value `token ...` + (a read-only token is enough: `read:repository`). +2. Import each JSON, then replace the placeholders: + - `REPLACE_WITH_YOUR_CREDENTIAL_ID` is filled in when you pick the credential + in the node. + - `REPLACE_WITH_YOUR_OUTLINE_COLLECTION_ID` is the collection the incident + and digest documents are written into. Create a collection, open it, the id + is in `collections.info` or in the URL of the collection settings. + - `REPLACE_ON_IMPORT` is the webhook id; n8n generates a fresh one. + - `ops.example.com` appears in the two links that go into push messages. +3. Activate all three. + +## Wiring the senders + +Forgejo, per repository: Settings, Webhooks, Forgejo, URL +`http://n8n:5678/webhook/forgejo-push`, content type `application/json`, +event `Push`. + +Forgejo refuses to deliver to a private address unless you tell it not to. That +is the `FORGEJO__webhook__ALLOWED_HOST_LIST: "external,n8n"` line in +`compose.yml`. Without it the push is accepted, the workflow never runs, and the +only trace is one line in the Forgejo log: + +``` +webhook can only call allowed HTTP servers (check your webhook.ALLOWED_HOST_LIST setting), deny 'n8n(172.18.0.3:5678)' +``` + +Uptime Kuma: Settings, Notifications, add a **Webhook** notification, URL +`http://n8n:5678/webhook/kuma-incident`, content type `application/json`, then +attach it to the monitors you want incident documents for. Kuma sends one POST +per state change, with `heartbeat.status` 0 for down and 1 for up. + +## The one file that matters for backups + +`/home/node/.n8n/config` holds `encryptionKey`. Every credential in the database +is encrypted with it. The database alone is worth nothing: + +``` +Credentials could not be decrypted. The likely reason is that a different +"encryptionKey" was used to encrypt the data. +``` + +The workflows survive, the credentials do not, and n8n quietly writes a fresh +key on first start when the file is missing, so nothing looks broken until a +workflow runs. The episode 7 backup covers the whole `n8n_data` volume, so the +key is in the snapshot. Verify it rather than assume it: + +```bash +restic ls latest | grep n8n_data/_data/config +``` + +If you set `N8N_ENCRYPTION_KEY` in the environment instead, that value wins and +the file is never written; then the key belongs in your password manager, and it +has to survive independently of the box. + +## Notes + +- `c-nightly-ops-digest.json` reads `/backup/backup.log`, which `compose.yml` + bind-mounts read-only from episode 7. Without that file the digest reports + "no backup log found" and still writes the document. +- `N8N_RESTRICT_FILE_ACCESS_TO: /backup` is what allows that read. Without the + variable, 2.x answers "Access to the file is not allowed." for every path. + Do not widen it to `/`: a workflow could then read the encryption key. +- The digest matches commit responses to repositories by position. That holds + because the HTTP node returns exactly one item per input item when the + response format is text and errors continue. +- These workflows use plain HTTP Request nodes on purpose. A Forgejo node, an + Outline node and an ntfy node either do not exist or would add a dependency + for three URL calls. diff --git a/automation/a-deploy-notifier.json b/automation/a-deploy-notifier.json new file mode 100644 index 0000000..c3fae0f --- /dev/null +++ b/automation/a-deploy-notifier.json @@ -0,0 +1,107 @@ +{ + "name": "A - Deploy notifier", + "nodes": [ + { + "id": "a1", + "name": "Forgejo push", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [ + 0, + 0 + ], + "webhookId": "REPLACE_ON_IMPORT", + "parameters": { + "httpMethod": "POST", + "path": "forgejo-push", + "responseMode": "onReceived", + "options": {} + } + }, + { + "id": "a2", + "name": "Format message", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 240, + 0 + ], + "parameters": { + "jsCode": "\n// The Forgejo push payload, reduced to the five things a human wants to know.\nconst b = $json.body ?? $json;\nconst repo = b.repository?.full_name ?? 'unknown/unknown';\nconst branch = String(b.ref ?? '').replace('refs/heads/', '');\nconst commits = Array.isArray(b.commits) ? b.commits : [];\nconst head = commits[commits.length - 1] ?? {};\nconst author = head.author?.name ?? b.pusher?.username ?? b.sender?.username ?? 'unknown';\nconst subject = String(head.message ?? '').split('\\n')[0].slice(0, 120);\nconst short = String(head.id ?? '').slice(0, 8);\nconst link = head.url ?? b.compare_url ?? b.repository?.html_url ?? '';\n\nreturn [{\n json: {\n pushTitle: repo + ' / ' + branch,\n text: commits.length + ' commit(s) by ' + author + '\\n' + short + ' ' + subject,\n link,\n repo, branch, author, subject, short,\n commitCount: commits.length,\n },\n}];\n" + } + }, + { + "id": "a3", + "name": "Push to ntfy", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 480, + 0 + ], + "parameters": { + "method": "POST", + "url": "http://ntfy/deploys", + "authentication": "genericCredentialType", + "genericAuthType": "httpBasicAuth", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "X-Title", + "value": "={{ $json.pushTitle }}" + }, + { + "name": "X-Priority", + "value": "3" + }, + { + "name": "X-Click", + "value": "={{ $json.link }}" + } + ] + }, + "sendBody": true, + "contentType": "raw", + "rawContentType": "text/plain", + "body": "={{ $json.text }}", + "options": {} + }, + "credentials": { + "httpBasicAuth": { + "id": "REPLACE_WITH_YOUR_CREDENTIAL_ID", + "name": "ntfy (n8n user)" + } + } + } + ], + "connections": { + "Forgejo push": { + "main": [ + [ + { + "node": "Format message", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format message": { + "main": [ + [ + { + "node": "Push to ntfy", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "settings": { + "executionOrder": "v1" + }, + "active": false +} diff --git a/automation/b-incident-responder.json b/automation/b-incident-responder.json new file mode 100644 index 0000000..2eae50b --- /dev/null +++ b/automation/b-incident-responder.json @@ -0,0 +1,358 @@ +{ + "name": "B - Incident responder", + "nodes": [ + { + "id": "b1", + "name": "Kuma incident", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [ + -220, + 0 + ], + "webhookId": "REPLACE_ON_IMPORT", + "parameters": { + "httpMethod": "POST", + "path": "kuma-incident", + "responseMode": "onReceived", + "options": {} + } + }, + { + "id": "b2", + "name": "Read event", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 0, + 0 + ], + "parameters": { + "jsCode": "\n// Uptime Kuma posts one JSON body for every state change. Reduce it to the\n// four fields both branches need, so nothing below has to know Kuma's shape.\nconst b = $json.body ?? $json;\nconst hb = b.heartbeat ?? {};\nconst mon = b.monitor ?? {};\nconst down = Number(hb.status) === 0;\nreturn [{\n json: {\n monitor: mon.name ?? 'unknown monitor',\n monitorUrl: mon.url ?? '',\n down,\n at: hb.time ? new Date(hb.time.replace(' ', 'T') + 'Z').toISOString() : new Date().toISOString(),\n reason: hb.msg ?? b.msg ?? '',\n },\n}];\n" + } + }, + { + "id": "b3", + "name": "Down?", + "type": "n8n-nodes-base.if", + "typeVersion": 2.2, + "position": [ + 220, + 0 + ], + "parameters": { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict", + "version": 2 + }, + "conditions": [ + { + "id": "c1", + "leftValue": "={{ $json.down }}", + "rightValue": "", + "operator": { + "type": "boolean", + "operation": "true", + "singleValue": true + } + } + ], + "combinator": "and" + }, + "options": {} + } + }, + { + "id": "b4", + "name": "Build incident doc", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 460, + -140 + ], + "parameters": { + "jsCode": "\n// One incident document per outage. \"Status: open\" is the marker the UP\n// branch searches for later, so it stays machine readable on purpose.\nconst e = $json;\nconst stamp = e.at.replace('T', ' ').replace(/\\..*/, ' UTC');\nconst title = 'Incident: ' + e.monitor + ' (' + stamp + ')';\n// A markdown list, not plain lines: Outline collapses single newlines inside\n// a paragraph, which would turn the whole record into one unreadable blob.\nconst text = [\n '- Monitor: ' + e.monitor,\n '- Status: open',\n '- Down since: ' + e.at,\n '- Reason: ' + (e.reason || 'no detail from the monitor'),\n e.monitorUrl ? '- Checked URL: ' + e.monitorUrl : '',\n '',\n '---',\n '',\n 'Opened automatically by n8n. The resolution is appended when the monitor reports UP again.',\n].filter(Boolean).join('\\n');\nreturn [{ json: { title, payload: { title, text, collectionId: 'REPLACE_WITH_YOUR_OUTLINE_COLLECTION_ID', publish: true } } }];\n" + } + }, + { + "id": "13d2a02f", + "name": "Create incident doc", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 680, + -140 + ], + "parameters": { + "method": "POST", + "url": "http://outline:3000/api/documents.create", + "options": {}, + "authentication": "genericCredentialType", + "genericAuthType": "httpHeaderAuth", + "sendBody": true, + "contentType": "json", + "specifyBody": "json", + "jsonBody": "={{ JSON.stringify($json.payload) }}" + }, + "credentials": { + "httpHeaderAuth": { + "id": "REPLACE_WITH_YOUR_CREDENTIAL_ID", + "name": "Outline API" + } + } + }, + { + "id": "e28ddc1d", + "name": "Push alert", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 900, + -140 + ], + "parameters": { + "method": "POST", + "url": "http://ntfy/alerts", + "options": {}, + "authentication": "genericCredentialType", + "genericAuthType": "httpBasicAuth", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "X-Title", + "value": "=DOWN: {{ $('Read event').first().json.monitor }}" + }, + { + "name": "X-Priority", + "value": "5" + } + ] + }, + "sendBody": true, + "contentType": "raw", + "rawContentType": "text/plain", + "body": "={{ $('Read event').first().json.reason || 'monitor reports down' }}\nIncident doc: https://wiki.ops.example.com{{ $json.data.url }}" + }, + "credentials": { + "httpBasicAuth": { + "id": "REPLACE_WITH_YOUR_CREDENTIAL_ID", + "name": "ntfy (n8n user)" + } + } + }, + { + "id": "0ac14924", + "name": "List Ops Log", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 460, + 140 + ], + "parameters": { + "method": "POST", + "url": "http://outline:3000/api/documents.list", + "options": {}, + "authentication": "genericCredentialType", + "genericAuthType": "httpHeaderAuth", + "sendBody": true, + "contentType": "json", + "specifyBody": "json", + "jsonBody": "={{ JSON.stringify({ collectionId: \"REPLACE_WITH_YOUR_OUTLINE_COLLECTION_ID\", limit: 25 }) }}" + }, + "credentials": { + "httpHeaderAuth": { + "id": "REPLACE_WITH_YOUR_CREDENTIAL_ID", + "name": "Outline API" + } + } + }, + { + "id": "b8", + "name": "Find open incident", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 680, + 140 + ], + "parameters": { + "jsCode": "\n// Find the still open incident for this monitor and close it out.\nconst e = $('Read event').first().json;\nconst docs = $input.first().json.data ?? [];\nconst open = docs.find(d =>\n String(d.text ?? '').includes('Monitor: ' + e.monitor) &&\n String(d.text ?? '').includes('Status: open'));\n\nif (!open) {\n return [{ json: { found: false, monitor: e.monitor, duration: 'unknown',\n note: 'no open incident document for this monitor' } }];\n}\n\nconst m = String(open.text).match(/Down since:\\s*(\\S+)/);\nconst since = m ? new Date(m[1]) : null;\nconst back = new Date(e.at);\nconst secs = since ? Math.max(0, Math.round((back - since) / 1000)) : null;\nconst human = secs === null ? 'unknown'\n : secs < 90 ? secs + ' s'\n : Math.floor(secs / 60) + ' min ' + (secs % 60) + ' s';\n\nconst text = String(open.text).replace('Status: open', 'Status: resolved')\n + '\\n- Back up: ' + e.at + '\\n- Downtime: ' + human + '\\n';\n\nreturn [{ json: {\n found: true, monitor: e.monitor, duration: human, seconds: secs,\n docUrl: open.url ?? '',\n payload: { id: open.id, title: String(open.title).replace('Incident:', 'Resolved:'), text, publish: true },\n}}];\n" + } + }, + { + "id": "e402dfe1", + "name": "Close incident doc", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 900, + 140 + ], + "parameters": { + "method": "POST", + "url": "http://outline:3000/api/documents.update", + "options": {}, + "authentication": "genericCredentialType", + "genericAuthType": "httpHeaderAuth", + "sendBody": true, + "contentType": "json", + "specifyBody": "json", + "jsonBody": "={{ JSON.stringify($json.payload || {}) }}" + }, + "credentials": { + "httpHeaderAuth": { + "id": "REPLACE_WITH_YOUR_CREDENTIAL_ID", + "name": "Outline API" + } + }, + "onError": "continueRegularOutput" + }, + { + "id": "7b4981c4", + "name": "Push all clear", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 1120, + 140 + ], + "parameters": { + "method": "POST", + "url": "http://ntfy/alerts", + "options": {}, + "authentication": "genericCredentialType", + "genericAuthType": "httpBasicAuth", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "X-Title", + "value": "=UP again: {{ $('Read event').first().json.monitor }}" + }, + { + "name": "X-Priority", + "value": "3" + } + ] + }, + "sendBody": true, + "contentType": "raw", + "rawContentType": "text/plain", + "body": "=Downtime {{ $('Find open incident').first().json.duration }}. Incident closed." + }, + "credentials": { + "httpBasicAuth": { + "id": "REPLACE_WITH_YOUR_CREDENTIAL_ID", + "name": "ntfy (n8n user)" + } + } + } + ], + "connections": { + "Kuma incident": { + "main": [ + [ + { + "node": "Read event", + "type": "main", + "index": 0 + } + ] + ] + }, + "Read event": { + "main": [ + [ + { + "node": "Down?", + "type": "main", + "index": 0 + } + ] + ] + }, + "Down?": { + "main": [ + [ + { + "node": "Build incident doc", + "type": "main", + "index": 0 + } + ], + [ + { + "node": "List Ops Log", + "type": "main", + "index": 0 + } + ] + ] + }, + "Build incident doc": { + "main": [ + [ + { + "node": "Create incident doc", + "type": "main", + "index": 0 + } + ] + ] + }, + "Create incident doc": { + "main": [ + [ + { + "node": "Push alert", + "type": "main", + "index": 0 + } + ] + ] + }, + "List Ops Log": { + "main": [ + [ + { + "node": "Find open incident", + "type": "main", + "index": 0 + } + ] + ] + }, + "Find open incident": { + "main": [ + [ + { + "node": "Close incident doc", + "type": "main", + "index": 0 + } + ] + ] + }, + "Close incident doc": { + "main": [ + [ + { + "node": "Push all clear", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "settings": { + "executionOrder": "v1" + }, + "active": false +} diff --git a/automation/c-nightly-ops-digest.json b/automation/c-nightly-ops-digest.json new file mode 100644 index 0000000..7daa941 --- /dev/null +++ b/automation/c-nightly-ops-digest.json @@ -0,0 +1,356 @@ +{ + "name": "C - Nightly ops digest", + "nodes": [ + { + "id": "c1", + "name": "Every night at 07:00", + "type": "n8n-nodes-base.scheduleTrigger", + "typeVersion": 1.2, + "position": [ + -460, + 0 + ], + "parameters": { + "rule": { + "interval": [ + { + "field": "cronExpression", + "expression": "0 7 * * *" + } + ] + } + } + }, + { + "id": "60aab8c8", + "name": "Forgejo repos", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + -240, + 0 + ], + "parameters": { + "method": "GET", + "url": "http://forgejo:3000/api/v1/repos/search?limit=50", + "options": {}, + "authentication": "genericCredentialType", + "genericAuthType": "httpHeaderAuth" + }, + "credentials": { + "httpHeaderAuth": { + "id": "REPLACE_WITH_YOUR_CREDENTIAL_ID", + "name": "Forgejo API" + } + } + }, + { + "id": "c3", + "name": "Repo list", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + -20, + 0 + ], + "parameters": { + "jsCode": "\n// One item per repository, so the next node runs once per repo.\nconst list = $json.data ?? [];\nreturn list.map(r => ({ json: { owner: r.owner?.login ?? r.full_name.split('/')[0],\n repo: r.name, full: r.full_name } }));\n" + } + }, + { + "id": "567ab14b", + "name": "Commits per repo", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 200, + 0 + ], + "parameters": { + "method": "GET", + "url": "=http://forgejo:3000/api/v1/repos/{{ $json.owner }}/{{ $json.repo }}/commits?limit=50&stat=false", + "options": { + "response": { + "response": { + "responseFormat": "text", + "outputPropertyName": "data" + } + } + }, + "authentication": "genericCredentialType", + "genericAuthType": "httpHeaderAuth" + }, + "credentials": { + "httpHeaderAuth": { + "id": "REPLACE_WITH_YOUR_CREDENTIAL_ID", + "name": "Forgejo API" + } + }, + "onError": "continueRegularOutput" + }, + { + "id": "4b0e6311", + "name": "Kuma uptime", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 420, + 0 + ], + "parameters": { + "method": "GET", + "url": "http://uptime-kuma:3001/api/status-page/heartbeat/status", + "options": {} + }, + "executeOnce": true + }, + { + "id": "1bb848e9", + "name": "Kuma monitors", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 640, + 0 + ], + "parameters": { + "method": "GET", + "url": "http://uptime-kuma:3001/api/status-page/status", + "options": {} + }, + "executeOnce": true + }, + { + "id": "c7", + "name": "Read backup log", + "type": "n8n-nodes-base.readWriteFile", + "typeVersion": 1, + "position": [ + 860, + 0 + ], + "executeOnce": true, + "parameters": { + "fileSelector": "/backup/backup.log", + "options": {} + }, + "onError": "continueRegularOutput" + }, + { + "id": "c8", + "name": "Log to text", + "type": "n8n-nodes-base.extractFromFile", + "typeVersion": 1, + "position": [ + 1080, + 0 + ], + "parameters": { + "operation": "text", + "binaryPropertyName": "data", + "options": {} + }, + "onError": "continueRegularOutput" + }, + { + "id": "c9", + "name": "Build digest", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1300, + 0 + ], + "parameters": { + "jsCode": "\n// Everything below is read from the three services on this box. Nothing is\n// invented: if a source is missing the digest says so instead of guessing.\nconst repos = $('Repo list').all().map(i => i.json);\nconst bodies = $('Commits per repo').all().map(i => i.json);\nconst since = Date.now() - 24 * 60 * 60 * 1000;\n\nlet commitTotal = 0;\nconst perRepo = [];\nbodies.forEach((b, idx) => {\n let list = [];\n try { list = JSON.parse(b.data ?? '[]'); } catch (e) { list = []; }\n if (!Array.isArray(list)) list = [];\n const recent = list.filter(c => {\n const d = c.commit?.author?.date ?? c.commit?.committer?.date ?? c.created;\n return d && new Date(d).getTime() >= since;\n });\n commitTotal += recent.length;\n perRepo.push({\n repo: repos[idx]?.full ?? ('repo ' + idx),\n commits: recent.length,\n last: recent.length ? String(recent[0]?.commit?.message ?? '').split('\\n')[0].slice(0, 80) : '',\n });\n});\n\n// Uptime Kuma: the public status page carries both the names and the 24h numbers.\nconst hb = $('Kuma uptime').first().json ?? {};\nconst sp = $('Kuma monitors').first().json ?? {};\nconst names = {};\n(sp.publicGroupList ?? []).forEach(g => (g.monitorList ?? []).forEach(m => { names[String(m.id)] = m.name; }));\nconst uptime = Object.entries(hb.uptimeList ?? {}).map(([k, v]) => ({\n name: names[k.split('_')[0]] ?? ('monitor ' + k.split('_')[0]),\n pct: Math.round(Number(v) * 10000) / 100,\n}));\nconst worst = uptime.length ? Math.min(...uptime.map(u => u.pct)) : null;\n\n// Episode 7 writes one line per phase into the backup log. Read the last run.\nconst log = String($input.first().json.data ?? '');\nlet snap = null, files = null, size = null, took = null, when = null, beat = false;\nfor (const line of log.split('\\n')) {\n let m;\n if ((m = line.match(/^snapshot ([0-9a-f]+) saved/))) snap = m[1];\n if ((m = line.match(/processed (\\d+) files, ([\\d.]+ [KMG]iB) in/))) { files = m[1]; size = m[2]; }\n if ((m = line.match(/^(\\S+ \\S+) \\[5\\/5\\] done in (\\d+)s/))) { when = m[1]; took = m[2]; }\n if (line.includes('heartbeat sent to Uptime Kuma')) beat = true;\n}\nconst backup = snap\n ? 'snapshot `' + snap + '`, ' + files + ' files, ' + size + ', took ' + took + ' s, finished ' + when + (beat ? ', heartbeat sent' : ', NO heartbeat')\n : 'no backup log found on this box';\n\nconst today = new Date().toISOString().slice(0, 10);\nconst text = [\n '## Git',\n '',\n commitTotal + ' commit(s) in the last 24 hours across ' + perRepo.length + ' repositories.',\n '',\n ...perRepo.map(r => '- ' + r.repo + ': ' + r.commits + (r.last ? ', latest \"' + r.last + '\"' : '')),\n '',\n '## Availability (last 24 hours)',\n '',\n ...(uptime.length ? uptime.map(u => '- ' + u.name + ': ' + u.pct + ' %')\n : ['- no monitors on the public status page']),\n '',\n '## Backup',\n '',\n '- ' + backup,\n '',\n '---',\n '',\n 'Written by n8n, workflow \"C - Nightly ops digest\". Sources: Forgejo API,',\n 'Uptime Kuma status page API, the nightly restic log from episode 7.',\n].join('\\n');\n\nconst summary = commitTotal + ' commits, ' + (worst === null ? 'no uptime data' : 'uptime ' + worst + ' % worst') + ', backup ' + (snap ? 'ok' : 'MISSING');\n\nreturn [{ json: {\n commitTotal, perRepo, uptime, worst, backup, summary,\n payload: { title: 'Ops digest ' + today, text, collectionId: 'REPLACE_WITH_YOUR_OUTLINE_COLLECTION_ID', publish: true },\n}}];\n" + } + }, + { + "id": "b1f2cb7c", + "name": "Write digest doc", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 1520, + 0 + ], + "parameters": { + "method": "POST", + "url": "http://outline:3000/api/documents.create", + "options": {}, + "authentication": "genericCredentialType", + "genericAuthType": "httpHeaderAuth", + "sendBody": true, + "contentType": "json", + "specifyBody": "json", + "jsonBody": "={{ JSON.stringify($json.payload) }}" + }, + "credentials": { + "httpHeaderAuth": { + "id": "REPLACE_WITH_YOUR_CREDENTIAL_ID", + "name": "Outline API" + } + } + }, + { + "id": "fc737f63", + "name": "Push digest", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 1740, + 0 + ], + "parameters": { + "method": "POST", + "url": "http://ntfy/deploys", + "options": {}, + "authentication": "genericCredentialType", + "genericAuthType": "httpBasicAuth", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "X-Title", + "value": "Nightly ops digest" + }, + { + "name": "X-Priority", + "value": "2" + }, + { + "name": "X-Click", + "value": "=https://wiki.ops.example.com{{ $json.data.url }}" + } + ] + }, + "sendBody": true, + "contentType": "raw", + "rawContentType": "text/plain", + "body": "={{ $('Build digest').first().json.summary }}" + }, + "credentials": { + "httpBasicAuth": { + "id": "REPLACE_WITH_YOUR_CREDENTIAL_ID", + "name": "ntfy (n8n user)" + } + } + } + ], + "connections": { + "Every night at 07:00": { + "main": [ + [ + { + "node": "Forgejo repos", + "type": "main", + "index": 0 + } + ] + ] + }, + "Forgejo repos": { + "main": [ + [ + { + "node": "Repo list", + "type": "main", + "index": 0 + } + ] + ] + }, + "Repo list": { + "main": [ + [ + { + "node": "Commits per repo", + "type": "main", + "index": 0 + } + ] + ] + }, + "Commits per repo": { + "main": [ + [ + { + "node": "Kuma uptime", + "type": "main", + "index": 0 + } + ] + ] + }, + "Kuma uptime": { + "main": [ + [ + { + "node": "Kuma monitors", + "type": "main", + "index": 0 + } + ] + ] + }, + "Kuma monitors": { + "main": [ + [ + { + "node": "Read backup log", + "type": "main", + "index": 0 + } + ] + ] + }, + "Read backup log": { + "main": [ + [ + { + "node": "Log to text", + "type": "main", + "index": 0 + } + ] + ] + }, + "Log to text": { + "main": [ + [ + { + "node": "Build digest", + "type": "main", + "index": 0 + } + ] + ] + }, + "Build digest": { + "main": [ + [ + { + "node": "Write digest doc", + "type": "main", + "index": 0 + } + ] + ] + }, + "Write digest doc": { + "main": [ + [ + { + "node": "Push digest", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "settings": { + "executionOrder": "v1" + }, + "active": false +} diff --git a/compose.yml b/compose.yml index 580fe88..eb5985e 100644 --- a/compose.yml +++ b/compose.yml @@ -23,6 +23,12 @@ name: selfhostedops-stack networks: proxy: + # Pinned on purpose. Compose hands the subnets out in creation order, so a + # restore onto a fresh box can renumber them - and every NetBird route that + # points at the old range stops working, silently (episode 8). + ipam: + config: + - subnet: 172.18.0.0/16 driver_opts: # Docker 28+ firewalls direct access to container IPs from other # interfaces (raw PREROUTING drop per container). This lets the NetBird @@ -30,6 +36,9 @@ networks: com.docker.network.bridge.trusted_host_interfaces: "wt0" ci: internal: + ipam: + config: + - subnet: 172.19.0.0/16 volumes: caddy_data: @@ -77,6 +86,11 @@ services: FORGEJO__openid__ENABLE_OPENID_SIGNIN: "false" FORGEJO__actions__ENABLED: "true" FORGEJO__repository__DEFAULT_REPO_UNITS: "repo.code,repo.issues,repo.pulls,repo.releases,repo.packages,repo.actions" + # Forgejo refuses to deliver a webhook to a private address by default. + # Calling the automation hub over the docker network is exactly that, so + # the container name has to be listed. "external" keeps ordinary public + # webhook targets working (episode 8). + FORGEJO__webhook__ALLOWED_HOST_LIST: "external,n8n" ports: - "2222:22" # git over SSH volumes: @@ -182,18 +196,50 @@ services: - ntfy_cache:/var/cache/ntfy networks: [proxy] + # The automation hub (episode 8). The editor stays on the mesh (episode 5); + # Caddy publishes nothing but the webhook paths. Internal senders (forgejo, + # uptime-kuma) call http://n8n:5678/webhook/... over the docker network and + # never leave the box at all. n8n: - image: docker.n8n.io/n8nio/n8n:latest + # Pinned. With :latest every restore from backup is silently an upgrade, + # and an upgrade is a thing you schedule, not a thing a restore does to you. + image: docker.n8n.io/n8nio/n8n:2.37.7 restart: unless-stopped environment: N8N_HOST: n8n.${DOMAIN} N8N_PORT: "5678" N8N_PROTOCOL: https - WEBHOOK_URL: https://n8n.${DOMAIN}/ + # The address n8n prints under every webhook node. It must be the public + # one, otherwise you hand your senders a URL that resolves nowhere. + # (2.x renamed WEBHOOK_URL to this. The old name still works and prints a + # deprecation line on every boot.) + N8N_WEBHOOK_URL: https://n8n.${DOMAIN}/ GENERIC_TIMEZONE: ${TZ:-Europe/Berlin} TZ: ${TZ:-Europe/Berlin} + # The editor is opened over the mesh on plain http://:5678. + # With secure cookies on, the browser never sends the session cookie back + # and the login screen just reappears, with no error anywhere. + N8N_SECURE_COOKIE: "false" + # Do not let anyone install unverified community node packages here. + # The default flips to false in a later version anyway. + N8N_UNVERIFIED_PACKAGES_ENABLED: "false" + # File nodes may read exactly one directory and nothing else. Left wide + # open, a workflow could read /home/node/.n8n/config - the file that + # decrypts every credential in the instance. + N8N_RESTRICT_FILE_ACCESS_TO: /backup + # Optional: pin the credential encryption key yourself instead of letting + # n8n write one into the volume. See .env.example before you set this. + N8N_ENCRYPTION_KEY: ${N8N_ENCRYPTION_KEY:-} + # A company box does not need to phone home. + N8N_DIAGNOSTICS_ENABLED: "false" + N8N_VERSION_NOTIFICATIONS_ENABLED: "false" + N8N_HIRING_BANNER_ENABLED: "false" volumes: - n8n_data:/home/node/.n8n + # Read-only view of the nightly backup log (episode 7) so the digest + # workflow can report the last restic snapshot. Optional: drop this line + # and the digest says "no backup log found" instead of failing. + - ./backup/backup.log:/backup/backup.log:ro networks: [proxy] # Optional starter VPN — enable with: docker compose --profile vpn up -d