selfhostedops/selfhostedops-stack
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Episode 16: Kuma alerts sorted by rules, a local agent and a script

Browse source 
ops-readonly gets kuma_alerts (host export kuma-export.py, one more read-only
mount). kuma-triage.py sorts the same data without a model. Flow H runs the
agent with skill alert-triage.md. Flow B: one open incident per monitor, an
UP closes all of them, titles name start and end and stay under Outline's
limit. B and D accept Kuma only with a header token. Monitoring notes: ntfy
rejects Kuma messages for monitors without a URL, resend for long outages.
This commit is contained in:
Maximilian Grimm 2026-09-29 22:14:50 +02:00
parent 54ab43dac7
commit 1655d04d7d
10 changed files with 705 additions and 67 deletions
Download patch file Download diff file Expand all files Collapse all files

209
automation/b-incident-responder.json
View file

@ -15,7 +15,14 @@
"httpMethod": "POST",
"path": "kuma-incident",
"responseMode": "onReceived",
"options": {}
"options": {},
"authentication": "headerAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "",
"name": "Kuma webhook token"
}
}
},
{
@ -31,40 +38,6 @@
"jsCode": "\n// Uptime Kuma posts one JSON body for every state change. Reduce it to the\n// four fields both branches need, so nothing below has to know Kuma's shape.\nconst b = $json.body ?? $json;\nconst hb = b.heartbeat ?? {};\nconst mon = b.monitor ?? {};\nconst down = Number(hb.status) === 0;\nreturn [{\n json: {\n monitor: mon.name ?? 'unknown monitor',\n monitorUrl: mon.url ?? '',\n down,\n at: hb.time ? new Date(hb.time.replace(' ', 'T') + 'Z').toISOString() : new Date().toISOString(),\n reason: hb.msg ?? b.msg ?? '',\n },\n}];\n"
}
},
{
"id": "b3",
"name": "Down?",
"type": "n8n-nodes-base.if",
"typeVersion": 2.2,
"position": [
220,
0
],
"parameters": {
"conditions": {
"options": {
"caseSensitive": true,
"leftValue": "",
"typeValidation": "strict",
"version": 2
},
"conditions": [
{
"id": "c1",
"leftValue": "={{ $json.down }}",
"rightValue": "",
"operator": {
"type": "boolean",
"operation": "true",
"singleValue": true
}
}
],
"combinator": "and"
},
"options": {}
}
},
{
"id": "b4",
"name": "Build incident doc",
@ -75,7 +48,7 @@
-140
],
"parameters": {
"jsCode": "\n// One incident document per outage. \"Status: open\" is the marker the UP\n// branch searches for later, so it stays machine readable on purpose.\nconst e = $json;\nconst stamp = e.at.replace('T', ' ').replace(/\\..*/, ' UTC');\nconst title = 'Incident: ' + e.monitor + ' (' + stamp + ')';\n// A markdown list, not plain lines: Outline collapses single newlines inside\n// a paragraph, which would turn the whole record into one unreadable blob.\nconst text = [\n '- Monitor: ' + e.monitor,\n '- Status: open',\n '- Down since: ' + e.at,\n '- Reason: ' + (e.reason || 'no detail from the monitor'),\n e.monitorUrl ? '- Checked URL: ' + e.monitorUrl : '',\n '',\n '---',\n '',\n 'Opened automatically by n8n. The resolution is appended when the monitor reports UP again.',\n].filter(Boolean).join('\\n');\nreturn [{ json: { title, payload: { title, text, collectionId: 'REPLACE_WITH_YOUR_OUTLINE_COLLECTION_ID', publish: true } } }];\n"
"jsCode": "\n// One incident document per outage. \"Status: open\" is the marker the UP\n// branch searches for later, so it stays machine readable on purpose.\nconst e = $('Read event').first().json;\nconst stamp = e.at.replace('T', ' ').replace(/\\..*/, ' UTC');\nconst title = 'Incident: ' + e.monitor.slice(0, 60) + ' (' + stamp + ')';\n// A markdown list, not plain lines: Outline collapses single newlines inside\n// a paragraph, which would turn the whole record into one unreadable blob.\nconst text = [\n '- Monitor: ' + e.monitor,\n '- Status: open',\n '- Down since: ' + e.at,\n '- Reason: ' + (e.reason || 'no detail from the monitor'),\n e.monitorUrl ? '- Checked URL: ' + e.monitorUrl : '',\n '',\n '---',\n '',\n 'Opened automatically by n8n. The resolution is appended when the monitor reports UP again.',\n].filter(Boolean).join('\\n');\nreturn [{ json: { title, payload: { title, text, collectionId: 'REPLACE_WITH_YOUR_OUTLINE_COLLECTION_ID', publish: true } } }];\n"
}
},
{
@ -163,7 +136,7 @@
"sendBody": true,
"contentType": "json",
"specifyBody": "json",
"jsonBody": "={{ JSON.stringify({ collectionId: \"REPLACE_WITH_YOUR_OUTLINE_COLLECTION_ID\", limit: 25 }) }}"
"jsonBody": "={{ JSON.stringify({ collectionId: \"6352d2b9-48e9-41e0-9bdb-535db44b774a\", limit: 100 }) }}"
},
"credentials": {
"httpHeaderAuth": {
@ -174,7 +147,7 @@
},
{
"id": "b8",
"name": "Find open incident",
"name": "Decide",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
@ -182,7 +155,7 @@
140
],
"parameters": {
"jsCode": "\n// Find the still open incident for this monitor and close it out.\nconst e = $('Read event').first().json;\nconst docs = $input.first().json.data ?? [];\nconst open = docs.find(d =>\n String(d.text ?? '').includes('Monitor: ' + e.monitor) &&\n String(d.text ?? '').includes('Status: open'));\n\nif (!open) {\n return [{ json: { found: false, monitor: e.monitor, duration: 'unknown',\n note: 'no open incident document for this monitor' } }];\n}\n\nconst m = String(open.text).match(/Down since:\\s*(\\S+)/);\nconst since = m ? new Date(m[1]) : null;\nconst back = new Date(e.at);\nconst secs = since ? Math.max(0, Math.round((back - since) / 1000)) : null;\nconst human = secs === null ? 'unknown'\n : secs < 90 ? secs + ' s'\n : Math.floor(secs / 60) + ' min ' + (secs % 60) + ' s';\n\nconst text = String(open.text).replace('Status: open', 'Status: resolved')\n + '\\n- Back up: ' + e.at + '\\n- Downtime: ' + human + '\\n';\n\nreturn [{ json: {\n found: true, monitor: e.monitor, duration: human, seconds: secs,\n docUrl: open.url ?? '',\n payload: { id: open.id, title: String(open.title).replace('Incident:', 'Resolved:'), text, publish: true },\n}}];\n"
"jsCode": "\n// Episode 16. Kuma repeats a DOWN after its own restart (PENDING -> DOWN is\n// \"important\"). Before this fix every DOWN opened a new incident and an UP\n// closed only one, so two incidents stayed \"open\" forever. Now: one open\n// incident per monitor, and an UP closes all of them.\nconst e = $('Read event').first().json;\nconst docs = $input.first().json.data ?? [];\nconst open = docs.filter(d =>\n String(d.text ?? '').includes('Monitor: ' + e.monitor) &&\n String(d.text ?? '').includes('Status: open'));\nconst fmt = s => String(s).replace('T', ' ').replace(/:\\d\\d(\\.\\d+)?Z$/, ' UTC');\nif (e.down) {\n if (open.length) return [{ json: { action: 'skip', monitor: e.monitor, note: 'already open: ' + open[0].title } }];\n return [{ json: { action: 'open', ...e } }];\n}\nif (!open.length) return [{ json: { action: 'skip', monitor: e.monitor, note: 'UP without an open incident' } }];\n// One title per document with its own start: an orphan from an older outage\n// must not borrow a start date from another one (lab16 first fix did, #385).\nconst fmt2 = d => fmt(d.toISOString());\nreturn open.map(d => {\n const m = String(d.text).match(/Down since:\\s*(\\S+)/); const since = m ? new Date(m[1]) : null;\n const secs = since ? Math.max(0, Math.round((new Date(e.at) - since) / 1000)) : null;\n const human = secs === null ? 'unknown' : secs < 90 ? secs + ' s' : Math.floor(secs / 60) + ' min ' + (secs % 60) + ' s';\n return { json: { action: 'close', monitor: e.monitor, duration: human, payload: { id: d.id, publish: true,\n title: 'Resolved: ' + e.monitor.slice(0, 40) + ' (down ' + (since ? fmt2(since) : '?') + ', back ' + fmt(e.at) + ')',\n text: String(d.text).replace('Status: open', 'Status: resolved') + '\\n- Back up: ' + e.at + '\\n- Downtime: ' + human + '\\n' } } };\n});\n"
}
},
{
@ -244,14 +217,113 @@
"sendBody": true,
"contentType": "raw",
"rawContentType": "text/plain",
"body": "=Downtime {{ $('Find open incident').first().json.duration }}. Incident closed."
"body": "=Downtime {{ $('Decide').first().json.duration }}. Incident closed."
},
"credentials": {
"httpBasicAuth": {
"id": "REPLACE_WITH_YOUR_CREDENTIAL_ID",
"name": "ntfy (n8n user)"
}
},
"executeOnce": true
},
{
"name": "Route",
"type": "n8n-nodes-base.switch",
"typeVersion": 3.2,
"position": [
700,
0
],
"parameters": {
"rules": {
"values": [
{
"conditions": {
"options": {
"caseSensitive": true,
"leftValue": "",
"typeValidation": "strict",
"version": 2
},
"combinator": "and",
"conditions": [
{
"id": "r0",
"leftValue": "={{ $json.action }}",
"rightValue": "open",
"operator": {
"type": "string",
"operation": "equals"
}
}
]
},
"renameOutput": true,
"outputKey": "open"
},
{
"conditions": {
"options": {
"caseSensitive": true,
"leftValue": "",
"typeValidation": "strict",
"version": 2
},
"combinator": "and",
"conditions": [
{
"id": "r1",
"leftValue": "={{ $json.action }}",
"rightValue": "close",
"operator": {
"type": "string",
"operation": "equals"
}
}
]
},
"renameOutput": true,
"outputKey": "close"
},
{
"conditions": {
"options": {
"caseSensitive": true,
"leftValue": "",
"typeValidation": "strict",
"version": 2
},
"combinator": "and",
"conditions": [
{
"id": "r2",
"leftValue": "={{ $json.action }}",
"rightValue": "skip",
"operator": {
"type": "string",
"operation": "equals"
}
}
]
},
"renameOutput": true,
"outputKey": "skip"
}
]
},
"options": {}
}
},
{
"name": "Duplicate or nothing open",
"type": "n8n-nodes-base.noOp",
"typeVersion": 1,
"position": [
950,
250
],
"parameters": {}
}
],
"connections": {
@ -270,14 +342,36 @@
"main": [
[
{
"node": "Down?",
"node": "List Ops Log",
"type": "main",
"index": 0
}
]
]
},
"Down?": {
"List Ops Log": {
"main": [
[
{
"node": "Decide",
"type": "main",
"index": 0
}
]
]
},
"Decide": {
"main": [
[
{
"node": "Route",
"type": "main",
"index": 0
}
]
]
},
"Route": {
"main": [
[
{
@ -288,7 +382,14 @@
],
[
{
"node": "List Ops Log",
"node": "Close incident doc",
"type": "main",
"index": 0
}
],
[
{
"node": "Duplicate or nothing open",
"type": "main",
"index": 0
}
@ -317,28 +418,6 @@
]
]
},
"List Ops Log": {
"main": [
[
{
"node": "Find open incident",
"type": "main",
"index": 0
}
]
]
},
"Find open incident": {
"main": [
[
{
"node": "Close incident doc",
"type": "main",
"index": 0
}
]
]
},
"Close incident doc": {
"main": [
[