selfhostedops/selfhostedops-stack
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
selfhostedops-stack/automation/b-incident-responder.json
Maximilian Grimm 1655d04d7d Episode 16: Kuma alerts sorted by rules, a local agent and a script 
ops-readonly gets kuma_alerts (host export kuma-export.py, one more read-only
mount). kuma-triage.py sorts the same data without a model. Flow H runs the
agent with skill alert-triage.md. Flow B: one open incident per monitor, an
UP closes all of them, titles name start and end and stay under Outline's
limit. B and D accept Kuma only with a header token. Monitoring notes: ntfy
rejects Kuma messages for monitors without a URL, resend for long outages.
2026-09-29 22:14:57 +02:00

437 lines
13 KiB
JSON
Raw Blame History

{
"name": "B - Incident responder",
"nodes": [
{
"id": "b1",
"name": "Kuma incident",
"type": "n8n-nodes-base.webhook",
"typeVersion": 2,
"position": [
-220,
0
],
"webhookId": "REPLACE_ON_IMPORT",
"parameters": {
"httpMethod": "POST",
"path": "kuma-incident",
"responseMode": "onReceived",
"options": {},
"authentication": "headerAuth"
},
"credentials": {
"httpHeaderAuth": {
"id": "",
"name": "Kuma webhook token"
}
}
},
{
"id": "b2",
"name": "Read event",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
0,
0
],
"parameters": {
"jsCode": "\n// Uptime Kuma posts one JSON body for every state change. Reduce it to the\n// four fields both branches need, so nothing below has to know Kuma's shape.\nconst b = $json.body ?? $json;\nconst hb = b.heartbeat ?? {};\nconst mon = b.monitor ?? {};\nconst down = Number(hb.status) === 0;\nreturn [{\n json: {\n monitor: mon.name ?? 'unknown monitor',\n monitorUrl: mon.url ?? '',\n down,\n at: hb.time ? new Date(hb.time.replace(' ', 'T') + 'Z').toISOString() : new Date().toISOString(),\n reason: hb.msg ?? b.msg ?? '',\n },\n}];\n"
}
},
{
"id": "b4",
"name": "Build incident doc",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
460,
-140
],
"parameters": {
"jsCode": "\n// One incident document per outage. \"Status: open\" is the marker the UP\n// branch searches for later, so it stays machine readable on purpose.\nconst e = $('Read event').first().json;\nconst stamp = e.at.replace('T', ' ').replace(/\\..*/, ' UTC');\nconst title = 'Incident: ' + e.monitor.slice(0, 60) + ' (' + stamp + ')';\n// A markdown list, not plain lines: Outline collapses single newlines inside\n// a paragraph, which would turn the whole record into one unreadable blob.\nconst text = [\n '- Monitor: ' + e.monitor,\n '- Status: open',\n '- Down since: ' + e.at,\n '- Reason: ' + (e.reason || 'no detail from the monitor'),\n e.monitorUrl ? '- Checked URL: ' + e.monitorUrl : '',\n '',\n '---',\n '',\n 'Opened automatically by n8n. The resolution is appended when the monitor reports UP again.',\n].filter(Boolean).join('\\n');\nreturn [{ json: { title, payload: { title, text, collectionId: 'REPLACE_WITH_YOUR_OUTLINE_COLLECTION_ID', publish: true } } }];\n"
}
},
{
"id": "13d2a02f",
"name": "Create incident doc",
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.2,
"position": [
680,
-140
],
"parameters": {
"method": "POST",
"url": "http://outline:3000/api/documents.create",
"options": {},
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth",
"sendBody": true,
"contentType": "json",
"specifyBody": "json",
"jsonBody": "={{ JSON.stringify($json.payload) }}"
},
"credentials": {
"httpHeaderAuth": {
"id": "REPLACE_WITH_YOUR_CREDENTIAL_ID",
"name": "Outline API"
}
}
},
{
"id": "e28ddc1d",
"name": "Push alert",
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.2,
"position": [
900,
-140
],
"parameters": {
"method": "POST",
"url": "http://ntfy/alerts",
"options": {},
"authentication": "genericCredentialType",
"genericAuthType": "httpBasicAuth",
"sendHeaders": true,
"headerParameters": {
"parameters": [
{
"name": "X-Title",
"value": "=DOWN: {{ $('Read event').first().json.monitor }}"
},
{
"name": "X-Priority",
"value": "5"
}
]
},
"sendBody": true,
"contentType": "raw",
"rawContentType": "text/plain",
"body": "={{ $('Read event').first().json.reason || 'monitor reports down' }}\nIncident doc: https://wiki.ops.example.com{{ $json.data.url }}"
},
"credentials": {
"httpBasicAuth": {
"id": "REPLACE_WITH_YOUR_CREDENTIAL_ID",
"name": "ntfy (n8n user)"
}
}
},
{
"id": "0ac14924",
"name": "List Ops Log",
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.2,
"position": [
460,
140
],
"parameters": {
"method": "POST",
"url": "http://outline:3000/api/documents.list",
"options": {},
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth",
"sendBody": true,
"contentType": "json",
"specifyBody": "json",
"jsonBody": "={{ JSON.stringify({ collectionId: \"6352d2b9-48e9-41e0-9bdb-535db44b774a\", limit: 100 }) }}"
},
"credentials": {
"httpHeaderAuth": {
"id": "REPLACE_WITH_YOUR_CREDENTIAL_ID",
"name": "Outline API"
}
}
},
{
"id": "b8",
"name": "Decide",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
680,
140
],
"parameters": {
"jsCode": "\n// Episode 16. Kuma repeats a DOWN after its own restart (PENDING -> DOWN is\n// \"important\"). Before this fix every DOWN opened a new incident and an UP\n// closed only one, so two incidents stayed \"open\" forever. Now: one open\n// incident per monitor, and an UP closes all of them.\nconst e = $('Read event').first().json;\nconst docs = $input.first().json.data ?? [];\nconst open = docs.filter(d =>\n String(d.text ?? '').includes('Monitor: ' + e.monitor) &&\n String(d.text ?? '').includes('Status: open'));\nconst fmt = s => String(s).replace('T', ' ').replace(/:\\d\\d(\\.\\d+)?Z$/, ' UTC');\nif (e.down) {\n if (open.length) return [{ json: { action: 'skip', monitor: e.monitor, note: 'already open: ' + open[0].title } }];\n return [{ json: { action: 'open', ...e } }];\n}\nif (!open.length) return [{ json: { action: 'skip', monitor: e.monitor, note: 'UP without an open incident' } }];\n// One title per document with its own start: an orphan from an older outage\n// must not borrow a start date from another one (lab16 first fix did, #385).\nconst fmt2 = d => fmt(d.toISOString());\nreturn open.map(d => {\n const m = String(d.text).match(/Down since:\\s*(\\S+)/); const since = m ? new Date(m[1]) : null;\n const secs = since ? Math.max(0, Math.round((new Date(e.at) - since) / 1000)) : null;\n const human = secs === null ? 'unknown' : secs < 90 ? secs + ' s' : Math.floor(secs / 60) + ' min ' + (secs % 60) + ' s';\n return { json: { action: 'close', monitor: e.monitor, duration: human, payload: { id: d.id, publish: true,\n title: 'Resolved: ' + e.monitor.slice(0, 40) + ' (down ' + (since ? fmt2(since) : '?') + ', back ' + fmt(e.at) + ')',\n text: String(d.text).replace('Status: open', 'Status: resolved') + '\\n- Back up: ' + e.at + '\\n- Downtime: ' + human + '\\n' } } };\n});\n"
}
},
{
"id": "e402dfe1",
"name": "Close incident doc",
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.2,
"position": [
900,
140
],
"parameters": {
"method": "POST",
"url": "http://outline:3000/api/documents.update",
"options": {},
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth",
"sendBody": true,
"contentType": "json",
"specifyBody": "json",
"jsonBody": "={{ JSON.stringify($json.payload || {}) }}"
},
"credentials": {
"httpHeaderAuth": {
"id": "REPLACE_WITH_YOUR_CREDENTIAL_ID",
"name": "Outline API"
}
},
"onError": "continueRegularOutput"
},
{
"id": "7b4981c4",
"name": "Push all clear",
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.2,
"position": [
1120,
140
],
"parameters": {
"method": "POST",
"url": "http://ntfy/alerts",
"options": {},
"authentication": "genericCredentialType",
"genericAuthType": "httpBasicAuth",
"sendHeaders": true,
"headerParameters": {
"parameters": [
{
"name": "X-Title",
"value": "=UP again: {{ $('Read event').first().json.monitor }}"
},
{
"name": "X-Priority",
"value": "3"
}
]
},
"sendBody": true,
"contentType": "raw",
"rawContentType": "text/plain",
"body": "=Downtime {{ $('Decide').first().json.duration }}. Incident closed."
},
"credentials": {
"httpBasicAuth": {
"id": "REPLACE_WITH_YOUR_CREDENTIAL_ID",
"name": "ntfy (n8n user)"
}
},
"executeOnce": true
},
{
"name": "Route",
"type": "n8n-nodes-base.switch",
"typeVersion": 3.2,
"position": [
700,
0
],
"parameters": {
"rules": {
"values": [
{
"conditions": {
"options": {
"caseSensitive": true,
"leftValue": "",
"typeValidation": "strict",
"version": 2
},
"combinator": "and",
"conditions": [
{
"id": "r0",
"leftValue": "={{ $json.action }}",
"rightValue": "open",
"operator": {
"type": "string",
"operation": "equals"
}
}
]
},
"renameOutput": true,
"outputKey": "open"
},
{
"conditions": {
"options": {
"caseSensitive": true,
"leftValue": "",
"typeValidation": "strict",
"version": 2
},
"combinator": "and",
"conditions": [
{
"id": "r1",
"leftValue": "={{ $json.action }}",
"rightValue": "close",
"operator": {
"type": "string",
"operation": "equals"
}
}
]
},
"renameOutput": true,
"outputKey": "close"
},
{
"conditions": {
"options": {
"caseSensitive": true,
"leftValue": "",
"typeValidation": "strict",
"version": 2
},
"combinator": "and",
"conditions": [
{
"id": "r2",
"leftValue": "={{ $json.action }}",
"rightValue": "skip",
"operator": {
"type": "string",
"operation": "equals"
}
}
]
},
"renameOutput": true,
"outputKey": "skip"
}
]
},
"options": {}
}
},
{
"name": "Duplicate or nothing open",
"type": "n8n-nodes-base.noOp",
"typeVersion": 1,
"position": [
950,
250
],
"parameters": {}
}
],
"connections": {
"Kuma incident": {
"main": [
[
{
"node": "Read event",
"type": "main",
"index": 0
}
]
]
},
"Read event": {
"main": [
[
{
"node": "List Ops Log",
"type": "main",
"index": 0
}
]
]
},
"List Ops Log": {
"main": [
[
{
"node": "Decide",
"type": "main",
"index": 0
}
]
]
},
"Decide": {
"main": [
[
{
"node": "Route",
"type": "main",
"index": 0
}
]
]
},
"Route": {
"main": [
[
{
"node": "Build incident doc",
"type": "main",
"index": 0
}
],
[
{
"node": "Close incident doc",
"type": "main",
"index": 0
}
],
[
{
"node": "Duplicate or nothing open",
"type": "main",
"index": 0
}
]
]
},
"Build incident doc": {
"main": [
[
{
"node": "Create incident doc",
"type": "main",
"index": 0
}
]
]
},
"Create incident doc": {
"main": [
[
{
"node": "Push alert",
"type": "main",
"index": 0
}
]
]
},
"Close incident doc": {
"main": [
[
{
"node": "Push all clear",
"type": "main",
"index": 0
}
]
]
}
},
"settings": {
"executionOrder": "v1"
},
"active": false
}
Reference in a new issue View git blame Copy permalink