selfhostedops-stack/automation/h-kuma-alert-triage.json
Maximilian Grimm 1655d04d7d Episode 16: Kuma alerts sorted by rules, a local agent and a script
ops-readonly gets kuma_alerts (host export kuma-export.py, one more read-only
mount). kuma-triage.py sorts the same data without a model. Flow H runs the
agent with skill alert-triage.md. Flow B: one open incident per monitor, an
UP closes all of them, titles name start and end and stay under Outline's
limit. B and D accept Kuma only with a header token. Monitoring notes: ntfy
rejects Kuma messages for monitors without a URL, resend for long outages.
2026-09-29 22:14:57 +02:00

354 lines
No EOL
11 KiB
JSON

{
"name": "H - Kuma alert triage",
"nodes": [
{
"name": "Start",
"type": "n8n-nodes-base.webhook",
"typeVersion": 2,
"position": [
0,
0
],
"webhookId": "REPLACE_ON_IMPORT",
"credentials": {
"httpHeaderAuth": {
"id": "",
"name": "Kuma triage trigger"
}
},
"parameters": {
"httpMethod": "POST",
"path": "kuma-alert-triage",
"authentication": "headerAuth",
"responseMode": "onReceived",
"options": {}
}
},
{
"name": "Load skill",
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.2,
"position": [
220,
0
],
"credentials": {
"httpHeaderAuth": {
"id": "",
"name": "Forgejo API"
}
},
"parameters": {
"method": "GET",
"url": "http://forgejo:3000/api/v1/repos/opsadmin/selfhostedops-stack/raw/agents/skills/alert-triage.md",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth",
"options": {
"response": {
"response": {
"responseFormat": "text"
}
}
}
}
},
{
"name": "Build prompts",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
440,
0
],
"parameters": {
"jsCode": "\n// The skill file is markdown with a small frontmatter block for humans; the\n// model gets everything below it. Model tags come from a fixed list, never\n// from the caller.\nfunction body(md) { return String(md ?? '').replace(/^---[\\s\\S]*?\\n---\\s*/, '').trim(); }\nconst b = $('Start').first().json.body ?? {};\nconst MODELS = ['qwen2.5-advisor:3b', 'qwen3-advisor:4b'];\nconst model = MODELS.includes(b.model) ? b.model : MODELS[0];\n// qwen3 thinks before it answers: at 3000 tokens it never reached the JSON (lab16 runs #380, #402, #410)\nconst numPredict = model.startsWith('qwen3') ? 8000 : 3000;\nreturn [{ json: {\n system: body($json.data),\n task: 'Call kuma_alerts once, then answer with the JSON object described in your instructions.',\n model, numPredict, label: String(b.label ?? '').slice(0, 40), deliver: b.deliver === true,\n} }];\n"
}
},
{
"name": "Alert triage agent",
"type": "@n8n/n8n-nodes-langchain.agent",
"typeVersion": 2.2,
"position": [
680,
0
],
"parameters": {
"promptType": "define",
"text": "={{ $json.task }}",
"hasOutputParser": false,
"options": {
"systemMessage": "={{ $json.system }}",
"maxIterations": 4,
"returnIntermediateSteps": true,
"enableStreaming": false
}
}
},
{
"name": "Ollama over the mesh",
"type": "@n8n/n8n-nodes-langchain.lmChatOllama",
"typeVersion": 1,
"position": [
600,
240
],
"credentials": {
"ollamaApi": {
"id": "",
"name": "Ollama (home lab over mesh)"
}
},
"parameters": {
"model": "={{ $('Build prompts').first().json.model }}",
"options": {
"temperature": 0,
"numCtx": 12288,
"numPredict": "={{ $('Build prompts').first().json.numPredict }}",
"keepAlive": "30m",
"think": false
}
}
},
{
"name": "kuma_alerts",
"type": "@n8n/n8n-nodes-langchain.toolHttpRequest",
"typeVersion": 1.1,
"position": [
800,
240
],
"credentials": {
"httpHeaderAuth": {
"id": "",
"name": "ops-readonly token"
}
},
"parameters": {
"toolDescription": "Every Uptime Kuma monitor for the current window: state now and since when, each status change in the window, heartbeat counts, TLS days left. Read only, no arguments. Call it once, first.",
"method": "GET",
"url": "http://ops-readonly:8098/read/kuma_alerts",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
}
},
{
"name": "Check alerts",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
1020,
0
],
"parameters": {
"jsCode": "\nfunction readObject(s) {\n if (s && typeof s === 'object') return s;\n const t = String(s ?? '');\n const a = t.indexOf('{'); const b = t.lastIndexOf('}');\n if (a === -1 || b <= a) return {};\n for (let end = b; end > a; end = t.lastIndexOf('}', end - 1)) {\n try { return JSON.parse(t.slice(a, end + 1)); } catch (e) { /* keep trying */ }\n }\n return {};\n}\n// Deterministic. Every alert is checked against what kuma_alerts returned in\n// THIS run. It filters, it never repairs: a missing alert stays missing.\nconst meta = $('Build prompts').first().json;\nconst raw = String($json.output ?? '');\nlet tool = null;\nfor (const s of ($json.intermediateSteps ?? [])) {\n if (s?.action?.tool === 'kuma_alerts') { try { tool = JSON.parse(s.observation); } catch (e) { /* not json */ } }\n}\nconst mons = new Map((tool?.monitors ?? []).map(m => [m.name, m]));\nconst KINDS = ['still_down', 'outage', 'recovered', 'flapping', 'cert_expiring'];\nconst got = readObject(raw);\nconst list = Array.isArray(got.alerts) ? got.alerts : [];\nconst accepted = []; const rejected = {};\nconst drop = r => { rejected[r] = (rejected[r] ?? 0) + 1; };\nif (!tool) drop('no kuma_alerts call in this run');\nfor (const a of (tool ? list : [])) {\n const names = Array.isArray(a?.monitors) ? a.monitors.map(String) : [];\n if (!KINDS.includes(a?.kind)) { drop('unknown kind'); continue; }\n if (!names.length || names.some(n => !mons.has(n))) { drop('monitor not in the tool result'); continue; }\n if (a.kind === 'recovered' && names.some(n => mons.get(n).now.status === 'DOWN')) { drop('recovered, but DOWN now'); continue; }\n accepted.push({ monitors: names, kind: a.kind });\n}\nreturn [{ json: { model: meta.model, label: meta.label, deliver: meta.deliver, tool_called: !!tool, raw, accepted, rejected } }];\n"
}
},
{
"name": "Build message",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
1240,
0
],
"parameters": {
"jsCode": "\n// Code writes the message, never the model: kind and monitor names only. The\n// model's summary text is not sent, it is a channel for whatever the monitored\n// pages said.\nconst j = $json;\nconst L = { still_down: 'STILL DOWN', outage: 'DOWN', recovered: 'was down, up again', flapping: 'FLAPPING', cert_expiring: 'cert expiring' };\nconst lines = j.accepted.map(a => L[a.kind] + ': ' + a.monitors.map(n => n.slice(0, 60)).join(', '));\nconst dropped = Object.entries(j.rejected).map(([k, v]) => v + 'x ' + k).join('; ');\nreturn [{ json: { ...j, title: j.accepted.length + ' alert(s) from the triage agent (' + j.model + ')',\n message: (lines.join('\\n') || 'nothing needs a human') + (dropped ? '\\n(dropped: ' + dropped + ')' : '') } }];\n"
}
},
{
"name": "Deliver?",
"type": "n8n-nodes-base.if",
"typeVersion": 2,
"position": [
1460,
0
],
"parameters": {
"conditions": {
"options": {
"caseSensitive": true,
"leftValue": "",
"typeValidation": "loose",
"version": 2
},
"conditions": [
{
"id": "c1",
"leftValue": "={{ $json.deliver }}",
"rightValue": "",
"operator": {
"type": "boolean",
"operation": "true",
"singleValue": true
}
}
],
"combinator": "and"
},
"options": {}
}
},
{
"name": "Push to lab16",
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.2,
"position": [
1680,
-100
],
"credentials": {
"httpBasicAuth": {
"id": "",
"name": "ntfy (n8n user)"
}
},
"parameters": {
"method": "POST",
"url": "http://ntfy/lab16",
"authentication": "genericCredentialType",
"genericAuthType": "httpBasicAuth",
"sendHeaders": true,
"headerParameters": {
"parameters": [
{
"name": "X-Title",
"value": "={{ $json.title }}"
},
{
"name": "X-Priority",
"value": "4"
}
]
},
"sendBody": true,
"contentType": "raw",
"rawContentType": "text/plain",
"body": "={{ $json.message }}",
"options": {}
}
},
{
"name": "Not delivered (measurement run)",
"type": "n8n-nodes-base.noOp",
"typeVersion": 1,
"position": [
1680,
100
],
"parameters": {}
}
],
"connections": {
"Start": {
"main": [
[
{
"node": "Load skill",
"type": "main",
"index": 0
}
]
]
},
"Load skill": {
"main": [
[
{
"node": "Build prompts",
"type": "main",
"index": 0
}
]
]
},
"Build prompts": {
"main": [
[
{
"node": "Alert triage agent",
"type": "main",
"index": 0
}
]
]
},
"Alert triage agent": {
"main": [
[
{
"node": "Check alerts",
"type": "main",
"index": 0
}
]
]
},
"Check alerts": {
"main": [
[
{
"node": "Build message",
"type": "main",
"index": 0
}
]
]
},
"Build message": {
"main": [
[
{
"node": "Deliver?",
"type": "main",
"index": 0
}
]
]
},
"Deliver?": {
"main": [
[
{
"node": "Push to lab16",
"type": "main",
"index": 0
}
],
[
{
"node": "Not delivered (measurement run)",
"type": "main",
"index": 0
}
]
]
},
"Ollama over the mesh": {
"ai_languageModel": [
[
{
"node": "Alert triage agent",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"kuma_alerts": {
"ai_tool": [
[
{
"node": "Alert triage agent",
"type": "ai_tool",
"index": 0
}
]
]
}
},
"settings": {
"executionOrder": "v1",
"executionTimeout": 600
},
"active": false
}