Episode 16: Kuma alerts sorted by rules, a local agent and a script
ops-readonly gets kuma_alerts (host export kuma-export.py, one more read-only mount). kuma-triage.py sorts the same data without a model. Flow H runs the agent with skill alert-triage.md. Flow B: one open incident per monitor, an UP closes all of them, titles name start and end and stay under Outline's limit. B and D accept Kuma only with a header token. Monitoring notes: ntfy rejects Kuma messages for monitors without a URL, resend for long outages.
This commit is contained in:
Maximilian Grimm
parent
54ab43dac7
commit
1655d04d7d
10 changed files with 705 additions and 67 deletions
354
automation/h-kuma-alert-triage.json
Normal file
354
automation/h-kuma-alert-triage.json
Normal file
|
|
@ -0,0 +1,354 @@
|
|||
{
|
||||
"name": "H - Kuma alert triage",
|
||||
"nodes": [
|
||||
{
|
||||
"name": "Start",
|
||||
"type": "n8n-nodes-base.webhook",
|
||||
"typeVersion": 2,
|
||||
"position": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"webhookId": "REPLACE_ON_IMPORT",
|
||||
"credentials": {
|
||||
"httpHeaderAuth": {
|
||||
"id": "",
|
||||
"name": "Kuma triage trigger"
|
||||
}
|
||||
},
|
||||
"parameters": {
|
||||
"httpMethod": "POST",
|
||||
"path": "kuma-alert-triage",
|
||||
"authentication": "headerAuth",
|
||||
"responseMode": "onReceived",
|
||||
"options": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Load skill",
|
||||
"type": "n8n-nodes-base.httpRequest",
|
||||
"typeVersion": 4.2,
|
||||
"position": [
|
||||
220,
|
||||
0
|
||||
],
|
||||
"credentials": {
|
||||
"httpHeaderAuth": {
|
||||
"id": "",
|
||||
"name": "Forgejo API"
|
||||
}
|
||||
},
|
||||
"parameters": {
|
||||
"method": "GET",
|
||||
"url": "http://forgejo:3000/api/v1/repos/opsadmin/selfhostedops-stack/raw/agents/skills/alert-triage.md",
|
||||
"authentication": "genericCredentialType",
|
||||
"genericAuthType": "httpHeaderAuth",
|
||||
"options": {
|
||||
"response": {
|
||||
"response": {
|
||||
"responseFormat": "text"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Build prompts",
|
||||
"type": "n8n-nodes-base.code",
|
||||
"typeVersion": 2,
|
||||
"position": [
|
||||
440,
|
||||
0
|
||||
],
|
||||
"parameters": {
|
||||
"jsCode": "\n// The skill file is markdown with a small frontmatter block for humans; the\n// model gets everything below it. Model tags come from a fixed list, never\n// from the caller.\nfunction body(md) { return String(md ?? '').replace(/^---[\\s\\S]*?\\n---\\s*/, '').trim(); }\nconst b = $('Start').first().json.body ?? {};\nconst MODELS = ['qwen2.5-advisor:3b', 'qwen3-advisor:4b'];\nconst model = MODELS.includes(b.model) ? b.model : MODELS[0];\n// qwen3 thinks before it answers: at 3000 tokens it never reached the JSON (lab16 runs #380, #402, #410)\nconst numPredict = model.startsWith('qwen3') ? 8000 : 3000;\nreturn [{ json: {\n system: body($json.data),\n task: 'Call kuma_alerts once, then answer with the JSON object described in your instructions.',\n model, numPredict, label: String(b.label ?? '').slice(0, 40), deliver: b.deliver === true,\n} }];\n"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Alert triage agent",
|
||||
"type": "@n8n/n8n-nodes-langchain.agent",
|
||||
"typeVersion": 2.2,
|
||||
"position": [
|
||||
680,
|
||||
0
|
||||
],
|
||||
"parameters": {
|
||||
"promptType": "define",
|
||||
"text": "={{ $json.task }}",
|
||||
"hasOutputParser": false,
|
||||
"options": {
|
||||
"systemMessage": "={{ $json.system }}",
|
||||
"maxIterations": 4,
|
||||
"returnIntermediateSteps": true,
|
||||
"enableStreaming": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Ollama over the mesh",
|
||||
"type": "@n8n/n8n-nodes-langchain.lmChatOllama",
|
||||
"typeVersion": 1,
|
||||
"position": [
|
||||
600,
|
||||
240
|
||||
],
|
||||
"credentials": {
|
||||
"ollamaApi": {
|
||||
"id": "",
|
||||
"name": "Ollama (home lab over mesh)"
|
||||
}
|
||||
},
|
||||
"parameters": {
|
||||
"model": "={{ $('Build prompts').first().json.model }}",
|
||||
"options": {
|
||||
"temperature": 0,
|
||||
"numCtx": 12288,
|
||||
"numPredict": "={{ $('Build prompts').first().json.numPredict }}",
|
||||
"keepAlive": "30m",
|
||||
"think": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "kuma_alerts",
|
||||
"type": "@n8n/n8n-nodes-langchain.toolHttpRequest",
|
||||
"typeVersion": 1.1,
|
||||
"position": [
|
||||
800,
|
||||
240
|
||||
],
|
||||
"credentials": {
|
||||
"httpHeaderAuth": {
|
||||
"id": "",
|
||||
"name": "ops-readonly token"
|
||||
}
|
||||
},
|
||||
"parameters": {
|
||||
"toolDescription": "Every Uptime Kuma monitor for the current window: state now and since when, each status change in the window, heartbeat counts, TLS days left. Read only, no arguments. Call it once, first.",
|
||||
"method": "GET",
|
||||
"url": "http://ops-readonly:8098/read/kuma_alerts",
|
||||
"authentication": "genericCredentialType",
|
||||
"genericAuthType": "httpHeaderAuth"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Check alerts",
|
||||
"type": "n8n-nodes-base.code",
|
||||
"typeVersion": 2,
|
||||
"position": [
|
||||
1020,
|
||||
0
|
||||
],
|
||||
"parameters": {
|
||||
"jsCode": "\nfunction readObject(s) {\n if (s && typeof s === 'object') return s;\n const t = String(s ?? '');\n const a = t.indexOf('{'); const b = t.lastIndexOf('}');\n if (a === -1 || b <= a) return {};\n for (let end = b; end > a; end = t.lastIndexOf('}', end - 1)) {\n try { return JSON.parse(t.slice(a, end + 1)); } catch (e) { /* keep trying */ }\n }\n return {};\n}\n// Deterministic. Every alert is checked against what kuma_alerts returned in\n// THIS run. It filters, it never repairs: a missing alert stays missing.\nconst meta = $('Build prompts').first().json;\nconst raw = String($json.output ?? '');\nlet tool = null;\nfor (const s of ($json.intermediateSteps ?? [])) {\n if (s?.action?.tool === 'kuma_alerts') { try { tool = JSON.parse(s.observation); } catch (e) { /* not json */ } }\n}\nconst mons = new Map((tool?.monitors ?? []).map(m => [m.name, m]));\nconst KINDS = ['still_down', 'outage', 'recovered', 'flapping', 'cert_expiring'];\nconst got = readObject(raw);\nconst list = Array.isArray(got.alerts) ? got.alerts : [];\nconst accepted = []; const rejected = {};\nconst drop = r => { rejected[r] = (rejected[r] ?? 0) + 1; };\nif (!tool) drop('no kuma_alerts call in this run');\nfor (const a of (tool ? list : [])) {\n const names = Array.isArray(a?.monitors) ? a.monitors.map(String) : [];\n if (!KINDS.includes(a?.kind)) { drop('unknown kind'); continue; }\n if (!names.length || names.some(n => !mons.has(n))) { drop('monitor not in the tool result'); continue; }\n if (a.kind === 'recovered' && names.some(n => mons.get(n).now.status === 'DOWN')) { drop('recovered, but DOWN now'); continue; }\n accepted.push({ monitors: names, kind: a.kind });\n}\nreturn [{ json: { model: meta.model, label: meta.label, deliver: meta.deliver, tool_called: !!tool, raw, accepted, rejected } }];\n"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Build message",
|
||||
"type": "n8n-nodes-base.code",
|
||||
"typeVersion": 2,
|
||||
"position": [
|
||||
1240,
|
||||
0
|
||||
],
|
||||
"parameters": {
|
||||
"jsCode": "\n// Code writes the message, never the model: kind and monitor names only. The\n// model's summary text is not sent, it is a channel for whatever the monitored\n// pages said.\nconst j = $json;\nconst L = { still_down: 'STILL DOWN', outage: 'DOWN', recovered: 'was down, up again', flapping: 'FLAPPING', cert_expiring: 'cert expiring' };\nconst lines = j.accepted.map(a => L[a.kind] + ': ' + a.monitors.map(n => n.slice(0, 60)).join(', '));\nconst dropped = Object.entries(j.rejected).map(([k, v]) => v + 'x ' + k).join('; ');\nreturn [{ json: { ...j, title: j.accepted.length + ' alert(s) from the triage agent (' + j.model + ')',\n message: (lines.join('\\n') || 'nothing needs a human') + (dropped ? '\\n(dropped: ' + dropped + ')' : '') } }];\n"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Deliver?",
|
||||
"type": "n8n-nodes-base.if",
|
||||
"typeVersion": 2,
|
||||
"position": [
|
||||
1460,
|
||||
0
|
||||
],
|
||||
"parameters": {
|
||||
"conditions": {
|
||||
"options": {
|
||||
"caseSensitive": true,
|
||||
"leftValue": "",
|
||||
"typeValidation": "loose",
|
||||
"version": 2
|
||||
},
|
||||
"conditions": [
|
||||
{
|
||||
"id": "c1",
|
||||
"leftValue": "={{ $json.deliver }}",
|
||||
"rightValue": "",
|
||||
"operator": {
|
||||
"type": "boolean",
|
||||
"operation": "true",
|
||||
"singleValue": true
|
||||
}
|
||||
}
|
||||
],
|
||||
"combinator": "and"
|
||||
},
|
||||
"options": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Push to lab16",
|
||||
"type": "n8n-nodes-base.httpRequest",
|
||||
"typeVersion": 4.2,
|
||||
"position": [
|
||||
1680,
|
||||
-100
|
||||
],
|
||||
"credentials": {
|
||||
"httpBasicAuth": {
|
||||
"id": "",
|
||||
"name": "ntfy (n8n user)"
|
||||
}
|
||||
},
|
||||
"parameters": {
|
||||
"method": "POST",
|
||||
"url": "http://ntfy/lab16",
|
||||
"authentication": "genericCredentialType",
|
||||
"genericAuthType": "httpBasicAuth",
|
||||
"sendHeaders": true,
|
||||
"headerParameters": {
|
||||
"parameters": [
|
||||
{
|
||||
"name": "X-Title",
|
||||
"value": "={{ $json.title }}"
|
||||
},
|
||||
{
|
||||
"name": "X-Priority",
|
||||
"value": "4"
|
||||
}
|
||||
]
|
||||
},
|
||||
"sendBody": true,
|
||||
"contentType": "raw",
|
||||
"rawContentType": "text/plain",
|
||||
"body": "={{ $json.message }}",
|
||||
"options": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Not delivered (measurement run)",
|
||||
"type": "n8n-nodes-base.noOp",
|
||||
"typeVersion": 1,
|
||||
"position": [
|
||||
1680,
|
||||
100
|
||||
],
|
||||
"parameters": {}
|
||||
}
|
||||
],
|
||||
"connections": {
|
||||
"Start": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Load skill",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Load skill": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Build prompts",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Build prompts": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Alert triage agent",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Alert triage agent": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Check alerts",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Check alerts": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Build message",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Build message": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Deliver?",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Deliver?": {
|
||||
"main": [
|
||||
[
|
||||
{
|
||||
"node": "Push to lab16",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"node": "Not delivered (measurement run)",
|
||||
"type": "main",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"Ollama over the mesh": {
|
||||
"ai_languageModel": [
|
||||
[
|
||||
{
|
||||
"node": "Alert triage agent",
|
||||
"type": "ai_languageModel",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
},
|
||||
"kuma_alerts": {
|
||||
"ai_tool": [
|
||||
[
|
||||
{
|
||||
"node": "Alert triage agent",
|
||||
"type": "ai_tool",
|
||||
"index": 0
|
||||
}
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
"settings": {
|
||||
"executionOrder": "v1",
|
||||
"executionTimeout": 600
|
||||
},
|
||||
"active": false
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue