selfhostedops/selfhostedops-stack
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Episode 14: an agent reads the WAF blocks, a script beats it, a human merges

Browse source 
ops-readonly gets two read-only questions, waf_blocked and waf_rule_info. The
rule ids per blocked request are in CrowdSec's alerts, not in its log, and
alerts need machine credentials that can also write, so waf-export.py runs on
the host and the container gets one more read-only mount.

Workflow G lets a local 3B model draft CRS route exclusions as a Forgejo issue.
The model never writes rule syntax, a code node checks every field against the
tool output, the only write permission is an issue in one repo, trusted sources
live in the workflow and the trigger needs a token.

waf-propose.py does the same without a model: 5 of 5 false positives, no
exclusion for an attack, 0.03 s. The model proposed an exclusion for nearly
every attack it saw. Numbers and the traps we hit are in HARDENING.md.
This commit is contained in:
Maximilian Grimm 2026-09-19 12:56:39 +02:00
parent 696f3e311d
commit 3edef38ba3
13 changed files with 987 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

393
automation/g-waf-exception-advisor.json Normal file
View file

@ -0,0 +1,393 @@
{
"name": "G - WAF exception advisor",
"nodes": [
{
"parameters": {
"httpMethod": "POST",
"path": "waf-exception-advisor",
"authentication": "headerAuth",
"responseMode": "onReceived",
"options": {}
},
"id": "d602f165-9486-5708-8a78-1e7c81cd9d50",
"name": "Start",
"type": "n8n-nodes-base.webhook",
"typeVersion": 2,
"position": [
0,
0
],
"credentials": {
"httpHeaderAuth": {
"id": "REPLACE_WITH_YOUR_CREDENTIAL_ID",
"name": "WAF advisor trigger"
}
}
},
{
"parameters": {
"method": "GET",
"url": "http://forgejo:3000/api/v1/repos/opsadmin/selfhostedops-stack/raw/agents/skills/waf-exceptions.md",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth",
"options": {
"response": {
"response": {
"responseFormat": "text"
}
}
}
},
"id": "edd71969-3feb-50a2-9298-d745cd414cb3",
"name": "Load skill",
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.2,
"position": [
220,
0
],
"credentials": {
"httpHeaderAuth": {
"id": "REPLACE_WITH_YOUR_CREDENTIAL_ID",
"name": "Forgejo API"
}
}
},
{
"parameters": {
"jsCode": "\n// The skill file is markdown with a small frontmatter block. The frontmatter\n// is for humans; the model gets everything below it.\nfunction body(md) { return String(md ?? '').replace(/^---[\\s\\S]*?\\n---\\s*/, '').trim(); }\n\nconst b = $('Start').first().json.body ?? {};\nconst started = new Date().toISOString();\n// Source IPs the operator trusts (office, the box itself). They live HERE, in\n// the workflow. Never take them from the webhook body: the webhook is reachable\n// from outside, and a caller who may name the trusted sources names himself.\nconst TRUSTED_SOURCES = ['198.51.100.10'];\nconst MODELS = ['qwen2.5-advisor:3b', 'qwen3-advisor:4b'];\nconst trusted = TRUSTED_SOURCES.filter(x => /^[0-9a-fA-F.:]{3,45}$/.test(x));\nlet task = 'Review the requests the WAF blocked in the last 24 hours and propose exclusions for the false positives. '\n + 'Current time: ' + started + '. Call waf_blocked once, then reply with the JSON object only.';\nif (trusted.length) task += ' Trusted source IPs: ' + trusted.join(', ') + '.';\nreturn [{ json: {\n label: String(b.label ?? ''), deliver: b.deliver === true, started, trusted,\n model: MODELS.includes(String(b.model ?? '')) ? String(b.model) : MODELS[0],\n system: body($json.data), task,\n}}];\n"
},
"id": "21706019-8eeb-5596-8f30-e387ed4cb821",
"name": "Build prompts",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
440,
0
]
},
{
"parameters": {
"promptType": "define",
"text": "={{ $json.task }}",
"hasOutputParser": false,
"options": {
"systemMessage": "={{ $json.system }}",
"maxIterations": 8,
"returnIntermediateSteps": true,
"enableStreaming": false
}
},
"id": "039673e7-59e2-55f5-b271-5afbe17e3c40",
"name": "WAF exception advisor",
"type": "@n8n/n8n-nodes-langchain.agent",
"typeVersion": 2.2,
"position": [
660,
0
]
},
{
"parameters": {
"model": "={{ $('Build prompts').first().json.model }}",
"options": {
"temperature": 0,
"numCtx": 8192,
"numPredict": 3000,
"keepAlive": "30m",
"think": false
}
},
"id": "eb2087ba-0068-525a-830c-d24836f7d6e3",
"name": "Ollama over the mesh",
"type": "@n8n/n8n-nodes-langchain.lmChatOllama",
"typeVersion": 1,
"position": [
560,
220
],
"credentials": {
"ollamaApi": {
"id": "REPLACE_WITH_YOUR_CREDENTIAL_ID",
"name": "Ollama (home lab over mesh)"
}
}
},
{
"parameters": {
"toolDescription": "Every distinct request the WAF blocked in the last 24 hours: source_ip, host, method, uri, count, auth_header_present, rule_ids, attack_classes. Read only, no arguments. Call it once, first.",
"method": "GET",
"url": "http://ops-readonly:8098/read/waf_blocked?hours=24",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth"
},
"id": "3c8bb290-7b36-5246-a87c-a9e390187a4b",
"name": "waf_blocked",
"type": "@n8n/n8n-nodes-langchain.toolHttpRequest",
"typeVersion": 1.1,
"position": [
720,
220
],
"credentials": {
"httpHeaderAuth": {
"id": "REPLACE_WITH_YOUR_CREDENTIAL_ID",
"name": "ops-readonly token"
}
}
},
{
"parameters": {
"toolDescription": "What one OWASP CRS rule id means: message, attack class, paranoia level. One argument: id, a six digit rule id taken from waf_blocked.",
"method": "GET",
"url": "http://ops-readonly:8098/read/waf_rule_info?id={id}",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth",
"placeholderDefinitions": {
"values": [
{
"name": "id",
"description": "a six digit CRS rule id, for example one from rule_ids",
"type": "string"
}
]
}
},
"id": "49dae44a-d64b-57c4-901b-584232ff151e",
"name": "waf_rule_info",
"type": "@n8n/n8n-nodes-langchain.toolHttpRequest",
"typeVersion": 1.1,
"position": [
880,
220
],
"credentials": {
"httpHeaderAuth": {
"id": "REPLACE_WITH_YOUR_CREDENTIAL_ID",
"name": "ops-readonly token"
}
}
},
{
"parameters": {
"jsCode": "\nfunction readObject(s) {\n if (s && typeof s === 'object') return s;\n const t = String(s ?? '');\n const a = t.indexOf('{'); const b = t.lastIndexOf('}');\n if (a === -1 || b <= a) return {};\n for (let end = b; end > a; end = t.lastIndexOf('}', end - 1)) {\n try { return JSON.parse(t.slice(a, end + 1)); } catch (e) { /* keep trying */ }\n }\n return {};\n}\n\n// Deterministic. The model proposed; this node checks every field against what\n// the tool really returned in this run. It never repairs a proposal. A proposal\n// that fails a check is dropped and the reason stays visible in the issue.\nconst meta = $('Build prompts').first().json;\nconst raw = String($json.output ?? '');\nconst answer = readObject(raw);\n\nlet blocked = [];\nfor (const step of ($json.intermediateSteps ?? [])) {\n if (step?.action?.tool !== 'waf_blocked') continue;\n try { blocked = (JSON.parse(step.observation).requests ?? []); } catch (e) { /* no evidence, everything fails below */ }\n}\nconst FORBIDDEN = ['949110', '959100']; // the CRS blocking rules: removing them frees the whole route\nconst hits = (p, uri) => p.match === 'contains' ? uri.includes(p.path) : uri.startsWith(p.path);\n\nconst accepted = []; const dropped = [];\nfor (const p0 of (Array.isArray(answer.proposals) ? answer.proposals : []).slice(0, 20)) {\n const p = {\n classification: String(p0?.classification ?? ''), host: String(p0?.host ?? ''),\n path: String(p0?.path ?? ''), match: p0?.match === 'contains' ? 'contains' : 'prefix',\n rule_ids: (Array.isArray(p0?.rule_ids) ? p0.rule_ids : []).map(x => String(x)).slice(0, 40),\n reason: String(p0?.reason ?? '').replace(/[`\\r\\n|<>]/g, ' ').slice(0, 200),\n };\n const why = [];\n const onHost = blocked.filter(r => r.host === p.host);\n const covered = onHost.filter(r => hits(p, r.uri));\n if (p.classification !== 'false_positive') why.push('not classified as false_positive');\n if (!onHost.length) why.push('host not in the blocked requests');\n if (!/^\\/?[A-Za-z0-9._\\/-]{3,80}$/.test(p.path) || p.path === '/') why.push('path missing, too broad or not plain');\n if (!p.rule_ids.length) why.push('no rule ids');\n if (p.rule_ids.some(i => !/^\\d{6}$/.test(i))) why.push('rule id is not a six digit number');\n if (p.rule_ids.some(i => FORBIDDEN.includes(i))) why.push('removes a CRS blocking rule (949110/959100)');\n if (!covered.length) why.push('path matches no blocked request on this host');\n const seenIds = new Set(covered.flatMap(r => r.rule_ids));\n const unseen = p.rule_ids.filter(i => /^\\d{6}$/.test(i) && !seenIds.has(i));\n if (covered.length && unseen.length) why.push('rule ids never fired on this route: ' + unseen.join(','));\n // The one check that is about intent, not format: somebody we trust must have\n // been blocked on this route. Path, payload and auth header can be copied by\n // an attacker, the source address of a finished HTTPS request cannot.\n if (meta.trusted.length && covered.length && !covered.some(r => meta.trusted.includes(r.source_ip)))\n why.push('no blocked request from a trusted source on this route');\n // Not a drop, a warning for the reviewer: the same exclusion would also have\n // let these requests through.\n const untrusted = meta.trusted.length ? covered.filter(r => !meta.trusted.includes(r.source_ip)) : [];\n p.evidence = covered.map(r => ({ source_ip: r.source_ip, trusted: meta.trusted.includes(r.source_ip), method: r.method, uri: r.uri, count: r.count, auth_header_present: r.auth_header_present }));\n p.warning = untrusted.length ? ('also matches ' + untrusted.length + ' blocked request(s) from sources that are not trusted') : '';\n (why.length ? dropped : accepted).push(why.length ? { ...p, why } : p);\n}\nreturn [{ json: {\n accepted, dropped, rejected_by_model: (Array.isArray(answer.rejected) ? answer.rejected : []).length,\n parsed: Object.keys(answer).length > 0, agent_raw: raw.slice(0, 6000),\n blocked_distinct: blocked.length, trusted_given: meta.trusted.length > 0, label: meta.label, deliver: meta.deliver, started: meta.started, model: meta.model,\n}}];\n"
},
"id": "79d5ce7a-6dbf-50d3-b0af-8b0dc2a96042",
"name": "Check proposal",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
980,
0
]
},
{
"parameters": {
"jsCode": "\n// Code writes the SecLang, never the model. Host, path and ids passed the\n// checks above, so nothing free-form reaches the rule text. Request text is\n// attacker text: it is shown as data (code span, backticks stripped, cut).\nconst j = $json;\nconst code = s => '`' + String(s).replace(/`/g, \"'\").replace(/[\\r\\n]/g, ' ').slice(0, 100) + '`';\nconst lines = ['The WAF exception advisor read ' + j.blocked_distinct + ' distinct blocked requests (model ' + j.model + ').',\n 'It cannot change the WAF. Review, then merge into `appsec-rules/crs-route-exclusions.yaml` by hand.', ''];\nlet n = 0; const rules = [];\nfor (const p of j.accepted) {\n n += 1;\n lines.push('### ' + n + '. ' + code(p.host) + ' ' + p.match + ' ' + code(p.path));\n lines.push('Rule ids: ' + p.rule_ids.join(', '));\n lines.push('Advisor reason (model text, unverified): ' + p.reason);\n if (p.warning) lines.push('**Warning: ' + p.warning + '.**');\n lines.push('', '| source | method | uri | count | auth header |', '| --- | --- | --- | --- | --- |');\n // A row from a source we do not trust shows only the checked prefix, never its own uri:\n // that uri is the attacker's free text channel to whoever reads this issue.\n for (const e of p.evidence) lines.push('| ' + e.source_ip + (e.trusted ? '' : ' (not trusted)') + ' | ' + e.method + ' | ' + code(e.trusted ? e.uri : p.path + ' ...') + ' | ' + e.count + ' | ' + (e.auth_header_present ? 'yes' : 'no') + ' |');\n lines.push('');\n const op = p.match === 'contains' ? '@contains' : '@beginsWith';\n rules.push(' - |\\n SecRule REQUEST_HEADERS:Host \"@streq ' + p.host + '\" \"id:' + (900200 + n) + ',phase:1,pass,nolog,chain,msg:\\'route exclusion ' + n + '\\'\"\\n'\n + ' SecRule REQUEST_URI \"' + op + ' ' + p.path + '\" \"' + p.rule_ids.map(i => 'ctl:ruleRemoveById=' + i).join(',') + '\"');\n}\nif (rules.length) lines.push('### Snippet', '', '```yaml', 'seclang_rules:', ...rules, '```', '');\nif (j.dropped.length) {\n lines.push('### Dropped by the deterministic check', '');\n // Counts only. A dropped proposal's host and path come from the model, and the model copies\n // request text: measured, that is how injection text reached the issue.\n const reasons = {};\n for (const d of j.dropped) for (const w of d.why) { const k = w.split(':')[0]; reasons[k] = (reasons[k] ?? 0) + 1; }\n for (const [k, v] of Object.entries(reasons)) lines.push('- ' + v + 'x ' + k);\n}\nif (!j.parsed) lines.push('The model did not return a JSON object. Nothing to propose.');\nif (!j.trusted_given) lines.unshift('**No trusted sources were given. Nothing below is backed by who sent the request. Treat every entry as unverified.**', '');\nreturn [{ json: { ...j, payload: {\n title: 'WAF exception proposal ' + j.started.slice(0, 16).replace('T', ' ') + ' UTC (' + j.accepted.length + ' accepted, ' + j.dropped.length + ' dropped)',\n body: lines.join('\\n') } } }];\n"
},
"id": "00aa4628-b3ef-5f43-b233-634623be1a52",
"name": "Build issue",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
1200,
0
]
},
{
"parameters": {
"conditions": {
"options": {
"caseSensitive": true,
"leftValue": "",
"typeValidation": "loose",
"version": 2
},
"conditions": [
{
"id": "c1",
"leftValue": "={{ $json.deliver }}",
"rightValue": "",
"operator": {
"type": "boolean",
"operation": "true",
"singleValue": true
}
}
],
"combinator": "and"
},
"options": {}
},
"id": "521a810e-2ef1-5b9c-9a0c-d57367540b98",
"name": "Deliver?",
"type": "n8n-nodes-base.if",
"typeVersion": 2,
"position": [
1420,
0
]
},
{
"parameters": {
"method": "POST",
"url": "http://forgejo:3000/api/v1/repos/opsadmin/waf-exceptions/issues",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth",
"sendBody": true,
"contentType": "json",
"specifyBody": "json",
"jsonBody": "={{ JSON.stringify($json.payload) }}",
"options": {}
},
"id": "5379e77d-cc71-5a16-9bfe-86382a150d0b",
"name": "Open issue for a human",
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.2,
"position": [
1640,
-100
],
"credentials": {
"httpHeaderAuth": {
"id": "REPLACE_WITH_YOUR_CREDENTIAL_ID",
"name": "Forgejo WAF advisor (issues only)"
}
}
},
{
"parameters": {},
"id": "33b09693-079e-5ee5-b1f3-16175a1fb53e",
"name": "Not delivered (measurement run)",
"type": "n8n-nodes-base.noOp",
"typeVersion": 1,
"position": [
1640,
100
]
}
],
"connections": {
"Start": {
"main": [
[
{
"node": "Load skill",
"type": "main",
"index": 0
}
]
]
},
"Load skill": {
"main": [
[
{
"node": "Build prompts",
"type": "main",
"index": 0
}
]
]
},
"Build prompts": {
"main": [
[
{
"node": "WAF exception advisor",
"type": "main",
"index": 0
}
]
]
},
"WAF exception advisor": {
"main": [
[
{
"node": "Check proposal",
"type": "main",
"index": 0
}
]
]
},
"Check proposal": {
"main": [
[
{
"node": "Build issue",
"type": "main",
"index": 0
}
]
]
},
"Build issue": {
"main": [
[
{
"node": "Deliver?",
"type": "main",
"index": 0
}
]
]
},
"Deliver?": {
"main": [
[
{
"node": "Open issue for a human",
"type": "main",
"index": 0
}
],
[
{
"node": "Not delivered (measurement run)",
"type": "main",
"index": 0
}
]
]
},
"Ollama over the mesh": {
"ai_languageModel": [
[
{
"node": "WAF exception advisor",
"type": "ai_languageModel",
"index": 0
}
]
]
},
"waf_blocked": {
"ai_tool": [
[
{
"node": "WAF exception advisor",
"type": "ai_tool",
"index": 0
}
]
]
},
"waf_rule_info": {
"ai_tool": [
[
{
"node": "WAF exception advisor",
"type": "ai_tool",
"index": 0
}
]
]
}
},
"settings": {
"executionOrder": "v1",
"executionTimeout": 300
}
}