Episode 15: Pocket ID as identity provider, tinyauth gate, offboarding script

Pocket ID replaces Forgejo as the OIDC provider for Outline, Vaultwarden and
NetBird; Forgejo logs in through it. Uptime Kuma sits behind tinyauth via
Caddy forward_auth (group ops), the public status page stays public.

offboarding/offboard.py switches one person off in every service with an API
and prints what a human still has to do: the shared passwords they could read.
Measured: SSO lock alone left 10 of 13 ways open for 45 minutes; the script
leaves 1 of 14, the vault copy on their laptop.

HARDENING.md: the ufw hairpin that broke Vaultwarden SSO and Kuma, the mesh
route into the docker bridges, and a fail2ban exception for Pocket ID.
This commit is contained in:
Maximilian Grimm 2026-09-23 11:52:17 +02:00
parent 3edef38ba3
commit 12542ecb19
9 changed files with 490 additions and 21 deletions

View file

@ -14,8 +14,8 @@ TZ=Europe/Berlin
OUTLINE_PG_PASSWORD=CHANGE_ME_openssl_rand_hex_32 OUTLINE_PG_PASSWORD=CHANGE_ME_openssl_rand_hex_32
OUTLINE_SECRET_KEY=CHANGE_ME_openssl_rand_hex_32 OUTLINE_SECRET_KEY=CHANGE_ME_openssl_rand_hex_32
OUTLINE_UTILS_SECRET=CHANGE_ME_openssl_rand_hex_32 OUTLINE_UTILS_SECRET=CHANGE_ME_openssl_rand_hex_32
# From Forgejo: Site Administration -> Applications -> OAuth2 # From Pocket ID (episode 15): Administration -> OIDC Clients, allowed group staff
# Redirect URI: https://wiki.<DOMAIN>/auth/oidc.callback # Callback URL: https://wiki.<DOMAIN>/auth/oidc.callback
OUTLINE_OIDC_CLIENT_ID= OUTLINE_OIDC_CLIENT_ID=
OUTLINE_OIDC_CLIENT_SECRET= OUTLINE_OIDC_CLIENT_SECRET=
@ -24,9 +24,9 @@ OUTLINE_OIDC_CLIENT_SECRET=
VAULTWARDEN_ADMIN_TOKEN=CHANGE_ME_openssl_rand_hex_32 VAULTWARDEN_ADMIN_TOKEN=CHANGE_ME_openssl_rand_hex_32
# Flip to false once your team has signed up: # Flip to false once your team has signed up:
VAULTWARDEN_SIGNUPS_ALLOWED=true VAULTWARDEN_SIGNUPS_ALLOWED=true
# Single sign-on against your own Forgejo (episode 4). # Single sign-on (episode 4 with Forgejo, since episode 15 Pocket ID).
# From Forgejo: Site Administration -> Applications -> OAuth2 Application # From Pocket ID: Administration -> OIDC Clients, allowed group staff
# Redirect URI must be EXACTLY: https://vault.<DOMAIN>/identity/connect/oidc-signin # Callback URL must be EXACTLY: https://vault.<DOMAIN>/identity/connect/oidc-signin
VAULTWARDEN_SSO_CLIENT_ID= VAULTWARDEN_SSO_CLIENT_ID=
VAULTWARDEN_SSO_CLIENT_SECRET= VAULTWARDEN_SSO_CLIENT_SECRET=
@ -84,3 +84,13 @@ OPS_ACTIONS_TOKEN=
# cscli bouncers add caddy-bouncer -o raw # cscli bouncers add caddy-bouncer -o raw
# Without it Caddy still starts, but blocks nothing and logs auth errors. # Without it Caddy still starts, but blocks nothing and logs auth errors.
CROWDSEC_BOUNCER_KEY= CROWDSEC_BOUNCER_KEY=
# --- Episode 15: Pocket ID + tinyauth ---------------------------------------
# Encrypts Pocket ID's signing keys at rest: openssl rand -base64 32
# Then open https://id.<DOMAIN>/setup IMMEDIATELY: until the first admin
# exists, that page belongs to whoever finds it first (scanners did, in seconds).
POCKET_ID_ENCRYPTION_KEY=
# The OIDC client "Tinyauth" in Pocket ID
# (callback https://auth.<DOMAIN>/api/oauth/callback/pocketid, allowed group ops).
TINYAUTH_CLIENT_ID=
TINYAUTH_CLIENT_SECRET=

2
.gitignore vendored
View file

@ -6,3 +6,5 @@ backup/backup.log
.playwright-mcp/ .playwright-mcp/
logs/ logs/
ops-readonly/token.env ops-readonly/token.env
offboarding/offboard.env
__pycache__/

View file

@ -85,8 +85,26 @@ vault.{$DOMAIN} {
status.{$DOMAIN} { status.{$DOMAIN} {
import common import common
import crowdsec import crowdsec
# episode 15: the Kuma UI only behind the login gate (tinyauth, group ops).
# Status pages, badges and push heartbeats stay public.
@public path /status/* /api/status-page/* /api/badge/* /api/push/* /assets/* /upload/* /icon.svg /favicon.ico /manifest.json
handle @public {
reverse_proxy uptime-kuma:3001 reverse_proxy uptime-kuma:3001
} }
handle {
forward_auth tinyauth:3000 {
uri /api/auth/caddy
copy_headers Remote-User Remote-Email Remote-Groups
# tinyauth v5 answers 401 and names its login page in a header;
# without this the browser only ever sees {"message":"Unauthorized"}
@login status 401
handle_response @login {
redir {rp.header.X-Tinyauth-Location} 302
}
}
reverse_proxy uptime-kuma:3001
}
}
# Push notifications (episode 6). Websockets and long polling pass through as-is. # Push notifications (episode 6). Websockets and long polling pass through as-is.
ntfy.{$DOMAIN} { ntfy.{$DOMAIN} {
@ -125,3 +143,17 @@ vpn.{$DOMAIN} {
# Dashboard (everything else) # Dashboard (everything else)
reverse_proxy /* netbird-dashboard:80 reverse_proxy /* netbird-dashboard:80
} }
# Identity provider (episode 15). Everyone logs in here, with a passkey.
id.{$DOMAIN} {
import common
import crowdsec
reverse_proxy pocket-id:1411
}
# Login gate for services without their own SSO (episode 15), used by forward_auth.
auth.{$DOMAIN} {
import common
import crowdsec
reverse_proxy tinyauth:3000
}

View file

@ -558,3 +558,47 @@ live run the model delivered 4 of 5 routes in 30 s, the script 5 of 5 in
and `git push` failed, because it had exempted `git-upload-pack` only. The one and `git push` failed, because it had exempted `git-upload-pack` only. The one
thing a model could add, the explanation, read "known attack pattern for RCE" thing a model could add, the explanation, read "known attack pattern for RCE"
on a request it had just classified as a false positive. on a request it had just classified as a false positive.
## Episode 15: your own identity provider, and three things the firewall broke quietly
**Containers could not reach the box's own public name.** Since ufw went on in
episode 10, every container on the `proxy` network that called
`https://<service>.<DOMAIN>` hit `[UFW BLOCK] ... DST=<box-ip> DPT=443`. Vaultwarden
single sign-on died with "Failed to discover OpenID provider", and Uptime Kuma
listed git, wiki and vault as down for twelve days straight, about 800 down
heartbeats a day. Kuma only notifies on a status change, and the last one was on
the day ufw went on.
Every OIDC client has to reach the provider, so this is not optional:
```sh
ufw allow proto tcp from 172.18.0.0/16 to <box-ip> port 443 comment "hairpin: containers to OIDC provider"
```
**The mesh could not reach the containers either.** Docker's own `DOCKER` chain
drops traffic into a bridge unless a port is published, and NetBird's ACCEPT in
`FORWARD` comes after it. `trusted_host_interfaces` (episode 5) only fixes the
raw table. Put this into the `DOCKER-USER` block of `/etc/ufw/after.rules`,
before the eth0 lines:
```
-A DOCKER-USER -i wt0 -o br-+ -j ACCEPT
```
**fail2ban locked out people logging in.** The `caddy-probes` jail counts
401/403/404. Pocket ID's login page asks `/api/users/me` (401 without a session)
and the logo (404) on every load. Ten page loads in a minute and the employee is
banned for an hour. Narrow exception in `filter.d/caddy-probes.conf`:
```
ignoreregex = "uri":"/api/users/me"
"uri":"/api/application-images/
```
**Pocket ID's `/setup` belongs to whoever opens it first.** A scanner fetched
the new host within seconds of the certificate. Create the admin right after
`docker compose up -d pocket-id`, before anything else.
**The login gate keeps its own session.** tinyauth checks Pocket ID once and
then trusts its cookie for 24 hours (`TINYAUTH_AUTH_SESSIONEXPIRY`), whatever
happens to the account. `offboarding/offboard.py` deletes those sessions. By hand: delete the person's
rows from the `sessions` table in `tinyauth.db`, or shorten the expiry.

View file

@ -189,6 +189,20 @@ people. Note that joining is three steps, not two: you invite, they log in,
and then an owner has to **confirm** them. Until that confirmation, a member and then an owner has to **confirm** them. Until that confirmation, a member
sees no shared items at all. sees no shared items at all.
## One identity provider, and an offboarding that ends at zero (Episode 15)
Since episode 15 everyone logs in through **Pocket ID** (`id.<DOMAIN>`, passkeys,
groups): Forgejo, Outline, Vaultwarden and NetBird are its OIDC clients, and
Uptime Kuma sits behind **tinyauth** (`auth.<DOMAIN>`) via Caddy `forward_auth`.
Group `staff` gets the apps, group `ops` gets the monitoring UI; the public status
page stays public. Both together need about 50 MB of RAM.
Switching someone off in the provider is not the same as offboarding them:
[`offboarding/`](offboarding/) has the numbers and the script that goes through
every service in about a second, and ends with the list of passwords a human
still has to change. The three firewall traps this uncovered are in
[HARDENING.md](HARDENING.md#episode-15-your-own-identity-provider-and-three-things-the-firewall-broke-quietly).
## Alerts that reach you (Episode 6) ## Alerts that reach you (Episode 6)
A monitor nobody hears is a dashboard. Episode 6 adds a push channel you own A monitor nobody hears is a dashboard. Episode 6 adds a push channel you own

View file

@ -3,8 +3,10 @@
# #
# One server. One file. Your whole company stack: # One server. One file. Your whole company stack:
# caddy reverse proxy + automatic HTTPS # caddy reverse proxy + automatic HTTPS
# forgejo git hosting + CI (also acts as SSO/OIDC provider) # forgejo git hosting + CI (was the SSO/OIDC provider until episode 15)
# outline team wiki (logs in via Forgejo SSO) # outline team wiki (logs in via Pocket ID)
# pocket-id identity provider, passkeys + groups (episode 15)
# tinyauth login gate for services without SSO (episode 15)
# vaultwarden password manager (Bitwarden-compatible) # vaultwarden password manager (Bitwarden-compatible)
# uptime-kuma monitoring + alerts # uptime-kuma monitoring + alerts
# ntfy push notifications for those alerts (episode 6) # ntfy push notifications for those alerts (episode 6)
@ -51,6 +53,8 @@ volumes:
ntfy_data: ntfy_data:
ntfy_cache: ntfy_cache:
n8n_data: n8n_data:
pocket_id_data:
tinyauth_data:
wg_data: wg_data:
docker_certs: docker_certs:
dind_data: dind_data:
@ -96,6 +100,12 @@ services:
# the container name has to be listed. "external" keeps ordinary public # the container name has to be listed. "external" keeps ordinary public
# webhook targets working (episode 8). # webhook targets working (episode 8).
FORGEJO__webhook__ALLOWED_HOST_LIST: "external,n8n" FORGEJO__webhook__ALLOWED_HOST_LIST: "external,n8n"
# Episode 15: people log in with Pocket ID (auth source "PocketID", added
# once with: forgejo admin auth add-oauth ...). An SSO login is linked to
# the existing account with the same mail address instead of a duplicate.
FORGEJO__oauth2_client__ENABLE_AUTO_REGISTRATION: "true"
FORGEJO__oauth2_client__ACCOUNT_LINKING: auto
FORGEJO__oauth2_client__USERNAME: nickname
ports: ports:
- "2222:22" # git over SSH - "2222:22" # git over SSH
volumes: volumes:
@ -133,14 +143,14 @@ services:
FILE_STORAGE: local FILE_STORAGE: local
FILE_STORAGE_LOCAL_ROOT_DIR: /var/lib/outline/data FILE_STORAGE_LOCAL_ROOT_DIR: /var/lib/outline/data
FORCE_HTTPS: "false" # TLS terminates at caddy FORCE_HTTPS: "false" # TLS terminates at caddy
# --- SSO via your own Forgejo (set up the OAuth2 app first, see README) --- # --- SSO via Pocket ID (episode 15; Forgejo was the provider until then) ---
OIDC_CLIENT_ID: ${OUTLINE_OIDC_CLIENT_ID} OIDC_CLIENT_ID: ${OUTLINE_OIDC_CLIENT_ID}
OIDC_CLIENT_SECRET: ${OUTLINE_OIDC_CLIENT_SECRET} OIDC_CLIENT_SECRET: ${OUTLINE_OIDC_CLIENT_SECRET}
OIDC_AUTH_URI: https://git.${DOMAIN}/login/oauth/authorize OIDC_AUTH_URI: https://id.${DOMAIN}/authorize
OIDC_TOKEN_URI: https://git.${DOMAIN}/login/oauth/access_token OIDC_TOKEN_URI: https://id.${DOMAIN}/api/oidc/token
OIDC_USERINFO_URI: https://git.${DOMAIN}/login/oauth/userinfo OIDC_USERINFO_URI: https://id.${DOMAIN}/api/oidc/userinfo
OIDC_LOGOUT_URI: https://git.${DOMAIN}/logout OIDC_LOGOUT_URI: https://id.${DOMAIN}/api/oidc/end-session
OIDC_DISPLAY_NAME: Forgejo OIDC_DISPLAY_NAME: Pocket ID
OIDC_SCOPES: openid profile email OIDC_SCOPES: openid profile email
volumes: volumes:
- outline_data:/var/lib/outline/data - outline_data:/var/lib/outline/data
@ -153,20 +163,22 @@ services:
DOMAIN: https://vault.${DOMAIN} DOMAIN: https://vault.${DOMAIN}
ADMIN_TOKEN: ${VAULTWARDEN_ADMIN_TOKEN} ADMIN_TOKEN: ${VAULTWARDEN_ADMIN_TOKEN}
SIGNUPS_ALLOWED: "${VAULTWARDEN_SIGNUPS_ALLOWED:-true}" # set to false after your team joined! SIGNUPS_ALLOWED: "${VAULTWARDEN_SIGNUPS_ALLOWED:-true}" # set to false after your team joined!
# --- Single sign-on via your own Forgejo (episode 4) --- # --- Single sign-on (episode 4 with Forgejo, since episode 15 Pocket ID) ---
# Create the OAuth2 app in Forgejo FIRST: Site Administration -> # Create the OIDC client in Pocket ID FIRST. Its callback URL must be exactly:
# Applications -> OAuth2. Its redirect URI must be exactly:
# https://vault.${DOMAIN}/identity/connect/oidc-signin # https://vault.${DOMAIN}/identity/connect/oidc-signin
# That path is not documented in the UI. Getting it wrong fails only # That path is not documented in the UI. Getting it wrong fails only
# AFTER the Forgejo login, which is why it costs people twenty minutes. # AFTER the provider login, which is why it costs people twenty minutes.
SSO_ENABLED: "true" SSO_ENABLED: "true"
# Just the base URL. Vaultwarden reads Forgejo's discovery document # Just the base URL. Vaultwarden reads the discovery document
# (/.well-known/openid-configuration) from here on the first login. # (/.well-known/openid-configuration) from here on the first login.
SSO_AUTHORITY: https://git.${DOMAIN} # SSO_AUTH_ONLY_NOT_SESSION stays false (default): every token refresh
# asks the provider again, so a person disabled there is out at the next
# refresh. Episode 15 measured why that alone was not enough with Forgejo.
SSO_AUTHORITY: https://id.${DOMAIN}
SSO_CLIENT_ID: ${VAULTWARDEN_SSO_CLIENT_ID} SSO_CLIENT_ID: ${VAULTWARDEN_SSO_CLIENT_ID}
SSO_CLIENT_SECRET: ${VAULTWARDEN_SSO_CLIENT_SECRET} SSO_CLIENT_SECRET: ${VAULTWARDEN_SSO_CLIENT_SECRET}
# Vaultwarden prepends "openid" itself, so listing it here again shows up # Vaultwarden prepends "openid" itself, so listing it here again shows up
# duplicated on the Forgejo consent screen. Harmless, but untidy. # duplicated on the consent screen. Harmless, but untidy.
SSO_SCOPES: "profile email" SSO_SCOPES: "profile email"
# Attach an SSO login to the existing account with the same mail address # Attach an SSO login to the existing account with the same mail address
# instead of creating a duplicate. Turn this on BEFORE people log in. # instead of creating a duplicate. Turn this on BEFORE people log in.
@ -247,6 +259,54 @@ services:
- ./backup/backup.log:/backup/backup.log:ro - ./backup/backup.log:/backup/backup.log:ro
networks: [proxy] networks: [proxy]
# Identity provider (episode 15). Forgejo used to be the OIDC provider for
# outline, vaultwarden and netbird. Now everyone logs in here, with a
# passkey, and switching one person off happens in one place. Pocket ID is
# small enough for a 4 GB box; groups decide who gets which service.
pocket-id:
image: ghcr.io/pocket-id/pocket-id:v2.16.0
restart: unless-stopped
environment:
APP_URL: https://id.${DOMAIN}
# Caddy sits in front, so trust X-Forwarded-For for the audit log.
TRUST_PROXY: "true"
# Encrypts the signing keys at rest. Generate once: openssl rand -base64 32
ENCRYPTION_KEY: ${POCKET_ID_ENCRYPTION_KEY}
ANALYTICS_DISABLED: "true"
volumes:
- pocket_id_data:/app/data
networks: [proxy]
# Login gate for services without their own SSO (episode 15). Caddy asks
# tinyauth on every request (forward_auth); tinyauth sends people to Pocket
# ID and lets through only members of the right group. No docker socket,
# apps are configured below, not via labels.
tinyauth:
image: ghcr.io/tinyauthapp/tinyauth:v5.2.0
restart: unless-stopped
environment:
TINYAUTH_APPURL: https://auth.${DOMAIN}
TINYAUTH_LABELPROVIDER: none
TINYAUTH_ANALYTICS_ENABLED: "false"
TINYAUTH_DATABASE_PATH: /data/tinyauth.db
TINYAUTH_AUTH_SECURECOOKIE: "true"
TINYAUTH_AUTH_TRUSTEDPROXIES: 172.18.0.0/16
TINYAUTH_OAUTH_AUTOREDIRECT: pocketid
TINYAUTH_OAUTH_PROVIDERS_POCKETID_NAME: Pocket ID
TINYAUTH_OAUTH_PROVIDERS_POCKETID_CLIENTID: ${TINYAUTH_CLIENT_ID}
TINYAUTH_OAUTH_PROVIDERS_POCKETID_CLIENTSECRET: ${TINYAUTH_CLIENT_SECRET}
TINYAUTH_OAUTH_PROVIDERS_POCKETID_AUTHURL: https://id.${DOMAIN}/authorize
TINYAUTH_OAUTH_PROVIDERS_POCKETID_TOKENURL: https://id.${DOMAIN}/api/oidc/token
TINYAUTH_OAUTH_PROVIDERS_POCKETID_USERINFOURL: https://id.${DOMAIN}/api/oidc/userinfo
TINYAUTH_OAUTH_PROVIDERS_POCKETID_REDIRECTURL: https://auth.${DOMAIN}/api/oauth/callback/pocketid
TINYAUTH_OAUTH_PROVIDERS_POCKETID_SCOPES: openid email profile groups
# Uptime Kuma: only the Pocket ID group "ops" gets past the gate.
TINYAUTH_APPS_KUMA_CONFIG_DOMAIN: status.${DOMAIN}
TINYAUTH_APPS_KUMA_OAUTH_GROUPS: ops
volumes:
- tinyauth_data:/data
networks: [proxy]
# Optional starter VPN — enable with: docker compose --profile vpn up -d # Optional starter VPN — enable with: docker compose --profile vpn up -d
# (The proper Netbird mesh deployment is Episode 5.) # (The proper Netbird mesh deployment is Episode 5.)
wg-easy: wg-easy:

78
offboarding/README.md Normal file
View file

@ -0,0 +1,78 @@
# Offboarding (episode 15)
Someone leaves. Switching them off in the single sign-on is not offboarding.
Measured on the demo box, with a test employee who had the access a real one has:
| After | Ways still open |
| --- | --- |
| Forgejo login prohibited (Forgejo was the SSO provider) | **10 of 13**, unchanged for 45 minutes |
| Pocket ID account disabled, nothing else | **12 of 14** |
| `./offboard.py <user>` (about one second) | **1 of 14**: the vault copy on their laptop |
What stayed open with only the SSO switched off: the wiki session (valid for
three months) and its API key, the vault sync (Forgejo kept handing out fresh
tokens to the locked account), a new vault login with the master password, the
local vault copy, the mesh peer (expires after 24 h), the automation account,
both notification logins, and the monitoring password everybody knows.
## Run it
On the box, as root:
```sh
cd /opt/selfhostedops-stack/offboarding
cp offboard.env.example offboard.env && chmod 600 offboard.env # fill in once
./offboard.py sam --dry-run # looks, changes nothing
./offboard.py sam
```
It disables the person in Pocket ID first (no new logins, refresh tokens
refused), then per service: deletes the tinyauth gate sessions, prohibits the
Forgejo login and deletes the SSH keys, suspends the Outline user, disables the
Vaultwarden user (which also logs out every device), deletes the NetBird user
with its peers, deletes the n8n user, deletes the ntfy user with its tokens.
Every step can run twice. A step that fails says why and the others still run.
Then it prints what only a human can do. The important line is this one:
```
Vaultwarden: remove the user from organization 'Northwind Labs' (needs an org
owner) and CHANGE THE 6 SHARED PASSWORDS they could read. A copy on their
laptop still opens with their master password, offline, forever.
```
That is not a figure of speech. After the script, on the test laptop, with the
network namespace cut off, `bw unlock` still opened all six shared entries in
clear text. No server-side switch reaches a file on someone else's disk.
Rotation is the offboarding step for everything they could read.
## What the keys in offboard.env may do
| Key | Scope | Why |
| --- | --- | --- |
| `POCKET_ID_API_KEY` | admin API, set an expiry | disable the user |
| `FORGEJO_ADMIN_TOKEN` | `write:admin`, `write:user` | prohibit login, delete keys, list tokens |
| `OUTLINE_API_KEY` | an admin's key | suspend |
| `NETBIRD_PAT` | service user, role admin, 30 days | delete user and peers |
| `N8N_API_KEY` | only `user:list`, `user:read`, `user:delete` | delete the user |
This file is the most powerful thing on the box after root. Keep it at 600,
give every key an expiry, and renew them on purpose.
## Traps we hit while building it
- **Vaultwarden admin routes want JSON.** `POST /admin/users/<id>/disable`
without a JSON content type is not a 403, it is a 404, and it looks exactly
like "user not found".
- **Vaultwarden rate-limits the admin login** (a handful per five minutes). Two
test runs back to back and the third one gets 429. The script says so.
- **An old n8n API key cannot delete users.** Keys created before scopes
existed, or with the wrong ones, get 403. Create one with `user:delete`.
- **Forgejo tokens:** an admin can list a user's tokens (`Sudo:` header) but
deleting one needs the owner's password. `prohibit_login` blocks them, so the
script names them instead.
- **Outline 1.10** lists an admin only their own API keys. Suspending the user
refuses their keys and browser sessions anyway (measured: 401 and 403).
- **Group restriction in Pocket ID is a separate switch.** Setting allowed
groups on a client via the API leaves `isGroupRestricted` false, and the UI
then says "Any user can sign in to this client". Set both.

View file

@ -0,0 +1,13 @@
# offboard.env: copy to offboard.env (chmod 600), fill in, never commit.
# DOMAIN and VAULTWARDEN_ADMIN_TOKEN are read from the stack's .env.
# Pocket ID: Settings -> Admin -> API Keys (set an expiry)
POCKET_ID_API_KEY=
# Forgejo: forgejo admin user generate-access-token --username <admin> --token-name offboarding --scopes write:admin --raw
FORGEJO_ADMIN_TOKEN=
# Outline: API key of a workspace admin (Settings -> API)
OUTLINE_API_KEY=
# NetBird: service user with role admin -> personal access token
NETBIRD_PAT=
# n8n: Settings -> n8n API (owner)
N8N_API_KEY=

216
offboarding/offboard.py Normal file
View file

@ -0,0 +1,216 @@
#!/usr/bin/env python3
"""Offboarding for the selfhostedops stack (episode 15).
./offboard.py <username> [--dry-run]
Run it on the box, as root, from this directory. It switches one person off in
every service that has an API for it, one step after the other, and prints
what it did. At the end it prints what only a human can do: the shared
passwords that person could read are not safe until someone changes them.
Nothing clever happens here. Each block is one service, one or two API calls.
Credentials come from offboard.env (see offboard.env.example), the
Vaultwarden admin token from the stack's own .env.
Order matters a little: the identity provider goes first, so no new login is
possible while the rest is still running.
"""
import http.cookiejar, json, os, sqlite3, subprocess, sys, time, urllib.parse, urllib.request
HERE = os.path.dirname(os.path.abspath(__file__))
STACK = os.path.dirname(HERE)
def load_env(path):
env = {}
for line in open(path):
line = line.strip()
if line and not line.startswith("#") and "=" in line:
k, v = line.split("=", 1)
env[k] = v.strip().strip('"')
return env
CFG = {**load_env(os.path.join(STACK, ".env")), **load_env(os.path.join(HERE, "offboard.env"))}
DOMAIN = CFG["DOMAIN"]
todo, report, started = [], [], time.time()
def ip(service):
# Admin endpoints that Caddy keeps closed (vaultwarden /admin, the n8n API) are
# called on the container itself. The box is on the docker bridge, so this works.
out = subprocess.run(["docker", "inspect", "-f", "{{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}",
f"selfhostedops-stack-{service}-1"], capture_output=True, text=True).stdout.split()
return out[0]
def volume(name):
return subprocess.run(["docker", "volume", "inspect", "-f", "{{.Mountpoint}}", f"selfhostedops-stack_{name}"],
capture_output=True, text=True).stdout.strip()
def call(method, url, body=None, headers=None, form=False, opener=None):
data = None
headers = dict(headers or {})
if body is not None and form:
data = urllib.parse.urlencode(body).encode()
headers["Content-Type"] = "application/x-www-form-urlencoded"
elif body is not None:
data = json.dumps(body).encode()
headers["Content-Type"] = "application/json"
req = urllib.request.Request(url, data=data, headers=headers, method=method)
try:
with (opener or urllib.request.build_opener()).open(req, timeout=20) as r:
raw = r.read()
return r.status, json.loads(raw) if raw[:1] in (b"{", b"[") else raw.decode(errors="replace")
except urllib.error.HTTPError as e:
return e.code, e.read().decode(errors="replace")[:200]
def step(service, what, ok, detail=""):
report.append((service, what, ok))
print(f"{time.time() - started:6.1f}s [{'ok' if ok else '!!'}] {service:12} {what} {detail}".rstrip(), flush=True)
def pocket_id(user):
h = {"X-API-KEY": CFG["POCKET_ID_API_KEY"]}
base = f"https://id.{DOMAIN}/api"
s, j = call("GET", f"{base}/users?search={urllib.parse.quote(user)}", headers=h)
hits = [u for u in (j.get("data", []) if s == 200 else []) if u["username"] == user]
if not hits:
return step("pocket-id", "user not found", False)
u = hits[0]
dto = {k: u[k] for k in ("username", "email", "firstName", "lastName", "isAdmin", "locale")}
if not DRY:
s, _ = call("PUT", f"{base}/users/{u['id']}", {**dto, "disabled": True}, h)
step("pocket-id", "disabled (no new login, refresh tokens refused)", DRY or s == 200)
return u["email"]
def forgejo(user):
h = {"Authorization": "token " + CFG["FORGEJO_ADMIN_TOKEN"]}
base = f"https://git.{DOMAIN}/api/v1"
if not DRY:
s, _ = call("PATCH", f"{base}/admin/users/{user}", {"login_name": user, "source_id": 0, "prohibit_login": True}, h)
step("forgejo", "prohibit_login (web, git over ssh and https, api)", s == 200)
s, keys = call("GET", f"{base}/users/{user}/keys", headers=h)
for k in keys if s == 200 else []:
if not DRY:
s2, _ = call("DELETE", f"{base}/admin/users/{user}/keys/{k['id']}", headers=h)
step("forgejo", f"ssh key '{k['title']}' deleted", DRY or s2 == 204)
# An admin may list a user's access tokens (Sudo header), but deleting one needs the
# owner's password (basic auth). prohibit_login already blocks them, so: name them.
s, toks = call("GET", f"{base}/users/{user}/tokens", headers={**h, "Sudo": user})
names = [t["name"] for t in toks] if s == 200 else []
if names:
todo.append(f"Forgejo: access tokens {names} are blocked by prohibit_login but still exist; "
"delete them if the account is ever re-enabled (only the owner can)")
def outline(email):
h = {"Authorization": "Bearer " + CFG["OUTLINE_API_KEY"]}
base = f"https://wiki.{DOMAIN}/api"
s, j = call("POST", f"{base}/users.list", {"query": email, "filter": "all"}, h)
hits = [u for u in (j.get("data", []) if s == 200 else []) if u.get("email") == email]
if not hits:
return step("outline", "user not found", False)
uid = hits[0]["id"]
if not DRY:
s, _ = call("POST", f"{base}/users.suspend", {"id": uid}, h)
step("outline", "suspended (browser session and api keys refused)", DRY or s == 200)
# Outline 1.10 lists an admin only their own API keys, so the user's keys cannot be
# deleted from here. A suspended user's keys and browser sessions are refused anyway.
def vaultwarden(email):
jar = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar()))
base = f"http://{ip('vaultwarden')}"
s, _ = call("POST", f"{base}/admin", {"token": CFG["VAULTWARDEN_ADMIN_TOKEN"]}, form=True, opener=jar)
if s != 200: # 429: the admin login allows only a few tries per 5 minutes
return step("vaultwarden", f"admin login refused (HTTP {s}), run again in 5 minutes", False)
s, users = call("GET", f"{base}/admin/users", opener=jar)
hits = [u for u in (users if s == 200 else []) if u["email"] == email]
if not hits:
return step("vaultwarden", "user not found", False)
uid = hits[0]["id"]
if not DRY:
# The admin routes only match with a JSON content type, without a body they answer 404.
s, _ = call("POST", f"{base}/admin/users/{uid}/disable", {}, opener=jar)
step("vaultwarden", "disabled + all devices logged out", DRY or s == 200)
# What could this person read? The item names are encrypted, the count is not.
db = volume("vaultwarden_data") + "/db.sqlite3"
q = ("SELECT o.name, COUNT(DISTINCT cc.cipher_uuid) FROM users_organizations uo "
"JOIN organizations o ON o.uuid = uo.org_uuid "
"JOIN collections c ON c.org_uuid = uo.org_uuid "
"LEFT JOIN users_collections ucl ON ucl.collection_uuid = c.uuid AND ucl.user_uuid = uo.user_uuid "
"JOIN ciphers_collections cc ON cc.collection_uuid = c.uuid "
"WHERE uo.user_uuid = ? AND (uo.access_all = 1 OR ucl.user_uuid IS NOT NULL) GROUP BY o.name")
# ponytail: direct collection grants only; add collections_groups if you use Vaultwarden groups
rows = sqlite3.connect(f"file:{db}?mode=ro", uri=True).execute(q, (uid,)).fetchall()
for org, n in rows:
todo.append(f"Vaultwarden: remove the user from organization '{org}' (needs an org owner) and "
f"CHANGE THE {n} SHARED PASSWORDS they could read. A copy on their laptop still opens "
f"with their master password, offline, forever.")
def netbird(email):
h = {"Authorization": "Token " + CFG["NETBIRD_PAT"]}
base = f"https://vpn.{DOMAIN}/api"
s, users = call("GET", f"{base}/users", headers=h)
s2, peers = call("GET", f"{base}/peers", headers=h)
for u in [u for u in (users if s == 200 else []) if u.get("email") == email]:
mine = [p["name"] for p in (peers if s2 == 200 else []) if p.get("user_id") == u["id"]]
if not DRY:
s3, _ = call("DELETE", f"{base}/users/{u['id']}", headers=h)
step("netbird", f"user deleted with its peers {mine}", DRY or s3 == 200)
def n8n(email):
h = {"X-N8N-API-KEY": CFG["N8N_API_KEY"]}
base = f"http://{ip('n8n')}:5678/api/v1"
s, u = call("GET", f"{base}/users/{urllib.parse.quote(email)}", headers=h)
if s == 404:
return step("n8n", "no such user (already gone)", True)
if s != 200:
return step("n8n", f"lookup failed (HTTP {s})", False)
if not DRY:
s, _ = call("DELETE", f"{base}/users/{u['id']}", headers=h)
step("n8n", "user deleted (workflows stay with the instance owner)", DRY or s in (200, 204), "" if DRY else f"HTTP {s}")
def ntfy(user):
cmd = ["docker", "compose", "-f", os.path.join(STACK, "compose.yml"), "exec", "-T", "ntfy", "ntfy", "user", "del", user]
r = subprocess.run(cmd, capture_output=True, text=True) if not DRY else None
gone = DRY or r.returncode == 0 or "does not exist" in r.stderr + r.stdout # a second run is fine
step("ntfy", "user and its tokens deleted", gone, "" if DRY else (r.stdout + r.stderr).strip())
def tinyauth(email):
# tinyauth keeps its own session for a day and does not ask Pocket ID again.
n = 0
if not DRY:
with sqlite3.connect(volume("tinyauth_data") + "/tinyauth.db") as db:
n = db.execute("DELETE FROM sessions WHERE email = ?", (email,)).rowcount
step("tinyauth", f"{n} gate sessions deleted", True)
if __name__ == "__main__":
if len(sys.argv) < 2 or not sys.argv[1].replace("-", "").isalnum():
sys.exit(__doc__)
USER, DRY = sys.argv[1], "--dry-run" in sys.argv
print(f"offboarding {USER}" + (" (DRY RUN: only looks, changes nothing)" if DRY else ""))
email = pocket_id(USER) or f"{USER}@{DOMAIN}"
for fn, arg in ((tinyauth, email), (forgejo, USER), (outline, email), (vaultwarden, email),
(netbird, email), (n8n, email), (ntfy, USER)):
try:
fn(arg)
except Exception as e: # one broken service must not stop the others
step(fn.__name__, f"FAILED: {e}", False)
todo.append("Uptime Kuma and anything else with a shared login: change that password now")
todo.append("Get the laptop back. Until then assume every local copy is readable.")
failed = [r for r in report if not r[2]]
print(f"\n{len(report) - len(failed)} of {len(report)} steps done in {time.time() - started:.1f} s"
+ (f", {len(failed)} FAILED, check above" if failed else ""))
print("\nStill for a human:")
for t in todo:
print(" - " + t)