selfhostedops/selfhostedops-stack
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
selfhostedops-stack/updates/health.sh
Maximilian Grimm 55b2fe7646 Episode 17: updates one service at a time, a health check that asks real questions, every tag pinned 
updates/health.sh: 43 checks across all services (API answers, git refs, a push
message round trip, SSO redirects and token endpoint, mesh state, backup age).
updates/update.sh: per service pull or build, snapshot, recreate, wait for the
health check, put image and data back on red and stop the run.
backup/backup.sh: online dumps for n8n, Pocket ID and tinyauth; BACKUP_TAG so
pre-update snapshots are not removed by the nightly keep-daily rule.
compose.yml and caddy/Dockerfile: exact versions instead of latest, 16, 7-alpine
and caddy:2-alpine.
2026-10-08 22:04:29 +02:00

162 lines
11 KiB
Bash
Executable file
Raw Permalink Blame History

#!/usr/bin/env bash
# ~$ selfhostedops - health check for the whole stack (episode 17)
#
# "The container is up" is not a health check. Every line below asks a service
# to do the thing it is there for: answer its API, hand out git refs, deliver
# a push message, start an SSO login, keep the mesh connected.
#
# updates/health.sh all checks, exit code = number of failures
# updates/health.sh forgejo n8n only these groups
# updates/health.sh --data also print row counts (to compare before/after)
#
# Runs on the box itself. Public hostnames are resolved to 127.0.0.1, so the
# request still goes through Caddy and its certificates, without leaving the box.
# No check asks for an answer of 401, 403 or 404 through Caddy: fail2ban counts
# those (episode 10) and would end up banning the box's own bridge address.
# Optional credentials in updates/health.env (see health.env.example):
# FORGEJO_HEALTH_TOKEN read-only token, for the git ref check on a private repo
# FORGEJO_HEALTH_REPO owner/name
# NTFY_HEALTH_AUTH user:password with read-write on the topic "healthcheck"
set -uo pipefail
STACK_DIR=${STACK_DIR:-/opt/selfhostedops-stack}
P=${PROJECT:-selfhostedops-stack}
set -a
. "$STACK_DIR/.env"
[ -f "$STACK_DIR/updates/health.env" ] && . "$STACK_DIR/updates/health.env"
set +a
D=$DOMAIN
SQLITE_IMAGE=${SQLITE_IMAGE:-keinos/sqlite3:latest}
DATA=0; GROUPS_WANTED=()
for a in "$@"; do [ "$a" = "--data" ] && DATA=1 || GROUPS_WANTED+=("$a"); done
want() { [ ${#GROUPS_WANTED[@]} -eq 0 ] || [[ " ${GROUPS_WANTED[*]} " == *" $1 "* ]]; }
FAILS=0; TOTAL=0
ok() { TOTAL=$((TOTAL+1)); printf 'PASS %-12s %s\n' "$1" "$2"; }
bad() { TOTAL=$((TOTAL+1)); FAILS=$((FAILS+1)); printf 'FAIL %-12s %s\n' "$1" "$2"; }
# check <group> <description> <actual> <expected regex>
check() { if [[ "$3" =~ $4 ]]; then ok "$1" "$2"; else bad "$1" "$2 (got: $(printf '%s' "$3" | tr '\n' ' ' | cut -c1-90))"; fi; }
pub() { local host=$1 path=$2; shift 2; curl -sS -m 8 --resolve "$host.$D:443:127.0.0.1" "$@" "https://$host.$D$path" 2>&1; }
code() { pub "$1" "$2" -o /dev/null -w '%{http_code}' "${@:3}"; }
loc() { pub "$1" "$2" -o /dev/null -w '%{redirect_url}' "${@:3}"; }
cip() { docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}' "$1" 2>/dev/null | cut -d' ' -f1; }
sq() { docker run --rm --user 0 -v "$1:/src:ro" "$SQLITE_IMAGE" sqlite3 "file:/src/$2?mode=ro" "$3" 2>&1; }
dx() { docker exec "$@" 2>&1; }
if want caddy; then
check caddy "config is valid" "$(dx $P-caddy-1 caddy validate --config /etc/caddy/Caddyfile | grep -o 'Valid configuration')" 'Valid configuration'
check caddy "plugins compiled in: rate_limit, crowdsec, appsec" "$(dx $P-caddy-1 caddy list-modules | grep -cE '^http.handlers.(rate_limit|crowdsec|appsec)$')" '^3$'
check caddy "certificate for git.$D is valid" "$(code git /api/healthz)" '^200$'
if command -v cscli >/dev/null; then
age=$(( $(date +%s) - $(date -d "$(cscli bouncers list -o json | jq -r '.[]|select(.type=="caddy-cs-bouncer")|.last_pull')" +%s 2>/dev/null || echo 0) ))
check caddy "CrowdSec bouncer pulled decisions in the last 60 s" "$age" '^([0-9]|[1-5][0-9])$'
fi
fi
if want forgejo; then
check forgejo "API healthz: database and cache pass" "$(pub git /api/healthz | jq -r '.status' 2>&1)" '^pass$'
if [ -n "${FORGEJO_HEALTH_TOKEN:-}" ]; then
check forgejo "git over HTTPS: refs of ${FORGEJO_HEALTH_REPO}" "$(pub git "/${FORGEJO_HEALTH_REPO}.git/info/refs?service=git-upload-pack" -u "health:$FORGEJO_HEALTH_TOKEN" | tr -d '\0' | grep -ac 'refs/heads/')" '^[1-9]'
fi
check forgejo "git over SSH: port 2222 answers" "$(timeout 3 bash -c 'exec 3<>/dev/tcp/127.0.0.1/2222; head -c 8 <&3' 2>&1)" '^SSH-2.0'
check forgejo "SSO: login button leads to id.$D" "$(loc git /user/oauth2/PocketID)" "^https://id\.$D/authorize"
fi
if want outline; then
check outline "postgres accepts connections" "$(dx $P-outline-postgres-1 pg_isready -U outline)" 'accepting connections'
check outline "redis answers PING" "$(dx $P-outline-redis-1 redis-cli ping)" '^PONG'
check outline "_health" "$(code wiki /_health)" '^200$'
check outline "API auth.config lists Pocket ID" "$(pub wiki /api/auth.config -X POST | jq -r '.data.providers[0].name' 2>&1)" '^Pocket ID$'
check outline "SSO: login leads to id.$D" "$(loc wiki /auth/oidc)" "^https://id\.$D/authorize"
check outline "database answers a query" "$(dx $P-outline-postgres-1 psql -U outline -tAc 'select count(*) from documents')" '^[0-9]+$'
fi
if want vaultwarden; then
check vaultwarden "alive" "$(code vault /alive)" '^200$'
check vaultwarden "API config reports a version" "$(pub vault /api/config | jq -r '.version' 2>&1)" '^20[0-9][0-9]\.'
check vaultwarden "SSO: prevalidate hands out a token" "$(pub vault '/identity/sso/prevalidate?domainHint=x' | jq -r '.token|length>20' 2>&1)" '^true$'
fi
if want kuma; then
# The status page API is asked on the container directly, not through Caddy: while Kuma migrates
# its database (1.x -> 2.x took five minutes here) it answers 404, and a health check that polls
# 404s through Caddy gets the box's own bridge address banned by fail2ban.
k="http://$(cip $P-uptime-kuma-1):3001"
sp=$(curl -sS -m 8 "$k/api/status-page/status" 2>&1)
check kuma "public status page API" "$(jq -r '.config.slug' <<<"$sp" 2>&1)" '^status$'
check kuma "status page shows 4 public monitors" "$(jq -r '[.publicGroupList[].monitorList[]]|length' <<<"$sp" 2>&1)" '^4$'
hb=$(curl -sS -m 8 "$k/api/status-page/heartbeat/status" 2>&1 | jq -r '[.heartbeatList[]|last]|"\(length) \([.[]|select(.status==1)]|length) \(now - ([.[].time|sub("\\.[0-9]+$";"")|sub(" ";"T")+"Z"|fromdate]|min)|floor)"' 2>&1)
check kuma "every public monitor: last heartbeat UP and younger than 180 s (monitors up age)" "$hb" '^([0-9]+) \1 ([0-9]|[1-9][0-9]|1[0-7][0-9])$'
check kuma "dashboard is behind the login gate" "$(loc status /)" "^https://auth\.$D/login"
down=$(sq ${P}_kuma_data kuma.db "select count(*) from monitor m where active=1 and (select status from heartbeat h where h.monitor_id=m.id order by time desc limit 1) is not 1")
check kuma "no active monitor is DOWN in the database" "$down" '^0$'
fi
if want ntfy; then
check ntfy "health" "$(pub ntfy /v1/health | jq -r '.healthy' 2>&1)" '^true$'
if [ -n "${NTFY_HEALTH_AUTH:-}" ]; then
id="health-$(date +%s)-$RANDOM"
pub ntfy /healthcheck -u "$NTFY_HEALTH_AUTH" -d "$id" -o /dev/null
check ntfy "a published message can be read back" "$(pub ntfy '/healthcheck/json?poll=1&since=1m' -u "$NTFY_HEALTH_AUTH" | grep -c "$id")" '^1$'
fi
fi
if want n8n; then
n=$(cip $P-n8n-1)
check n8n "readiness (database reachable, migrations done)" "$(curl -sS -m 8 http://$n:5678/healthz/readiness 2>&1)" '"status":"ok"'
check n8n "active workflows (at least ${N8N_EXPECT_ACTIVE:-1})" "$(dx $P-n8n-1 n8n list:workflow --active=true | grep -c '|')" "^(${N8N_EXPECT_ACTIVE:-[1-9][0-9]*})$"
# 403 = the webhook is registered and asks for its token. 404 = workflow not active.
check n8n "webhook kuma-incident is registered and asks for its token" "$(curl -sS -m 8 -o /dev/null -w '%{http_code}' -X POST http://$n:5678/webhook/kuma-incident -d '{}' 2>&1)" '^403$'
fi
if want pocket-id; then
check pocket-id "healthz" "$(code id /healthz)" '^20[04]$'
check pocket-id "discovery document names the right issuer" "$(pub id /.well-known/openid-configuration | jq -r '.issuer' 2>&1)" "^https://id\.$D$"
check pocket-id "signing keys are published" "$(pub id /.well-known/jwks.json | jq -r '.keys|length' 2>&1)" '^[1-9]'
# A made-up code with the real client secret must fail as invalid_grant. invalid_client would mean the client is gone.
check pocket-id "token endpoint still knows the Outline client" "$(curl -sS -m 8 -X POST http://$(cip $P-pocket-id-1):1411/api/oidc/token -d grant_type=authorization_code -d code=health -d client_id=$OUTLINE_OIDC_CLIENT_ID -d client_secret=$OUTLINE_OIDC_CLIENT_SECRET -d redirect_uri=https://wiki.$D/auth/oidc.callback 2>&1 | jq -r '.error' 2>&1)" '^invalid_grant$'
fi
if want tinyauth; then
check tinyauth "healthz" "$(pub auth /api/healthz | jq -r '.message' 2>&1)" '^Healthy$'
check tinyauth "login is handed to id.$D" "$(pub auth /api/oauth/url/pocketid | jq -r '.url' 2>&1)" "^https://id\.$D/authorize"
fi
if want netbird; then
check netbird "dashboard" "$(code vpn /)" '^200$'
check netbird "embedded identity provider discovery" "$(pub vpn /oauth2/.well-known/openid-configuration | jq -r '.issuer' 2>&1)" "^https://vpn\.$D"
check netbird "management API alive and asks for a login" "$(curl -sS -m 8 -o /dev/null -w '%{http_code}' http://127.0.0.1:8081/api/users 2>&1)" '^401$'
check netbird "this host: management and signal connected" "$(netbird status 2>&1 | grep -cE '^(Management|Signal): Connected')" '^2$'
check netbird "this host: at least one peer connected" "$(netbird status 2>&1 | grep -oE 'Peers count: [0-9]+' | grep -oE '[0-9]+$')" '^[1-9]'
if docker inspect mesh-client >/dev/null 2>&1; then
check netbird "mesh-client container: management connected" "$(dx mesh-client netbird status | grep -c '^Management: Connected')" '^1$'
fi
fi
if want ops; then
check ops "ops-readonly healthz" "$(curl -sS -m 8 http://$(cip $P-ops-readonly-1):8098/healthz 2>&1 | jq -r '.status' 2>&1)" '^ok$'
check ops "ops-actions healthz" "$(curl -sS -m 8 http://$(cip $P-ops-actions-1):8099/healthz 2>&1 | jq -r '.status' 2>&1)" '^ok$'
fi
if want backup; then
check backup "nightly timer is scheduled" "$(systemctl is-active selfhostedops-backup.timer 2>&1)" '^active$'
last=$(grep '=== backup end ===' "$STACK_DIR/backup/backup.log" | tail -1 | cut -c1-19)
check backup "last finished backup is younger than 26 h ($last)" "$(( ( $(date +%s) - $(date -d "$last" +%s 2>/dev/null || echo 0) ) / 3600 ))" '^([0-9]|1[0-9]|2[0-5])$'
fi
if [ $DATA = 1 ]; then
echo
echo "DATA forgejo repositories $(sq ${P}_forgejo_data gitea/gitea.db 'select count(*) from repository'), users $(sq ${P}_forgejo_data gitea/gitea.db 'select count(*) from user where type=0')"
echo "DATA outline documents $(dx $P-outline-postgres-1 psql -U outline -tAc 'select count(*) from documents'), users $(dx $P-outline-postgres-1 psql -U outline -tAc 'select count(*) from users'), migrations $(dx $P-outline-postgres-1 psql -U outline -tAc 'select count(*) from "SequelizeMeta"')"
echo "DATA vaultwarden users $(sq ${P}_vaultwarden_data db.sqlite3 'select count(*) from users'), ciphers $(sq ${P}_vaultwarden_data db.sqlite3 'select count(*) from ciphers')"
echo "DATA kuma monitors $(sq ${P}_kuma_data kuma.db 'select count(*) from monitor'), notifications $(sq ${P}_kuma_data kuma.db 'select count(*) from notification'), heartbeats $(sq ${P}_kuma_data kuma.db 'select count(*) from heartbeat')"
echo "DATA n8n workflows $(sq ${P}_n8n_data database.sqlite 'select count(*) from workflow_entity'), credentials $(sq ${P}_n8n_data database.sqlite 'select count(*) from credentials_entity'), executions $(sq ${P}_n8n_data database.sqlite 'select count(*) from execution_entity'), migrations $(sq ${P}_n8n_data database.sqlite 'select count(*) from migrations')"
echo "DATA pocket-id users $(sq ${P}_pocket_id_data pocket-id.db 'select count(*) from users'), clients $(sq ${P}_pocket_id_data pocket-id.db 'select count(*) from oidc_clients')"
echo "DATA netbird peers $(sq netbird_netbird_data store.db 'select count(*) from peers')"
fi
echo
echo "$((TOTAL-FAILS)) of $TOTAL checks passed, $FAILS failed"
exit $FAILS
Reference in a new issue View git blame Copy permalink