Episode 17: updates one service at a time, a health check that asks real questions, every tag pinned
updates/health.sh: 43 checks across all services (API answers, git refs, a push message round trip, SSO redirects and token endpoint, mesh state, backup age). updates/update.sh: per service pull or build, snapshot, recreate, wait for the health check, put image and data back on red and stop the run. backup/backup.sh: online dumps for n8n, Pocket ID and tinyauth; BACKUP_TAG so pre-update snapshots are not removed by the nightly keep-daily rule. compose.yml and caddy/Dockerfile: exact versions instead of latest, 16, 7-alpine and caddy:2-alpine.
This commit is contained in:
Maximilian Grimm
parent
1655d04d7d
commit
55b2fe7646
7 changed files with 380 additions and 19 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -8,3 +8,4 @@ logs/
|
||||||
ops-readonly/token.env
|
ops-readonly/token.env
|
||||||
offboarding/offboard.env
|
offboarding/offboard.env
|
||||||
__pycache__/
|
__pycache__/
|
||||||
|
updates/health.env
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,8 @@
|
||||||
#
|
#
|
||||||
# Application-consistent, without stopping anything:
|
# Application-consistent, without stopping anything:
|
||||||
# 1. pg_dump of Outline's Postgres -> backup/dumps/outline.sql.gz
|
# 1. pg_dump of Outline's Postgres -> backup/dumps/outline.sql.gz
|
||||||
# 2. online SQLite backups (Forgejo, Vaultwarden, Uptime Kuma, ntfy, NetBird)
|
# 2. online SQLite backups (Forgejo, Vaultwarden, Uptime Kuma, ntfy, n8n,
|
||||||
|
# Pocket ID, tinyauth, NetBird)
|
||||||
# -> backup/dumps/*.db
|
# -> backup/dumps/*.db
|
||||||
# 3. restic backup of the stack directory, /opt/netbird and every stateful
|
# 3. restic backup of the stack directory, /opt/netbird and every stateful
|
||||||
# docker volume (the raw files, plus the dumps from 1 and 2)
|
# docker volume (the raw files, plus the dumps from 1 and 2)
|
||||||
|
|
@ -22,6 +23,12 @@ LOG_FILE="$BACKUP_DIR/backup.log"
|
||||||
PROJECT=${PROJECT:-selfhostedops-stack}
|
PROJECT=${PROJECT:-selfhostedops-stack}
|
||||||
VOLUMES=/var/lib/docker/volumes
|
VOLUMES=/var/lib/docker/volumes
|
||||||
|
|
||||||
|
# Tag of the snapshot. The nightly timer uses "nightly". updates/update.sh
|
||||||
|
# runs this script with BACKUP_TAG=pre-update: those snapshots are left alone
|
||||||
|
# by forget below. Without that, "keep 7 daily" would drop the snapshot from
|
||||||
|
# before the update as soon as a second backup runs on the same day (episode 17).
|
||||||
|
BACKUP_TAG=${BACKUP_TAG:-nightly}
|
||||||
|
|
||||||
# Small alpine image with a current sqlite3, used for the online SQLite copies.
|
# Small alpine image with a current sqlite3, used for the online SQLite copies.
|
||||||
SQLITE_IMAGE=${SQLITE_IMAGE:-keinos/sqlite3:latest}
|
SQLITE_IMAGE=${SQLITE_IMAGE:-keinos/sqlite3:latest}
|
||||||
|
|
||||||
|
|
@ -77,6 +84,9 @@ sqlite_backup "${PROJECT}_vaultwarden_data" db.sqlite3 vaultwarden.sqlite3
|
||||||
sqlite_backup "${PROJECT}_kuma_data" kuma.db kuma.db
|
sqlite_backup "${PROJECT}_kuma_data" kuma.db kuma.db
|
||||||
sqlite_backup "${PROJECT}_ntfy_data" user.db ntfy-user.db
|
sqlite_backup "${PROJECT}_ntfy_data" user.db ntfy-user.db
|
||||||
sqlite_backup "${PROJECT}_ntfy_cache" cache.db ntfy-cache.db
|
sqlite_backup "${PROJECT}_ntfy_cache" cache.db ntfy-cache.db
|
||||||
|
sqlite_backup "${PROJECT}_n8n_data" database.sqlite n8n.sqlite
|
||||||
|
sqlite_backup "${PROJECT}_pocket_id_data" pocket-id.db pocket-id.db
|
||||||
|
sqlite_backup "${PROJECT}_tinyauth_data" tinyauth.db tinyauth.db
|
||||||
sqlite_backup "netbird_netbird_data" store.db netbird-store.db
|
sqlite_backup "netbird_netbird_data" store.db netbird-store.db
|
||||||
sqlite_backup "netbird_netbird_data" idp.db netbird-idp.db
|
sqlite_backup "netbird_netbird_data" idp.db netbird-idp.db
|
||||||
sqlite_backup "netbird_netbird_data" events.db netbird-events.db
|
sqlite_backup "netbird_netbird_data" events.db netbird-events.db
|
||||||
|
|
@ -91,12 +101,16 @@ EXCLUDES=(--exclude "$VOLUMES/metadata.db" --exclude "$VOLUMES/backingFsBlockDev
|
||||||
for v in "${EXCLUDE_VOLUMES[@]}"; do
|
for v in "${EXCLUDE_VOLUMES[@]}"; do
|
||||||
EXCLUDES+=(--exclude "$VOLUMES/${PROJECT}_$v")
|
EXCLUDES+=(--exclude "$VOLUMES/${PROJECT}_$v")
|
||||||
done
|
done
|
||||||
restic backup --tag nightly "${EXCLUDES[@]}" \
|
restic backup --tag "$BACKUP_TAG" "${EXCLUDES[@]}" \
|
||||||
"$STACK_DIR" "$NETBIRD_DIR" "$VOLUMES"
|
"$STACK_DIR" "$NETBIRD_DIR" "$VOLUMES"
|
||||||
|
|
||||||
|
if [ "$BACKUP_TAG" = nightly ]; then
|
||||||
log "[4/5] restic forget + prune"
|
log "[4/5] restic forget + prune"
|
||||||
restic forget --tag nightly --group-by host,tags \
|
restic forget --tag nightly --group-by host,tags \
|
||||||
--keep-daily 7 --keep-weekly 4 --keep-monthly 6 --prune
|
--keep-daily 7 --keep-weekly 4 --keep-monthly 6 --prune
|
||||||
|
else
|
||||||
|
log "[4/5] skipped: snapshots tagged $BACKUP_TAG are kept until you remove them"
|
||||||
|
fi
|
||||||
|
|
||||||
END=$(date +%s)
|
END=$(date +%s)
|
||||||
DURATION=$((END - START))
|
DURATION=$((END - START))
|
||||||
|
|
|
||||||
|
|
@ -3,11 +3,15 @@
|
||||||
# in with xcaddy - there is no runtime "enable" for it.
|
# in with xcaddy - there is no runtime "enable" for it.
|
||||||
# episode 12: same trick for CrowdSec. The bouncer (http) and the AppSec
|
# episode 12: same trick for CrowdSec. The bouncer (http) and the AppSec
|
||||||
# handler (appsec) are plugins too, pinned to a release tag instead of @main.
|
# handler (appsec) are plugins too, pinned to a release tag instead of @main.
|
||||||
FROM caddy:2-builder-alpine AS builder
|
# episode 17: every version is written down. "caddy:2-alpine" plus an unpinned
|
||||||
|
# plugin means the next rebuild is an update nobody decided on. docker compose
|
||||||
|
# pull never touches this image, it only changes when you run a build, so the
|
||||||
|
# compose file carries the same version in the image tag.
|
||||||
|
FROM caddy:2.11.7-builder-alpine AS builder
|
||||||
RUN xcaddy build \
|
RUN xcaddy build \
|
||||||
--with github.com/mholt/caddy-ratelimit \
|
--with github.com/mholt/caddy-ratelimit@v0.1.0 \
|
||||||
--with github.com/hslatman/caddy-crowdsec-bouncer/http@v0.14.1 \
|
--with github.com/hslatman/caddy-crowdsec-bouncer/http@v0.14.1 \
|
||||||
--with github.com/hslatman/caddy-crowdsec-bouncer/appsec@v0.14.1
|
--with github.com/hslatman/caddy-crowdsec-bouncer/appsec@v0.14.1
|
||||||
|
|
||||||
FROM caddy:2-alpine
|
FROM caddy:2.11.7-alpine
|
||||||
COPY --from=builder /usr/bin/caddy /usr/bin/caddy
|
COPY --from=builder /usr/bin/caddy /usr/bin/caddy
|
||||||
|
|
|
||||||
22
compose.yml
22
compose.yml
|
|
@ -62,7 +62,7 @@ volumes:
|
||||||
|
|
||||||
services:
|
services:
|
||||||
caddy:
|
caddy:
|
||||||
image: selfhostedops/caddy:crowdsec # episode 12: caddy-ratelimit + CrowdSec bouncer + AppSec
|
image: selfhostedops/caddy:2.11.7-crowdsec0.14.1 # episode 12: caddy-ratelimit + CrowdSec bouncer + AppSec
|
||||||
build: ./caddy
|
build: ./caddy
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
ports:
|
ports:
|
||||||
|
|
@ -83,7 +83,7 @@ services:
|
||||||
networks: [proxy]
|
networks: [proxy]
|
||||||
|
|
||||||
forgejo:
|
forgejo:
|
||||||
image: codeberg.org/forgejo/forgejo:16
|
image: codeberg.org/forgejo/forgejo:16.0.5
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
environment:
|
environment:
|
||||||
USER_UID: "1000"
|
USER_UID: "1000"
|
||||||
|
|
@ -113,7 +113,7 @@ services:
|
||||||
networks: [proxy]
|
networks: [proxy]
|
||||||
|
|
||||||
outline-postgres:
|
outline-postgres:
|
||||||
image: postgres:16-alpine
|
image: postgres:16.15-alpine
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
environment:
|
environment:
|
||||||
POSTGRES_USER: outline
|
POSTGRES_USER: outline
|
||||||
|
|
@ -124,12 +124,12 @@ services:
|
||||||
networks: [internal]
|
networks: [internal]
|
||||||
|
|
||||||
outline-redis:
|
outline-redis:
|
||||||
image: redis:7-alpine
|
image: redis:7.4.11-alpine
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
networks: [internal]
|
networks: [internal]
|
||||||
|
|
||||||
outline:
|
outline:
|
||||||
image: outlinewiki/outline:latest
|
image: outlinewiki/outline:1.10.1
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
depends_on: [outline-postgres, outline-redis]
|
depends_on: [outline-postgres, outline-redis]
|
||||||
environment:
|
environment:
|
||||||
|
|
@ -157,7 +157,7 @@ services:
|
||||||
networks: [proxy, internal]
|
networks: [proxy, internal]
|
||||||
|
|
||||||
vaultwarden:
|
vaultwarden:
|
||||||
image: vaultwarden/server:latest
|
image: vaultwarden/server:1.37.4
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
environment:
|
environment:
|
||||||
DOMAIN: https://vault.${DOMAIN}
|
DOMAIN: https://vault.${DOMAIN}
|
||||||
|
|
@ -188,7 +188,7 @@ services:
|
||||||
networks: [proxy]
|
networks: [proxy]
|
||||||
|
|
||||||
uptime-kuma:
|
uptime-kuma:
|
||||||
image: louislam/uptime-kuma:1
|
image: louislam/uptime-kuma:2.5.5
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
volumes:
|
volumes:
|
||||||
- kuma_data:/app/data
|
- kuma_data:/app/data
|
||||||
|
|
@ -197,7 +197,7 @@ services:
|
||||||
# Push notifications for the monitoring alerts (episode 6).
|
# Push notifications for the monitoring alerts (episode 6).
|
||||||
# Uptime Kuma publishes to the topic "alerts", your phone subscribes to it.
|
# Uptime Kuma publishes to the topic "alerts", your phone subscribes to it.
|
||||||
ntfy:
|
ntfy:
|
||||||
image: binwiederhier/ntfy:v2.28.0
|
image: binwiederhier/ntfy:v2.29.0
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
command: serve
|
command: serve
|
||||||
environment:
|
environment:
|
||||||
|
|
@ -220,7 +220,7 @@ services:
|
||||||
n8n:
|
n8n:
|
||||||
# Pinned. With :latest every restore from backup is silently an upgrade,
|
# Pinned. With :latest every restore from backup is silently an upgrade,
|
||||||
# and an upgrade is a thing you schedule, not a thing a restore does to you.
|
# and an upgrade is a thing you schedule, not a thing a restore does to you.
|
||||||
image: docker.n8n.io/n8nio/n8n:2.37.7
|
image: docker.n8n.io/n8nio/n8n:2.42.5
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
environment:
|
environment:
|
||||||
N8N_HOST: n8n.${DOMAIN}
|
N8N_HOST: n8n.${DOMAIN}
|
||||||
|
|
@ -264,7 +264,7 @@ services:
|
||||||
# passkey, and switching one person off happens in one place. Pocket ID is
|
# passkey, and switching one person off happens in one place. Pocket ID is
|
||||||
# small enough for a 4 GB box; groups decide who gets which service.
|
# small enough for a 4 GB box; groups decide who gets which service.
|
||||||
pocket-id:
|
pocket-id:
|
||||||
image: ghcr.io/pocket-id/pocket-id:v2.16.0
|
image: ghcr.io/pocket-id/pocket-id:v2.18.0
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
environment:
|
environment:
|
||||||
APP_URL: https://id.${DOMAIN}
|
APP_URL: https://id.${DOMAIN}
|
||||||
|
|
@ -335,7 +335,7 @@ services:
|
||||||
#
|
#
|
||||||
# Start it with: docker compose --profile agent up -d
|
# Start it with: docker compose --profile agent up -d
|
||||||
ops-actions:
|
ops-actions:
|
||||||
image: docker:cli
|
image: docker:29.8.2-cli
|
||||||
profiles: [agent]
|
profiles: [agent]
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
environment:
|
environment:
|
||||||
|
|
|
||||||
9
updates/health.env.example
Normal file
9
updates/health.env.example
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
# Optional credentials for updates/health.sh. Copy to health.env (mode 600).
|
||||||
|
# Read-only Forgejo token: forgejo admin user generate-access-token -u <user> -t health --scopes read:repository --raw
|
||||||
|
FORGEJO_HEALTH_TOKEN=
|
||||||
|
FORGEJO_HEALTH_REPO=owner/repo
|
||||||
|
# ntfy user with read-write on the topic "healthcheck":
|
||||||
|
# ntfy user add health && ntfy access health healthcheck rw
|
||||||
|
NTFY_HEALTH_AUTH=health:password
|
||||||
|
# Number of active n8n workflows you expect (leave empty for "at least one")
|
||||||
|
N8N_EXPECT_ACTIVE=
|
||||||
162
updates/health.sh
Executable file
162
updates/health.sh
Executable file
|
|
@ -0,0 +1,162 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# ~$ selfhostedops - health check for the whole stack (episode 17)
|
||||||
|
#
|
||||||
|
# "The container is up" is not a health check. Every line below asks a service
|
||||||
|
# to do the thing it is there for: answer its API, hand out git refs, deliver
|
||||||
|
# a push message, start an SSO login, keep the mesh connected.
|
||||||
|
#
|
||||||
|
# updates/health.sh all checks, exit code = number of failures
|
||||||
|
# updates/health.sh forgejo n8n only these groups
|
||||||
|
# updates/health.sh --data also print row counts (to compare before/after)
|
||||||
|
#
|
||||||
|
# Runs on the box itself. Public hostnames are resolved to 127.0.0.1, so the
|
||||||
|
# request still goes through Caddy and its certificates, without leaving the box.
|
||||||
|
# No check asks for an answer of 401, 403 or 404 through Caddy: fail2ban counts
|
||||||
|
# those (episode 10) and would end up banning the box's own bridge address.
|
||||||
|
# Optional credentials in updates/health.env (see health.env.example):
|
||||||
|
# FORGEJO_HEALTH_TOKEN read-only token, for the git ref check on a private repo
|
||||||
|
# FORGEJO_HEALTH_REPO owner/name
|
||||||
|
# NTFY_HEALTH_AUTH user:password with read-write on the topic "healthcheck"
|
||||||
|
set -uo pipefail
|
||||||
|
|
||||||
|
STACK_DIR=${STACK_DIR:-/opt/selfhostedops-stack}
|
||||||
|
P=${PROJECT:-selfhostedops-stack}
|
||||||
|
set -a
|
||||||
|
. "$STACK_DIR/.env"
|
||||||
|
[ -f "$STACK_DIR/updates/health.env" ] && . "$STACK_DIR/updates/health.env"
|
||||||
|
set +a
|
||||||
|
D=$DOMAIN
|
||||||
|
SQLITE_IMAGE=${SQLITE_IMAGE:-keinos/sqlite3:latest}
|
||||||
|
|
||||||
|
DATA=0; GROUPS_WANTED=()
|
||||||
|
for a in "$@"; do [ "$a" = "--data" ] && DATA=1 || GROUPS_WANTED+=("$a"); done
|
||||||
|
want() { [ ${#GROUPS_WANTED[@]} -eq 0 ] || [[ " ${GROUPS_WANTED[*]} " == *" $1 "* ]]; }
|
||||||
|
|
||||||
|
FAILS=0; TOTAL=0
|
||||||
|
ok() { TOTAL=$((TOTAL+1)); printf 'PASS %-12s %s\n' "$1" "$2"; }
|
||||||
|
bad() { TOTAL=$((TOTAL+1)); FAILS=$((FAILS+1)); printf 'FAIL %-12s %s\n' "$1" "$2"; }
|
||||||
|
# check <group> <description> <actual> <expected regex>
|
||||||
|
check() { if [[ "$3" =~ $4 ]]; then ok "$1" "$2"; else bad "$1" "$2 (got: $(printf '%s' "$3" | tr '\n' ' ' | cut -c1-90))"; fi; }
|
||||||
|
|
||||||
|
pub() { local host=$1 path=$2; shift 2; curl -sS -m 8 --resolve "$host.$D:443:127.0.0.1" "$@" "https://$host.$D$path" 2>&1; }
|
||||||
|
code() { pub "$1" "$2" -o /dev/null -w '%{http_code}' "${@:3}"; }
|
||||||
|
loc() { pub "$1" "$2" -o /dev/null -w '%{redirect_url}' "${@:3}"; }
|
||||||
|
cip() { docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}' "$1" 2>/dev/null | cut -d' ' -f1; }
|
||||||
|
sq() { docker run --rm --user 0 -v "$1:/src:ro" "$SQLITE_IMAGE" sqlite3 "file:/src/$2?mode=ro" "$3" 2>&1; }
|
||||||
|
dx() { docker exec "$@" 2>&1; }
|
||||||
|
|
||||||
|
if want caddy; then
|
||||||
|
check caddy "config is valid" "$(dx $P-caddy-1 caddy validate --config /etc/caddy/Caddyfile | grep -o 'Valid configuration')" 'Valid configuration'
|
||||||
|
check caddy "plugins compiled in: rate_limit, crowdsec, appsec" "$(dx $P-caddy-1 caddy list-modules | grep -cE '^http.handlers.(rate_limit|crowdsec|appsec)$')" '^3$'
|
||||||
|
check caddy "certificate for git.$D is valid" "$(code git /api/healthz)" '^200$'
|
||||||
|
if command -v cscli >/dev/null; then
|
||||||
|
age=$(( $(date +%s) - $(date -d "$(cscli bouncers list -o json | jq -r '.[]|select(.type=="caddy-cs-bouncer")|.last_pull')" +%s 2>/dev/null || echo 0) ))
|
||||||
|
check caddy "CrowdSec bouncer pulled decisions in the last 60 s" "$age" '^([0-9]|[1-5][0-9])$'
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if want forgejo; then
|
||||||
|
check forgejo "API healthz: database and cache pass" "$(pub git /api/healthz | jq -r '.status' 2>&1)" '^pass$'
|
||||||
|
if [ -n "${FORGEJO_HEALTH_TOKEN:-}" ]; then
|
||||||
|
check forgejo "git over HTTPS: refs of ${FORGEJO_HEALTH_REPO}" "$(pub git "/${FORGEJO_HEALTH_REPO}.git/info/refs?service=git-upload-pack" -u "health:$FORGEJO_HEALTH_TOKEN" | tr -d '\0' | grep -ac 'refs/heads/')" '^[1-9]'
|
||||||
|
fi
|
||||||
|
check forgejo "git over SSH: port 2222 answers" "$(timeout 3 bash -c 'exec 3<>/dev/tcp/127.0.0.1/2222; head -c 8 <&3' 2>&1)" '^SSH-2.0'
|
||||||
|
check forgejo "SSO: login button leads to id.$D" "$(loc git /user/oauth2/PocketID)" "^https://id\.$D/authorize"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if want outline; then
|
||||||
|
check outline "postgres accepts connections" "$(dx $P-outline-postgres-1 pg_isready -U outline)" 'accepting connections'
|
||||||
|
check outline "redis answers PING" "$(dx $P-outline-redis-1 redis-cli ping)" '^PONG'
|
||||||
|
check outline "_health" "$(code wiki /_health)" '^200$'
|
||||||
|
check outline "API auth.config lists Pocket ID" "$(pub wiki /api/auth.config -X POST | jq -r '.data.providers[0].name' 2>&1)" '^Pocket ID$'
|
||||||
|
check outline "SSO: login leads to id.$D" "$(loc wiki /auth/oidc)" "^https://id\.$D/authorize"
|
||||||
|
check outline "database answers a query" "$(dx $P-outline-postgres-1 psql -U outline -tAc 'select count(*) from documents')" '^[0-9]+$'
|
||||||
|
fi
|
||||||
|
|
||||||
|
if want vaultwarden; then
|
||||||
|
check vaultwarden "alive" "$(code vault /alive)" '^200$'
|
||||||
|
check vaultwarden "API config reports a version" "$(pub vault /api/config | jq -r '.version' 2>&1)" '^20[0-9][0-9]\.'
|
||||||
|
check vaultwarden "SSO: prevalidate hands out a token" "$(pub vault '/identity/sso/prevalidate?domainHint=x' | jq -r '.token|length>20' 2>&1)" '^true$'
|
||||||
|
fi
|
||||||
|
|
||||||
|
if want kuma; then
|
||||||
|
# The status page API is asked on the container directly, not through Caddy: while Kuma migrates
|
||||||
|
# its database (1.x -> 2.x took five minutes here) it answers 404, and a health check that polls
|
||||||
|
# 404s through Caddy gets the box's own bridge address banned by fail2ban.
|
||||||
|
k="http://$(cip $P-uptime-kuma-1):3001"
|
||||||
|
sp=$(curl -sS -m 8 "$k/api/status-page/status" 2>&1)
|
||||||
|
check kuma "public status page API" "$(jq -r '.config.slug' <<<"$sp" 2>&1)" '^status$'
|
||||||
|
check kuma "status page shows 4 public monitors" "$(jq -r '[.publicGroupList[].monitorList[]]|length' <<<"$sp" 2>&1)" '^4$'
|
||||||
|
hb=$(curl -sS -m 8 "$k/api/status-page/heartbeat/status" 2>&1 | jq -r '[.heartbeatList[]|last]|"\(length) \([.[]|select(.status==1)]|length) \(now - ([.[].time|sub("\\.[0-9]+$";"")|sub(" ";"T")+"Z"|fromdate]|min)|floor)"' 2>&1)
|
||||||
|
check kuma "every public monitor: last heartbeat UP and younger than 180 s (monitors up age)" "$hb" '^([0-9]+) \1 ([0-9]|[1-9][0-9]|1[0-7][0-9])$'
|
||||||
|
check kuma "dashboard is behind the login gate" "$(loc status /)" "^https://auth\.$D/login"
|
||||||
|
down=$(sq ${P}_kuma_data kuma.db "select count(*) from monitor m where active=1 and (select status from heartbeat h where h.monitor_id=m.id order by time desc limit 1) is not 1")
|
||||||
|
check kuma "no active monitor is DOWN in the database" "$down" '^0$'
|
||||||
|
fi
|
||||||
|
|
||||||
|
if want ntfy; then
|
||||||
|
check ntfy "health" "$(pub ntfy /v1/health | jq -r '.healthy' 2>&1)" '^true$'
|
||||||
|
if [ -n "${NTFY_HEALTH_AUTH:-}" ]; then
|
||||||
|
id="health-$(date +%s)-$RANDOM"
|
||||||
|
pub ntfy /healthcheck -u "$NTFY_HEALTH_AUTH" -d "$id" -o /dev/null
|
||||||
|
check ntfy "a published message can be read back" "$(pub ntfy '/healthcheck/json?poll=1&since=1m' -u "$NTFY_HEALTH_AUTH" | grep -c "$id")" '^1$'
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if want n8n; then
|
||||||
|
n=$(cip $P-n8n-1)
|
||||||
|
check n8n "readiness (database reachable, migrations done)" "$(curl -sS -m 8 http://$n:5678/healthz/readiness 2>&1)" '"status":"ok"'
|
||||||
|
check n8n "active workflows (at least ${N8N_EXPECT_ACTIVE:-1})" "$(dx $P-n8n-1 n8n list:workflow --active=true | grep -c '|')" "^(${N8N_EXPECT_ACTIVE:-[1-9][0-9]*})$"
|
||||||
|
# 403 = the webhook is registered and asks for its token. 404 = workflow not active.
|
||||||
|
check n8n "webhook kuma-incident is registered and asks for its token" "$(curl -sS -m 8 -o /dev/null -w '%{http_code}' -X POST http://$n:5678/webhook/kuma-incident -d '{}' 2>&1)" '^403$'
|
||||||
|
fi
|
||||||
|
|
||||||
|
if want pocket-id; then
|
||||||
|
check pocket-id "healthz" "$(code id /healthz)" '^20[04]$'
|
||||||
|
check pocket-id "discovery document names the right issuer" "$(pub id /.well-known/openid-configuration | jq -r '.issuer' 2>&1)" "^https://id\.$D$"
|
||||||
|
check pocket-id "signing keys are published" "$(pub id /.well-known/jwks.json | jq -r '.keys|length' 2>&1)" '^[1-9]'
|
||||||
|
# A made-up code with the real client secret must fail as invalid_grant. invalid_client would mean the client is gone.
|
||||||
|
check pocket-id "token endpoint still knows the Outline client" "$(curl -sS -m 8 -X POST http://$(cip $P-pocket-id-1):1411/api/oidc/token -d grant_type=authorization_code -d code=health -d client_id=$OUTLINE_OIDC_CLIENT_ID -d client_secret=$OUTLINE_OIDC_CLIENT_SECRET -d redirect_uri=https://wiki.$D/auth/oidc.callback 2>&1 | jq -r '.error' 2>&1)" '^invalid_grant$'
|
||||||
|
fi
|
||||||
|
|
||||||
|
if want tinyauth; then
|
||||||
|
check tinyauth "healthz" "$(pub auth /api/healthz | jq -r '.message' 2>&1)" '^Healthy$'
|
||||||
|
check tinyauth "login is handed to id.$D" "$(pub auth /api/oauth/url/pocketid | jq -r '.url' 2>&1)" "^https://id\.$D/authorize"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if want netbird; then
|
||||||
|
check netbird "dashboard" "$(code vpn /)" '^200$'
|
||||||
|
check netbird "embedded identity provider discovery" "$(pub vpn /oauth2/.well-known/openid-configuration | jq -r '.issuer' 2>&1)" "^https://vpn\.$D"
|
||||||
|
check netbird "management API alive and asks for a login" "$(curl -sS -m 8 -o /dev/null -w '%{http_code}' http://127.0.0.1:8081/api/users 2>&1)" '^401$'
|
||||||
|
check netbird "this host: management and signal connected" "$(netbird status 2>&1 | grep -cE '^(Management|Signal): Connected')" '^2$'
|
||||||
|
check netbird "this host: at least one peer connected" "$(netbird status 2>&1 | grep -oE 'Peers count: [0-9]+' | grep -oE '[0-9]+$')" '^[1-9]'
|
||||||
|
if docker inspect mesh-client >/dev/null 2>&1; then
|
||||||
|
check netbird "mesh-client container: management connected" "$(dx mesh-client netbird status | grep -c '^Management: Connected')" '^1$'
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if want ops; then
|
||||||
|
check ops "ops-readonly healthz" "$(curl -sS -m 8 http://$(cip $P-ops-readonly-1):8098/healthz 2>&1 | jq -r '.status' 2>&1)" '^ok$'
|
||||||
|
check ops "ops-actions healthz" "$(curl -sS -m 8 http://$(cip $P-ops-actions-1):8099/healthz 2>&1 | jq -r '.status' 2>&1)" '^ok$'
|
||||||
|
fi
|
||||||
|
|
||||||
|
if want backup; then
|
||||||
|
check backup "nightly timer is scheduled" "$(systemctl is-active selfhostedops-backup.timer 2>&1)" '^active$'
|
||||||
|
last=$(grep '=== backup end ===' "$STACK_DIR/backup/backup.log" | tail -1 | cut -c1-19)
|
||||||
|
check backup "last finished backup is younger than 26 h ($last)" "$(( ( $(date +%s) - $(date -d "$last" +%s 2>/dev/null || echo 0) ) / 3600 ))" '^([0-9]|1[0-9]|2[0-5])$'
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ $DATA = 1 ]; then
|
||||||
|
echo
|
||||||
|
echo "DATA forgejo repositories $(sq ${P}_forgejo_data gitea/gitea.db 'select count(*) from repository'), users $(sq ${P}_forgejo_data gitea/gitea.db 'select count(*) from user where type=0')"
|
||||||
|
echo "DATA outline documents $(dx $P-outline-postgres-1 psql -U outline -tAc 'select count(*) from documents'), users $(dx $P-outline-postgres-1 psql -U outline -tAc 'select count(*) from users'), migrations $(dx $P-outline-postgres-1 psql -U outline -tAc 'select count(*) from "SequelizeMeta"')"
|
||||||
|
echo "DATA vaultwarden users $(sq ${P}_vaultwarden_data db.sqlite3 'select count(*) from users'), ciphers $(sq ${P}_vaultwarden_data db.sqlite3 'select count(*) from ciphers')"
|
||||||
|
echo "DATA kuma monitors $(sq ${P}_kuma_data kuma.db 'select count(*) from monitor'), notifications $(sq ${P}_kuma_data kuma.db 'select count(*) from notification'), heartbeats $(sq ${P}_kuma_data kuma.db 'select count(*) from heartbeat')"
|
||||||
|
echo "DATA n8n workflows $(sq ${P}_n8n_data database.sqlite 'select count(*) from workflow_entity'), credentials $(sq ${P}_n8n_data database.sqlite 'select count(*) from credentials_entity'), executions $(sq ${P}_n8n_data database.sqlite 'select count(*) from execution_entity'), migrations $(sq ${P}_n8n_data database.sqlite 'select count(*) from migrations')"
|
||||||
|
echo "DATA pocket-id users $(sq ${P}_pocket_id_data pocket-id.db 'select count(*) from users'), clients $(sq ${P}_pocket_id_data pocket-id.db 'select count(*) from oidc_clients')"
|
||||||
|
echo "DATA netbird peers $(sq netbird_netbird_data store.db 'select count(*) from peers')"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "$((TOTAL-FAILS)) of $TOTAL checks passed, $FAILS failed"
|
||||||
|
exit $FAILS
|
||||||
171
updates/update.sh
Executable file
171
updates/update.sh
Executable file
|
|
@ -0,0 +1,171 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# ~$ selfhostedops - update one service at a time (episode 17)
|
||||||
|
#
|
||||||
|
# "docker compose pull && docker compose up -d" updates everything at once and
|
||||||
|
# tells you nothing about which of the eleven changes broke the login. This
|
||||||
|
# does the same work in order:
|
||||||
|
#
|
||||||
|
# for each service: pull (or build) the image named in the compose file
|
||||||
|
# -> snapshot (backup/backup.sh, tag pre-update)
|
||||||
|
# -> keep the running image under a name of its own
|
||||||
|
# -> recreate that one service
|
||||||
|
# -> wait for updates/health.sh to pass, first the group
|
||||||
|
# of the service, then everything
|
||||||
|
# -> red: put image and data back, stop the whole run
|
||||||
|
#
|
||||||
|
# updates/update.sh plan what would change, nothing is touched
|
||||||
|
# updates/update.sh run all services, in the order below
|
||||||
|
# updates/update.sh run n8n outline only these
|
||||||
|
# TIMEOUT=900 updates/update.sh run uptime-kuma wait longer for one service
|
||||||
|
#
|
||||||
|
# You pick the versions: write the new tag into the compose file first (and
|
||||||
|
# commit it). The script never invents a version, it only rolls out what the
|
||||||
|
# file says. Runs on the box, as root. Needs backup/.restic-env (episode 7).
|
||||||
|
#
|
||||||
|
# Two things pile up and are yours to clear once you trust the new state:
|
||||||
|
# restic forget --tag pre-update --keep-last 3 --prune the snapshots
|
||||||
|
# docker image rm $(docker images -q 'selfhostedops-rollback/*') the old images
|
||||||
|
# A service that needs longer than TIMEOUT to come back counts as red and is
|
||||||
|
# put back. A database migration can take minutes (Uptime Kuma 1 -> 2: five).
|
||||||
|
set -uo pipefail
|
||||||
|
|
||||||
|
STACK_DIR=${STACK_DIR:-/opt/selfhostedops-stack}
|
||||||
|
NETBIRD_DIR=${NETBIRD_DIR:-/opt/netbird}
|
||||||
|
P=${PROJECT:-selfhostedops-stack}
|
||||||
|
VOLUMES=/var/lib/docker/volumes
|
||||||
|
TIMEOUT=${TIMEOUT:-180} # seconds a service gets to turn green
|
||||||
|
STAGE=/var/lib/selfhostedops-restore
|
||||||
|
ASIDE=/var/lib/selfhostedops-failed-update
|
||||||
|
LOG=$STACK_DIR/updates/update.log
|
||||||
|
|
||||||
|
# Least critical first, the monitoring last: it should watch the others move.
|
||||||
|
ORDER="ntfy forgejo outline-postgres outline-redis outline vaultwarden n8n pocket-id tinyauth netbird-server dashboard ops-actions ops-readonly caddy uptime-kuma"
|
||||||
|
|
||||||
|
# service -> compose directory | health groups | volumes | dump file=volume/path ...
|
||||||
|
info() {
|
||||||
|
case $1 in
|
||||||
|
ntfy) echo "$STACK_DIR|ntfy|${P}_ntfy_data ${P}_ntfy_cache|ntfy-user.db=${P}_ntfy_data/user.db ntfy-cache.db=${P}_ntfy_cache/cache.db" ;;
|
||||||
|
forgejo) echo "$STACK_DIR|forgejo|${P}_forgejo_data|forgejo.db=${P}_forgejo_data/gitea/gitea.db" ;;
|
||||||
|
outline-postgres) echo "$STACK_DIR|outline|${P}_outline_pg|" ;;
|
||||||
|
outline-redis) echo "$STACK_DIR|outline||" ;;
|
||||||
|
outline) echo "$STACK_DIR|outline|${P}_outline_data|" ;;
|
||||||
|
vaultwarden) echo "$STACK_DIR|vaultwarden|${P}_vaultwarden_data|vaultwarden.sqlite3=${P}_vaultwarden_data/db.sqlite3" ;;
|
||||||
|
n8n) echo "$STACK_DIR|n8n|${P}_n8n_data|n8n.sqlite=${P}_n8n_data/database.sqlite" ;;
|
||||||
|
pocket-id) echo "$STACK_DIR|pocket-id tinyauth|${P}_pocket_id_data|pocket-id.db=${P}_pocket_id_data/pocket-id.db" ;;
|
||||||
|
tinyauth) echo "$STACK_DIR|tinyauth kuma|${P}_tinyauth_data|tinyauth.db=${P}_tinyauth_data/tinyauth.db" ;;
|
||||||
|
uptime-kuma) echo "$STACK_DIR|kuma|${P}_kuma_data|kuma.db=${P}_kuma_data/kuma.db" ;;
|
||||||
|
caddy) echo "$STACK_DIR|caddy||" ;;
|
||||||
|
ops-actions|ops-readonly) echo "$STACK_DIR|ops||" ;;
|
||||||
|
netbird-server) echo "$NETBIRD_DIR|netbird|netbird_netbird_data|netbird-store.db=netbird_netbird_data/store.db netbird-idp.db=netbird_netbird_data/idp.db netbird-events.db=netbird_netbird_data/events.db" ;;
|
||||||
|
dashboard) echo "$NETBIRD_DIR|netbird||" ;;
|
||||||
|
*) return 1 ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
log() { printf '%s %s\n' "$(date -u '+%H:%M:%S')" "$*" | tee -a "$LOG"; }
|
||||||
|
# profiles: ops-actions and ops-readonly only exist with --profile agent
|
||||||
|
dc() { ( cd "$1" && shift && docker compose --profile agent "$@" ); }
|
||||||
|
cid() { dc "$1" ps -q "$2" 2>/dev/null | head -1; }
|
||||||
|
running_image() { local c; c=$(cid "$1" "$2"); [ -n "$c" ] && docker inspect -f '{{.Image}}' "$c"; }
|
||||||
|
target_ref() { dc "$1" config --format json 2>/dev/null | jq -r --arg s "$2" '.services[$s].image'; }
|
||||||
|
has_build() { dc "$1" config --format json 2>/dev/null | jq -e --arg s "$2" '.services[$s].build' >/dev/null; }
|
||||||
|
|
||||||
|
# Fetch the image the compose file names, without touching the running container.
|
||||||
|
fetch() {
|
||||||
|
if has_build "$1" "$2"; then dc "$1" build --pull "$2" >>"$LOG" 2>&1; else dc "$1" pull -q "$2" >>"$LOG" 2>&1; fi
|
||||||
|
}
|
||||||
|
|
||||||
|
wait_green() { # wait_green <seconds> <health groups...>
|
||||||
|
local until=$(( $(date +%s) + $1 )); shift
|
||||||
|
while :; do
|
||||||
|
bash "$STACK_DIR/updates/health.sh" "$@" >"$STACK_DIR/updates/.last-health" 2>&1 && return 0
|
||||||
|
[ "$(date +%s)" -ge "$until" ] && return 1
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
put_back() { # put_back <service> <dir> <old image id> <snapshot id>
|
||||||
|
local svc=$1 dir=$2 old=$3 snap=$4 vols dumps v rel dst src
|
||||||
|
IFS='|' read -r _ _ vols dumps <<<"$(info "$svc")"
|
||||||
|
log " ROLLBACK $svc: stop, data from snapshot $snap, image $(echo "$old" | cut -c8-19)"
|
||||||
|
dc "$dir" stop "$svc" >>"$LOG" 2>&1
|
||||||
|
case $svc in outline|outline-postgres) dc "$dir" stop outline >>"$LOG" 2>&1 ;; esac
|
||||||
|
rm -rf "$STAGE"; mkdir -p "$STAGE" "$ASIDE"
|
||||||
|
local inc=(--include "$STACK_DIR/backup/dumps"); for v in $vols; do inc+=(--include "$VOLUMES/$v"); done
|
||||||
|
restic restore "$snap" --target "$STAGE" "${inc[@]}" >>"$LOG" 2>&1 || { log " restic restore FAILED, nothing changed on disk"; return 1; }
|
||||||
|
for v in $vols; do # the failed state is moved aside, not deleted
|
||||||
|
rm -rf "$ASIDE/$v"; mv "$VOLUMES/$v/_data" "$ASIDE/$v"; mv "$STAGE$VOLUMES/$v/_data" "$VOLUMES/$v/_data"
|
||||||
|
done
|
||||||
|
for v in $dumps; do # consistent online dump over the raw file copy
|
||||||
|
src="$STAGE$STACK_DIR/backup/dumps/${v%%=*}"; rel=${v#*=}; dst="$VOLUMES/${rel%%/*}/_data/${rel#*/}"
|
||||||
|
rm -f "$dst-wal" "$dst-shm"; cp "$src" "$dst.new"; chown --reference="$dst" "$dst.new"; chmod --reference="$dst" "$dst.new"; mv "$dst.new" "$dst"
|
||||||
|
done
|
||||||
|
docker tag "$old" "selfhostedops-rollback/$svc:previous"
|
||||||
|
printf 'services:\n %s:\n image: selfhostedops-rollback/%s:previous\n' "$svc" "$svc" > "$dir/compose.rollback.yml"
|
||||||
|
local files=(-f "$(ls "$dir"/compose.yml "$dir"/docker-compose.yml 2>/dev/null | head -1)" -f "$dir/compose.rollback.yml")
|
||||||
|
reload_pg() { # empty the schema and load the pg_dump from the snapshot
|
||||||
|
for _ in $(seq 60); do docker exec "$P-outline-postgres-1" pg_isready -U outline >/dev/null 2>&1 && break; sleep 1; done
|
||||||
|
docker exec "$P-outline-postgres-1" psql -U outline -q -c 'drop schema public cascade; create schema public;' outline >>"$LOG" 2>&1
|
||||||
|
zcat "$STAGE$STACK_DIR/backup/dumps/outline.sql.gz" | docker exec -i "$P-outline-postgres-1" psql -U outline -q outline >>"$LOG" 2>&1
|
||||||
|
}
|
||||||
|
case $svc in
|
||||||
|
outline) reload_pg; dc "$dir" "${files[@]}" up -d --no-deps outline >>"$LOG" 2>&1 ;;
|
||||||
|
outline-postgres) dc "$dir" "${files[@]}" up -d --no-deps outline-postgres >>"$LOG" 2>&1; reload_pg; dc "$dir" up -d --no-deps outline >>"$LOG" 2>&1 ;;
|
||||||
|
*) dc "$dir" "${files[@]}" up -d --no-deps "$svc" >>"$LOG" 2>&1 ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
update_one() {
|
||||||
|
local svc=$1 dir groups old new snap t0 t1
|
||||||
|
IFS='|' read -r dir groups _ _ <<<"$(info "$svc")" || { log "unknown service $svc"; return 2; }
|
||||||
|
[ -n "$(cid "$dir" "$svc")" ] || { log "$svc: not running, skipped"; return 0; }
|
||||||
|
old=$(running_image "$dir" "$svc")
|
||||||
|
log "$svc: fetching $(target_ref "$dir" "$svc")"
|
||||||
|
fetch "$dir" "$svc" || { log "$svc: FETCH FAILED, service untouched"; return 1; }
|
||||||
|
new=$(docker image inspect -f '{{.Id}}' "$(target_ref "$dir" "$svc")" 2>/dev/null)
|
||||||
|
if [ "$old" = "$new" ]; then log "$svc: already on that image, nothing to do"; return 0; fi
|
||||||
|
|
||||||
|
log "$svc: snapshot before the change"
|
||||||
|
BACKUP_TAG=pre-update bash "$STACK_DIR/backup/backup.sh" >>"$LOG" 2>&1 || { log "$svc: BACKUP FAILED, service untouched"; return 1; }
|
||||||
|
snap=$(grep -oE 'snapshot [0-9a-f]{8} saved' "$STACK_DIR/backup/backup.log" | tail -1 | cut -d' ' -f2)
|
||||||
|
docker tag "$old" "selfhostedops-rollback/$svc:previous" # a name keeps the old image from being pruned
|
||||||
|
|
||||||
|
t0=$(date +%s)
|
||||||
|
log "$svc: recreate ($(echo "$old" | cut -c8-19) -> $(echo "$new" | cut -c8-19)), snapshot $snap"
|
||||||
|
dc "$dir" up -d --no-deps "$svc" >>"$LOG" 2>&1
|
||||||
|
# shellcheck disable=SC2086
|
||||||
|
if wait_green "$TIMEOUT" $groups && wait_green "$TIMEOUT"; then
|
||||||
|
t1=$(date +%s); log "$svc: GREEN after $((t1 - t0)) s ($(tail -1 "$STACK_DIR/updates/.last-health"))"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
t1=$(date +%s); log "$svc: RED after $((t1 - t0)) s"
|
||||||
|
grep '^FAIL' "$STACK_DIR/updates/.last-health" | tee -a "$LOG"
|
||||||
|
put_back "$svc" "$dir" "$old" "$snap"
|
||||||
|
if wait_green "$TIMEOUT"; then log "$svc: rolled back, health check green again after $(( $(date +%s) - t1 )) s"
|
||||||
|
else log "$svc: rolled back, but the health check is STILL RED"; grep '^FAIL' "$STACK_DIR/updates/.last-health" | tee -a "$LOG"; fi
|
||||||
|
log "$svc: the compose file still names the new tag. Set it back before the next 'up -d'."
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd=${1:-plan}; shift || true
|
||||||
|
set -a; . "$STACK_DIR/backup/.restic-env"; set +a
|
||||||
|
case $cmd in
|
||||||
|
plan)
|
||||||
|
# No registry lookup: with exact tags in the compose file, a different tag is the update.
|
||||||
|
printf '%-18s %-46s %s\n' SERVICE RUNNING TARGET
|
||||||
|
for svc in ${*:-$ORDER}; do
|
||||||
|
IFS='|' read -r dir _ _ _ <<<"$(info "$svc")"; c=$(cid "$dir" "$svc"); [ -n "$c" ] || continue
|
||||||
|
ref=$(target_ref "$dir" "$svc"); cur=$(docker inspect -f '{{.Config.Image}}' "$c")
|
||||||
|
case $cur in sha256:*) cur="(untagged) $(echo "$cur" | cut -c8-19)" ;; esac
|
||||||
|
note=""; [ "$cur" != "$ref" ] && note=" <- update"; has_build "$dir" "$svc" && note=" <- local build, rebuilt on run"
|
||||||
|
printf '%-18s %-46s %s%s\n' "$svc" "$cur" "$ref" "$note"
|
||||||
|
done ;;
|
||||||
|
run)
|
||||||
|
bash "$STACK_DIR/updates/health.sh" >/dev/null 2>&1 || { log "health check is red before the start, not updating anything"; bash "$STACK_DIR/updates/health.sh" | grep '^FAIL'; exit 1; }
|
||||||
|
start=$(date +%s); log "=== update run: ${*:-$ORDER} (timeout per service $TIMEOUT s) ==="
|
||||||
|
for svc in ${*:-$ORDER}; do
|
||||||
|
update_one "$svc" || { log "=== STOPPED at $svc after $(( $(date +%s) - start )) s, later services untouched ==="; exit 1; }
|
||||||
|
done
|
||||||
|
log "=== all done in $(( $(date +%s) - start )) s ===" ;;
|
||||||
|
*) sed -n '2,24p' "$0"; exit 2 ;;
|
||||||
|
esac
|
||||||
Loading…
Add table
Add a link
Reference in a new issue