diff --git a/.gitignore b/.gitignore index 828475f..237bfe5 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ logs/ ops-readonly/token.env offboarding/offboard.env __pycache__/ +updates/health.env diff --git a/backup/backup.sh b/backup/backup.sh index 6bcfb8d..3eb2b69 100755 --- a/backup/backup.sh +++ b/backup/backup.sh @@ -3,7 +3,8 @@ # # Application-consistent, without stopping anything: # 1. pg_dump of Outline's Postgres -> backup/dumps/outline.sql.gz -# 2. online SQLite backups (Forgejo, Vaultwarden, Uptime Kuma, ntfy, NetBird) +# 2. online SQLite backups (Forgejo, Vaultwarden, Uptime Kuma, ntfy, n8n, +# Pocket ID, tinyauth, NetBird) # -> backup/dumps/*.db # 3. restic backup of the stack directory, /opt/netbird and every stateful # docker volume (the raw files, plus the dumps from 1 and 2) @@ -22,6 +23,12 @@ LOG_FILE="$BACKUP_DIR/backup.log" PROJECT=${PROJECT:-selfhostedops-stack} VOLUMES=/var/lib/docker/volumes +# Tag of the snapshot. The nightly timer uses "nightly". updates/update.sh +# runs this script with BACKUP_TAG=pre-update: those snapshots are left alone +# by forget below. Without that, "keep 7 daily" would drop the snapshot from +# before the update as soon as a second backup runs on the same day (episode 17). +BACKUP_TAG=${BACKUP_TAG:-nightly} + # Small alpine image with a current sqlite3, used for the online SQLite copies. SQLITE_IMAGE=${SQLITE_IMAGE:-keinos/sqlite3:latest} @@ -77,6 +84,9 @@ sqlite_backup "${PROJECT}_vaultwarden_data" db.sqlite3 vaultwarden.sqlite3 sqlite_backup "${PROJECT}_kuma_data" kuma.db kuma.db sqlite_backup "${PROJECT}_ntfy_data" user.db ntfy-user.db sqlite_backup "${PROJECT}_ntfy_cache" cache.db ntfy-cache.db +sqlite_backup "${PROJECT}_n8n_data" database.sqlite n8n.sqlite +sqlite_backup "${PROJECT}_pocket_id_data" pocket-id.db pocket-id.db +sqlite_backup "${PROJECT}_tinyauth_data" tinyauth.db tinyauth.db sqlite_backup "netbird_netbird_data" store.db netbird-store.db sqlite_backup "netbird_netbird_data" idp.db netbird-idp.db sqlite_backup "netbird_netbird_data" events.db netbird-events.db @@ -91,12 +101,16 @@ EXCLUDES=(--exclude "$VOLUMES/metadata.db" --exclude "$VOLUMES/backingFsBlockDev for v in "${EXCLUDE_VOLUMES[@]}"; do EXCLUDES+=(--exclude "$VOLUMES/${PROJECT}_$v") done -restic backup --tag nightly "${EXCLUDES[@]}" \ +restic backup --tag "$BACKUP_TAG" "${EXCLUDES[@]}" \ "$STACK_DIR" "$NETBIRD_DIR" "$VOLUMES" -log "[4/5] restic forget + prune" -restic forget --tag nightly --group-by host,tags \ - --keep-daily 7 --keep-weekly 4 --keep-monthly 6 --prune +if [ "$BACKUP_TAG" = nightly ]; then + log "[4/5] restic forget + prune" + restic forget --tag nightly --group-by host,tags \ + --keep-daily 7 --keep-weekly 4 --keep-monthly 6 --prune +else + log "[4/5] skipped: snapshots tagged $BACKUP_TAG are kept until you remove them" +fi END=$(date +%s) DURATION=$((END - START)) diff --git a/caddy/Dockerfile b/caddy/Dockerfile index e3f943e..51248ba 100644 --- a/caddy/Dockerfile +++ b/caddy/Dockerfile @@ -3,11 +3,15 @@ # in with xcaddy - there is no runtime "enable" for it. # episode 12: same trick for CrowdSec. The bouncer (http) and the AppSec # handler (appsec) are plugins too, pinned to a release tag instead of @main. -FROM caddy:2-builder-alpine AS builder +# episode 17: every version is written down. "caddy:2-alpine" plus an unpinned +# plugin means the next rebuild is an update nobody decided on. docker compose +# pull never touches this image, it only changes when you run a build, so the +# compose file carries the same version in the image tag. +FROM caddy:2.11.7-builder-alpine AS builder RUN xcaddy build \ - --with github.com/mholt/caddy-ratelimit \ + --with github.com/mholt/caddy-ratelimit@v0.1.0 \ --with github.com/hslatman/caddy-crowdsec-bouncer/http@v0.14.1 \ --with github.com/hslatman/caddy-crowdsec-bouncer/appsec@v0.14.1 -FROM caddy:2-alpine +FROM caddy:2.11.7-alpine COPY --from=builder /usr/bin/caddy /usr/bin/caddy diff --git a/compose.yml b/compose.yml index 8cd4975..e0a601a 100644 --- a/compose.yml +++ b/compose.yml @@ -62,7 +62,7 @@ volumes: services: caddy: - image: selfhostedops/caddy:crowdsec # episode 12: caddy-ratelimit + CrowdSec bouncer + AppSec + image: selfhostedops/caddy:2.11.7-crowdsec0.14.1 # episode 12: caddy-ratelimit + CrowdSec bouncer + AppSec build: ./caddy restart: unless-stopped ports: @@ -83,7 +83,7 @@ services: networks: [proxy] forgejo: - image: codeberg.org/forgejo/forgejo:16 + image: codeberg.org/forgejo/forgejo:16.0.5 restart: unless-stopped environment: USER_UID: "1000" @@ -113,7 +113,7 @@ services: networks: [proxy] outline-postgres: - image: postgres:16-alpine + image: postgres:16.15-alpine restart: unless-stopped environment: POSTGRES_USER: outline @@ -124,12 +124,12 @@ services: networks: [internal] outline-redis: - image: redis:7-alpine + image: redis:7.4.11-alpine restart: unless-stopped networks: [internal] outline: - image: outlinewiki/outline:latest + image: outlinewiki/outline:1.10.1 restart: unless-stopped depends_on: [outline-postgres, outline-redis] environment: @@ -157,7 +157,7 @@ services: networks: [proxy, internal] vaultwarden: - image: vaultwarden/server:latest + image: vaultwarden/server:1.37.4 restart: unless-stopped environment: DOMAIN: https://vault.${DOMAIN} @@ -188,7 +188,7 @@ services: networks: [proxy] uptime-kuma: - image: louislam/uptime-kuma:1 + image: louislam/uptime-kuma:2.5.5 restart: unless-stopped volumes: - kuma_data:/app/data @@ -197,7 +197,7 @@ services: # Push notifications for the monitoring alerts (episode 6). # Uptime Kuma publishes to the topic "alerts", your phone subscribes to it. ntfy: - image: binwiederhier/ntfy:v2.28.0 + image: binwiederhier/ntfy:v2.29.0 restart: unless-stopped command: serve environment: @@ -220,7 +220,7 @@ services: n8n: # Pinned. With :latest every restore from backup is silently an upgrade, # and an upgrade is a thing you schedule, not a thing a restore does to you. - image: docker.n8n.io/n8nio/n8n:2.37.7 + image: docker.n8n.io/n8nio/n8n:2.42.5 restart: unless-stopped environment: N8N_HOST: n8n.${DOMAIN} @@ -264,7 +264,7 @@ services: # passkey, and switching one person off happens in one place. Pocket ID is # small enough for a 4 GB box; groups decide who gets which service. pocket-id: - image: ghcr.io/pocket-id/pocket-id:v2.16.0 + image: ghcr.io/pocket-id/pocket-id:v2.18.0 restart: unless-stopped environment: APP_URL: https://id.${DOMAIN} @@ -335,7 +335,7 @@ services: # # Start it with: docker compose --profile agent up -d ops-actions: - image: docker:cli + image: docker:29.8.2-cli profiles: [agent] restart: unless-stopped environment: diff --git a/updates/health.env.example b/updates/health.env.example new file mode 100644 index 0000000..550b5bb --- /dev/null +++ b/updates/health.env.example @@ -0,0 +1,9 @@ +# Optional credentials for updates/health.sh. Copy to health.env (mode 600). +# Read-only Forgejo token: forgejo admin user generate-access-token -u -t health --scopes read:repository --raw +FORGEJO_HEALTH_TOKEN= +FORGEJO_HEALTH_REPO=owner/repo +# ntfy user with read-write on the topic "healthcheck": +# ntfy user add health && ntfy access health healthcheck rw +NTFY_HEALTH_AUTH=health:password +# Number of active n8n workflows you expect (leave empty for "at least one") +N8N_EXPECT_ACTIVE= diff --git a/updates/health.sh b/updates/health.sh new file mode 100755 index 0000000..a814ee1 --- /dev/null +++ b/updates/health.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +# ~$ selfhostedops - health check for the whole stack (episode 17) +# +# "The container is up" is not a health check. Every line below asks a service +# to do the thing it is there for: answer its API, hand out git refs, deliver +# a push message, start an SSO login, keep the mesh connected. +# +# updates/health.sh all checks, exit code = number of failures +# updates/health.sh forgejo n8n only these groups +# updates/health.sh --data also print row counts (to compare before/after) +# +# Runs on the box itself. Public hostnames are resolved to 127.0.0.1, so the +# request still goes through Caddy and its certificates, without leaving the box. +# No check asks for an answer of 401, 403 or 404 through Caddy: fail2ban counts +# those (episode 10) and would end up banning the box's own bridge address. +# Optional credentials in updates/health.env (see health.env.example): +# FORGEJO_HEALTH_TOKEN read-only token, for the git ref check on a private repo +# FORGEJO_HEALTH_REPO owner/name +# NTFY_HEALTH_AUTH user:password with read-write on the topic "healthcheck" +set -uo pipefail + +STACK_DIR=${STACK_DIR:-/opt/selfhostedops-stack} +P=${PROJECT:-selfhostedops-stack} +set -a +. "$STACK_DIR/.env" +[ -f "$STACK_DIR/updates/health.env" ] && . "$STACK_DIR/updates/health.env" +set +a +D=$DOMAIN +SQLITE_IMAGE=${SQLITE_IMAGE:-keinos/sqlite3:latest} + +DATA=0; GROUPS_WANTED=() +for a in "$@"; do [ "$a" = "--data" ] && DATA=1 || GROUPS_WANTED+=("$a"); done +want() { [ ${#GROUPS_WANTED[@]} -eq 0 ] || [[ " ${GROUPS_WANTED[*]} " == *" $1 "* ]]; } + +FAILS=0; TOTAL=0 +ok() { TOTAL=$((TOTAL+1)); printf 'PASS %-12s %s\n' "$1" "$2"; } +bad() { TOTAL=$((TOTAL+1)); FAILS=$((FAILS+1)); printf 'FAIL %-12s %s\n' "$1" "$2"; } +# check +check() { if [[ "$3" =~ $4 ]]; then ok "$1" "$2"; else bad "$1" "$2 (got: $(printf '%s' "$3" | tr '\n' ' ' | cut -c1-90))"; fi; } + +pub() { local host=$1 path=$2; shift 2; curl -sS -m 8 --resolve "$host.$D:443:127.0.0.1" "$@" "https://$host.$D$path" 2>&1; } +code() { pub "$1" "$2" -o /dev/null -w '%{http_code}' "${@:3}"; } +loc() { pub "$1" "$2" -o /dev/null -w '%{redirect_url}' "${@:3}"; } +cip() { docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}' "$1" 2>/dev/null | cut -d' ' -f1; } +sq() { docker run --rm --user 0 -v "$1:/src:ro" "$SQLITE_IMAGE" sqlite3 "file:/src/$2?mode=ro" "$3" 2>&1; } +dx() { docker exec "$@" 2>&1; } + +if want caddy; then + check caddy "config is valid" "$(dx $P-caddy-1 caddy validate --config /etc/caddy/Caddyfile | grep -o 'Valid configuration')" 'Valid configuration' + check caddy "plugins compiled in: rate_limit, crowdsec, appsec" "$(dx $P-caddy-1 caddy list-modules | grep -cE '^http.handlers.(rate_limit|crowdsec|appsec)$')" '^3$' + check caddy "certificate for git.$D is valid" "$(code git /api/healthz)" '^200$' + if command -v cscli >/dev/null; then + age=$(( $(date +%s) - $(date -d "$(cscli bouncers list -o json | jq -r '.[]|select(.type=="caddy-cs-bouncer")|.last_pull')" +%s 2>/dev/null || echo 0) )) + check caddy "CrowdSec bouncer pulled decisions in the last 60 s" "$age" '^([0-9]|[1-5][0-9])$' + fi +fi + +if want forgejo; then + check forgejo "API healthz: database and cache pass" "$(pub git /api/healthz | jq -r '.status' 2>&1)" '^pass$' + if [ -n "${FORGEJO_HEALTH_TOKEN:-}" ]; then + check forgejo "git over HTTPS: refs of ${FORGEJO_HEALTH_REPO}" "$(pub git "/${FORGEJO_HEALTH_REPO}.git/info/refs?service=git-upload-pack" -u "health:$FORGEJO_HEALTH_TOKEN" | tr -d '\0' | grep -ac 'refs/heads/')" '^[1-9]' + fi + check forgejo "git over SSH: port 2222 answers" "$(timeout 3 bash -c 'exec 3<>/dev/tcp/127.0.0.1/2222; head -c 8 <&3' 2>&1)" '^SSH-2.0' + check forgejo "SSO: login button leads to id.$D" "$(loc git /user/oauth2/PocketID)" "^https://id\.$D/authorize" +fi + +if want outline; then + check outline "postgres accepts connections" "$(dx $P-outline-postgres-1 pg_isready -U outline)" 'accepting connections' + check outline "redis answers PING" "$(dx $P-outline-redis-1 redis-cli ping)" '^PONG' + check outline "_health" "$(code wiki /_health)" '^200$' + check outline "API auth.config lists Pocket ID" "$(pub wiki /api/auth.config -X POST | jq -r '.data.providers[0].name' 2>&1)" '^Pocket ID$' + check outline "SSO: login leads to id.$D" "$(loc wiki /auth/oidc)" "^https://id\.$D/authorize" + check outline "database answers a query" "$(dx $P-outline-postgres-1 psql -U outline -tAc 'select count(*) from documents')" '^[0-9]+$' +fi + +if want vaultwarden; then + check vaultwarden "alive" "$(code vault /alive)" '^200$' + check vaultwarden "API config reports a version" "$(pub vault /api/config | jq -r '.version' 2>&1)" '^20[0-9][0-9]\.' + check vaultwarden "SSO: prevalidate hands out a token" "$(pub vault '/identity/sso/prevalidate?domainHint=x' | jq -r '.token|length>20' 2>&1)" '^true$' +fi + +if want kuma; then + # The status page API is asked on the container directly, not through Caddy: while Kuma migrates + # its database (1.x -> 2.x took five minutes here) it answers 404, and a health check that polls + # 404s through Caddy gets the box's own bridge address banned by fail2ban. + k="http://$(cip $P-uptime-kuma-1):3001" + sp=$(curl -sS -m 8 "$k/api/status-page/status" 2>&1) + check kuma "public status page API" "$(jq -r '.config.slug' <<<"$sp" 2>&1)" '^status$' + check kuma "status page shows 4 public monitors" "$(jq -r '[.publicGroupList[].monitorList[]]|length' <<<"$sp" 2>&1)" '^4$' + hb=$(curl -sS -m 8 "$k/api/status-page/heartbeat/status" 2>&1 | jq -r '[.heartbeatList[]|last]|"\(length) \([.[]|select(.status==1)]|length) \(now - ([.[].time|sub("\\.[0-9]+$";"")|sub(" ";"T")+"Z"|fromdate]|min)|floor)"' 2>&1) + check kuma "every public monitor: last heartbeat UP and younger than 180 s (monitors up age)" "$hb" '^([0-9]+) \1 ([0-9]|[1-9][0-9]|1[0-7][0-9])$' + check kuma "dashboard is behind the login gate" "$(loc status /)" "^https://auth\.$D/login" + down=$(sq ${P}_kuma_data kuma.db "select count(*) from monitor m where active=1 and (select status from heartbeat h where h.monitor_id=m.id order by time desc limit 1) is not 1") + check kuma "no active monitor is DOWN in the database" "$down" '^0$' +fi + +if want ntfy; then + check ntfy "health" "$(pub ntfy /v1/health | jq -r '.healthy' 2>&1)" '^true$' + if [ -n "${NTFY_HEALTH_AUTH:-}" ]; then + id="health-$(date +%s)-$RANDOM" + pub ntfy /healthcheck -u "$NTFY_HEALTH_AUTH" -d "$id" -o /dev/null + check ntfy "a published message can be read back" "$(pub ntfy '/healthcheck/json?poll=1&since=1m' -u "$NTFY_HEALTH_AUTH" | grep -c "$id")" '^1$' + fi +fi + +if want n8n; then + n=$(cip $P-n8n-1) + check n8n "readiness (database reachable, migrations done)" "$(curl -sS -m 8 http://$n:5678/healthz/readiness 2>&1)" '"status":"ok"' + check n8n "active workflows (at least ${N8N_EXPECT_ACTIVE:-1})" "$(dx $P-n8n-1 n8n list:workflow --active=true | grep -c '|')" "^(${N8N_EXPECT_ACTIVE:-[1-9][0-9]*})$" + # 403 = the webhook is registered and asks for its token. 404 = workflow not active. + check n8n "webhook kuma-incident is registered and asks for its token" "$(curl -sS -m 8 -o /dev/null -w '%{http_code}' -X POST http://$n:5678/webhook/kuma-incident -d '{}' 2>&1)" '^403$' +fi + +if want pocket-id; then + check pocket-id "healthz" "$(code id /healthz)" '^20[04]$' + check pocket-id "discovery document names the right issuer" "$(pub id /.well-known/openid-configuration | jq -r '.issuer' 2>&1)" "^https://id\.$D$" + check pocket-id "signing keys are published" "$(pub id /.well-known/jwks.json | jq -r '.keys|length' 2>&1)" '^[1-9]' + # A made-up code with the real client secret must fail as invalid_grant. invalid_client would mean the client is gone. + check pocket-id "token endpoint still knows the Outline client" "$(curl -sS -m 8 -X POST http://$(cip $P-pocket-id-1):1411/api/oidc/token -d grant_type=authorization_code -d code=health -d client_id=$OUTLINE_OIDC_CLIENT_ID -d client_secret=$OUTLINE_OIDC_CLIENT_SECRET -d redirect_uri=https://wiki.$D/auth/oidc.callback 2>&1 | jq -r '.error' 2>&1)" '^invalid_grant$' +fi + +if want tinyauth; then + check tinyauth "healthz" "$(pub auth /api/healthz | jq -r '.message' 2>&1)" '^Healthy$' + check tinyauth "login is handed to id.$D" "$(pub auth /api/oauth/url/pocketid | jq -r '.url' 2>&1)" "^https://id\.$D/authorize" +fi + +if want netbird; then + check netbird "dashboard" "$(code vpn /)" '^200$' + check netbird "embedded identity provider discovery" "$(pub vpn /oauth2/.well-known/openid-configuration | jq -r '.issuer' 2>&1)" "^https://vpn\.$D" + check netbird "management API alive and asks for a login" "$(curl -sS -m 8 -o /dev/null -w '%{http_code}' http://127.0.0.1:8081/api/users 2>&1)" '^401$' + check netbird "this host: management and signal connected" "$(netbird status 2>&1 | grep -cE '^(Management|Signal): Connected')" '^2$' + check netbird "this host: at least one peer connected" "$(netbird status 2>&1 | grep -oE 'Peers count: [0-9]+' | grep -oE '[0-9]+$')" '^[1-9]' + if docker inspect mesh-client >/dev/null 2>&1; then + check netbird "mesh-client container: management connected" "$(dx mesh-client netbird status | grep -c '^Management: Connected')" '^1$' + fi +fi + +if want ops; then + check ops "ops-readonly healthz" "$(curl -sS -m 8 http://$(cip $P-ops-readonly-1):8098/healthz 2>&1 | jq -r '.status' 2>&1)" '^ok$' + check ops "ops-actions healthz" "$(curl -sS -m 8 http://$(cip $P-ops-actions-1):8099/healthz 2>&1 | jq -r '.status' 2>&1)" '^ok$' +fi + +if want backup; then + check backup "nightly timer is scheduled" "$(systemctl is-active selfhostedops-backup.timer 2>&1)" '^active$' + last=$(grep '=== backup end ===' "$STACK_DIR/backup/backup.log" | tail -1 | cut -c1-19) + check backup "last finished backup is younger than 26 h ($last)" "$(( ( $(date +%s) - $(date -d "$last" +%s 2>/dev/null || echo 0) ) / 3600 ))" '^([0-9]|1[0-9]|2[0-5])$' +fi + +if [ $DATA = 1 ]; then + echo + echo "DATA forgejo repositories $(sq ${P}_forgejo_data gitea/gitea.db 'select count(*) from repository'), users $(sq ${P}_forgejo_data gitea/gitea.db 'select count(*) from user where type=0')" + echo "DATA outline documents $(dx $P-outline-postgres-1 psql -U outline -tAc 'select count(*) from documents'), users $(dx $P-outline-postgres-1 psql -U outline -tAc 'select count(*) from users'), migrations $(dx $P-outline-postgres-1 psql -U outline -tAc 'select count(*) from "SequelizeMeta"')" + echo "DATA vaultwarden users $(sq ${P}_vaultwarden_data db.sqlite3 'select count(*) from users'), ciphers $(sq ${P}_vaultwarden_data db.sqlite3 'select count(*) from ciphers')" + echo "DATA kuma monitors $(sq ${P}_kuma_data kuma.db 'select count(*) from monitor'), notifications $(sq ${P}_kuma_data kuma.db 'select count(*) from notification'), heartbeats $(sq ${P}_kuma_data kuma.db 'select count(*) from heartbeat')" + echo "DATA n8n workflows $(sq ${P}_n8n_data database.sqlite 'select count(*) from workflow_entity'), credentials $(sq ${P}_n8n_data database.sqlite 'select count(*) from credentials_entity'), executions $(sq ${P}_n8n_data database.sqlite 'select count(*) from execution_entity'), migrations $(sq ${P}_n8n_data database.sqlite 'select count(*) from migrations')" + echo "DATA pocket-id users $(sq ${P}_pocket_id_data pocket-id.db 'select count(*) from users'), clients $(sq ${P}_pocket_id_data pocket-id.db 'select count(*) from oidc_clients')" + echo "DATA netbird peers $(sq netbird_netbird_data store.db 'select count(*) from peers')" +fi + +echo +echo "$((TOTAL-FAILS)) of $TOTAL checks passed, $FAILS failed" +exit $FAILS diff --git a/updates/update.sh b/updates/update.sh new file mode 100755 index 0000000..efad6e7 --- /dev/null +++ b/updates/update.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +# ~$ selfhostedops - update one service at a time (episode 17) +# +# "docker compose pull && docker compose up -d" updates everything at once and +# tells you nothing about which of the eleven changes broke the login. This +# does the same work in order: +# +# for each service: pull (or build) the image named in the compose file +# -> snapshot (backup/backup.sh, tag pre-update) +# -> keep the running image under a name of its own +# -> recreate that one service +# -> wait for updates/health.sh to pass, first the group +# of the service, then everything +# -> red: put image and data back, stop the whole run +# +# updates/update.sh plan what would change, nothing is touched +# updates/update.sh run all services, in the order below +# updates/update.sh run n8n outline only these +# TIMEOUT=900 updates/update.sh run uptime-kuma wait longer for one service +# +# You pick the versions: write the new tag into the compose file first (and +# commit it). The script never invents a version, it only rolls out what the +# file says. Runs on the box, as root. Needs backup/.restic-env (episode 7). +# +# Two things pile up and are yours to clear once you trust the new state: +# restic forget --tag pre-update --keep-last 3 --prune the snapshots +# docker image rm $(docker images -q 'selfhostedops-rollback/*') the old images +# A service that needs longer than TIMEOUT to come back counts as red and is +# put back. A database migration can take minutes (Uptime Kuma 1 -> 2: five). +set -uo pipefail + +STACK_DIR=${STACK_DIR:-/opt/selfhostedops-stack} +NETBIRD_DIR=${NETBIRD_DIR:-/opt/netbird} +P=${PROJECT:-selfhostedops-stack} +VOLUMES=/var/lib/docker/volumes +TIMEOUT=${TIMEOUT:-180} # seconds a service gets to turn green +STAGE=/var/lib/selfhostedops-restore +ASIDE=/var/lib/selfhostedops-failed-update +LOG=$STACK_DIR/updates/update.log + +# Least critical first, the monitoring last: it should watch the others move. +ORDER="ntfy forgejo outline-postgres outline-redis outline vaultwarden n8n pocket-id tinyauth netbird-server dashboard ops-actions ops-readonly caddy uptime-kuma" + +# service -> compose directory | health groups | volumes | dump file=volume/path ... +info() { + case $1 in + ntfy) echo "$STACK_DIR|ntfy|${P}_ntfy_data ${P}_ntfy_cache|ntfy-user.db=${P}_ntfy_data/user.db ntfy-cache.db=${P}_ntfy_cache/cache.db" ;; + forgejo) echo "$STACK_DIR|forgejo|${P}_forgejo_data|forgejo.db=${P}_forgejo_data/gitea/gitea.db" ;; + outline-postgres) echo "$STACK_DIR|outline|${P}_outline_pg|" ;; + outline-redis) echo "$STACK_DIR|outline||" ;; + outline) echo "$STACK_DIR|outline|${P}_outline_data|" ;; + vaultwarden) echo "$STACK_DIR|vaultwarden|${P}_vaultwarden_data|vaultwarden.sqlite3=${P}_vaultwarden_data/db.sqlite3" ;; + n8n) echo "$STACK_DIR|n8n|${P}_n8n_data|n8n.sqlite=${P}_n8n_data/database.sqlite" ;; + pocket-id) echo "$STACK_DIR|pocket-id tinyauth|${P}_pocket_id_data|pocket-id.db=${P}_pocket_id_data/pocket-id.db" ;; + tinyauth) echo "$STACK_DIR|tinyauth kuma|${P}_tinyauth_data|tinyauth.db=${P}_tinyauth_data/tinyauth.db" ;; + uptime-kuma) echo "$STACK_DIR|kuma|${P}_kuma_data|kuma.db=${P}_kuma_data/kuma.db" ;; + caddy) echo "$STACK_DIR|caddy||" ;; + ops-actions|ops-readonly) echo "$STACK_DIR|ops||" ;; + netbird-server) echo "$NETBIRD_DIR|netbird|netbird_netbird_data|netbird-store.db=netbird_netbird_data/store.db netbird-idp.db=netbird_netbird_data/idp.db netbird-events.db=netbird_netbird_data/events.db" ;; + dashboard) echo "$NETBIRD_DIR|netbird||" ;; + *) return 1 ;; + esac +} + +log() { printf '%s %s\n' "$(date -u '+%H:%M:%S')" "$*" | tee -a "$LOG"; } +# profiles: ops-actions and ops-readonly only exist with --profile agent +dc() { ( cd "$1" && shift && docker compose --profile agent "$@" ); } +cid() { dc "$1" ps -q "$2" 2>/dev/null | head -1; } +running_image() { local c; c=$(cid "$1" "$2"); [ -n "$c" ] && docker inspect -f '{{.Image}}' "$c"; } +target_ref() { dc "$1" config --format json 2>/dev/null | jq -r --arg s "$2" '.services[$s].image'; } +has_build() { dc "$1" config --format json 2>/dev/null | jq -e --arg s "$2" '.services[$s].build' >/dev/null; } + +# Fetch the image the compose file names, without touching the running container. +fetch() { + if has_build "$1" "$2"; then dc "$1" build --pull "$2" >>"$LOG" 2>&1; else dc "$1" pull -q "$2" >>"$LOG" 2>&1; fi +} + +wait_green() { # wait_green + local until=$(( $(date +%s) + $1 )); shift + while :; do + bash "$STACK_DIR/updates/health.sh" "$@" >"$STACK_DIR/updates/.last-health" 2>&1 && return 0 + [ "$(date +%s)" -ge "$until" ] && return 1 + sleep 5 + done +} + +put_back() { # put_back + local svc=$1 dir=$2 old=$3 snap=$4 vols dumps v rel dst src + IFS='|' read -r _ _ vols dumps <<<"$(info "$svc")" + log " ROLLBACK $svc: stop, data from snapshot $snap, image $(echo "$old" | cut -c8-19)" + dc "$dir" stop "$svc" >>"$LOG" 2>&1 + case $svc in outline|outline-postgres) dc "$dir" stop outline >>"$LOG" 2>&1 ;; esac + rm -rf "$STAGE"; mkdir -p "$STAGE" "$ASIDE" + local inc=(--include "$STACK_DIR/backup/dumps"); for v in $vols; do inc+=(--include "$VOLUMES/$v"); done + restic restore "$snap" --target "$STAGE" "${inc[@]}" >>"$LOG" 2>&1 || { log " restic restore FAILED, nothing changed on disk"; return 1; } + for v in $vols; do # the failed state is moved aside, not deleted + rm -rf "$ASIDE/$v"; mv "$VOLUMES/$v/_data" "$ASIDE/$v"; mv "$STAGE$VOLUMES/$v/_data" "$VOLUMES/$v/_data" + done + for v in $dumps; do # consistent online dump over the raw file copy + src="$STAGE$STACK_DIR/backup/dumps/${v%%=*}"; rel=${v#*=}; dst="$VOLUMES/${rel%%/*}/_data/${rel#*/}" + rm -f "$dst-wal" "$dst-shm"; cp "$src" "$dst.new"; chown --reference="$dst" "$dst.new"; chmod --reference="$dst" "$dst.new"; mv "$dst.new" "$dst" + done + docker tag "$old" "selfhostedops-rollback/$svc:previous" + printf 'services:\n %s:\n image: selfhostedops-rollback/%s:previous\n' "$svc" "$svc" > "$dir/compose.rollback.yml" + local files=(-f "$(ls "$dir"/compose.yml "$dir"/docker-compose.yml 2>/dev/null | head -1)" -f "$dir/compose.rollback.yml") + reload_pg() { # empty the schema and load the pg_dump from the snapshot + for _ in $(seq 60); do docker exec "$P-outline-postgres-1" pg_isready -U outline >/dev/null 2>&1 && break; sleep 1; done + docker exec "$P-outline-postgres-1" psql -U outline -q -c 'drop schema public cascade; create schema public;' outline >>"$LOG" 2>&1 + zcat "$STAGE$STACK_DIR/backup/dumps/outline.sql.gz" | docker exec -i "$P-outline-postgres-1" psql -U outline -q outline >>"$LOG" 2>&1 + } + case $svc in + outline) reload_pg; dc "$dir" "${files[@]}" up -d --no-deps outline >>"$LOG" 2>&1 ;; + outline-postgres) dc "$dir" "${files[@]}" up -d --no-deps outline-postgres >>"$LOG" 2>&1; reload_pg; dc "$dir" up -d --no-deps outline >>"$LOG" 2>&1 ;; + *) dc "$dir" "${files[@]}" up -d --no-deps "$svc" >>"$LOG" 2>&1 ;; + esac +} + +update_one() { + local svc=$1 dir groups old new snap t0 t1 + IFS='|' read -r dir groups _ _ <<<"$(info "$svc")" || { log "unknown service $svc"; return 2; } + [ -n "$(cid "$dir" "$svc")" ] || { log "$svc: not running, skipped"; return 0; } + old=$(running_image "$dir" "$svc") + log "$svc: fetching $(target_ref "$dir" "$svc")" + fetch "$dir" "$svc" || { log "$svc: FETCH FAILED, service untouched"; return 1; } + new=$(docker image inspect -f '{{.Id}}' "$(target_ref "$dir" "$svc")" 2>/dev/null) + if [ "$old" = "$new" ]; then log "$svc: already on that image, nothing to do"; return 0; fi + + log "$svc: snapshot before the change" + BACKUP_TAG=pre-update bash "$STACK_DIR/backup/backup.sh" >>"$LOG" 2>&1 || { log "$svc: BACKUP FAILED, service untouched"; return 1; } + snap=$(grep -oE 'snapshot [0-9a-f]{8} saved' "$STACK_DIR/backup/backup.log" | tail -1 | cut -d' ' -f2) + docker tag "$old" "selfhostedops-rollback/$svc:previous" # a name keeps the old image from being pruned + + t0=$(date +%s) + log "$svc: recreate ($(echo "$old" | cut -c8-19) -> $(echo "$new" | cut -c8-19)), snapshot $snap" + dc "$dir" up -d --no-deps "$svc" >>"$LOG" 2>&1 + # shellcheck disable=SC2086 + if wait_green "$TIMEOUT" $groups && wait_green "$TIMEOUT"; then + t1=$(date +%s); log "$svc: GREEN after $((t1 - t0)) s ($(tail -1 "$STACK_DIR/updates/.last-health"))" + return 0 + fi + t1=$(date +%s); log "$svc: RED after $((t1 - t0)) s" + grep '^FAIL' "$STACK_DIR/updates/.last-health" | tee -a "$LOG" + put_back "$svc" "$dir" "$old" "$snap" + if wait_green "$TIMEOUT"; then log "$svc: rolled back, health check green again after $(( $(date +%s) - t1 )) s" + else log "$svc: rolled back, but the health check is STILL RED"; grep '^FAIL' "$STACK_DIR/updates/.last-health" | tee -a "$LOG"; fi + log "$svc: the compose file still names the new tag. Set it back before the next 'up -d'." + return 1 +} + +cmd=${1:-plan}; shift || true +set -a; . "$STACK_DIR/backup/.restic-env"; set +a +case $cmd in + plan) + # No registry lookup: with exact tags in the compose file, a different tag is the update. + printf '%-18s %-46s %s\n' SERVICE RUNNING TARGET + for svc in ${*:-$ORDER}; do + IFS='|' read -r dir _ _ _ <<<"$(info "$svc")"; c=$(cid "$dir" "$svc"); [ -n "$c" ] || continue + ref=$(target_ref "$dir" "$svc"); cur=$(docker inspect -f '{{.Config.Image}}' "$c") + case $cur in sha256:*) cur="(untagged) $(echo "$cur" | cut -c8-19)" ;; esac + note=""; [ "$cur" != "$ref" ] && note=" <- update"; has_build "$dir" "$svc" && note=" <- local build, rebuilt on run" + printf '%-18s %-46s %s%s\n' "$svc" "$cur" "$ref" "$note" + done ;; + run) + bash "$STACK_DIR/updates/health.sh" >/dev/null 2>&1 || { log "health check is red before the start, not updating anything"; bash "$STACK_DIR/updates/health.sh" | grep '^FAIL'; exit 1; } + start=$(date +%s); log "=== update run: ${*:-$ORDER} (timeout per service $TIMEOUT s) ===" + for svc in ${*:-$ORDER}; do + update_one "$svc" || { log "=== STOPPED at $svc after $(( $(date +%s) - start )) s, later services untouched ==="; exit 1; } + done + log "=== all done in $(( $(date +%s) - start )) s ===" ;; + *) sed -n '2,24p' "$0"; exit 2 ;; +esac