Episode 17: updates one service at a time, a health check that asks real questions, every tag pinned
updates/health.sh: 43 checks across all services (API answers, git refs, a push message round trip, SSO redirects and token endpoint, mesh state, backup age). updates/update.sh: per service pull or build, snapshot, recreate, wait for the health check, put image and data back on red and stop the run. backup/backup.sh: online dumps for n8n, Pocket ID and tinyauth; BACKUP_TAG so pre-update snapshots are not removed by the nightly keep-daily rule. compose.yml and caddy/Dockerfile: exact versions instead of latest, 16, 7-alpine and caddy:2-alpine.
This commit is contained in:
Maximilian Grimm
parent
1655d04d7d
commit
55b2fe7646
7 changed files with 380 additions and 19 deletions
9
updates/health.env.example
Normal file
9
updates/health.env.example
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
# Optional credentials for updates/health.sh. Copy to health.env (mode 600).
|
||||
# Read-only Forgejo token: forgejo admin user generate-access-token -u <user> -t health --scopes read:repository --raw
|
||||
FORGEJO_HEALTH_TOKEN=
|
||||
FORGEJO_HEALTH_REPO=owner/repo
|
||||
# ntfy user with read-write on the topic "healthcheck":
|
||||
# ntfy user add health && ntfy access health healthcheck rw
|
||||
NTFY_HEALTH_AUTH=health:password
|
||||
# Number of active n8n workflows you expect (leave empty for "at least one")
|
||||
N8N_EXPECT_ACTIVE=
|
||||
162
updates/health.sh
Executable file
162
updates/health.sh
Executable file
|
|
@ -0,0 +1,162 @@
|
|||
#!/usr/bin/env bash
|
||||
# ~$ selfhostedops - health check for the whole stack (episode 17)
|
||||
#
|
||||
# "The container is up" is not a health check. Every line below asks a service
|
||||
# to do the thing it is there for: answer its API, hand out git refs, deliver
|
||||
# a push message, start an SSO login, keep the mesh connected.
|
||||
#
|
||||
# updates/health.sh all checks, exit code = number of failures
|
||||
# updates/health.sh forgejo n8n only these groups
|
||||
# updates/health.sh --data also print row counts (to compare before/after)
|
||||
#
|
||||
# Runs on the box itself. Public hostnames are resolved to 127.0.0.1, so the
|
||||
# request still goes through Caddy and its certificates, without leaving the box.
|
||||
# No check asks for an answer of 401, 403 or 404 through Caddy: fail2ban counts
|
||||
# those (episode 10) and would end up banning the box's own bridge address.
|
||||
# Optional credentials in updates/health.env (see health.env.example):
|
||||
# FORGEJO_HEALTH_TOKEN read-only token, for the git ref check on a private repo
|
||||
# FORGEJO_HEALTH_REPO owner/name
|
||||
# NTFY_HEALTH_AUTH user:password with read-write on the topic "healthcheck"
|
||||
set -uo pipefail
|
||||
|
||||
STACK_DIR=${STACK_DIR:-/opt/selfhostedops-stack}
|
||||
P=${PROJECT:-selfhostedops-stack}
|
||||
set -a
|
||||
. "$STACK_DIR/.env"
|
||||
[ -f "$STACK_DIR/updates/health.env" ] && . "$STACK_DIR/updates/health.env"
|
||||
set +a
|
||||
D=$DOMAIN
|
||||
SQLITE_IMAGE=${SQLITE_IMAGE:-keinos/sqlite3:latest}
|
||||
|
||||
DATA=0; GROUPS_WANTED=()
|
||||
for a in "$@"; do [ "$a" = "--data" ] && DATA=1 || GROUPS_WANTED+=("$a"); done
|
||||
want() { [ ${#GROUPS_WANTED[@]} -eq 0 ] || [[ " ${GROUPS_WANTED[*]} " == *" $1 "* ]]; }
|
||||
|
||||
FAILS=0; TOTAL=0
|
||||
ok() { TOTAL=$((TOTAL+1)); printf 'PASS %-12s %s\n' "$1" "$2"; }
|
||||
bad() { TOTAL=$((TOTAL+1)); FAILS=$((FAILS+1)); printf 'FAIL %-12s %s\n' "$1" "$2"; }
|
||||
# check <group> <description> <actual> <expected regex>
|
||||
check() { if [[ "$3" =~ $4 ]]; then ok "$1" "$2"; else bad "$1" "$2 (got: $(printf '%s' "$3" | tr '\n' ' ' | cut -c1-90))"; fi; }
|
||||
|
||||
pub() { local host=$1 path=$2; shift 2; curl -sS -m 8 --resolve "$host.$D:443:127.0.0.1" "$@" "https://$host.$D$path" 2>&1; }
|
||||
code() { pub "$1" "$2" -o /dev/null -w '%{http_code}' "${@:3}"; }
|
||||
loc() { pub "$1" "$2" -o /dev/null -w '%{redirect_url}' "${@:3}"; }
|
||||
cip() { docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}' "$1" 2>/dev/null | cut -d' ' -f1; }
|
||||
sq() { docker run --rm --user 0 -v "$1:/src:ro" "$SQLITE_IMAGE" sqlite3 "file:/src/$2?mode=ro" "$3" 2>&1; }
|
||||
dx() { docker exec "$@" 2>&1; }
|
||||
|
||||
if want caddy; then
|
||||
check caddy "config is valid" "$(dx $P-caddy-1 caddy validate --config /etc/caddy/Caddyfile | grep -o 'Valid configuration')" 'Valid configuration'
|
||||
check caddy "plugins compiled in: rate_limit, crowdsec, appsec" "$(dx $P-caddy-1 caddy list-modules | grep -cE '^http.handlers.(rate_limit|crowdsec|appsec)$')" '^3$'
|
||||
check caddy "certificate for git.$D is valid" "$(code git /api/healthz)" '^200$'
|
||||
if command -v cscli >/dev/null; then
|
||||
age=$(( $(date +%s) - $(date -d "$(cscli bouncers list -o json | jq -r '.[]|select(.type=="caddy-cs-bouncer")|.last_pull')" +%s 2>/dev/null || echo 0) ))
|
||||
check caddy "CrowdSec bouncer pulled decisions in the last 60 s" "$age" '^([0-9]|[1-5][0-9])$'
|
||||
fi
|
||||
fi
|
||||
|
||||
if want forgejo; then
|
||||
check forgejo "API healthz: database and cache pass" "$(pub git /api/healthz | jq -r '.status' 2>&1)" '^pass$'
|
||||
if [ -n "${FORGEJO_HEALTH_TOKEN:-}" ]; then
|
||||
check forgejo "git over HTTPS: refs of ${FORGEJO_HEALTH_REPO}" "$(pub git "/${FORGEJO_HEALTH_REPO}.git/info/refs?service=git-upload-pack" -u "health:$FORGEJO_HEALTH_TOKEN" | tr -d '\0' | grep -ac 'refs/heads/')" '^[1-9]'
|
||||
fi
|
||||
check forgejo "git over SSH: port 2222 answers" "$(timeout 3 bash -c 'exec 3<>/dev/tcp/127.0.0.1/2222; head -c 8 <&3' 2>&1)" '^SSH-2.0'
|
||||
check forgejo "SSO: login button leads to id.$D" "$(loc git /user/oauth2/PocketID)" "^https://id\.$D/authorize"
|
||||
fi
|
||||
|
||||
if want outline; then
|
||||
check outline "postgres accepts connections" "$(dx $P-outline-postgres-1 pg_isready -U outline)" 'accepting connections'
|
||||
check outline "redis answers PING" "$(dx $P-outline-redis-1 redis-cli ping)" '^PONG'
|
||||
check outline "_health" "$(code wiki /_health)" '^200$'
|
||||
check outline "API auth.config lists Pocket ID" "$(pub wiki /api/auth.config -X POST | jq -r '.data.providers[0].name' 2>&1)" '^Pocket ID$'
|
||||
check outline "SSO: login leads to id.$D" "$(loc wiki /auth/oidc)" "^https://id\.$D/authorize"
|
||||
check outline "database answers a query" "$(dx $P-outline-postgres-1 psql -U outline -tAc 'select count(*) from documents')" '^[0-9]+$'
|
||||
fi
|
||||
|
||||
if want vaultwarden; then
|
||||
check vaultwarden "alive" "$(code vault /alive)" '^200$'
|
||||
check vaultwarden "API config reports a version" "$(pub vault /api/config | jq -r '.version' 2>&1)" '^20[0-9][0-9]\.'
|
||||
check vaultwarden "SSO: prevalidate hands out a token" "$(pub vault '/identity/sso/prevalidate?domainHint=x' | jq -r '.token|length>20' 2>&1)" '^true$'
|
||||
fi
|
||||
|
||||
if want kuma; then
|
||||
# The status page API is asked on the container directly, not through Caddy: while Kuma migrates
|
||||
# its database (1.x -> 2.x took five minutes here) it answers 404, and a health check that polls
|
||||
# 404s through Caddy gets the box's own bridge address banned by fail2ban.
|
||||
k="http://$(cip $P-uptime-kuma-1):3001"
|
||||
sp=$(curl -sS -m 8 "$k/api/status-page/status" 2>&1)
|
||||
check kuma "public status page API" "$(jq -r '.config.slug' <<<"$sp" 2>&1)" '^status$'
|
||||
check kuma "status page shows 4 public monitors" "$(jq -r '[.publicGroupList[].monitorList[]]|length' <<<"$sp" 2>&1)" '^4$'
|
||||
hb=$(curl -sS -m 8 "$k/api/status-page/heartbeat/status" 2>&1 | jq -r '[.heartbeatList[]|last]|"\(length) \([.[]|select(.status==1)]|length) \(now - ([.[].time|sub("\\.[0-9]+$";"")|sub(" ";"T")+"Z"|fromdate]|min)|floor)"' 2>&1)
|
||||
check kuma "every public monitor: last heartbeat UP and younger than 180 s (monitors up age)" "$hb" '^([0-9]+) \1 ([0-9]|[1-9][0-9]|1[0-7][0-9])$'
|
||||
check kuma "dashboard is behind the login gate" "$(loc status /)" "^https://auth\.$D/login"
|
||||
down=$(sq ${P}_kuma_data kuma.db "select count(*) from monitor m where active=1 and (select status from heartbeat h where h.monitor_id=m.id order by time desc limit 1) is not 1")
|
||||
check kuma "no active monitor is DOWN in the database" "$down" '^0$'
|
||||
fi
|
||||
|
||||
if want ntfy; then
|
||||
check ntfy "health" "$(pub ntfy /v1/health | jq -r '.healthy' 2>&1)" '^true$'
|
||||
if [ -n "${NTFY_HEALTH_AUTH:-}" ]; then
|
||||
id="health-$(date +%s)-$RANDOM"
|
||||
pub ntfy /healthcheck -u "$NTFY_HEALTH_AUTH" -d "$id" -o /dev/null
|
||||
check ntfy "a published message can be read back" "$(pub ntfy '/healthcheck/json?poll=1&since=1m' -u "$NTFY_HEALTH_AUTH" | grep -c "$id")" '^1$'
|
||||
fi
|
||||
fi
|
||||
|
||||
if want n8n; then
|
||||
n=$(cip $P-n8n-1)
|
||||
check n8n "readiness (database reachable, migrations done)" "$(curl -sS -m 8 http://$n:5678/healthz/readiness 2>&1)" '"status":"ok"'
|
||||
check n8n "active workflows (at least ${N8N_EXPECT_ACTIVE:-1})" "$(dx $P-n8n-1 n8n list:workflow --active=true | grep -c '|')" "^(${N8N_EXPECT_ACTIVE:-[1-9][0-9]*})$"
|
||||
# 403 = the webhook is registered and asks for its token. 404 = workflow not active.
|
||||
check n8n "webhook kuma-incident is registered and asks for its token" "$(curl -sS -m 8 -o /dev/null -w '%{http_code}' -X POST http://$n:5678/webhook/kuma-incident -d '{}' 2>&1)" '^403$'
|
||||
fi
|
||||
|
||||
if want pocket-id; then
|
||||
check pocket-id "healthz" "$(code id /healthz)" '^20[04]$'
|
||||
check pocket-id "discovery document names the right issuer" "$(pub id /.well-known/openid-configuration | jq -r '.issuer' 2>&1)" "^https://id\.$D$"
|
||||
check pocket-id "signing keys are published" "$(pub id /.well-known/jwks.json | jq -r '.keys|length' 2>&1)" '^[1-9]'
|
||||
# A made-up code with the real client secret must fail as invalid_grant. invalid_client would mean the client is gone.
|
||||
check pocket-id "token endpoint still knows the Outline client" "$(curl -sS -m 8 -X POST http://$(cip $P-pocket-id-1):1411/api/oidc/token -d grant_type=authorization_code -d code=health -d client_id=$OUTLINE_OIDC_CLIENT_ID -d client_secret=$OUTLINE_OIDC_CLIENT_SECRET -d redirect_uri=https://wiki.$D/auth/oidc.callback 2>&1 | jq -r '.error' 2>&1)" '^invalid_grant$'
|
||||
fi
|
||||
|
||||
if want tinyauth; then
|
||||
check tinyauth "healthz" "$(pub auth /api/healthz | jq -r '.message' 2>&1)" '^Healthy$'
|
||||
check tinyauth "login is handed to id.$D" "$(pub auth /api/oauth/url/pocketid | jq -r '.url' 2>&1)" "^https://id\.$D/authorize"
|
||||
fi
|
||||
|
||||
if want netbird; then
|
||||
check netbird "dashboard" "$(code vpn /)" '^200$'
|
||||
check netbird "embedded identity provider discovery" "$(pub vpn /oauth2/.well-known/openid-configuration | jq -r '.issuer' 2>&1)" "^https://vpn\.$D"
|
||||
check netbird "management API alive and asks for a login" "$(curl -sS -m 8 -o /dev/null -w '%{http_code}' http://127.0.0.1:8081/api/users 2>&1)" '^401$'
|
||||
check netbird "this host: management and signal connected" "$(netbird status 2>&1 | grep -cE '^(Management|Signal): Connected')" '^2$'
|
||||
check netbird "this host: at least one peer connected" "$(netbird status 2>&1 | grep -oE 'Peers count: [0-9]+' | grep -oE '[0-9]+$')" '^[1-9]'
|
||||
if docker inspect mesh-client >/dev/null 2>&1; then
|
||||
check netbird "mesh-client container: management connected" "$(dx mesh-client netbird status | grep -c '^Management: Connected')" '^1$'
|
||||
fi
|
||||
fi
|
||||
|
||||
if want ops; then
|
||||
check ops "ops-readonly healthz" "$(curl -sS -m 8 http://$(cip $P-ops-readonly-1):8098/healthz 2>&1 | jq -r '.status' 2>&1)" '^ok$'
|
||||
check ops "ops-actions healthz" "$(curl -sS -m 8 http://$(cip $P-ops-actions-1):8099/healthz 2>&1 | jq -r '.status' 2>&1)" '^ok$'
|
||||
fi
|
||||
|
||||
if want backup; then
|
||||
check backup "nightly timer is scheduled" "$(systemctl is-active selfhostedops-backup.timer 2>&1)" '^active$'
|
||||
last=$(grep '=== backup end ===' "$STACK_DIR/backup/backup.log" | tail -1 | cut -c1-19)
|
||||
check backup "last finished backup is younger than 26 h ($last)" "$(( ( $(date +%s) - $(date -d "$last" +%s 2>/dev/null || echo 0) ) / 3600 ))" '^([0-9]|1[0-9]|2[0-5])$'
|
||||
fi
|
||||
|
||||
if [ $DATA = 1 ]; then
|
||||
echo
|
||||
echo "DATA forgejo repositories $(sq ${P}_forgejo_data gitea/gitea.db 'select count(*) from repository'), users $(sq ${P}_forgejo_data gitea/gitea.db 'select count(*) from user where type=0')"
|
||||
echo "DATA outline documents $(dx $P-outline-postgres-1 psql -U outline -tAc 'select count(*) from documents'), users $(dx $P-outline-postgres-1 psql -U outline -tAc 'select count(*) from users'), migrations $(dx $P-outline-postgres-1 psql -U outline -tAc 'select count(*) from "SequelizeMeta"')"
|
||||
echo "DATA vaultwarden users $(sq ${P}_vaultwarden_data db.sqlite3 'select count(*) from users'), ciphers $(sq ${P}_vaultwarden_data db.sqlite3 'select count(*) from ciphers')"
|
||||
echo "DATA kuma monitors $(sq ${P}_kuma_data kuma.db 'select count(*) from monitor'), notifications $(sq ${P}_kuma_data kuma.db 'select count(*) from notification'), heartbeats $(sq ${P}_kuma_data kuma.db 'select count(*) from heartbeat')"
|
||||
echo "DATA n8n workflows $(sq ${P}_n8n_data database.sqlite 'select count(*) from workflow_entity'), credentials $(sq ${P}_n8n_data database.sqlite 'select count(*) from credentials_entity'), executions $(sq ${P}_n8n_data database.sqlite 'select count(*) from execution_entity'), migrations $(sq ${P}_n8n_data database.sqlite 'select count(*) from migrations')"
|
||||
echo "DATA pocket-id users $(sq ${P}_pocket_id_data pocket-id.db 'select count(*) from users'), clients $(sq ${P}_pocket_id_data pocket-id.db 'select count(*) from oidc_clients')"
|
||||
echo "DATA netbird peers $(sq netbird_netbird_data store.db 'select count(*) from peers')"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "$((TOTAL-FAILS)) of $TOTAL checks passed, $FAILS failed"
|
||||
exit $FAILS
|
||||
171
updates/update.sh
Executable file
171
updates/update.sh
Executable file
|
|
@ -0,0 +1,171 @@
|
|||
#!/usr/bin/env bash
|
||||
# ~$ selfhostedops - update one service at a time (episode 17)
|
||||
#
|
||||
# "docker compose pull && docker compose up -d" updates everything at once and
|
||||
# tells you nothing about which of the eleven changes broke the login. This
|
||||
# does the same work in order:
|
||||
#
|
||||
# for each service: pull (or build) the image named in the compose file
|
||||
# -> snapshot (backup/backup.sh, tag pre-update)
|
||||
# -> keep the running image under a name of its own
|
||||
# -> recreate that one service
|
||||
# -> wait for updates/health.sh to pass, first the group
|
||||
# of the service, then everything
|
||||
# -> red: put image and data back, stop the whole run
|
||||
#
|
||||
# updates/update.sh plan what would change, nothing is touched
|
||||
# updates/update.sh run all services, in the order below
|
||||
# updates/update.sh run n8n outline only these
|
||||
# TIMEOUT=900 updates/update.sh run uptime-kuma wait longer for one service
|
||||
#
|
||||
# You pick the versions: write the new tag into the compose file first (and
|
||||
# commit it). The script never invents a version, it only rolls out what the
|
||||
# file says. Runs on the box, as root. Needs backup/.restic-env (episode 7).
|
||||
#
|
||||
# Two things pile up and are yours to clear once you trust the new state:
|
||||
# restic forget --tag pre-update --keep-last 3 --prune the snapshots
|
||||
# docker image rm $(docker images -q 'selfhostedops-rollback/*') the old images
|
||||
# A service that needs longer than TIMEOUT to come back counts as red and is
|
||||
# put back. A database migration can take minutes (Uptime Kuma 1 -> 2: five).
|
||||
set -uo pipefail
|
||||
|
||||
STACK_DIR=${STACK_DIR:-/opt/selfhostedops-stack}
|
||||
NETBIRD_DIR=${NETBIRD_DIR:-/opt/netbird}
|
||||
P=${PROJECT:-selfhostedops-stack}
|
||||
VOLUMES=/var/lib/docker/volumes
|
||||
TIMEOUT=${TIMEOUT:-180} # seconds a service gets to turn green
|
||||
STAGE=/var/lib/selfhostedops-restore
|
||||
ASIDE=/var/lib/selfhostedops-failed-update
|
||||
LOG=$STACK_DIR/updates/update.log
|
||||
|
||||
# Least critical first, the monitoring last: it should watch the others move.
|
||||
ORDER="ntfy forgejo outline-postgres outline-redis outline vaultwarden n8n pocket-id tinyauth netbird-server dashboard ops-actions ops-readonly caddy uptime-kuma"
|
||||
|
||||
# service -> compose directory | health groups | volumes | dump file=volume/path ...
|
||||
info() {
|
||||
case $1 in
|
||||
ntfy) echo "$STACK_DIR|ntfy|${P}_ntfy_data ${P}_ntfy_cache|ntfy-user.db=${P}_ntfy_data/user.db ntfy-cache.db=${P}_ntfy_cache/cache.db" ;;
|
||||
forgejo) echo "$STACK_DIR|forgejo|${P}_forgejo_data|forgejo.db=${P}_forgejo_data/gitea/gitea.db" ;;
|
||||
outline-postgres) echo "$STACK_DIR|outline|${P}_outline_pg|" ;;
|
||||
outline-redis) echo "$STACK_DIR|outline||" ;;
|
||||
outline) echo "$STACK_DIR|outline|${P}_outline_data|" ;;
|
||||
vaultwarden) echo "$STACK_DIR|vaultwarden|${P}_vaultwarden_data|vaultwarden.sqlite3=${P}_vaultwarden_data/db.sqlite3" ;;
|
||||
n8n) echo "$STACK_DIR|n8n|${P}_n8n_data|n8n.sqlite=${P}_n8n_data/database.sqlite" ;;
|
||||
pocket-id) echo "$STACK_DIR|pocket-id tinyauth|${P}_pocket_id_data|pocket-id.db=${P}_pocket_id_data/pocket-id.db" ;;
|
||||
tinyauth) echo "$STACK_DIR|tinyauth kuma|${P}_tinyauth_data|tinyauth.db=${P}_tinyauth_data/tinyauth.db" ;;
|
||||
uptime-kuma) echo "$STACK_DIR|kuma|${P}_kuma_data|kuma.db=${P}_kuma_data/kuma.db" ;;
|
||||
caddy) echo "$STACK_DIR|caddy||" ;;
|
||||
ops-actions|ops-readonly) echo "$STACK_DIR|ops||" ;;
|
||||
netbird-server) echo "$NETBIRD_DIR|netbird|netbird_netbird_data|netbird-store.db=netbird_netbird_data/store.db netbird-idp.db=netbird_netbird_data/idp.db netbird-events.db=netbird_netbird_data/events.db" ;;
|
||||
dashboard) echo "$NETBIRD_DIR|netbird||" ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
log() { printf '%s %s\n' "$(date -u '+%H:%M:%S')" "$*" | tee -a "$LOG"; }
|
||||
# profiles: ops-actions and ops-readonly only exist with --profile agent
|
||||
dc() { ( cd "$1" && shift && docker compose --profile agent "$@" ); }
|
||||
cid() { dc "$1" ps -q "$2" 2>/dev/null | head -1; }
|
||||
running_image() { local c; c=$(cid "$1" "$2"); [ -n "$c" ] && docker inspect -f '{{.Image}}' "$c"; }
|
||||
target_ref() { dc "$1" config --format json 2>/dev/null | jq -r --arg s "$2" '.services[$s].image'; }
|
||||
has_build() { dc "$1" config --format json 2>/dev/null | jq -e --arg s "$2" '.services[$s].build' >/dev/null; }
|
||||
|
||||
# Fetch the image the compose file names, without touching the running container.
|
||||
fetch() {
|
||||
if has_build "$1" "$2"; then dc "$1" build --pull "$2" >>"$LOG" 2>&1; else dc "$1" pull -q "$2" >>"$LOG" 2>&1; fi
|
||||
}
|
||||
|
||||
wait_green() { # wait_green <seconds> <health groups...>
|
||||
local until=$(( $(date +%s) + $1 )); shift
|
||||
while :; do
|
||||
bash "$STACK_DIR/updates/health.sh" "$@" >"$STACK_DIR/updates/.last-health" 2>&1 && return 0
|
||||
[ "$(date +%s)" -ge "$until" ] && return 1
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
|
||||
put_back() { # put_back <service> <dir> <old image id> <snapshot id>
|
||||
local svc=$1 dir=$2 old=$3 snap=$4 vols dumps v rel dst src
|
||||
IFS='|' read -r _ _ vols dumps <<<"$(info "$svc")"
|
||||
log " ROLLBACK $svc: stop, data from snapshot $snap, image $(echo "$old" | cut -c8-19)"
|
||||
dc "$dir" stop "$svc" >>"$LOG" 2>&1
|
||||
case $svc in outline|outline-postgres) dc "$dir" stop outline >>"$LOG" 2>&1 ;; esac
|
||||
rm -rf "$STAGE"; mkdir -p "$STAGE" "$ASIDE"
|
||||
local inc=(--include "$STACK_DIR/backup/dumps"); for v in $vols; do inc+=(--include "$VOLUMES/$v"); done
|
||||
restic restore "$snap" --target "$STAGE" "${inc[@]}" >>"$LOG" 2>&1 || { log " restic restore FAILED, nothing changed on disk"; return 1; }
|
||||
for v in $vols; do # the failed state is moved aside, not deleted
|
||||
rm -rf "$ASIDE/$v"; mv "$VOLUMES/$v/_data" "$ASIDE/$v"; mv "$STAGE$VOLUMES/$v/_data" "$VOLUMES/$v/_data"
|
||||
done
|
||||
for v in $dumps; do # consistent online dump over the raw file copy
|
||||
src="$STAGE$STACK_DIR/backup/dumps/${v%%=*}"; rel=${v#*=}; dst="$VOLUMES/${rel%%/*}/_data/${rel#*/}"
|
||||
rm -f "$dst-wal" "$dst-shm"; cp "$src" "$dst.new"; chown --reference="$dst" "$dst.new"; chmod --reference="$dst" "$dst.new"; mv "$dst.new" "$dst"
|
||||
done
|
||||
docker tag "$old" "selfhostedops-rollback/$svc:previous"
|
||||
printf 'services:\n %s:\n image: selfhostedops-rollback/%s:previous\n' "$svc" "$svc" > "$dir/compose.rollback.yml"
|
||||
local files=(-f "$(ls "$dir"/compose.yml "$dir"/docker-compose.yml 2>/dev/null | head -1)" -f "$dir/compose.rollback.yml")
|
||||
reload_pg() { # empty the schema and load the pg_dump from the snapshot
|
||||
for _ in $(seq 60); do docker exec "$P-outline-postgres-1" pg_isready -U outline >/dev/null 2>&1 && break; sleep 1; done
|
||||
docker exec "$P-outline-postgres-1" psql -U outline -q -c 'drop schema public cascade; create schema public;' outline >>"$LOG" 2>&1
|
||||
zcat "$STAGE$STACK_DIR/backup/dumps/outline.sql.gz" | docker exec -i "$P-outline-postgres-1" psql -U outline -q outline >>"$LOG" 2>&1
|
||||
}
|
||||
case $svc in
|
||||
outline) reload_pg; dc "$dir" "${files[@]}" up -d --no-deps outline >>"$LOG" 2>&1 ;;
|
||||
outline-postgres) dc "$dir" "${files[@]}" up -d --no-deps outline-postgres >>"$LOG" 2>&1; reload_pg; dc "$dir" up -d --no-deps outline >>"$LOG" 2>&1 ;;
|
||||
*) dc "$dir" "${files[@]}" up -d --no-deps "$svc" >>"$LOG" 2>&1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
update_one() {
|
||||
local svc=$1 dir groups old new snap t0 t1
|
||||
IFS='|' read -r dir groups _ _ <<<"$(info "$svc")" || { log "unknown service $svc"; return 2; }
|
||||
[ -n "$(cid "$dir" "$svc")" ] || { log "$svc: not running, skipped"; return 0; }
|
||||
old=$(running_image "$dir" "$svc")
|
||||
log "$svc: fetching $(target_ref "$dir" "$svc")"
|
||||
fetch "$dir" "$svc" || { log "$svc: FETCH FAILED, service untouched"; return 1; }
|
||||
new=$(docker image inspect -f '{{.Id}}' "$(target_ref "$dir" "$svc")" 2>/dev/null)
|
||||
if [ "$old" = "$new" ]; then log "$svc: already on that image, nothing to do"; return 0; fi
|
||||
|
||||
log "$svc: snapshot before the change"
|
||||
BACKUP_TAG=pre-update bash "$STACK_DIR/backup/backup.sh" >>"$LOG" 2>&1 || { log "$svc: BACKUP FAILED, service untouched"; return 1; }
|
||||
snap=$(grep -oE 'snapshot [0-9a-f]{8} saved' "$STACK_DIR/backup/backup.log" | tail -1 | cut -d' ' -f2)
|
||||
docker tag "$old" "selfhostedops-rollback/$svc:previous" # a name keeps the old image from being pruned
|
||||
|
||||
t0=$(date +%s)
|
||||
log "$svc: recreate ($(echo "$old" | cut -c8-19) -> $(echo "$new" | cut -c8-19)), snapshot $snap"
|
||||
dc "$dir" up -d --no-deps "$svc" >>"$LOG" 2>&1
|
||||
# shellcheck disable=SC2086
|
||||
if wait_green "$TIMEOUT" $groups && wait_green "$TIMEOUT"; then
|
||||
t1=$(date +%s); log "$svc: GREEN after $((t1 - t0)) s ($(tail -1 "$STACK_DIR/updates/.last-health"))"
|
||||
return 0
|
||||
fi
|
||||
t1=$(date +%s); log "$svc: RED after $((t1 - t0)) s"
|
||||
grep '^FAIL' "$STACK_DIR/updates/.last-health" | tee -a "$LOG"
|
||||
put_back "$svc" "$dir" "$old" "$snap"
|
||||
if wait_green "$TIMEOUT"; then log "$svc: rolled back, health check green again after $(( $(date +%s) - t1 )) s"
|
||||
else log "$svc: rolled back, but the health check is STILL RED"; grep '^FAIL' "$STACK_DIR/updates/.last-health" | tee -a "$LOG"; fi
|
||||
log "$svc: the compose file still names the new tag. Set it back before the next 'up -d'."
|
||||
return 1
|
||||
}
|
||||
|
||||
cmd=${1:-plan}; shift || true
|
||||
set -a; . "$STACK_DIR/backup/.restic-env"; set +a
|
||||
case $cmd in
|
||||
plan)
|
||||
# No registry lookup: with exact tags in the compose file, a different tag is the update.
|
||||
printf '%-18s %-46s %s\n' SERVICE RUNNING TARGET
|
||||
for svc in ${*:-$ORDER}; do
|
||||
IFS='|' read -r dir _ _ _ <<<"$(info "$svc")"; c=$(cid "$dir" "$svc"); [ -n "$c" ] || continue
|
||||
ref=$(target_ref "$dir" "$svc"); cur=$(docker inspect -f '{{.Config.Image}}' "$c")
|
||||
case $cur in sha256:*) cur="(untagged) $(echo "$cur" | cut -c8-19)" ;; esac
|
||||
note=""; [ "$cur" != "$ref" ] && note=" <- update"; has_build "$dir" "$svc" && note=" <- local build, rebuilt on run"
|
||||
printf '%-18s %-46s %s%s\n' "$svc" "$cur" "$ref" "$note"
|
||||
done ;;
|
||||
run)
|
||||
bash "$STACK_DIR/updates/health.sh" >/dev/null 2>&1 || { log "health check is red before the start, not updating anything"; bash "$STACK_DIR/updates/health.sh" | grep '^FAIL'; exit 1; }
|
||||
start=$(date +%s); log "=== update run: ${*:-$ORDER} (timeout per service $TIMEOUT s) ==="
|
||||
for svc in ${*:-$ORDER}; do
|
||||
update_one "$svc" || { log "=== STOPPED at $svc after $(( $(date +%s) - start )) s, later services untouched ==="; exit 1; }
|
||||
done
|
||||
log "=== all done in $(( $(date +%s) - start )) s ===" ;;
|
||||
*) sed -n '2,24p' "$0"; exit 2 ;;
|
||||
esac
|
||||
Loading…
Add table
Add a link
Reference in a new issue