Episode 17: updates one service at a time, a health check that asks real questions, every tag pinned
updates/health.sh: 43 checks across all services (API answers, git refs, a push message round trip, SSO redirects and token endpoint, mesh state, backup age). updates/update.sh: per service pull or build, snapshot, recreate, wait for the health check, put image and data back on red and stop the run. backup/backup.sh: online dumps for n8n, Pocket ID and tinyauth; BACKUP_TAG so pre-update snapshots are not removed by the nightly keep-daily rule. compose.yml and caddy/Dockerfile: exact versions instead of latest, 16, 7-alpine and caddy:2-alpine.
This commit is contained in:
Maximilian Grimm
parent
1655d04d7d
commit
55b2fe7646
7 changed files with 380 additions and 19 deletions
171
updates/update.sh
Executable file
171
updates/update.sh
Executable file
|
|
@ -0,0 +1,171 @@
|
|||
#!/usr/bin/env bash
|
||||
# ~$ selfhostedops - update one service at a time (episode 17)
|
||||
#
|
||||
# "docker compose pull && docker compose up -d" updates everything at once and
|
||||
# tells you nothing about which of the eleven changes broke the login. This
|
||||
# does the same work in order:
|
||||
#
|
||||
# for each service: pull (or build) the image named in the compose file
|
||||
# -> snapshot (backup/backup.sh, tag pre-update)
|
||||
# -> keep the running image under a name of its own
|
||||
# -> recreate that one service
|
||||
# -> wait for updates/health.sh to pass, first the group
|
||||
# of the service, then everything
|
||||
# -> red: put image and data back, stop the whole run
|
||||
#
|
||||
# updates/update.sh plan what would change, nothing is touched
|
||||
# updates/update.sh run all services, in the order below
|
||||
# updates/update.sh run n8n outline only these
|
||||
# TIMEOUT=900 updates/update.sh run uptime-kuma wait longer for one service
|
||||
#
|
||||
# You pick the versions: write the new tag into the compose file first (and
|
||||
# commit it). The script never invents a version, it only rolls out what the
|
||||
# file says. Runs on the box, as root. Needs backup/.restic-env (episode 7).
|
||||
#
|
||||
# Two things pile up and are yours to clear once you trust the new state:
|
||||
# restic forget --tag pre-update --keep-last 3 --prune the snapshots
|
||||
# docker image rm $(docker images -q 'selfhostedops-rollback/*') the old images
|
||||
# A service that needs longer than TIMEOUT to come back counts as red and is
|
||||
# put back. A database migration can take minutes (Uptime Kuma 1 -> 2: five).
|
||||
set -uo pipefail
|
||||
|
||||
STACK_DIR=${STACK_DIR:-/opt/selfhostedops-stack}
|
||||
NETBIRD_DIR=${NETBIRD_DIR:-/opt/netbird}
|
||||
P=${PROJECT:-selfhostedops-stack}
|
||||
VOLUMES=/var/lib/docker/volumes
|
||||
TIMEOUT=${TIMEOUT:-180} # seconds a service gets to turn green
|
||||
STAGE=/var/lib/selfhostedops-restore
|
||||
ASIDE=/var/lib/selfhostedops-failed-update
|
||||
LOG=$STACK_DIR/updates/update.log
|
||||
|
||||
# Least critical first, the monitoring last: it should watch the others move.
|
||||
ORDER="ntfy forgejo outline-postgres outline-redis outline vaultwarden n8n pocket-id tinyauth netbird-server dashboard ops-actions ops-readonly caddy uptime-kuma"
|
||||
|
||||
# service -> compose directory | health groups | volumes | dump file=volume/path ...
|
||||
info() {
|
||||
case $1 in
|
||||
ntfy) echo "$STACK_DIR|ntfy|${P}_ntfy_data ${P}_ntfy_cache|ntfy-user.db=${P}_ntfy_data/user.db ntfy-cache.db=${P}_ntfy_cache/cache.db" ;;
|
||||
forgejo) echo "$STACK_DIR|forgejo|${P}_forgejo_data|forgejo.db=${P}_forgejo_data/gitea/gitea.db" ;;
|
||||
outline-postgres) echo "$STACK_DIR|outline|${P}_outline_pg|" ;;
|
||||
outline-redis) echo "$STACK_DIR|outline||" ;;
|
||||
outline) echo "$STACK_DIR|outline|${P}_outline_data|" ;;
|
||||
vaultwarden) echo "$STACK_DIR|vaultwarden|${P}_vaultwarden_data|vaultwarden.sqlite3=${P}_vaultwarden_data/db.sqlite3" ;;
|
||||
n8n) echo "$STACK_DIR|n8n|${P}_n8n_data|n8n.sqlite=${P}_n8n_data/database.sqlite" ;;
|
||||
pocket-id) echo "$STACK_DIR|pocket-id tinyauth|${P}_pocket_id_data|pocket-id.db=${P}_pocket_id_data/pocket-id.db" ;;
|
||||
tinyauth) echo "$STACK_DIR|tinyauth kuma|${P}_tinyauth_data|tinyauth.db=${P}_tinyauth_data/tinyauth.db" ;;
|
||||
uptime-kuma) echo "$STACK_DIR|kuma|${P}_kuma_data|kuma.db=${P}_kuma_data/kuma.db" ;;
|
||||
caddy) echo "$STACK_DIR|caddy||" ;;
|
||||
ops-actions|ops-readonly) echo "$STACK_DIR|ops||" ;;
|
||||
netbird-server) echo "$NETBIRD_DIR|netbird|netbird_netbird_data|netbird-store.db=netbird_netbird_data/store.db netbird-idp.db=netbird_netbird_data/idp.db netbird-events.db=netbird_netbird_data/events.db" ;;
|
||||
dashboard) echo "$NETBIRD_DIR|netbird||" ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
log() { printf '%s %s\n' "$(date -u '+%H:%M:%S')" "$*" | tee -a "$LOG"; }
|
||||
# profiles: ops-actions and ops-readonly only exist with --profile agent
|
||||
dc() { ( cd "$1" && shift && docker compose --profile agent "$@" ); }
|
||||
cid() { dc "$1" ps -q "$2" 2>/dev/null | head -1; }
|
||||
running_image() { local c; c=$(cid "$1" "$2"); [ -n "$c" ] && docker inspect -f '{{.Image}}' "$c"; }
|
||||
target_ref() { dc "$1" config --format json 2>/dev/null | jq -r --arg s "$2" '.services[$s].image'; }
|
||||
has_build() { dc "$1" config --format json 2>/dev/null | jq -e --arg s "$2" '.services[$s].build' >/dev/null; }
|
||||
|
||||
# Fetch the image the compose file names, without touching the running container.
|
||||
fetch() {
|
||||
if has_build "$1" "$2"; then dc "$1" build --pull "$2" >>"$LOG" 2>&1; else dc "$1" pull -q "$2" >>"$LOG" 2>&1; fi
|
||||
}
|
||||
|
||||
wait_green() { # wait_green <seconds> <health groups...>
|
||||
local until=$(( $(date +%s) + $1 )); shift
|
||||
while :; do
|
||||
bash "$STACK_DIR/updates/health.sh" "$@" >"$STACK_DIR/updates/.last-health" 2>&1 && return 0
|
||||
[ "$(date +%s)" -ge "$until" ] && return 1
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
|
||||
put_back() { # put_back <service> <dir> <old image id> <snapshot id>
|
||||
local svc=$1 dir=$2 old=$3 snap=$4 vols dumps v rel dst src
|
||||
IFS='|' read -r _ _ vols dumps <<<"$(info "$svc")"
|
||||
log " ROLLBACK $svc: stop, data from snapshot $snap, image $(echo "$old" | cut -c8-19)"
|
||||
dc "$dir" stop "$svc" >>"$LOG" 2>&1
|
||||
case $svc in outline|outline-postgres) dc "$dir" stop outline >>"$LOG" 2>&1 ;; esac
|
||||
rm -rf "$STAGE"; mkdir -p "$STAGE" "$ASIDE"
|
||||
local inc=(--include "$STACK_DIR/backup/dumps"); for v in $vols; do inc+=(--include "$VOLUMES/$v"); done
|
||||
restic restore "$snap" --target "$STAGE" "${inc[@]}" >>"$LOG" 2>&1 || { log " restic restore FAILED, nothing changed on disk"; return 1; }
|
||||
for v in $vols; do # the failed state is moved aside, not deleted
|
||||
rm -rf "$ASIDE/$v"; mv "$VOLUMES/$v/_data" "$ASIDE/$v"; mv "$STAGE$VOLUMES/$v/_data" "$VOLUMES/$v/_data"
|
||||
done
|
||||
for v in $dumps; do # consistent online dump over the raw file copy
|
||||
src="$STAGE$STACK_DIR/backup/dumps/${v%%=*}"; rel=${v#*=}; dst="$VOLUMES/${rel%%/*}/_data/${rel#*/}"
|
||||
rm -f "$dst-wal" "$dst-shm"; cp "$src" "$dst.new"; chown --reference="$dst" "$dst.new"; chmod --reference="$dst" "$dst.new"; mv "$dst.new" "$dst"
|
||||
done
|
||||
docker tag "$old" "selfhostedops-rollback/$svc:previous"
|
||||
printf 'services:\n %s:\n image: selfhostedops-rollback/%s:previous\n' "$svc" "$svc" > "$dir/compose.rollback.yml"
|
||||
local files=(-f "$(ls "$dir"/compose.yml "$dir"/docker-compose.yml 2>/dev/null | head -1)" -f "$dir/compose.rollback.yml")
|
||||
reload_pg() { # empty the schema and load the pg_dump from the snapshot
|
||||
for _ in $(seq 60); do docker exec "$P-outline-postgres-1" pg_isready -U outline >/dev/null 2>&1 && break; sleep 1; done
|
||||
docker exec "$P-outline-postgres-1" psql -U outline -q -c 'drop schema public cascade; create schema public;' outline >>"$LOG" 2>&1
|
||||
zcat "$STAGE$STACK_DIR/backup/dumps/outline.sql.gz" | docker exec -i "$P-outline-postgres-1" psql -U outline -q outline >>"$LOG" 2>&1
|
||||
}
|
||||
case $svc in
|
||||
outline) reload_pg; dc "$dir" "${files[@]}" up -d --no-deps outline >>"$LOG" 2>&1 ;;
|
||||
outline-postgres) dc "$dir" "${files[@]}" up -d --no-deps outline-postgres >>"$LOG" 2>&1; reload_pg; dc "$dir" up -d --no-deps outline >>"$LOG" 2>&1 ;;
|
||||
*) dc "$dir" "${files[@]}" up -d --no-deps "$svc" >>"$LOG" 2>&1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
update_one() {
|
||||
local svc=$1 dir groups old new snap t0 t1
|
||||
IFS='|' read -r dir groups _ _ <<<"$(info "$svc")" || { log "unknown service $svc"; return 2; }
|
||||
[ -n "$(cid "$dir" "$svc")" ] || { log "$svc: not running, skipped"; return 0; }
|
||||
old=$(running_image "$dir" "$svc")
|
||||
log "$svc: fetching $(target_ref "$dir" "$svc")"
|
||||
fetch "$dir" "$svc" || { log "$svc: FETCH FAILED, service untouched"; return 1; }
|
||||
new=$(docker image inspect -f '{{.Id}}' "$(target_ref "$dir" "$svc")" 2>/dev/null)
|
||||
if [ "$old" = "$new" ]; then log "$svc: already on that image, nothing to do"; return 0; fi
|
||||
|
||||
log "$svc: snapshot before the change"
|
||||
BACKUP_TAG=pre-update bash "$STACK_DIR/backup/backup.sh" >>"$LOG" 2>&1 || { log "$svc: BACKUP FAILED, service untouched"; return 1; }
|
||||
snap=$(grep -oE 'snapshot [0-9a-f]{8} saved' "$STACK_DIR/backup/backup.log" | tail -1 | cut -d' ' -f2)
|
||||
docker tag "$old" "selfhostedops-rollback/$svc:previous" # a name keeps the old image from being pruned
|
||||
|
||||
t0=$(date +%s)
|
||||
log "$svc: recreate ($(echo "$old" | cut -c8-19) -> $(echo "$new" | cut -c8-19)), snapshot $snap"
|
||||
dc "$dir" up -d --no-deps "$svc" >>"$LOG" 2>&1
|
||||
# shellcheck disable=SC2086
|
||||
if wait_green "$TIMEOUT" $groups && wait_green "$TIMEOUT"; then
|
||||
t1=$(date +%s); log "$svc: GREEN after $((t1 - t0)) s ($(tail -1 "$STACK_DIR/updates/.last-health"))"
|
||||
return 0
|
||||
fi
|
||||
t1=$(date +%s); log "$svc: RED after $((t1 - t0)) s"
|
||||
grep '^FAIL' "$STACK_DIR/updates/.last-health" | tee -a "$LOG"
|
||||
put_back "$svc" "$dir" "$old" "$snap"
|
||||
if wait_green "$TIMEOUT"; then log "$svc: rolled back, health check green again after $(( $(date +%s) - t1 )) s"
|
||||
else log "$svc: rolled back, but the health check is STILL RED"; grep '^FAIL' "$STACK_DIR/updates/.last-health" | tee -a "$LOG"; fi
|
||||
log "$svc: the compose file still names the new tag. Set it back before the next 'up -d'."
|
||||
return 1
|
||||
}
|
||||
|
||||
cmd=${1:-plan}; shift || true
|
||||
set -a; . "$STACK_DIR/backup/.restic-env"; set +a
|
||||
case $cmd in
|
||||
plan)
|
||||
# No registry lookup: with exact tags in the compose file, a different tag is the update.
|
||||
printf '%-18s %-46s %s\n' SERVICE RUNNING TARGET
|
||||
for svc in ${*:-$ORDER}; do
|
||||
IFS='|' read -r dir _ _ _ <<<"$(info "$svc")"; c=$(cid "$dir" "$svc"); [ -n "$c" ] || continue
|
||||
ref=$(target_ref "$dir" "$svc"); cur=$(docker inspect -f '{{.Config.Image}}' "$c")
|
||||
case $cur in sha256:*) cur="(untagged) $(echo "$cur" | cut -c8-19)" ;; esac
|
||||
note=""; [ "$cur" != "$ref" ] && note=" <- update"; has_build "$dir" "$svc" && note=" <- local build, rebuilt on run"
|
||||
printf '%-18s %-46s %s%s\n' "$svc" "$cur" "$ref" "$note"
|
||||
done ;;
|
||||
run)
|
||||
bash "$STACK_DIR/updates/health.sh" >/dev/null 2>&1 || { log "health check is red before the start, not updating anything"; bash "$STACK_DIR/updates/health.sh" | grep '^FAIL'; exit 1; }
|
||||
start=$(date +%s); log "=== update run: ${*:-$ORDER} (timeout per service $TIMEOUT s) ==="
|
||||
for svc in ${*:-$ORDER}; do
|
||||
update_one "$svc" || { log "=== STOPPED at $svc after $(( $(date +%s) - start )) s, later services untouched ==="; exit 1; }
|
||||
done
|
||||
log "=== all done in $(( $(date +%s) - start )) s ===" ;;
|
||||
*) sed -n '2,24p' "$0"; exit 2 ;;
|
||||
esac
|
||||
Loading…
Add table
Add a link
Reference in a new issue