55b2fe7646
updates/health.sh: 43 checks across all services (API answers, git refs, a push message round trip, SSO redirects and token endpoint, mesh state, backup age). updates/update.sh: per service pull or build, snapshot, recreate, wait for the health check, put image and data back on red and stop the run. backup/backup.sh: online dumps for n8n, Pocket ID and tinyauth; BACKUP_TAG so pre-update snapshots are not removed by the nightly keep-daily rule. compose.yml and caddy/Dockerfile: exact versions instead of latest, 16, 7-alpine and caddy:2-alpine.
139 lines
5.9 KiB
Bash
Executable file
139 lines
5.9 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# ~$ selfhostedops - nightly backup of the whole stack (episode 7)
|
|
#
|
|
# Application-consistent, without stopping anything:
|
|
# 1. pg_dump of Outline's Postgres -> backup/dumps/outline.sql.gz
|
|
# 2. online SQLite backups (Forgejo, Vaultwarden, Uptime Kuma, ntfy, n8n,
|
|
# Pocket ID, tinyauth, NetBird)
|
|
# -> backup/dumps/*.db
|
|
# 3. restic backup of the stack directory, /opt/netbird and every stateful
|
|
# docker volume (the raw files, plus the dumps from 1 and 2)
|
|
# 4. restic forget + prune (7 daily, 4 weekly, 6 monthly)
|
|
# 5. one heartbeat to the Uptime Kuma push monitor "Nightly backup"
|
|
#
|
|
# Credentials live in backup/.restic-env (mode 600), see .restic-env.example.
|
|
# Runs from systemd: backup/selfhostedops-backup.timer (03:00 Europe/Berlin).
|
|
set -Eeuo pipefail
|
|
|
|
STACK_DIR=${STACK_DIR:-/opt/selfhostedops-stack}
|
|
NETBIRD_DIR=${NETBIRD_DIR:-/opt/netbird}
|
|
BACKUP_DIR="$STACK_DIR/backup"
|
|
DUMP_DIR="$BACKUP_DIR/dumps"
|
|
LOG_FILE="$BACKUP_DIR/backup.log"
|
|
PROJECT=${PROJECT:-selfhostedops-stack}
|
|
VOLUMES=/var/lib/docker/volumes
|
|
|
|
# Tag of the snapshot. The nightly timer uses "nightly". updates/update.sh
|
|
# runs this script with BACKUP_TAG=pre-update: those snapshots are left alone
|
|
# by forget below. Without that, "keep 7 daily" would drop the snapshot from
|
|
# before the update as soon as a second backup runs on the same day (episode 17).
|
|
BACKUP_TAG=${BACKUP_TAG:-nightly}
|
|
|
|
# Small alpine image with a current sqlite3, used for the online SQLite copies.
|
|
SQLITE_IMAGE=${SQLITE_IMAGE:-keinos/sqlite3:latest}
|
|
|
|
# Uptime Kuma push monitor (episode 6). Leave empty to skip the heartbeat.
|
|
# Can also be set in .restic-env so this file stays free of instance data.
|
|
# KUMA_PUSH_URL=
|
|
KUMA_PUSH_URL=${KUMA_PUSH_URL:-}
|
|
|
|
# Volumes that are rebuilt from scratch on restore and therefore not backed up:
|
|
# dind_data the CI runner's own docker image cache (1.8 GB of pulled layers)
|
|
# runner_data runner registration, re-created with one forgejo-runner register
|
|
# docker_certs TLS certs between runner and dind, regenerated on first start
|
|
EXCLUDE_VOLUMES=(dind_data runner_data docker_certs)
|
|
|
|
log() { printf '%s %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*"; }
|
|
|
|
on_error() {
|
|
log "BACKUP FAILED (line $1)"
|
|
if [ -n "$KUMA_PUSH_URL" ]; then
|
|
curl -fsS -o /dev/null "${KUMA_PUSH_URL}?status=down&msg=backup+failed" || true
|
|
fi
|
|
}
|
|
compose() { docker compose --project-directory "$STACK_DIR" "$@"; }
|
|
|
|
# sqlite_backup <volume> <path inside volume> <dump name>
|
|
# Uses sqlite's online backup API, so the copy is consistent even while the
|
|
# application keeps writing (a plain cp of a WAL-mode database is not).
|
|
sqlite_backup() {
|
|
local vol="$1" db="$2" out="$3"
|
|
if ! docker volume inspect "$vol" >/dev/null 2>&1; then
|
|
log " skip $out: volume $vol does not exist"
|
|
return 0
|
|
fi
|
|
# --user 0: the image defaults to an unprivileged user that can read neither
|
|
# the application databases nor write into dumps/.
|
|
docker run --rm --user 0 -v "$vol:/src" -v "$DUMP_DIR:/dumps" "$SQLITE_IMAGE" \
|
|
sh -c "if [ -f '/src/$db' ]; then sqlite3 '/src/$db' '.backup /dumps/$out'; else echo ' skip $out: /src/$db not found'; fi"
|
|
}
|
|
|
|
main() {
|
|
trap 'on_error $LINENO' ERR
|
|
START=$(date +%s)
|
|
log "=== backup start ==="
|
|
|
|
log "[1/5] pg_dump outline"
|
|
compose exec -T outline-postgres pg_dump -U outline --clean --if-exists outline \
|
|
| gzip -6 > "$DUMP_DIR/outline.sql.gz"
|
|
log " $(du -h "$DUMP_DIR/outline.sql.gz" | cut -f1) outline.sql.gz"
|
|
|
|
log "[2/5] sqlite online backups"
|
|
sqlite_backup "${PROJECT}_forgejo_data" gitea/gitea.db forgejo.db
|
|
sqlite_backup "${PROJECT}_vaultwarden_data" db.sqlite3 vaultwarden.sqlite3
|
|
sqlite_backup "${PROJECT}_kuma_data" kuma.db kuma.db
|
|
sqlite_backup "${PROJECT}_ntfy_data" user.db ntfy-user.db
|
|
sqlite_backup "${PROJECT}_ntfy_cache" cache.db ntfy-cache.db
|
|
sqlite_backup "${PROJECT}_n8n_data" database.sqlite n8n.sqlite
|
|
sqlite_backup "${PROJECT}_pocket_id_data" pocket-id.db pocket-id.db
|
|
sqlite_backup "${PROJECT}_tinyauth_data" tinyauth.db tinyauth.db
|
|
sqlite_backup "netbird_netbird_data" store.db netbird-store.db
|
|
sqlite_backup "netbird_netbird_data" idp.db netbird-idp.db
|
|
sqlite_backup "netbird_netbird_data" events.db netbird-events.db
|
|
log " $(ls "$DUMP_DIR" | wc -l) dump files, $(du -sh "$DUMP_DIR" | cut -f1) total"
|
|
|
|
log "[3/5] restic backup"
|
|
# The whole volumes directory goes in, so a volume added by a later episode
|
|
# is backed up without anyone remembering to add it here. Docker's own
|
|
# volume registry (metadata.db) belongs to the docker on THIS box, not to
|
|
# the backup: on a restore box, `docker volume create` rebuilds it.
|
|
EXCLUDES=(--exclude "$VOLUMES/metadata.db" --exclude "$VOLUMES/backingFsBlockDev" --exclude "$LOG_FILE")
|
|
for v in "${EXCLUDE_VOLUMES[@]}"; do
|
|
EXCLUDES+=(--exclude "$VOLUMES/${PROJECT}_$v")
|
|
done
|
|
restic backup --tag "$BACKUP_TAG" "${EXCLUDES[@]}" \
|
|
"$STACK_DIR" "$NETBIRD_DIR" "$VOLUMES"
|
|
|
|
if [ "$BACKUP_TAG" = nightly ]; then
|
|
log "[4/5] restic forget + prune"
|
|
restic forget --tag nightly --group-by host,tags \
|
|
--keep-daily 7 --keep-weekly 4 --keep-monthly 6 --prune
|
|
else
|
|
log "[4/5] skipped: snapshots tagged $BACKUP_TAG are kept until you remove them"
|
|
fi
|
|
|
|
END=$(date +%s)
|
|
DURATION=$((END - START))
|
|
log "[5/5] done in ${DURATION}s"
|
|
|
|
if [ -n "$KUMA_PUSH_URL" ]; then
|
|
curl -fsS -o /dev/null "${KUMA_PUSH_URL}?status=up&msg=OK&ping=${DURATION}" \
|
|
&& log " heartbeat sent to Uptime Kuma" \
|
|
|| log " heartbeat to Uptime Kuma failed (backup itself is fine)"
|
|
fi
|
|
|
|
log "=== backup end ==="
|
|
}
|
|
|
|
# Only one run at a time (the timer and a manual run must not overlap).
|
|
exec 9>/run/lock/selfhostedops-backup.lock
|
|
flock -n 9 || { log "another backup is still running, exiting"; exit 1; }
|
|
|
|
set -a
|
|
# shellcheck disable=SC1091
|
|
. "$BACKUP_DIR/.restic-env"
|
|
set +a
|
|
|
|
mkdir -p "$DUMP_DIR"
|
|
main 2>&1 | tee -a "$LOG_FILE"
|
|
exit "${PIPESTATUS[0]}"
|