From cf374770c4fff56f4ad6290f85b3c107b8469e91 Mon Sep 17 00:00:00 2001 From: Maximilian Grimm Date: Wed, 2 Sep 2026 19:51:35 +0200 Subject: [PATCH] Episode 7: nightly restic backup to S3 with consistent dumps, heartbeat, and a restore script for a fresh box Claude-Session: https://claude.ai/code/session_01Y1BTehJjJ4Chr6YscVRXmn --- .gitignore | 3 + backup/.restic-env.example | 18 +++ backup/README.md | 180 ++++++++++++++++++++++++++++ backup/backup.sh | 125 +++++++++++++++++++ backup/restore.sh | 96 +++++++++++++++ backup/selfhostedops-backup.service | 12 ++ backup/selfhostedops-backup.timer | 10 ++ 7 files changed, 444 insertions(+) create mode 100644 backup/.restic-env.example create mode 100644 backup/README.md create mode 100755 backup/backup.sh create mode 100644 backup/restore.sh create mode 100644 backup/selfhostedops-backup.service create mode 100644 backup/selfhostedops-backup.timer diff --git a/.gitignore b/.gitignore index ba509d4..e3c9429 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ .env runner-config.yml +backup/.restic-env +backup/dumps/ +backup/backup.log diff --git a/backup/.restic-env.example b/backup/.restic-env.example new file mode 100644 index 0000000..fd68c2d --- /dev/null +++ b/backup/.restic-env.example @@ -0,0 +1,18 @@ +# ~$ selfhostedops - restic credentials for backup/backup.sh +# Copy to backup/.restic-env and chmod 600 it. The file is sourced by the +# backup script and by you on a restore box (set -a; . ./.restic-env; set +a). + +# Any S3-compatible bucket: Hetzner Object Storage, Backblaze B2, Wasabi, +# MinIO or Garage on a second box. Use a key that can only see this bucket. +RESTIC_REPOSITORY=s3:https://s3.example.com/selfhostedops-backup +AWS_ACCESS_KEY_ID=GK0000000000000000000000 +AWS_SECRET_ACCESS_KEY=change-me +AWS_DEFAULT_REGION=garage + +# Encrypts everything in the repository. Generate with: openssl rand -hex 32 +# Store it in Vaultwarden AND somewhere outside the stack (paper, second +# password manager): without it the backup is a pile of random bytes. +RESTIC_PASSWORD=change-me + +# Optional: Uptime Kuma push monitor "Nightly backup" (episode 6). +# KUMA_PUSH_URL=https://status.example.com/api/push/XXXXXXXXXX diff --git a/backup/README.md b/backup/README.md new file mode 100644 index 0000000..f67f928 --- /dev/null +++ b/backup/README.md @@ -0,0 +1,180 @@ +# Backup and restore - lose the whole stack once and get it back (episode 7) + +One script, one systemd timer, one S3 bucket. Every night the stack dumps +its databases, restic ships everything to the bucket, old snapshots are +pruned, and Uptime Kuma gets a heartbeat. The restore is a second script +that turns an empty Debian box into the running stack in a few minutes. + +Measured on the demo box (2 vCPU, 4 GB, ~590 MB of data): first backup +11 s, every night after that about 5 s and a few MB. Restore on a fresh +box: 10 s install, 7 s restore, 68 s `docker compose up` (image pulls), +services answering with their old certificates after about two minutes. + +## What goes into the backup + +| Path | Why | +| --- | --- | +| `/opt/selfhostedops-stack` | `compose.yml`, `Caddyfile`, `.env`, this folder incl. `dumps/` | +| `/opt/netbird` | NetBird compose, `config.yaml`, `dashboard.env` | +| `/var/lib/docker/volumes` | every named volume: Forgejo, Outline (Postgres + uploads), Vaultwarden, Uptime Kuma, ntfy, n8n, NetBird, **and Caddy's `caddy_data`** (the Let's Encrypt certificates and account, so the restored box serves valid HTTPS before DNS has moved) | + +Excluded, because a restore rebuilds them for free: + +| Volume | Why it is out | +| --- | --- | +| `dind_data` | the CI runner's private docker image cache, 1.8 GB of pulled layers | +| `runner_data` | the runner registration, one `forgejo-runner register` recreates it | +| `docker_certs` | TLS between runner and dind, generated on first start | +| `metadata.db` | docker's own volume registry; belongs to the docker on that box | + +The whole volumes directory is the target, so a volume a later episode adds +is backed up without anyone editing the script. Forgetting to add a new +volume to a hand-maintained list is the classic way backups rot. + +## Application-consistent without downtime + +A raw copy of a running database file is a coin flip. So before restic runs: + +- Outline: `pg_dump` through `docker compose exec outline-postgres` + -> `dumps/outline.sql.gz` +- Forgejo, Vaultwarden, Uptime Kuma, ntfy, NetBird (all SQLite): + `sqlite3 ".backup "` in a throwaway `keinos/sqlite3` container + -> `dumps/*.db`. That is SQLite's online backup API, safe while the app + keeps writing (a `cp` of a WAL-mode database is not). + +The raw volume files are still in the snapshot. In the normal case they +come back fine (the restore rehearsal below used exactly those). The dumps +are the fallback for the day they do not. + +## Install (on the box) + +```bash +apt-get install -y restic # Debian 13 ships 0.18.0 +cd /opt/selfhostedops-stack +mkdir -p backup && cp backup/.restic-env.example backup/.restic-env +chmod 600 backup/.restic-env && nano backup/.restic-env # bucket, key, password +docker pull keinos/sqlite3:latest + +set -a; . backup/.restic-env; set +a +restic init +``` + +Any S3-compatible target works: Hetzner Object Storage, Backblaze B2, +Wasabi, MinIO or Garage on a second box. Give the key access to **this one +bucket only**: the key sits on the box it protects, and a box that is +compromised must not be able to reach the other buckets. + +Put the restic password in Vaultwarden **and** somewhere outside the stack. +Vaultwarden is inside the backup; if the box is gone, so is the vault that +holds the password to the backup of the vault. + +Then the timer: + +```bash +ln -s /opt/selfhostedops-stack/backup/selfhostedops-backup.service /etc/systemd/system/ +ln -s /opt/selfhostedops-stack/backup/selfhostedops-backup.timer /etc/systemd/system/ +systemctl daemon-reload +systemctl enable --now selfhostedops-backup.timer +systemctl list-timers selfhostedops-backup.timer +``` + +`OnCalendar=*-*-* 03:00:00 Europe/Berlin` with `Persistent=true`: a box that +was down at 03:00 runs the backup as soon as it is back. + +Run one by hand first and read the summary: + +```bash +./backup.sh +restic snapshots +restic stats --mode raw-data +restic check +``` + +Retention is `--keep-daily 7 --keep-weekly 4 --keep-monthly 6`, applied +with `--prune` after every run. Two runs on the same day count as one day: +the policy keeps the last one of the day (and the oldest snapshot overall), +the one in between is pruned. + +### Heartbeat + +The last line of the script hits the Uptime Kuma push monitor from episode +6 with `?status=up&ping=`. If the script fails it sends +`status=down`; if it never runs, Kuma raises the alarm itself after the +monitor's grace period. Set `KUMA_PUSH_URL` in `.restic-env`. + +## Restore runbook + +Rehearsed on a fresh Hetzner cpx22 with Debian 13. Everything below is +`backup/restore.sh`; the phases are listed so you can do them by hand. + +```bash +# 0. fresh box, as root. Bring the one file that is not in the backup: +mkdir -p /opt/selfhostedops-stack/backup +scp .restic-env root@NEWBOX:/opt/selfhostedops-stack/backup/ # from your vault +curl -fsSLO https://git.example.com/you/selfhostedops-stack/raw/branch/main/backup/restore.sh + +# 1. install docker, compose plugin, restic (measured: 10 s) +bash restore.sh install + +# 2. named volumes first, then the files (measured: 7 s) +bash restore.sh restore +# = restic ls latest /var/lib/docker/volumes | ... | docker volume create +# restic restore latest --target / + +# 3. start (measured: 68 s) +bash restore.sh up +# = cd /opt/selfhostedops-stack && docker compose up -d +# cd /opt/netbird && docker compose up -d +``` + +Verify before touching DNS. The old names still point at the old box, so +pin them to the new IP for curl: + +```bash +NEW=203.0.113.10 +for h in git wiki vault status vpn ntfy; do + curl -sS -o /dev/null -w "%{http_code} tls=%{ssl_verify_result} $h\n" \ + --resolve $h.ops.example.com:443:$NEW https://$h.ops.example.com/ +done +curl -sS --resolve git.ops.example.com:443:$NEW \ + 'https://git.ops.example.com/api/v1/repos/search' | jq '.data[].full_name' +curl -sS --resolve wiki.ops.example.com:443:$NEW https://wiki.ops.example.com/_health +curl -sS --resolve vault.ops.example.com:443:$NEW https://vault.ops.example.com/alive +``` + +`tls=0` on every line means the certificates came out of `caddy_data` and +are the originals. Then move the DNS record, delete the old box. + +The restored box has `.restic-env` but no timer. Install the timer there +the same way once it is the box; two hosts writing to one repository works +but muddles the snapshot list. + +## Gotchas we hit + +- **A function named `install` in a bash script calls itself.** The first + restore script had `install() { ... install -m 0755 -d /etc/apt/keyrings ... }` + and looped for ten minutes before anyone looked. Prefix your phase + functions or call `/usr/bin/install`. +- **`keinos/sqlite3` runs as an unprivileged user.** It can neither read + the application databases nor write into `dumps/`: `Error: cannot open + "/dumps/forgejo.db"`. `docker run --user 0` fixes it. +- **`trap ... ERR` does not fire inside functions** unless the script sets + `-E` (`set -Eeuo pipefail`). Without it the failure heartbeat never goes out. +- **`docker compose up` warns about every restored volume:** `volume + "selfhostedops-stack_forgejo_data" already exists but was not created by + Docker Compose`. It is a warning, compose uses the volume. Do not `down -v` + to make it go away. +- **`:latest` images mean a restore is also an upgrade.** Outline ran two + schema migrations on the restored database because the fresh pull was + newer than the image on the old box. It worked here; pin tags if you want + a restore that changes nothing but the box. +- **Ownership comes back with the files** (restic restores uid/gid), but + only if you restore as root with `--target /`. Restoring into a scratch + directory and copying by hand loses the `git` user on Forgejo's data. +- **The restored Uptime Kuma starts monitoring and alerting immediately**, + in parallel with the old box until DNS moves. Expect duplicate + notifications for that window. +- **Excluded targets still show up in the snapshot's path list** if you + pass them as backup arguments and exclude them at the same time. Pass the + parent directory and let `--exclude` do the work; the path list stays + honest. diff --git a/backup/backup.sh b/backup/backup.sh new file mode 100755 index 0000000..6bcfb8d --- /dev/null +++ b/backup/backup.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# ~$ selfhostedops - nightly backup of the whole stack (episode 7) +# +# Application-consistent, without stopping anything: +# 1. pg_dump of Outline's Postgres -> backup/dumps/outline.sql.gz +# 2. online SQLite backups (Forgejo, Vaultwarden, Uptime Kuma, ntfy, NetBird) +# -> backup/dumps/*.db +# 3. restic backup of the stack directory, /opt/netbird and every stateful +# docker volume (the raw files, plus the dumps from 1 and 2) +# 4. restic forget + prune (7 daily, 4 weekly, 6 monthly) +# 5. one heartbeat to the Uptime Kuma push monitor "Nightly backup" +# +# Credentials live in backup/.restic-env (mode 600), see .restic-env.example. +# Runs from systemd: backup/selfhostedops-backup.timer (03:00 Europe/Berlin). +set -Eeuo pipefail + +STACK_DIR=${STACK_DIR:-/opt/selfhostedops-stack} +NETBIRD_DIR=${NETBIRD_DIR:-/opt/netbird} +BACKUP_DIR="$STACK_DIR/backup" +DUMP_DIR="$BACKUP_DIR/dumps" +LOG_FILE="$BACKUP_DIR/backup.log" +PROJECT=${PROJECT:-selfhostedops-stack} +VOLUMES=/var/lib/docker/volumes + +# Small alpine image with a current sqlite3, used for the online SQLite copies. +SQLITE_IMAGE=${SQLITE_IMAGE:-keinos/sqlite3:latest} + +# Uptime Kuma push monitor (episode 6). Leave empty to skip the heartbeat. +# Can also be set in .restic-env so this file stays free of instance data. +# KUMA_PUSH_URL= +KUMA_PUSH_URL=${KUMA_PUSH_URL:-} + +# Volumes that are rebuilt from scratch on restore and therefore not backed up: +# dind_data the CI runner's own docker image cache (1.8 GB of pulled layers) +# runner_data runner registration, re-created with one forgejo-runner register +# docker_certs TLS certs between runner and dind, regenerated on first start +EXCLUDE_VOLUMES=(dind_data runner_data docker_certs) + +log() { printf '%s %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*"; } + +on_error() { + log "BACKUP FAILED (line $1)" + if [ -n "$KUMA_PUSH_URL" ]; then + curl -fsS -o /dev/null "${KUMA_PUSH_URL}?status=down&msg=backup+failed" || true + fi +} +compose() { docker compose --project-directory "$STACK_DIR" "$@"; } + +# sqlite_backup +# Uses sqlite's online backup API, so the copy is consistent even while the +# application keeps writing (a plain cp of a WAL-mode database is not). +sqlite_backup() { + local vol="$1" db="$2" out="$3" + if ! docker volume inspect "$vol" >/dev/null 2>&1; then + log " skip $out: volume $vol does not exist" + return 0 + fi + # --user 0: the image defaults to an unprivileged user that can read neither + # the application databases nor write into dumps/. + docker run --rm --user 0 -v "$vol:/src" -v "$DUMP_DIR:/dumps" "$SQLITE_IMAGE" \ + sh -c "if [ -f '/src/$db' ]; then sqlite3 '/src/$db' '.backup /dumps/$out'; else echo ' skip $out: /src/$db not found'; fi" +} + +main() { +trap 'on_error $LINENO' ERR +START=$(date +%s) +log "=== backup start ===" + +log "[1/5] pg_dump outline" +compose exec -T outline-postgres pg_dump -U outline --clean --if-exists outline \ + | gzip -6 > "$DUMP_DIR/outline.sql.gz" +log " $(du -h "$DUMP_DIR/outline.sql.gz" | cut -f1) outline.sql.gz" + +log "[2/5] sqlite online backups" +sqlite_backup "${PROJECT}_forgejo_data" gitea/gitea.db forgejo.db +sqlite_backup "${PROJECT}_vaultwarden_data" db.sqlite3 vaultwarden.sqlite3 +sqlite_backup "${PROJECT}_kuma_data" kuma.db kuma.db +sqlite_backup "${PROJECT}_ntfy_data" user.db ntfy-user.db +sqlite_backup "${PROJECT}_ntfy_cache" cache.db ntfy-cache.db +sqlite_backup "netbird_netbird_data" store.db netbird-store.db +sqlite_backup "netbird_netbird_data" idp.db netbird-idp.db +sqlite_backup "netbird_netbird_data" events.db netbird-events.db +log " $(ls "$DUMP_DIR" | wc -l) dump files, $(du -sh "$DUMP_DIR" | cut -f1) total" + +log "[3/5] restic backup" +# The whole volumes directory goes in, so a volume added by a later episode +# is backed up without anyone remembering to add it here. Docker's own +# volume registry (metadata.db) belongs to the docker on THIS box, not to +# the backup: on a restore box, `docker volume create` rebuilds it. +EXCLUDES=(--exclude "$VOLUMES/metadata.db" --exclude "$VOLUMES/backingFsBlockDev" --exclude "$LOG_FILE") +for v in "${EXCLUDE_VOLUMES[@]}"; do + EXCLUDES+=(--exclude "$VOLUMES/${PROJECT}_$v") +done +restic backup --tag nightly "${EXCLUDES[@]}" \ + "$STACK_DIR" "$NETBIRD_DIR" "$VOLUMES" + +log "[4/5] restic forget + prune" +restic forget --tag nightly --group-by host,tags \ + --keep-daily 7 --keep-weekly 4 --keep-monthly 6 --prune + +END=$(date +%s) +DURATION=$((END - START)) +log "[5/5] done in ${DURATION}s" + +if [ -n "$KUMA_PUSH_URL" ]; then + curl -fsS -o /dev/null "${KUMA_PUSH_URL}?status=up&msg=OK&ping=${DURATION}" \ + && log " heartbeat sent to Uptime Kuma" \ + || log " heartbeat to Uptime Kuma failed (backup itself is fine)" +fi + +log "=== backup end ===" +} + +# Only one run at a time (the timer and a manual run must not overlap). +exec 9>/run/lock/selfhostedops-backup.lock +flock -n 9 || { log "another backup is still running, exiting"; exit 1; } + +set -a +# shellcheck disable=SC1091 +. "$BACKUP_DIR/.restic-env" +set +a + +mkdir -p "$DUMP_DIR" +main 2>&1 | tee -a "$LOG_FILE" +exit "${PIPESTATUS[0]}" diff --git a/backup/restore.sh b/backup/restore.sh new file mode 100644 index 0000000..1cb81d4 --- /dev/null +++ b/backup/restore.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# ~$ selfhostedops - bring the whole stack back on an empty Debian box (episode 7) +# +# Usage on a fresh Debian 12/13 server, as root: +# mkdir -p /opt/selfhostedops-stack/backup +# (from the vault) +# curl -fsSLO https:///selfhostedops-stack/raw/branch/main/backup/restore.sh +# bash restore.sh install # docker, compose plugin, restic +# bash restore.sh restore # docker volumes + restic restore latest --target / +# bash restore.sh up # docker compose up -d for the stack and NetBird +# bash restore.sh all # the three in a row +# +# Each phase prints its own wall-clock time. Nothing here touches DNS: point +# the records at the new box when the services answer (test first with +# curl --resolve wiki.example.com:443: https://wiki.example.com/). +set -Eeuo pipefail + +STACK_DIR=${STACK_DIR:-/opt/selfhostedops-stack} +NETBIRD_DIR=${NETBIRD_DIR:-/opt/netbird} +ENV_FILE="$STACK_DIR/backup/.restic-env" +SNAPSHOT=${SNAPSHOT:-latest} + +log() { printf '%s %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*"; } +phase_start() { PHASE_T0=$(date +%s%N); log "=== phase $1 ==="; } +phase_end() { + local ms=$(( ($(date +%s%N) - PHASE_T0) / 1000000 )) + log "=== phase $1 done in $((ms / 1000)).$(printf '%03d' $((ms % 1000)))s ===" +} + +load_env() { + [ -f "$ENV_FILE" ] || { echo "missing $ENV_FILE (copy it from your vault first)"; exit 1; } + set -a + # shellcheck disable=SC1090 + . "$ENV_FILE" + set +a +} + +phase_install() { + phase_start install + export DEBIAN_FRONTEND=noninteractive + apt-get update -q + apt-get install -y -q ca-certificates curl gnupg restic + # (coreutils install, not a function of this script) + /usr/bin/install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc + chmod a+r /etc/apt/keyrings/docker.asc + # shellcheck disable=SC1091 + . /etc/os-release + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian $VERSION_CODENAME stable" \ + > /etc/apt/sources.list.d/docker.list + apt-get update -q + apt-get install -y -q docker-ce docker-ce-cli containerd.io docker-compose-plugin + systemctl enable --now docker + docker --version; docker compose version; restic version + phase_end install +} + +phase_restore() { + load_env + phase_start restore + log "snapshots in the repository:" + restic snapshots --compact + # Named volumes must exist in docker's registry before the files land in + # /var/lib/docker/volumes//_data, otherwise compose creates fresh + # empty ones on top. The names come straight out of the snapshot. + log "creating docker volumes" + restic ls "$SNAPSHOT" /var/lib/docker/volumes \ + | grep '^/var/lib/docker/volumes/[^/]*_' \ + | while read -r dir; do docker volume create "$(basename "$dir")"; done + log "restic restore $SNAPSHOT --target /" + restic restore "$SNAPSHOT" --target / + log "restored:" + du -sh "$STACK_DIR" "$NETBIRD_DIR" /var/lib/docker/volumes/*_*/_data + phase_end restore +} + +phase_up() { + phase_start up + cd "$STACK_DIR" + docker compose up -d + # NetBird joins the stack's proxy network, so it comes second. + if [ -d "$NETBIRD_DIR" ]; then + cd "$NETBIRD_DIR" + docker compose up -d + fi + docker ps --format 'table {{.Names}}\t{{.Status}}' + phase_end up +} + +case "${1:-}" in + install) phase_install ;; + restore) phase_restore ;; + up) phase_up ;; + all) phase_install; phase_restore; phase_up ;; + *) echo "usage: $0 install|restore|up|all"; exit 1 ;; +esac diff --git a/backup/selfhostedops-backup.service b/backup/selfhostedops-backup.service new file mode 100644 index 0000000..506c35b --- /dev/null +++ b/backup/selfhostedops-backup.service @@ -0,0 +1,12 @@ +[Unit] +Description=selfhostedops nightly backup (restic to S3) +After=docker.service network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/opt/selfhostedops-stack/backup +ExecStart=/opt/selfhostedops-stack/backup/backup.sh +Nice=10 +IOSchedulingClass=best-effort +IOSchedulingPriority=7 diff --git a/backup/selfhostedops-backup.timer b/backup/selfhostedops-backup.timer new file mode 100644 index 0000000..edf8f0f --- /dev/null +++ b/backup/selfhostedops-backup.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Run the selfhostedops backup every night at 03:00 Europe/Berlin + +[Timer] +OnCalendar=*-*-* 03:00:00 Europe/Berlin +# If the box was off at 03:00, run as soon as it is back. +Persistent=true + +[Install] +WantedBy=timers.target