#!/usr/bin/env bash
# ~$ selfhostedops - nightly backup of the whole stack (episode 7)
#
# Application-consistent, without stopping anything:
#   1. pg_dump of Outline's Postgres            -> backup/dumps/outline.sql.gz
#   2. online SQLite backups (Forgejo, Vaultwarden, Uptime Kuma, ntfy, NetBird)
#                                               -> backup/dumps/*.db
#   3. restic backup of the stack directory, /opt/netbird and every stateful
#      docker volume (the raw files, plus the dumps from 1 and 2)
#   4. restic forget + prune (7 daily, 4 weekly, 6 monthly)
#   5. one heartbeat to the Uptime Kuma push monitor "Nightly backup"
#
# Credentials live in backup/.restic-env (mode 600), see .restic-env.example.
# Runs from systemd: backup/selfhostedops-backup.timer (03:00 Europe/Berlin).
set -Eeuo pipefail

STACK_DIR=${STACK_DIR:-/opt/selfhostedops-stack}
NETBIRD_DIR=${NETBIRD_DIR:-/opt/netbird}
BACKUP_DIR="$STACK_DIR/backup"
DUMP_DIR="$BACKUP_DIR/dumps"
LOG_FILE="$BACKUP_DIR/backup.log"
PROJECT=${PROJECT:-selfhostedops-stack}
VOLUMES=/var/lib/docker/volumes

# Small alpine image with a current sqlite3, used for the online SQLite copies.
SQLITE_IMAGE=${SQLITE_IMAGE:-keinos/sqlite3:latest}

# Uptime Kuma push monitor (episode 6). Leave empty to skip the heartbeat.
# Can also be set in .restic-env so this file stays free of instance data.
# KUMA_PUSH_URL=
KUMA_PUSH_URL=${KUMA_PUSH_URL:-}

# Volumes that are rebuilt from scratch on restore and therefore not backed up:
#   dind_data    the CI runner's own docker image cache (1.8 GB of pulled layers)
#   runner_data  runner registration, re-created with one forgejo-runner register
#   docker_certs TLS certs between runner and dind, regenerated on first start
EXCLUDE_VOLUMES=(dind_data runner_data docker_certs)

log() { printf '%s %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*"; }

on_error() {
  log "BACKUP FAILED (line $1)"
  if [ -n "$KUMA_PUSH_URL" ]; then
    curl -fsS -o /dev/null "${KUMA_PUSH_URL}?status=down&msg=backup+failed" || true
  fi
}
compose() { docker compose --project-directory "$STACK_DIR" "$@"; }

# sqlite_backup <volume> <path inside volume> <dump name>
# Uses sqlite's online backup API, so the copy is consistent even while the
# application keeps writing (a plain cp of a WAL-mode database is not).
sqlite_backup() {
  local vol="$1" db="$2" out="$3"
  if ! docker volume inspect "$vol" >/dev/null 2>&1; then
    log "  skip $out: volume $vol does not exist"
    return 0
  fi
  # --user 0: the image defaults to an unprivileged user that can read neither
  # the application databases nor write into dumps/.
  docker run --rm --user 0 -v "$vol:/src" -v "$DUMP_DIR:/dumps" "$SQLITE_IMAGE" \
    sh -c "if [ -f '/src/$db' ]; then sqlite3 '/src/$db' '.backup /dumps/$out'; else echo '  skip $out: /src/$db not found'; fi"
}

main() {
trap 'on_error $LINENO' ERR
START=$(date +%s)
log "=== backup start ==="

log "[1/5] pg_dump outline"
compose exec -T outline-postgres pg_dump -U outline --clean --if-exists outline \
  | gzip -6 > "$DUMP_DIR/outline.sql.gz"
log "  $(du -h "$DUMP_DIR/outline.sql.gz" | cut -f1) outline.sql.gz"

log "[2/5] sqlite online backups"
sqlite_backup "${PROJECT}_forgejo_data"     gitea/gitea.db  forgejo.db
sqlite_backup "${PROJECT}_vaultwarden_data" db.sqlite3      vaultwarden.sqlite3
sqlite_backup "${PROJECT}_kuma_data"        kuma.db         kuma.db
sqlite_backup "${PROJECT}_ntfy_data"        user.db         ntfy-user.db
sqlite_backup "${PROJECT}_ntfy_cache"       cache.db        ntfy-cache.db
sqlite_backup "netbird_netbird_data"        store.db        netbird-store.db
sqlite_backup "netbird_netbird_data"        idp.db          netbird-idp.db
sqlite_backup "netbird_netbird_data"        events.db       netbird-events.db
log "  $(ls "$DUMP_DIR" | wc -l) dump files, $(du -sh "$DUMP_DIR" | cut -f1) total"

log "[3/5] restic backup"
# The whole volumes directory goes in, so a volume added by a later episode
# is backed up without anyone remembering to add it here. Docker's own
# volume registry (metadata.db) belongs to the docker on THIS box, not to
# the backup: on a restore box, `docker volume create` rebuilds it.
EXCLUDES=(--exclude "$VOLUMES/metadata.db" --exclude "$VOLUMES/backingFsBlockDev" --exclude "$LOG_FILE")
for v in "${EXCLUDE_VOLUMES[@]}"; do
  EXCLUDES+=(--exclude "$VOLUMES/${PROJECT}_$v")
done
restic backup --tag nightly "${EXCLUDES[@]}" \
  "$STACK_DIR" "$NETBIRD_DIR" "$VOLUMES"

log "[4/5] restic forget + prune"
restic forget --tag nightly --group-by host,tags \
  --keep-daily 7 --keep-weekly 4 --keep-monthly 6 --prune

END=$(date +%s)
DURATION=$((END - START))
log "[5/5] done in ${DURATION}s"

if [ -n "$KUMA_PUSH_URL" ]; then
  curl -fsS -o /dev/null "${KUMA_PUSH_URL}?status=up&msg=OK&ping=${DURATION}" \
    && log "  heartbeat sent to Uptime Kuma" \
    || log "  heartbeat to Uptime Kuma failed (backup itself is fine)"
fi

log "=== backup end ==="
}

# Only one run at a time (the timer and a manual run must not overlap).
exec 9>/run/lock/selfhostedops-backup.lock
flock -n 9 || { log "another backup is still running, exiting"; exit 1; }

set -a
# shellcheck disable=SC1091
. "$BACKUP_DIR/.restic-env"
set +a

mkdir -p "$DUMP_DIR"
main 2>&1 | tee -a "$LOG_FILE"
exit "${PIPESTATUS[0]}"