#!/usr/bin/env bash
# ~$ selfhostedops - update one service at a time (episode 17)
#
# "docker compose pull && docker compose up -d" updates everything at once and
# tells you nothing about which of the eleven changes broke the login. This
# does the same work in order:
#
#   for each service:  pull (or build) the image named in the compose file
#                      -> snapshot (backup/backup.sh, tag pre-update)
#                      -> keep the running image under a name of its own
#                      -> recreate that one service
#                      -> wait for updates/health.sh to pass, first the group
#                         of the service, then everything
#                      -> red: put image and data back, stop the whole run
#
#   updates/update.sh plan              what would change, nothing is touched
#   updates/update.sh run               all services, in the order below
#   updates/update.sh run n8n outline   only these
#   TIMEOUT=900 updates/update.sh run uptime-kuma    wait longer for one service
#
# You pick the versions: write the new tag into the compose file first (and
# commit it). The script never invents a version, it only rolls out what the
# file says. Runs on the box, as root. Needs backup/.restic-env (episode 7).
#
# Two things pile up and are yours to clear once you trust the new state:
#   restic forget --tag pre-update --keep-last 3 --prune     the snapshots
#   docker image rm $(docker images -q 'selfhostedops-rollback/*')   the old images
# A service that needs longer than TIMEOUT to come back counts as red and is
# put back. A database migration can take minutes (Uptime Kuma 1 -> 2: five).
set -uo pipefail

STACK_DIR=${STACK_DIR:-/opt/selfhostedops-stack}
NETBIRD_DIR=${NETBIRD_DIR:-/opt/netbird}
P=${PROJECT:-selfhostedops-stack}
VOLUMES=/var/lib/docker/volumes
TIMEOUT=${TIMEOUT:-180}          # seconds a service gets to turn green
STAGE=/var/lib/selfhostedops-restore
ASIDE=/var/lib/selfhostedops-failed-update
LOG=$STACK_DIR/updates/update.log

# Least critical first, the monitoring last: it should watch the others move.
ORDER="ntfy forgejo outline-postgres outline-redis outline vaultwarden n8n pocket-id tinyauth netbird-server dashboard ops-actions ops-readonly caddy uptime-kuma"

# service -> compose directory | health groups | volumes | dump file=volume/path ...
info() {
  case $1 in
    ntfy)             echo "$STACK_DIR|ntfy|${P}_ntfy_data ${P}_ntfy_cache|ntfy-user.db=${P}_ntfy_data/user.db ntfy-cache.db=${P}_ntfy_cache/cache.db" ;;
    forgejo)          echo "$STACK_DIR|forgejo|${P}_forgejo_data|forgejo.db=${P}_forgejo_data/gitea/gitea.db" ;;
    outline-postgres) echo "$STACK_DIR|outline|${P}_outline_pg|" ;;
    outline-redis)    echo "$STACK_DIR|outline||" ;;
    outline)          echo "$STACK_DIR|outline|${P}_outline_data|" ;;
    vaultwarden)      echo "$STACK_DIR|vaultwarden|${P}_vaultwarden_data|vaultwarden.sqlite3=${P}_vaultwarden_data/db.sqlite3" ;;
    n8n)              echo "$STACK_DIR|n8n|${P}_n8n_data|n8n.sqlite=${P}_n8n_data/database.sqlite" ;;
    pocket-id)        echo "$STACK_DIR|pocket-id tinyauth|${P}_pocket_id_data|pocket-id.db=${P}_pocket_id_data/pocket-id.db" ;;
    tinyauth)         echo "$STACK_DIR|tinyauth kuma|${P}_tinyauth_data|tinyauth.db=${P}_tinyauth_data/tinyauth.db" ;;
    uptime-kuma)      echo "$STACK_DIR|kuma|${P}_kuma_data|kuma.db=${P}_kuma_data/kuma.db" ;;
    caddy)            echo "$STACK_DIR|caddy||" ;;
    ops-actions|ops-readonly) echo "$STACK_DIR|ops||" ;;
    netbird-server)   echo "$NETBIRD_DIR|netbird|netbird_netbird_data|netbird-store.db=netbird_netbird_data/store.db netbird-idp.db=netbird_netbird_data/idp.db netbird-events.db=netbird_netbird_data/events.db" ;;
    dashboard)        echo "$NETBIRD_DIR|netbird||" ;;
    *) return 1 ;;
  esac
}

log() { printf '%s %s\n' "$(date -u '+%H:%M:%S')" "$*" | tee -a "$LOG"; }
# profiles: ops-actions and ops-readonly only exist with --profile agent
dc()  { ( cd "$1" && shift && docker compose --profile agent "$@" ); }
cid() { dc "$1" ps -q "$2" 2>/dev/null | head -1; }
running_image() { local c; c=$(cid "$1" "$2"); [ -n "$c" ] && docker inspect -f '{{.Image}}' "$c"; }
target_ref()    { dc "$1" config --format json 2>/dev/null | jq -r --arg s "$2" '.services[$s].image'; }
has_build()     { dc "$1" config --format json 2>/dev/null | jq -e --arg s "$2" '.services[$s].build' >/dev/null; }

# Fetch the image the compose file names, without touching the running container.
fetch() {
  if has_build "$1" "$2"; then dc "$1" build --pull "$2" >>"$LOG" 2>&1; else dc "$1" pull -q "$2" >>"$LOG" 2>&1; fi
}

wait_green() {   # wait_green <seconds> <health groups...>
  local until=$(( $(date +%s) + $1 )); shift
  while :; do
    bash "$STACK_DIR/updates/health.sh" "$@" >"$STACK_DIR/updates/.last-health" 2>&1 && return 0
    [ "$(date +%s)" -ge "$until" ] && return 1
    sleep 5
  done
}

put_back() {     # put_back <service> <dir> <old image id> <snapshot id>
  local svc=$1 dir=$2 old=$3 snap=$4 vols dumps v rel dst src
  IFS='|' read -r _ _ vols dumps <<<"$(info "$svc")"
  log "  ROLLBACK $svc: stop, data from snapshot $snap, image $(echo "$old" | cut -c8-19)"
  dc "$dir" stop "$svc" >>"$LOG" 2>&1
  case $svc in outline|outline-postgres) dc "$dir" stop outline >>"$LOG" 2>&1 ;; esac
  rm -rf "$STAGE"; mkdir -p "$STAGE" "$ASIDE"
  local inc=(--include "$STACK_DIR/backup/dumps"); for v in $vols; do inc+=(--include "$VOLUMES/$v"); done
  restic restore "$snap" --target "$STAGE" "${inc[@]}" >>"$LOG" 2>&1 || { log "  restic restore FAILED, nothing changed on disk"; return 1; }
  for v in $vols; do     # the failed state is moved aside, not deleted
    rm -rf "$ASIDE/$v"; mv "$VOLUMES/$v/_data" "$ASIDE/$v"; mv "$STAGE$VOLUMES/$v/_data" "$VOLUMES/$v/_data"
  done
  for v in $dumps; do    # consistent online dump over the raw file copy
    src="$STAGE$STACK_DIR/backup/dumps/${v%%=*}"; rel=${v#*=}; dst="$VOLUMES/${rel%%/*}/_data/${rel#*/}"
    rm -f "$dst-wal" "$dst-shm"; cp "$src" "$dst.new"; chown --reference="$dst" "$dst.new"; chmod --reference="$dst" "$dst.new"; mv "$dst.new" "$dst"
  done
  docker tag "$old" "selfhostedops-rollback/$svc:previous"
  printf 'services:\n  %s:\n    image: selfhostedops-rollback/%s:previous\n' "$svc" "$svc" > "$dir/compose.rollback.yml"
  local files=(-f "$(ls "$dir"/compose.yml "$dir"/docker-compose.yml 2>/dev/null | head -1)" -f "$dir/compose.rollback.yml")
  reload_pg() {          # empty the schema and load the pg_dump from the snapshot
    for _ in $(seq 60); do docker exec "$P-outline-postgres-1" pg_isready -U outline >/dev/null 2>&1 && break; sleep 1; done
    docker exec "$P-outline-postgres-1" psql -U outline -q -c 'drop schema public cascade; create schema public;' outline >>"$LOG" 2>&1
    zcat "$STAGE$STACK_DIR/backup/dumps/outline.sql.gz" | docker exec -i "$P-outline-postgres-1" psql -U outline -q outline >>"$LOG" 2>&1
  }
  case $svc in
    outline)          reload_pg; dc "$dir" "${files[@]}" up -d --no-deps outline >>"$LOG" 2>&1 ;;
    outline-postgres) dc "$dir" "${files[@]}" up -d --no-deps outline-postgres >>"$LOG" 2>&1; reload_pg; dc "$dir" up -d --no-deps outline >>"$LOG" 2>&1 ;;
    *)                dc "$dir" "${files[@]}" up -d --no-deps "$svc" >>"$LOG" 2>&1 ;;
  esac
}

update_one() {
  local svc=$1 dir groups old new snap t0 t1
  IFS='|' read -r dir groups _ _ <<<"$(info "$svc")" || { log "unknown service $svc"; return 2; }
  [ -n "$(cid "$dir" "$svc")" ] || { log "$svc: not running, skipped"; return 0; }
  old=$(running_image "$dir" "$svc")
  log "$svc: fetching $(target_ref "$dir" "$svc")"
  fetch "$dir" "$svc" || { log "$svc: FETCH FAILED, service untouched"; return 1; }
  new=$(docker image inspect -f '{{.Id}}' "$(target_ref "$dir" "$svc")" 2>/dev/null)
  if [ "$old" = "$new" ]; then log "$svc: already on that image, nothing to do"; return 0; fi

  log "$svc: snapshot before the change"
  BACKUP_TAG=pre-update bash "$STACK_DIR/backup/backup.sh" >>"$LOG" 2>&1 || { log "$svc: BACKUP FAILED, service untouched"; return 1; }
  snap=$(grep -oE 'snapshot [0-9a-f]{8} saved' "$STACK_DIR/backup/backup.log" | tail -1 | cut -d' ' -f2)
  docker tag "$old" "selfhostedops-rollback/$svc:previous"   # a name keeps the old image from being pruned

  t0=$(date +%s)
  log "$svc: recreate ($(echo "$old" | cut -c8-19) -> $(echo "$new" | cut -c8-19)), snapshot $snap"
  dc "$dir" up -d --no-deps "$svc" >>"$LOG" 2>&1
  # shellcheck disable=SC2086
  if wait_green "$TIMEOUT" $groups && wait_green "$TIMEOUT"; then
    t1=$(date +%s); log "$svc: GREEN after $((t1 - t0)) s ($(tail -1 "$STACK_DIR/updates/.last-health"))"
    return 0
  fi
  t1=$(date +%s); log "$svc: RED after $((t1 - t0)) s"
  grep '^FAIL' "$STACK_DIR/updates/.last-health" | tee -a "$LOG"
  put_back "$svc" "$dir" "$old" "$snap"
  if wait_green "$TIMEOUT"; then log "$svc: rolled back, health check green again after $(( $(date +%s) - t1 )) s"
  else log "$svc: rolled back, but the health check is STILL RED"; grep '^FAIL' "$STACK_DIR/updates/.last-health" | tee -a "$LOG"; fi
  log "$svc: the compose file still names the new tag. Set it back before the next 'up -d'."
  return 1
}

cmd=${1:-plan}; shift || true
set -a; . "$STACK_DIR/backup/.restic-env"; set +a
case $cmd in
  plan)
    # No registry lookup: with exact tags in the compose file, a different tag is the update.
    printf '%-18s %-46s %s\n' SERVICE RUNNING TARGET
    for svc in ${*:-$ORDER}; do
      IFS='|' read -r dir _ _ _ <<<"$(info "$svc")"; c=$(cid "$dir" "$svc"); [ -n "$c" ] || continue
      ref=$(target_ref "$dir" "$svc"); cur=$(docker inspect -f '{{.Config.Image}}' "$c")
      case $cur in sha256:*) cur="(untagged) $(echo "$cur" | cut -c8-19)" ;; esac
      note=""; [ "$cur" != "$ref" ] && note="   <- update"; has_build "$dir" "$svc" && note="   <- local build, rebuilt on run"
      printf '%-18s %-46s %s%s\n' "$svc" "$cur" "$ref" "$note"
    done ;;
  run)
    bash "$STACK_DIR/updates/health.sh" >/dev/null 2>&1 || { log "health check is red before the start, not updating anything"; bash "$STACK_DIR/updates/health.sh" | grep '^FAIL'; exit 1; }
    start=$(date +%s); log "=== update run: ${*:-$ORDER} (timeout per service $TIMEOUT s) ==="
    for svc in ${*:-$ORDER}; do
      update_one "$svc" || { log "=== STOPPED at $svc after $(( $(date +%s) - start )) s, later services untouched ==="; exit 1; }
    done
    log "=== all done in $(( $(date +%s) - start )) s ===" ;;
  *) sed -n '2,24p' "$0"; exit 2 ;;
esac