#!/usr/bin/env bash
# ~$ selfhostedops - health check for the whole stack (episode 17)
#
# "The container is up" is not a health check. Every line below asks a service
# to do the thing it is there for: answer its API, hand out git refs, deliver
# a push message, start an SSO login, keep the mesh connected.
#
#   updates/health.sh              all checks, exit code = number of failures
#   updates/health.sh forgejo n8n  only these groups
#   updates/health.sh --data       also print row counts (to compare before/after)
#
# Runs on the box itself. Public hostnames are resolved to 127.0.0.1, so the
# request still goes through Caddy and its certificates, without leaving the box.
# No check asks for an answer of 401, 403 or 404 through Caddy: fail2ban counts
# those (episode 10) and would end up banning the box's own bridge address.
# Optional credentials in updates/health.env (see health.env.example):
#   FORGEJO_HEALTH_TOKEN   read-only token, for the git ref check on a private repo
#   FORGEJO_HEALTH_REPO    owner/name
#   NTFY_HEALTH_AUTH       user:password with read-write on the topic "healthcheck"
set -uo pipefail

STACK_DIR=${STACK_DIR:-/opt/selfhostedops-stack}
P=${PROJECT:-selfhostedops-stack}
set -a
. "$STACK_DIR/.env"
[ -f "$STACK_DIR/updates/health.env" ] && . "$STACK_DIR/updates/health.env"
set +a
D=$DOMAIN
SQLITE_IMAGE=${SQLITE_IMAGE:-keinos/sqlite3:latest}

DATA=0; GROUPS_WANTED=()
for a in "$@"; do [ "$a" = "--data" ] && DATA=1 || GROUPS_WANTED+=("$a"); done
want() { [ ${#GROUPS_WANTED[@]} -eq 0 ] || [[ " ${GROUPS_WANTED[*]} " == *" $1 "* ]]; }

FAILS=0; TOTAL=0
ok()   { TOTAL=$((TOTAL+1)); printf 'PASS  %-12s %s\n' "$1" "$2"; }
bad()  { TOTAL=$((TOTAL+1)); FAILS=$((FAILS+1)); printf 'FAIL  %-12s %s\n' "$1" "$2"; }
# check <group> <description> <actual> <expected regex>
check() { if [[ "$3" =~ $4 ]]; then ok "$1" "$2"; else bad "$1" "$2  (got: $(printf '%s' "$3" | tr '\n' ' ' | cut -c1-90))"; fi; }

pub()  { local host=$1 path=$2; shift 2; curl -sS -m 8 --resolve "$host.$D:443:127.0.0.1" "$@" "https://$host.$D$path" 2>&1; }
code() { pub "$1" "$2" -o /dev/null -w '%{http_code}' "${@:3}"; }
loc()  { pub "$1" "$2" -o /dev/null -w '%{redirect_url}' "${@:3}"; }
cip()  { docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}' "$1" 2>/dev/null | cut -d' ' -f1; }
sq()   { docker run --rm --user 0 -v "$1:/src:ro" "$SQLITE_IMAGE" sqlite3 "file:/src/$2?mode=ro" "$3" 2>&1; }
dx()   { docker exec "$@" 2>&1; }

if want caddy; then
  check caddy "config is valid" "$(dx $P-caddy-1 caddy validate --config /etc/caddy/Caddyfile | grep -o 'Valid configuration')" 'Valid configuration'
  check caddy "plugins compiled in: rate_limit, crowdsec, appsec" "$(dx $P-caddy-1 caddy list-modules | grep -cE '^http.handlers.(rate_limit|crowdsec|appsec)$')" '^3$'
  check caddy "certificate for git.$D is valid" "$(code git /api/healthz)" '^200$'
  if command -v cscli >/dev/null; then
    age=$(( $(date +%s) - $(date -d "$(cscli bouncers list -o json | jq -r '.[]|select(.type=="caddy-cs-bouncer")|.last_pull')" +%s 2>/dev/null || echo 0) ))
    check caddy "CrowdSec bouncer pulled decisions in the last 60 s" "$age" '^([0-9]|[1-5][0-9])$'
  fi
fi

if want forgejo; then
  check forgejo "API healthz: database and cache pass" "$(pub git /api/healthz | jq -r '.status' 2>&1)" '^pass$'
  if [ -n "${FORGEJO_HEALTH_TOKEN:-}" ]; then
    check forgejo "git over HTTPS: refs of ${FORGEJO_HEALTH_REPO}" "$(pub git "/${FORGEJO_HEALTH_REPO}.git/info/refs?service=git-upload-pack" -u "health:$FORGEJO_HEALTH_TOKEN" | tr -d '\0' | grep -ac 'refs/heads/')" '^[1-9]'
  fi
  check forgejo "git over SSH: port 2222 answers" "$(timeout 3 bash -c 'exec 3<>/dev/tcp/127.0.0.1/2222; head -c 8 <&3' 2>&1)" '^SSH-2.0'
  check forgejo "SSO: login button leads to id.$D" "$(loc git /user/oauth2/PocketID)" "^https://id\.$D/authorize"
fi

if want outline; then
  check outline "postgres accepts connections" "$(dx $P-outline-postgres-1 pg_isready -U outline)" 'accepting connections'
  check outline "redis answers PING" "$(dx $P-outline-redis-1 redis-cli ping)" '^PONG'
  check outline "_health" "$(code wiki /_health)" '^200$'
  check outline "API auth.config lists Pocket ID" "$(pub wiki /api/auth.config -X POST | jq -r '.data.providers[0].name' 2>&1)" '^Pocket ID$'
  check outline "SSO: login leads to id.$D" "$(loc wiki /auth/oidc)" "^https://id\.$D/authorize"
  check outline "database answers a query" "$(dx $P-outline-postgres-1 psql -U outline -tAc 'select count(*) from documents')" '^[0-9]+$'
fi

if want vaultwarden; then
  check vaultwarden "alive" "$(code vault /alive)" '^200$'
  check vaultwarden "API config reports a version" "$(pub vault /api/config | jq -r '.version' 2>&1)" '^20[0-9][0-9]\.'
  check vaultwarden "SSO: prevalidate hands out a token" "$(pub vault '/identity/sso/prevalidate?domainHint=x' | jq -r '.token|length>20' 2>&1)" '^true$'
fi

if want kuma; then
  # The status page API is asked on the container directly, not through Caddy: while Kuma migrates
  # its database (1.x -> 2.x took five minutes here) it answers 404, and a health check that polls
  # 404s through Caddy gets the box's own bridge address banned by fail2ban.
  k="http://$(cip $P-uptime-kuma-1):3001"
  sp=$(curl -sS -m 8 "$k/api/status-page/status" 2>&1)
  check kuma "public status page API" "$(jq -r '.config.slug' <<<"$sp" 2>&1)" '^status$'
  check kuma "status page shows 4 public monitors" "$(jq -r '[.publicGroupList[].monitorList[]]|length' <<<"$sp" 2>&1)" '^4$'
  hb=$(curl -sS -m 8 "$k/api/status-page/heartbeat/status" 2>&1 | jq -r '[.heartbeatList[]|last]|"\(length) \([.[]|select(.status==1)]|length) \(now - ([.[].time|sub("\\.[0-9]+$";"")|sub(" ";"T")+"Z"|fromdate]|min)|floor)"' 2>&1)
  check kuma "every public monitor: last heartbeat UP and younger than 180 s (monitors up age)" "$hb" '^([0-9]+) \1 ([0-9]|[1-9][0-9]|1[0-7][0-9])$'
  check kuma "dashboard is behind the login gate" "$(loc status /)" "^https://auth\.$D/login"
  down=$(sq ${P}_kuma_data kuma.db "select count(*) from monitor m where active=1 and (select status from heartbeat h where h.monitor_id=m.id order by time desc limit 1) is not 1")
  check kuma "no active monitor is DOWN in the database" "$down" '^0$'
fi

if want ntfy; then
  check ntfy "health" "$(pub ntfy /v1/health | jq -r '.healthy' 2>&1)" '^true$'
  if [ -n "${NTFY_HEALTH_AUTH:-}" ]; then
    id="health-$(date +%s)-$RANDOM"
    pub ntfy /healthcheck -u "$NTFY_HEALTH_AUTH" -d "$id" -o /dev/null
    check ntfy "a published message can be read back" "$(pub ntfy '/healthcheck/json?poll=1&since=1m' -u "$NTFY_HEALTH_AUTH" | grep -c "$id")" '^1$'
  fi
fi

if want n8n; then
  n=$(cip $P-n8n-1)
  check n8n "readiness (database reachable, migrations done)" "$(curl -sS -m 8 http://$n:5678/healthz/readiness 2>&1)" '"status":"ok"'
  check n8n "active workflows (at least ${N8N_EXPECT_ACTIVE:-1})" "$(dx $P-n8n-1 n8n list:workflow --active=true | grep -c '|')" "^(${N8N_EXPECT_ACTIVE:-[1-9][0-9]*})$"
  # 403 = the webhook is registered and asks for its token. 404 = workflow not active.
  check n8n "webhook kuma-incident is registered and asks for its token" "$(curl -sS -m 8 -o /dev/null -w '%{http_code}' -X POST http://$n:5678/webhook/kuma-incident -d '{}' 2>&1)" '^403$'
fi

if want pocket-id; then
  check pocket-id "healthz" "$(code id /healthz)" '^20[04]$'
  check pocket-id "discovery document names the right issuer" "$(pub id /.well-known/openid-configuration | jq -r '.issuer' 2>&1)" "^https://id\.$D$"
  check pocket-id "signing keys are published" "$(pub id /.well-known/jwks.json | jq -r '.keys|length' 2>&1)" '^[1-9]'
  # A made-up code with the real client secret must fail as invalid_grant. invalid_client would mean the client is gone.
  check pocket-id "token endpoint still knows the Outline client" "$(curl -sS -m 8 -X POST http://$(cip $P-pocket-id-1):1411/api/oidc/token -d grant_type=authorization_code -d code=health -d client_id=$OUTLINE_OIDC_CLIENT_ID -d client_secret=$OUTLINE_OIDC_CLIENT_SECRET -d redirect_uri=https://wiki.$D/auth/oidc.callback 2>&1 | jq -r '.error' 2>&1)" '^invalid_grant$'
fi

if want tinyauth; then
  check tinyauth "healthz" "$(pub auth /api/healthz | jq -r '.message' 2>&1)" '^Healthy$'
  check tinyauth "login is handed to id.$D" "$(pub auth /api/oauth/url/pocketid | jq -r '.url' 2>&1)" "^https://id\.$D/authorize"
fi

if want netbird; then
  check netbird "dashboard" "$(code vpn /)" '^200$'
  check netbird "embedded identity provider discovery" "$(pub vpn /oauth2/.well-known/openid-configuration | jq -r '.issuer' 2>&1)" "^https://vpn\.$D"
  check netbird "management API alive and asks for a login" "$(curl -sS -m 8 -o /dev/null -w '%{http_code}' http://127.0.0.1:8081/api/users 2>&1)" '^401$'
  check netbird "this host: management and signal connected" "$(netbird status 2>&1 | grep -cE '^(Management|Signal): Connected')" '^2$'
  check netbird "this host: at least one peer connected" "$(netbird status 2>&1 | grep -oE 'Peers count: [0-9]+' | grep -oE '[0-9]+$')" '^[1-9]'
  if docker inspect mesh-client >/dev/null 2>&1; then
    check netbird "mesh-client container: management connected" "$(dx mesh-client netbird status | grep -c '^Management: Connected')" '^1$'
  fi
fi

if want ops; then
  check ops "ops-readonly healthz" "$(curl -sS -m 8 http://$(cip $P-ops-readonly-1):8098/healthz 2>&1 | jq -r '.status' 2>&1)" '^ok$'
  check ops "ops-actions healthz" "$(curl -sS -m 8 http://$(cip $P-ops-actions-1):8099/healthz 2>&1 | jq -r '.status' 2>&1)" '^ok$'
fi

if want backup; then
  check backup "nightly timer is scheduled" "$(systemctl is-active selfhostedops-backup.timer 2>&1)" '^active$'
  last=$(grep '=== backup end ===' "$STACK_DIR/backup/backup.log" | tail -1 | cut -c1-19)
  check backup "last finished backup is younger than 26 h ($last)" "$(( ( $(date +%s) - $(date -d "$last" +%s 2>/dev/null || echo 0) ) / 3600 ))" '^([0-9]|1[0-9]|2[0-5])$'
fi

if [ $DATA = 1 ]; then
  echo
  echo "DATA  forgejo      repositories $(sq ${P}_forgejo_data gitea/gitea.db 'select count(*) from repository'), users $(sq ${P}_forgejo_data gitea/gitea.db 'select count(*) from user where type=0')"
  echo "DATA  outline      documents $(dx $P-outline-postgres-1 psql -U outline -tAc 'select count(*) from documents'), users $(dx $P-outline-postgres-1 psql -U outline -tAc 'select count(*) from users'), migrations $(dx $P-outline-postgres-1 psql -U outline -tAc 'select count(*) from "SequelizeMeta"')"
  echo "DATA  vaultwarden  users $(sq ${P}_vaultwarden_data db.sqlite3 'select count(*) from users'), ciphers $(sq ${P}_vaultwarden_data db.sqlite3 'select count(*) from ciphers')"
  echo "DATA  kuma         monitors $(sq ${P}_kuma_data kuma.db 'select count(*) from monitor'), notifications $(sq ${P}_kuma_data kuma.db 'select count(*) from notification'), heartbeats $(sq ${P}_kuma_data kuma.db 'select count(*) from heartbeat')"
  echo "DATA  n8n          workflows $(sq ${P}_n8n_data database.sqlite 'select count(*) from workflow_entity'), credentials $(sq ${P}_n8n_data database.sqlite 'select count(*) from credentials_entity'), executions $(sq ${P}_n8n_data database.sqlite 'select count(*) from execution_entity'), migrations $(sq ${P}_n8n_data database.sqlite 'select count(*) from migrations')"
  echo "DATA  pocket-id    users $(sq ${P}_pocket_id_data pocket-id.db 'select count(*) from users'), clients $(sq ${P}_pocket_id_data pocket-id.db 'select count(*) from oidc_clients')"
  echo "DATA  netbird      peers $(sq netbird_netbird_data store.db 'select count(*) from peers')"
fi

echo
echo "$((TOTAL-FAILS)) of $TOTAL checks passed, $FAILS failed"
exit $FAILS