#!/usr/bin/env bash
#
# Nightly restic backup of mainframe's LOCAL data to the Hetzner Storage Box.
#
# Scope: this host only. The six directories under /home/connor that are NFS
# mounts from the NAS (audio books docs downloads photo video) are excluded
# automatically by --one-file-system and are backed up by the NAS's own copy
# of this script, straight off the XFS array. Pulling 500G over NFS every
# night to hand it back to the same machine would be absurd.
#
# Order matters: dump-databases-then-snapshot. backup-db-dump writes the
# Immich pg dump onto the NAS mount so the NAS's 03:30 run picks it up an hour
# later. If that run ever overtakes this one it snapshots yesterday's dump --
# degraded, not broken, and Immich also writes its own daily dump to the same
# directory.

set -euo pipefail
umask 077

export RESTIC_CACHE_DIR=/var/cache/restic
set -a; . /etc/restic/hetzner.env; set +a

# `hostname` is not installed on the NAS and is not on systemd's PATH in
# general; uname -n always is.
HOST=$(uname -n); HOST=${HOST%%.*}
EXCLUDES=/etc/restic/excludes.txt
NTFY_URL=https://ntfy.rcjohnstone.com/backup
NTFY_ENV=/etc/ntfy/publish.env
LOG=$(mktemp /tmp/restic-backup.XXXXXX)
# Keep the log when the run FAILS. Without this the EXIT trap deleted the only
# record of restic's actual error, leaving nothing to diagnose from but the 25
# lines that made it into the ntfy body -- which is exactly what happened on
# 2026-08-24 when forget/prune died and the cause could not be recovered.
for d in /var/log "${HOME:-/nonexistent}/.local/state" /tmp; do
    [ -d "$d" ] && [ -w "$d" ] && { FAILLOG=$d/restic-backup.failed.log; break; }
done
cleanup() {
    local rc=$?
    # Explicit if, not `[ $rc -ne 0 ] && cp ...`: a failing test as the last
    # statement of a trap is the kind of set -e landmine that has bitten this
    # codebase before.
    if [ "$rc" -ne 0 ] && [ -n "${FAILLOG:-}" ]; then
        cp -f "$LOG" "$FAILLOG" 2>/dev/null || true
    fi
    rm -f "$LOG"
}
trap cleanup EXIT

PATHS=(
    /etc
    /root
    # /usr/local rather than /usr/local/bin: shanty's live database is bind
    # mounted from /usr/local/shanty, not from anywhere under /home/connor.
    /usr/local
    /var/backups/db
    # Five named podman volumes live here. Four are tiny and one (model-cache)
    # is excluded. Compose bind-mounts almost everything from /home/connor, so
    # this is easy to forget -- and forgetting it silently drops syncthing's
    # and searxng's state from every snapshot.
    /var/lib/containers/storage/volumes
    /home/connor
)

log()  { printf '%s restic-backup: %s\n' "$(date -Is)" "$*" | tee -a "$LOG"; }

notify() {  # notify <priority> <tags> <title> <body>
    local pri=$1 tags=$2 title=$3 body=$4 u p
    [ -r "$NTFY_ENV" ] || return 0
    # PARSED, not sourced. The bot password contains ` and &, so `. $NTFY_ENV`
    # dies with a syntax error -- and it cannot simply be quoted either,
    # because movie_recs_notify reads the same file with a literal split on
    # "=" and would then send the quotes as part of the password.
    u=$(sed -n 's/^NTFY_USER=//p' "$NTFY_ENV" | head -1)
    p=$(sed -n 's/^NTFY_PASS=//p' "$NTFY_ENV" | head -1)
    [ -n "$u" ] && [ -n "$p" ] || return 0
    curl -fsS --max-time 20 \
         -u "$u:$p" \
         -H "Title: $title" -H "Priority: $pri" -H "Tags: $tags" \
         -d "$body" "$NTFY_URL" >/dev/null || true
}

fail() {
    log "FAILED: $1"
    notify urgent "rotating_light" "Backup FAILED on $HOST" \
        "$1"$'\n\n'"$(tail -n 25 "$LOG")"
    exit 1
}

# --- 1. consistent database dumps ------------------------------------------
log "dumping databases"
/usr/local/bin/backup-db-dump >>"$LOG" 2>&1 || fail "backup-db-dump failed"

# --- 2. snapshot -----------------------------------------------------------
log "backing up: ${PATHS[*]}"
rc=0
nice -n 10 ionice -c2 -n7 restic backup \
    --one-file-system \
    --exclude-file="$EXCLUDES" \
    --exclude-caches \
    --tag "$HOST" \
    --verbose=1 \
    "${PATHS[@]}" >>"$LOG" 2>&1 || rc=$?
# restic exits 3 when it could not read *some* files but the snapshot was
# still written. That is worth a warning, not a failure -- a nightly job that
# hard-fails on one transiently-locked file stops being a backup.
if [ "$rc" -ne 0 ]; then
    [ "$rc" -eq 3 ] || fail "restic backup exited $rc"
    log "WARN: restic exited 3 (some files unreadable); snapshot was written"
fi

# --- 3. retention ----------------------------------------------------------
log "forget + prune"
# --group-by host, NOT the default host+paths. With the default, changing the
# PATHS list above starts a fresh retention group and the snapshots taken under
# the old path list are kept forever -- every group gets its own
# daily/weekly/monthly/yearly allowance. One host per repo, so one group.
restic forget --prune \
    --group-by host \
    --tag "$HOST" \
    --keep-daily 14 --keep-weekly 8 --keep-monthly 12 --keep-yearly 3 \
    >>"$LOG" 2>&1 || fail "restic forget/prune failed"

# --- 4. report -------------------------------------------------------------
summary=$(grep -E '^(Added to the repository|processed|snapshot [0-9a-f]{8} saved)' "$LOG" | tail -3)
stats=$(restic stats --mode raw-data latest 2>/dev/null | grep -E 'Total Size' || true)
log "done"
notify default "floppy_disk" "Backup OK on $HOST" "${summary:-(no summary)}"$'\n'"$stats"
