#!/usr/bin/env bash
#
# Nightly restic backup of the NAS array to the Hetzner Storage Box.
#
# Runs as connor (see restic-backup.service, User=connor) because the storage
# box ssh key and the `storagebox` Host alias live in ~connor/.ssh.
#
# Scope. The Storage Box is 5TB; /nas is 6.0T used. So this is a choice, not
# an oversight:
#
#   photo 226G  IN   -- irreplaceable
#   docs   17G  IN   -- irreplaceable
#   books 1.6G  IN   -- trivially small
#   audio 297G  IN   -- ripped/purchased, not all re-downloadable
#   video 4.9T  OUT  -- re-acquirable, and does not fit
#   downloads 574G OUT -- transient torrent scratch
#
# ~542G of source data. If video ever needs to be covered it needs its own
# larger target, not a smaller exclude list here.
#
# Timing: this runs at 03:30, an hour after mainframe's 02:30 job, which
# writes the Immich postgres dump into /nas/photo/immich/backups over NFS.
# Snapshotting the photo blobs and the database that indexes them in one
# consistent set is the whole point of that arrangement.

set -euo pipefail
umask 077

export RESTIC_CACHE_DIR="$HOME/.cache/restic"
set -a; . "$HOME/.config/restic/hetzner.env"; set +a

# `hostname` is not installed here and is not on systemd's PATH in general;
# uname -n always is.
HOST=$(uname -n); HOST=${HOST%%.*}
EXCLUDES="$HOME/.config/restic/excludes.txt"
NTFY_URL=https://ntfy.rcjohnstone.com/backup
NTFY_ENV="$HOME/.config/ntfy/publish.env"
LOG=$(mktemp /tmp/restic-backup.XXXXXX)
# Keep the log when the run FAILS. Without this the EXIT trap deleted the only
# record of restic's actual error, leaving nothing to diagnose from but the 25
# lines that made it into the ntfy body -- which is exactly what happened on
# 2026-08-24 when forget/prune died and the cause could not be recovered.
for d in /var/log "$HOME/.local/state" /tmp; do
    [ -d "$d" ] && [ -w "$d" ] && { FAILLOG=$d/restic-backup.failed.log; break; }
done
cleanup() {
    local rc=$?
    # Explicit if, not `[ $rc -ne 0 ] && cp ...`: a failing test as the last
    # statement of a trap is the kind of set -e landmine that has bitten this
    # codebase before.
    if [ "$rc" -ne 0 ] && [ -n "${FAILLOG:-}" ]; then
        cp -f "$LOG" "$FAILLOG" 2>/dev/null || true
    fi
    rm -f "$LOG"
}
trap cleanup EXIT

PATHS=(
    /nas/photo
    /nas/docs
    /nas/books
    /nas/audio
    /etc
    /home/connor
)

log() { printf '%s restic-backup: %s\n' "$(date -Is)" "$*" | tee -a "$LOG"; }

notify() {  # notify <priority> <tags> <title> <body>
    local pri=$1 tags=$2 title=$3 body=$4 u p
    [ -r "$NTFY_ENV" ] || return 0
    # PARSED, not sourced. The bot password contains ` and &, so `. $NTFY_ENV`
    # dies with a syntax error -- and it cannot simply be quoted either,
    # because movie_recs_notify reads the same file with a literal split on
    # "=" and would then send the quotes as part of the password.
    u=$(sed -n 's/^NTFY_USER=//p' "$NTFY_ENV" | head -1)
    p=$(sed -n 's/^NTFY_PASS=//p' "$NTFY_ENV" | head -1)
    [ -n "$u" ] && [ -n "$p" ] || return 0
    curl -fsS --max-time 20 \
         -u "$u:$p" \
         -H "Title: $title" -H "Priority: $pri" -H "Tags: $tags" \
         -d "$body" "$NTFY_URL" >/dev/null || true
}

fail() {
    log "FAILED: $1"
    notify urgent "rotating_light" "Backup FAILED on $HOST" \
        "$1"$'\n\n'"$(tail -n 25 "$LOG")"
    exit 1
}

# --- 1. sanity: never snapshot an unmounted array --------------------------
# Without this, a failed mount turns into a successful backup of an empty
# directory, forget --prune ages out the real snapshots, and the loss is
# silent until the day it matters.
mountpoint -q /nas || fail "/nas is not mounted; refusing to snapshot"

# --- 2. snapshot -----------------------------------------------------------
log "backing up: ${PATHS[*]}"
rc=0
nice -n 10 ionice -c2 -n7 restic backup \
    --one-file-system \
    --exclude-file="$EXCLUDES" \
    --exclude-caches \
    --tag "$HOST" \
    --verbose=1 \
    "${PATHS[@]}" >>"$LOG" 2>&1 || rc=$?
if [ "$rc" -ne 0 ]; then
    [ "$rc" -eq 3 ] || fail "restic backup exited $rc"
    log "WARN: restic exited 3 (some files unreadable); snapshot was written"
fi

# --- 3. retention ----------------------------------------------------------
log "forget + prune"
# --group-by host, NOT the default host+paths. With the default, changing the
# PATHS list above starts a fresh retention group and the snapshots taken under
# the old path list are kept forever -- every group gets its own
# daily/weekly/monthly/yearly allowance. One host per repo, so one group.
restic forget --prune \
    --group-by host \
    --tag "$HOST" \
    --keep-daily 14 --keep-weekly 8 --keep-monthly 12 --keep-yearly 3 \
    >>"$LOG" 2>&1 || fail "restic forget/prune failed"

# --- 4. report -------------------------------------------------------------
summary=$(grep -E '^(Added to the repository|processed|snapshot [0-9a-f]{8} saved)' "$LOG" | tail -3)
stats=$(restic stats --mode raw-data latest 2>/dev/null | grep -E 'Total Size' || true)
log "done"
notify default "floppy_disk" "Backup OK on $HOST" "${summary:-(no summary)}"$'\n'"$stats"
