2026-09-14 14:24:37 -04:00
commit e6644d0616
462 changed files with 23524 additions and 0 deletions
@@ -0,0 +1,129 @@
#!/usr/bin/env bash
#
# Nightly restic backup of the NAS array to the Hetzner Storage Box.
#
# Runs as connor (see restic-backup.service, User=connor) because the storage
# box ssh key and the `storagebox` Host alias live in ~connor/.ssh.
#
# Scope. The Storage Box is 5TB; /nas is 6.0T used. So this is a choice, not
# an oversight:
#
#   photo 226G  IN   -- irreplaceable
#   docs   17G  IN   -- irreplaceable
#   books 1.6G  IN   -- trivially small
#   audio 297G  IN   -- ripped/purchased, not all re-downloadable
#   video 4.9T  OUT  -- re-acquirable, and does not fit
#   downloads 574G OUT -- transient torrent scratch
#
# ~542G of source data. If video ever needs to be covered it needs its own
# larger target, not a smaller exclude list here.
#
# Timing: this runs at 03:30, an hour after mainframe's 02:30 job, which
# writes the Immich postgres dump into /nas/photo/immich/backups over NFS.
# Snapshotting the photo blobs and the database that indexes them in one
# consistent set is the whole point of that arrangement.
set -euo pipefail
umask 077
export RESTIC_CACHE_DIR="$HOME/.cache/restic"
set -a; . "$HOME/.config/restic/hetzner.env"; set +a
# `hostname` is not installed here and is not on systemd's PATH in general;
# uname -n always is.
HOST=$(uname -n); HOST=${HOST%%.*}
EXCLUDES="$HOME/.config/restic/excludes.txt"
NTFY_URL=https://ntfy.rcjohnstone.com/backup
NTFY_ENV="$HOME/.config/ntfy/publish.env"
LOG=$(mktemp /tmp/restic-backup.XXXXXX)
# Keep the log when the run FAILS. Without this the EXIT trap deleted the only
# record of restic's actual error, leaving nothing to diagnose from but the 25
# lines that made it into the ntfy body -- which is exactly what happened on
# 2026-08-24 when forget/prune died and the cause could not be recovered.
for d in /var/log "$HOME/.local/state" /tmp; do
    [ -d "$d" ] && [ -w "$d" ] && { FAILLOG=$d/restic-backup.failed.log; break; }
done
cleanup() {
    local rc=$?
    # Explicit if, not `[ $rc -ne 0 ] && cp ...`: a failing test as the last
    # statement of a trap is the kind of set -e landmine that has bitten this
    # codebase before.
    if [ "$rc" -ne 0 ] && [ -n "${FAILLOG:-}" ]; then
        cp -f "$LOG" "$FAILLOG" 2>/dev/null || true
    fi
    rm -f "$LOG"
}
trap cleanup EXIT
PATHS=(
    /nas/photo
    /nas/docs
    /nas/books
    /nas/audio
    /etc
    /home/connor
)
log() { printf '%s restic-backup: %s\n' "$(date -Is)" "$*" | tee -a "$LOG"; }
notify() {  # notify <priority> <tags> <title> <body>
    local pri=$1 tags=$2 title=$3 body=$4 u p
    [ -r "$NTFY_ENV" ] || return 0
    # PARSED, not sourced. The bot password contains ` and &, so `. $NTFY_ENV`
    # dies with a syntax error -- and it cannot simply be quoted either,
    # because movie_recs_notify reads the same file with a literal split on
    # "=" and would then send the quotes as part of the password.
    u=$(sed -n 's/^NTFY_USER=//p' "$NTFY_ENV" | head -1)
    p=$(sed -n 's/^NTFY_PASS=//p' "$NTFY_ENV" | head -1)
    [ -n "$u" ] && [ -n "$p" ] || return 0
    curl -fsS --max-time 20 \
         -u "$u:$p" \
         -H "Title: $title" -H "Priority: $pri" -H "Tags: $tags" \
         -d "$body" "$NTFY_URL" >/dev/null || true
}
fail() {
    log "FAILED: $1"
    notify urgent "rotating_light" "Backup FAILED on $HOST" \
        "$1"$'\n\n'"$(tail -n 25 "$LOG")"
    exit 1
}
# --- 1. sanity: never snapshot an unmounted array --------------------------
# Without this, a failed mount turns into a successful backup of an empty
# directory, forget --prune ages out the real snapshots, and the loss is
# silent until the day it matters.
mountpoint -q /nas || fail "/nas is not mounted; refusing to snapshot"
# --- 2. snapshot -----------------------------------------------------------
log "backing up: ${PATHS[*]}"
rc=0
nice -n 10 ionice -c2 -n7 restic backup \
    --one-file-system \
    --exclude-file="$EXCLUDES" \
    --exclude-caches \
    --tag "$HOST" \
    --verbose=1 \
    "${PATHS[@]}" >>"$LOG" 2>&1 || rc=$?
if [ "$rc" -ne 0 ]; then
    [ "$rc" -eq 3 ] || fail "restic backup exited $rc"
    log "WARN: restic exited 3 (some files unreadable); snapshot was written"
fi
# --- 3. retention ----------------------------------------------------------
log "forget + prune"
# --group-by host, NOT the default host+paths. With the default, changing the
# PATHS list above starts a fresh retention group and the snapshots taken under
# the old path list are kept forever -- every group gets its own
# daily/weekly/monthly/yearly allowance. One host per repo, so one group.
restic forget --prune \
    --group-by host \
    --tag "$HOST" \
    --keep-daily 14 --keep-weekly 8 --keep-monthly 12 --keep-yearly 3 \
    >>"$LOG" 2>&1 || fail "restic forget/prune failed"
# --- 4. report -------------------------------------------------------------
summary=$(grep -E '^(Added to the repository|processed|snapshot [0-9a-f]{8} saved)' "$LOG" | tail -3)
stats=$(restic stats --mode raw-data latest 2>/dev/null | grep -E 'Total Size' || true)
log "done"
notify default "floppy_disk" "Backup OK on $HOST" "${summary:-(no summary)}"$'\n'"$stats"
@@ -0,0 +1,111 @@
#!/usr/bin/env bash
#
# ONE-TIME seed of the NAS restic repository.
#
# Why this exists rather than just running restic-backup: a single `restic
# backup` over 520G takes ~12h on this uplink and commits nothing until it
# finishes. Two attempts were already lost whole -- one to an edit of the
# running script, one to an unplanned reboot -- because an interrupted run
# leaves only unreferenced packs that the next run cannot reuse.
#
# So: back up in chunks, each of which commits its own snapshot. An
# interruption now costs one chunk, not the run. Re-running picks up where it
# left off via the state file, which survives reboots.
#
# The chunk list does NOT have to be exhaustive. Once it completes, the normal
# restic-backup.service run snapshots the full path set and dedupes against
# everything seeded here, so any file not covered by a chunk is picked up then
# -- in minutes rather than hours. Afterwards:
#
#     restic forget --tag seed --group-by '' && restic prune
#
# drops the seed snapshots; their data stays, referenced by the real snapshot.
set -euo pipefail
umask 077
export RESTIC_CACHE_DIR="$HOME/.cache/restic"
set -a; . "$HOME/.config/restic/hetzner.env"; set +a
EXCLUDES="$HOME/.config/restic/excludes.txt"
STATE="$HOME/.local/state/restic-seed.done"
BATCH_BYTES=$(( 15 * 1024 * 1024 * 1024 ))   # ~15G per chunk => ~20 min each
mkdir -p "$(dirname "$STATE")"; touch "$STATE"
log() { printf '%s restic-seed: %s\n' "$(date -Is)" "$*"; }
# Backed up whole -- each is small enough to be one chunk.
WHOLE=(
    /etc
    /home/connor
    /nas/books
    /nas/docs
    /nas/photo/takeout
    /nas/photo/immich/backups
    /nas/audio/podcast
)
# Too big for one chunk; split into batches of their immediate children.
# admin/ is immich's only library user and holds 185G across 22 year folders;
# music/ and shanty/ are ~190 artist directories each.
SPLIT=(
    /nas/photo/immich/library/admin
    /nas/audio/music
    /nas/audio/shanty
)
# run_chunk <path>...
# Keyed by the md5 of its path list so the state file survives re-ordering.
run_chunk() {
    local key; key=$(printf '%s\n' "$@" | md5sum | cut -c1-12)
    if grep -qx "$key" "$STATE"; then
        log "skip  [$key] $1 ${2:+(+$(($#-1)) more)}"
        return 0
    fi
    log "chunk [$key] $1 ${2:+(+$(($#-1)) more)}"
    # Exit 3 means some files were unreadable but the snapshot was written --
    # still progress, still worth recording.
    local rc=0
    nice -n 10 ionice -c2 -n7 restic backup \
        --one-file-system \
        --exclude-file="$EXCLUDES" \
        --exclude-caches \
        --tag nas --tag seed \
        "$@" || rc=$?
    if [ "$rc" -eq 0 ] || [ "$rc" -eq 3 ]; then
        printf '%s\n' "$key" >> "$STATE"
        # NOT `[ ... ] && log`: under `set -e` a false test makes that the
        # function's exit status and kills the whole seed on the first
        # SUCCESSFUL chunk.
        if [ "$rc" -eq 3 ]; then log "  (exit 3: some files unreadable)"; fi
    else
        log "  FAILED rc=$rc -- leaving unrecorded so a re-run retries it"
        return "$rc"
    fi
}
for p in "${WHOLE[@]}"; do
    [ -e "$p" ] || { log "missing, skipping: $p"; continue; }
    run_chunk "$p"
done
for root in "${SPLIT[@]}"; do
    [ -d "$root" ] || { log "missing, skipping: $root"; continue; }
    batch=(); size=0
    while IFS= read -r -d '' child; do
        csize=$(du -sb --apparent-size "$child" 2>/dev/null | cut -f1) || csize=0
        # Flush first if adding this child would overflow, so a single
        # oversized child still gets a chunk of its own rather than being
        # merged into a huge one.
        if [ "${#batch[@]}" -gt 0 ] && [ $(( size + csize )) -gt "$BATCH_BYTES" ]; then
            run_chunk "${batch[@]}"
            batch=(); size=0
        fi
        batch+=("$child"); size=$(( size + csize ))
    done < <(find "$root" -mindepth 1 -maxdepth 1 -print0 | sort -z)
    if [ "${#batch[@]}" -gt 0 ]; then run_chunk "${batch[@]}"; fi
done
log "seed complete -- $(wc -l < "$STATE") chunks recorded"
log "next: systemctl start restic-backup.service (full run, dedupes against these)"