<body> local pri=$1 tags=$2 title=$3 body=$4 u p [ -r "$NTFY_ENV" ] || return 0 # PARSED, not sourced. The bot password contains ` and &, so `. $NTFY_ENV` # dies with a syntax error -- and it cannot simply be quoted either, # because movie_recs_notify reads the same file with a literal split on # "=" and would then send the quotes as part of the password. u=$(sed -n 's/^NTFY_USER=//p' "$NTFY_ENV" | head -1) p=$(sed -n 's/^NTFY_PASS=//p' "$NTFY_ENV" | head -1) [ -n "$u" ] && [ -n "$p" ] || return 0 curl -fsS --max-time 20 \ -u "$u:$p" \ -H "Title: $title" -H "Priority: $pri" -H "Tags: $tags" \ -d "$body" "$NTFY_URL" >/dev/null || true } fail() { log "FAILED: $1" notify urgent "rotating_light" "Backup FAILED on $HOST" \ "$1"$'\n\n'"$(tail -n 25 "$LOG")" exit 1 } # --- 1. sanity: never snapshot an unmounted array -------------------------- # Without this, a failed mount turns into a successful backup of an empty # directory, forget --prune ages out the real snapshots, and the loss is # silent until the day it matters. mountpoint -q /nas || fail "/nas is not mounted; refusing to snapshot" # --- 2. snapshot ----------------------------------------------------------- log "backing up: ${PATHS[*]}" rc=0 nice -n 10 ionice -c2 -n7 restic backup \ --one-file-system \ --exclude-file="$EXCLUDES" \ --exclude-caches \ --tag "$HOST" \ --verbose=1 \ "${PATHS[@]}" >>"$LOG" 2>&1 || rc=$? if [ "$rc" -ne 0 ]; then [ "$rc" -eq 3 ] || fail "restic backup exited $rc" log "WARN: restic exited 3 (some files unreadable); snapshot was written" fi # --- 3. retention ---------------------------------------------------------- log "forget + prune" # --group-by host, NOT the default host+paths. With the default, changing the # PATHS list above starts a fresh retention group and the snapshots taken under # the old path list are kept forever -- every group gets its own # daily/weekly/monthly/yearly allowance. One host per repo, so one group. restic forget --prune \ --group-by host \ --tag "$HOST" \ --keep-daily 14 --keep-weekly 8 --keep-monthly 12 --keep-yearly 3 \ >>"$LOG" 2>&1 || fail "restic forget/prune failed" # --- 4. report ------------------------------------------------------------- summary=$(grep -E '^(Added to the repository|processed|snapshot [0-9a-f]{8} saved)' "$LOG" | tail -3) stats=$(restic stats --mode raw-data latest 2>/dev/null | grep -E 'Total Size' || true) log "done" notify default "floppy_disk" "Backup OK on $HOST" "${summary:-(no summary)}"$'\n'"$stats"

#!/usr/bin/env bash # # Nightly restic backup of the NAS array to the Hetzner Storage Box. # # Runs as connor (see restic-backup.service, User=connor) because the storage # box ssh key and the `storagebox` Host alias live in ~connor/.ssh. # # Scope. The Storage Box is 5TB; /nas is 6.0T used. So this is a choice, not # an oversight: # # photo 226G IN -- irreplaceable # docs 17G IN -- irreplaceable # books 1.6G IN -- trivially small # audio 297G IN -- ripped/purchased, not all re-downloadable # video 4.9T OUT -- re-acquirable, and does not fit # downloads 574G OUT -- transient torrent scratch # # ~542G of source data. If video ever needs to be covered it needs its own # larger target, not a smaller exclude list here. # # Timing: this runs at 03:30, an hour after mainframe's 02:30 job, which # writes the Immich postgres dump into /nas/photo/immich/backups over NFS. # Snapshotting the photo blobs and the database that indexes them in one # consistent set is the whole point of that arrangement. set -euo pipefail umask 077 export RESTIC_CACHE_DIR="$HOME/.cache/restic" set -a; . "$HOME/.config/restic/hetzner.env"; set +a # `hostname` is not installed here and is not on systemd's PATH in general; # uname -n always is. HOST=$(uname -n); HOST=${HOST%%.*} EXCLUDES="$HOME/.config/restic/excludes.txt" NTFY_URL=https://ntfy.rcjohnstone.com/backup NTFY_ENV="$HOME/.config/ntfy/publish.env" LOG=$(mktemp /tmp/restic-backup.XXXXXX) # Keep the log when the run FAILS. Without this the EXIT trap deleted the only # record of restic's actual error, leaving nothing to diagnose from but the 25 # lines that made it into the ntfy body -- which is exactly what happened on # 2026-08-24 when forget/prune died and the cause could not be recovered. for d in /var/log "$HOME/.local/state" /tmp; do [ -d "$d" ] && [ -w "$d" ] && { FAILLOG=$d/restic-backup.failed.log; break; } done cleanup() { local rc=$? # Explicit if, not `[ $rc -ne 0 ] && cp ...`: a failing test as the last # statement of a trap is the kind of set -e landmine that has bitten this # codebase before. if [ "$rc" -ne 0 ] && [ -n "${FAILLOG:-}" ]; then cp -f "$LOG" "$FAILLOG" 2>/dev/null || true fi rm -f "$LOG" } trap cleanup EXIT PATHS=( /nas/photo /nas/docs /nas/books /nas/audio /etc /home/connor ) log() { printf '%s restic-backup: %s\n' "$(date -Is)" "$*" | tee -a "$LOG"; } notify() { # notify <body> local pri=$1 tags=$2 title=$3 body=$4 u p [ -r "$NTFY_ENV" ] || return 0 # PARSED, not sourced. The bot password contains ` and &, so `. $NTFY_ENV` # dies with a syntax error -- and it cannot simply be quoted either, # because movie_recs_notify reads the same file with a literal split on # "=" and would then send the quotes as part of the password. u=$(sed -n 's/^NTFY_USER=//p' "$NTFY_ENV" | head -1) p=$(sed -n 's/^NTFY_PASS=//p' "$NTFY_ENV" | head -1) [ -n "$u" ] && [ -n "$p" ] || return 0 curl -fsS --max-time 20 \ -u "$u:$p" \ -H "Title: $title" -H "Priority: $pri" -H "Tags: $tags" \ -d "$body" "$NTFY_URL" >/dev/null || true } fail() { log "FAILED: $1" notify urgent "rotating_light" "Backup FAILED on $HOST" \ "$1"$'\n\n'"$(tail -n 25 "$LOG")" exit 1 } # --- 1. sanity: never snapshot an unmounted array -------------------------- # Without this, a failed mount turns into a successful backup of an empty # directory, forget --prune ages out the real snapshots, and the loss is # silent until the day it matters. mountpoint -q /nas || fail "/nas is not mounted; refusing to snapshot" # --- 2. snapshot ----------------------------------------------------------- log "backing up: ${PATHS[*]}" rc=0 nice -n 10 ionice -c2 -n7 restic backup \ --one-file-system \ --exclude-file="$EXCLUDES" \ --exclude-caches \ --tag "$HOST" \ --verbose=1 \ "${PATHS[@]}" >>"$LOG" 2>&1 || rc=$? if [ "$rc" -ne 0 ]; then [ "$rc" -eq 3 ] || fail "restic backup exited $rc" log "WARN: restic exited 3 (some files unreadable); snapshot was written" fi # --- 3. retention ---------------------------------------------------------- log "forget + prune" # --group-by host, NOT the default host+paths. With the default, changing the # PATHS list above starts a fresh retention group and the snapshots taken under # the old path list are kept forever -- every group gets its own # daily/weekly/monthly/yearly allowance. One host per repo, so one group. restic forget --prune \ --group-by host \ --tag "$HOST" \ --keep-daily 14 --keep-weekly 8 --keep-monthly 12 --keep-yearly 3 \ >>"$LOG" 2>&1 || fail "restic forget/prune failed" # --- 4. report ------------------------------------------------------------- summary=$(grep -E '^(Added to the repository|processed|snapshot [0-9a-f]{8} saved)' "$LOG" | tail -3) stats=$(restic stats --mode raw-data latest 2>/dev/null | grep -E 'Total Size' || true) log "done" notify default "floppy_disk" "Backup OK on $HOST" "${summary:-(no summary)}"$'\n'"$stats"