<body> local pri=$1 tags=$2 title=$3 body=$4 u p [ -r "$NTFY_ENV" ] || return 0 # PARSED, not sourced. The bot password contains ` and &, so `. $NTFY_ENV` # dies with a syntax error -- and it cannot simply be quoted either, # because movie_recs_notify reads the same file with a literal split on # "=" and would then send the quotes as part of the password. u=$(sed -n 's/^NTFY_USER=//p' "$NTFY_ENV" | head -1) p=$(sed -n 's/^NTFY_PASS=//p' "$NTFY_ENV" | head -1) [ -n "$u" ] && [ -n "$p" ] || return 0 curl -fsS --max-time 20 \ -u "$u:$p" \ -H "Title: $title" -H "Priority: $pri" -H "Tags: $tags" \ -d "$body" "$NTFY_URL" >/dev/null || true } fail() { log "FAILED: $1" notify urgent "rotating_light" "Backup FAILED on $HOST" \ "$1"$'\n\n'"$(tail -n 25 "$LOG")" exit 1 } # --- 1. consistent database dumps ------------------------------------------ log "dumping databases" /usr/local/bin/backup-db-dump >>"$LOG" 2>&1 || fail "backup-db-dump failed" # --- 2. snapshot ----------------------------------------------------------- log "backing up: ${PATHS[*]}" rc=0 nice -n 10 ionice -c2 -n7 restic backup \ --one-file-system \ --exclude-file="$EXCLUDES" \ --exclude-caches \ --tag "$HOST" \ --verbose=1 \ "${PATHS[@]}" >>"$LOG" 2>&1 || rc=$? # restic exits 3 when it could not read *some* files but the snapshot was # still written. That is worth a warning, not a failure -- a nightly job that # hard-fails on one transiently-locked file stops being a backup. if [ "$rc" -ne 0 ]; then [ "$rc" -eq 3 ] || fail "restic backup exited $rc" log "WARN: restic exited 3 (some files unreadable); snapshot was written" fi # --- 3. retention ---------------------------------------------------------- log "forget + prune" # --group-by host, NOT the default host+paths. With the default, changing the # PATHS list above starts a fresh retention group and the snapshots taken under # the old path list are kept forever -- every group gets its own # daily/weekly/monthly/yearly allowance. One host per repo, so one group. restic forget --prune \ --group-by host \ --tag "$HOST" \ --keep-daily 14 --keep-weekly 8 --keep-monthly 12 --keep-yearly 3 \ >>"$LOG" 2>&1 || fail "restic forget/prune failed" # --- 4. report ------------------------------------------------------------- summary=$(grep -E '^(Added to the repository|processed|snapshot [0-9a-f]{8} saved)' "$LOG" | tail -3) stats=$(restic stats --mode raw-data latest 2>/dev/null | grep -E 'Total Size' || true) log "done" notify default "floppy_disk" "Backup OK on $HOST" "${summary:-(no summary)}"$'\n'"$stats"

#!/usr/bin/env bash # # Nightly restic backup of mainframe's LOCAL data to the Hetzner Storage Box. # # Scope: this host only. The six directories under /home/connor that are NFS # mounts from the NAS (audio books docs downloads photo video) are excluded # automatically by --one-file-system and are backed up by the NAS's own copy # of this script, straight off the XFS array. Pulling 500G over NFS every # night to hand it back to the same machine would be absurd. # # Order matters: dump-databases-then-snapshot. backup-db-dump writes the # Immich pg dump onto the NAS mount so the NAS's 03:30 run picks it up an hour # later. If that run ever overtakes this one it snapshots yesterday's dump -- # degraded, not broken, and Immich also writes its own daily dump to the same # directory. set -euo pipefail umask 077 export RESTIC_CACHE_DIR=/var/cache/restic set -a; . /etc/restic/hetzner.env; set +a # `hostname` is not installed on the NAS and is not on systemd's PATH in # general; uname -n always is. HOST=$(uname -n); HOST=${HOST%%.*} EXCLUDES=/etc/restic/excludes.txt NTFY_URL=https://ntfy.rcjohnstone.com/backup NTFY_ENV=/etc/ntfy/publish.env LOG=$(mktemp /tmp/restic-backup.XXXXXX) # Keep the log when the run FAILS. Without this the EXIT trap deleted the only # record of restic's actual error, leaving nothing to diagnose from but the 25 # lines that made it into the ntfy body -- which is exactly what happened on # 2026-08-24 when forget/prune died and the cause could not be recovered. for d in /var/log "${HOME:-/nonexistent}/.local/state" /tmp; do [ -d "$d" ] && [ -w "$d" ] && { FAILLOG=$d/restic-backup.failed.log; break; } done cleanup() { local rc=$? # Explicit if, not `[ $rc -ne 0 ] && cp ...`: a failing test as the last # statement of a trap is the kind of set -e landmine that has bitten this # codebase before. if [ "$rc" -ne 0 ] && [ -n "${FAILLOG:-}" ]; then cp -f "$LOG" "$FAILLOG" 2>/dev/null || true fi rm -f "$LOG" } trap cleanup EXIT PATHS=( /etc /root # /usr/local rather than /usr/local/bin: shanty's live database is bind # mounted from /usr/local/shanty, not from anywhere under /home/connor. /usr/local /var/backups/db # Five named podman volumes live here. Four are tiny and one (model-cache) # is excluded. Compose bind-mounts almost everything from /home/connor, so # this is easy to forget -- and forgetting it silently drops syncthing's # and searxng's state from every snapshot. /var/lib/containers/storage/volumes /home/connor ) log() { printf '%s restic-backup: %s\n' "$(date -Is)" "$*" | tee -a "$LOG"; } notify() { # notify <body> local pri=$1 tags=$2 title=$3 body=$4 u p [ -r "$NTFY_ENV" ] || return 0 # PARSED, not sourced. The bot password contains ` and &, so `. $NTFY_ENV` # dies with a syntax error -- and it cannot simply be quoted either, # because movie_recs_notify reads the same file with a literal split on # "=" and would then send the quotes as part of the password. u=$(sed -n 's/^NTFY_USER=//p' "$NTFY_ENV" | head -1) p=$(sed -n 's/^NTFY_PASS=//p' "$NTFY_ENV" | head -1) [ -n "$u" ] && [ -n "$p" ] || return 0 curl -fsS --max-time 20 \ -u "$u:$p" \ -H "Title: $title" -H "Priority: $pri" -H "Tags: $tags" \ -d "$body" "$NTFY_URL" >/dev/null || true } fail() { log "FAILED: $1" notify urgent "rotating_light" "Backup FAILED on $HOST" \ "$1"$'\n\n'"$(tail -n 25 "$LOG")" exit 1 } # --- 1. consistent database dumps ------------------------------------------ log "dumping databases" /usr/local/bin/backup-db-dump >>"$LOG" 2>&1 || fail "backup-db-dump failed" # --- 2. snapshot ----------------------------------------------------------- log "backing up: ${PATHS[*]}" rc=0 nice -n 10 ionice -c2 -n7 restic backup \ --one-file-system \ --exclude-file="$EXCLUDES" \ --exclude-caches \ --tag "$HOST" \ --verbose=1 \ "${PATHS[@]}" >>"$LOG" 2>&1 || rc=$? # restic exits 3 when it could not read *some* files but the snapshot was # still written. That is worth a warning, not a failure -- a nightly job that # hard-fails on one transiently-locked file stops being a backup. if [ "$rc" -ne 0 ]; then [ "$rc" -eq 3 ] || fail "restic backup exited $rc" log "WARN: restic exited 3 (some files unreadable); snapshot was written" fi # --- 3. retention ---------------------------------------------------------- log "forget + prune" # --group-by host, NOT the default host+paths. With the default, changing the # PATHS list above starts a fresh retention group and the snapshots taken under # the old path list are kept forever -- every group gets its own # daily/weekly/monthly/yearly allowance. One host per repo, so one group. restic forget --prune \ --group-by host \ --tag "$HOST" \ --keep-daily 14 --keep-weekly 8 --keep-monthly 12 --keep-yearly 3 \ >>"$LOG" 2>&1 || fail "restic forget/prune failed" # --- 4. report ------------------------------------------------------------- summary=$(grep -E '^(Added to the repository|processed|snapshot [0-9a-f]{8} saved)' "$LOG" | tail -3) stats=$(restic stats --mode raw-data latest 2>/dev/null | grep -E 'Total Size' || true) log "done" notify default "floppy_disk" "Backup OK on $HOST" "${summary:-(no summary)}"$'\n'"$stats"