connor
e6644d0616
Replaces the previous repo, which had split into two histories that never met (mainframe on a dead GitLab remote, the laptops on Gitea) with 146 dirty files across three machines and the NAS never enrolled at all. Branch-per-machine is gone. One main, with host differences expressed as small files under hosts/<hostname>/ rather than as branches, so there is nothing to merge. The reconciled zsh layer reduces 15-33 line forks to 1-7 effective lines per host; distro differences (oh-my-zsh prefix, syntax-highlighting path, fd vs fdfind) are probed in common/ instead. Fresh history: the old one carried six plaintext credentials, 45 MB of mail caches, browser caches and vendored binaries. 5,096 tracked files and 144 MB become 462 files and 2.6 MB. The .gitignore is now an allowlist, which is what keeps that true. Root cause of the rot: ~/.local/bin was a symlink to scripts/ with GOPATH inside it, so every go install wrote into version control (2.2 GB on the work laptop). PATH now points at the repo instead of the reverse. Also: Hyprland replaces sway and is sourced in two halves so $browser is defined before use; singleton automations carry ConditionHost= alongside host-layer-only placement; ddns moves from cron to a guarded timer; package manifests and pkg-snapshot/pkg-restore replace the X11-era install_scripts/; networkmanager-dmenu added to system76 (the binding always existed, the package never did).
130 lines
5.0 KiB
Bash
Executable File
130 lines
5.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Nightly restic backup of the NAS array to the Hetzner Storage Box.
|
|
#
|
|
# Runs as connor (see restic-backup.service, User=connor) because the storage
|
|
# box ssh key and the `storagebox` Host alias live in ~connor/.ssh.
|
|
#
|
|
# Scope. The Storage Box is 5TB; /nas is 6.0T used. So this is a choice, not
|
|
# an oversight:
|
|
#
|
|
# photo 226G IN -- irreplaceable
|
|
# docs 17G IN -- irreplaceable
|
|
# books 1.6G IN -- trivially small
|
|
# audio 297G IN -- ripped/purchased, not all re-downloadable
|
|
# video 4.9T OUT -- re-acquirable, and does not fit
|
|
# downloads 574G OUT -- transient torrent scratch
|
|
#
|
|
# ~542G of source data. If video ever needs to be covered it needs its own
|
|
# larger target, not a smaller exclude list here.
|
|
#
|
|
# Timing: this runs at 03:30, an hour after mainframe's 02:30 job, which
|
|
# writes the Immich postgres dump into /nas/photo/immich/backups over NFS.
|
|
# Snapshotting the photo blobs and the database that indexes them in one
|
|
# consistent set is the whole point of that arrangement.
|
|
|
|
set -euo pipefail
|
|
umask 077
|
|
|
|
export RESTIC_CACHE_DIR="$HOME/.cache/restic"
|
|
set -a; . "$HOME/.config/restic/hetzner.env"; set +a
|
|
|
|
# `hostname` is not installed here and is not on systemd's PATH in general;
|
|
# uname -n always is.
|
|
HOST=$(uname -n); HOST=${HOST%%.*}
|
|
EXCLUDES="$HOME/.config/restic/excludes.txt"
|
|
NTFY_URL=https://ntfy.rcjohnstone.com/backup
|
|
NTFY_ENV="$HOME/.config/ntfy/publish.env"
|
|
LOG=$(mktemp /tmp/restic-backup.XXXXXX)
|
|
# Keep the log when the run FAILS. Without this the EXIT trap deleted the only
|
|
# record of restic's actual error, leaving nothing to diagnose from but the 25
|
|
# lines that made it into the ntfy body -- which is exactly what happened on
|
|
# 2026-08-24 when forget/prune died and the cause could not be recovered.
|
|
for d in /var/log "$HOME/.local/state" /tmp; do
|
|
[ -d "$d" ] && [ -w "$d" ] && { FAILLOG=$d/restic-backup.failed.log; break; }
|
|
done
|
|
cleanup() {
|
|
local rc=$?
|
|
# Explicit if, not `[ $rc -ne 0 ] && cp ...`: a failing test as the last
|
|
# statement of a trap is the kind of set -e landmine that has bitten this
|
|
# codebase before.
|
|
if [ "$rc" -ne 0 ] && [ -n "${FAILLOG:-}" ]; then
|
|
cp -f "$LOG" "$FAILLOG" 2>/dev/null || true
|
|
fi
|
|
rm -f "$LOG"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
PATHS=(
|
|
/nas/photo
|
|
/nas/docs
|
|
/nas/books
|
|
/nas/audio
|
|
/etc
|
|
/home/connor
|
|
)
|
|
|
|
log() { printf '%s restic-backup: %s\n' "$(date -Is)" "$*" | tee -a "$LOG"; }
|
|
|
|
notify() { # notify <priority> <tags> <title> <body>
|
|
local pri=$1 tags=$2 title=$3 body=$4 u p
|
|
[ -r "$NTFY_ENV" ] || return 0
|
|
# PARSED, not sourced. The bot password contains ` and &, so `. $NTFY_ENV`
|
|
# dies with a syntax error -- and it cannot simply be quoted either,
|
|
# because movie_recs_notify reads the same file with a literal split on
|
|
# "=" and would then send the quotes as part of the password.
|
|
u=$(sed -n 's/^NTFY_USER=//p' "$NTFY_ENV" | head -1)
|
|
p=$(sed -n 's/^NTFY_PASS=//p' "$NTFY_ENV" | head -1)
|
|
[ -n "$u" ] && [ -n "$p" ] || return 0
|
|
curl -fsS --max-time 20 \
|
|
-u "$u:$p" \
|
|
-H "Title: $title" -H "Priority: $pri" -H "Tags: $tags" \
|
|
-d "$body" "$NTFY_URL" >/dev/null || true
|
|
}
|
|
|
|
fail() {
|
|
log "FAILED: $1"
|
|
notify urgent "rotating_light" "Backup FAILED on $HOST" \
|
|
"$1"$'\n\n'"$(tail -n 25 "$LOG")"
|
|
exit 1
|
|
}
|
|
|
|
# --- 1. sanity: never snapshot an unmounted array --------------------------
|
|
# Without this, a failed mount turns into a successful backup of an empty
|
|
# directory, forget --prune ages out the real snapshots, and the loss is
|
|
# silent until the day it matters.
|
|
mountpoint -q /nas || fail "/nas is not mounted; refusing to snapshot"
|
|
|
|
# --- 2. snapshot -----------------------------------------------------------
|
|
log "backing up: ${PATHS[*]}"
|
|
rc=0
|
|
nice -n 10 ionice -c2 -n7 restic backup \
|
|
--one-file-system \
|
|
--exclude-file="$EXCLUDES" \
|
|
--exclude-caches \
|
|
--tag "$HOST" \
|
|
--verbose=1 \
|
|
"${PATHS[@]}" >>"$LOG" 2>&1 || rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
[ "$rc" -eq 3 ] || fail "restic backup exited $rc"
|
|
log "WARN: restic exited 3 (some files unreadable); snapshot was written"
|
|
fi
|
|
|
|
# --- 3. retention ----------------------------------------------------------
|
|
log "forget + prune"
|
|
# --group-by host, NOT the default host+paths. With the default, changing the
|
|
# PATHS list above starts a fresh retention group and the snapshots taken under
|
|
# the old path list are kept forever -- every group gets its own
|
|
# daily/weekly/monthly/yearly allowance. One host per repo, so one group.
|
|
restic forget --prune \
|
|
--group-by host \
|
|
--tag "$HOST" \
|
|
--keep-daily 14 --keep-weekly 8 --keep-monthly 12 --keep-yearly 3 \
|
|
>>"$LOG" 2>&1 || fail "restic forget/prune failed"
|
|
|
|
# --- 4. report -------------------------------------------------------------
|
|
summary=$(grep -E '^(Added to the repository|processed|snapshot [0-9a-f]{8} saved)' "$LOG" | tail -3)
|
|
stats=$(restic stats --mode raw-data latest 2>/dev/null | grep -E 'Total Size' || true)
|
|
log "done"
|
|
notify default "floppy_disk" "Backup OK on $HOST" "${summary:-(no summary)}"$'\n'"$stats"
|