connor
e6644d0616
Replaces the previous repo, which had split into two histories that never met (mainframe on a dead GitLab remote, the laptops on Gitea) with 146 dirty files across three machines and the NAS never enrolled at all. Branch-per-machine is gone. One main, with host differences expressed as small files under hosts/<hostname>/ rather than as branches, so there is nothing to merge. The reconciled zsh layer reduces 15-33 line forks to 1-7 effective lines per host; distro differences (oh-my-zsh prefix, syntax-highlighting path, fd vs fdfind) are probed in common/ instead. Fresh history: the old one carried six plaintext credentials, 45 MB of mail caches, browser caches and vendored binaries. 5,096 tracked files and 144 MB become 462 files and 2.6 MB. The .gitignore is now an allowlist, which is what keeps that true. Root cause of the rot: ~/.local/bin was a symlink to scripts/ with GOPATH inside it, so every go install wrote into version control (2.2 GB on the work laptop). PATH now points at the repo instead of the reverse. Also: Hyprland replaces sway and is sourced in two halves so $browser is defined before use; singleton automations carry ConditionHost= alongside host-layer-only placement; ddns moves from cron to a guarded timer; package manifests and pkg-snapshot/pkg-restore replace the X11-era install_scripts/; networkmanager-dmenu added to system76 (the binding always existed, the package never did).
128 lines
5.3 KiB
Bash
Executable File
128 lines
5.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Nightly restic backup of mainframe's LOCAL data to the Hetzner Storage Box.
|
|
#
|
|
# Scope: this host only. The six directories under /home/connor that are NFS
|
|
# mounts from the NAS (audio books docs downloads photo video) are excluded
|
|
# automatically by --one-file-system and are backed up by the NAS's own copy
|
|
# of this script, straight off the XFS array. Pulling 500G over NFS every
|
|
# night to hand it back to the same machine would be absurd.
|
|
#
|
|
# Order matters: dump-databases-then-snapshot. backup-db-dump writes the
|
|
# Immich pg dump onto the NAS mount so the NAS's 03:30 run picks it up an hour
|
|
# later. If that run ever overtakes this one it snapshots yesterday's dump --
|
|
# degraded, not broken, and Immich also writes its own daily dump to the same
|
|
# directory.
|
|
|
|
set -euo pipefail
|
|
umask 077
|
|
|
|
export RESTIC_CACHE_DIR=/var/cache/restic
|
|
set -a; . /etc/restic/hetzner.env; set +a
|
|
|
|
# `hostname` is not installed on the NAS and is not on systemd's PATH in
|
|
# general; uname -n always is.
|
|
HOST=$(uname -n); HOST=${HOST%%.*}
|
|
EXCLUDES=/etc/restic/excludes.txt
|
|
NTFY_URL=https://ntfy.rcjohnstone.com/backup
|
|
NTFY_ENV=/etc/ntfy/publish.env
|
|
LOG=$(mktemp /tmp/restic-backup.XXXXXX)
|
|
# Keep the log when the run FAILS. Without this the EXIT trap deleted the only
|
|
# record of restic's actual error, leaving nothing to diagnose from but the 25
|
|
# lines that made it into the ntfy body -- which is exactly what happened on
|
|
# 2026-08-24 when forget/prune died and the cause could not be recovered.
|
|
for d in /var/log "${HOME:-/nonexistent}/.local/state" /tmp; do
|
|
[ -d "$d" ] && [ -w "$d" ] && { FAILLOG=$d/restic-backup.failed.log; break; }
|
|
done
|
|
cleanup() {
|
|
local rc=$?
|
|
# Explicit if, not `[ $rc -ne 0 ] && cp ...`: a failing test as the last
|
|
# statement of a trap is the kind of set -e landmine that has bitten this
|
|
# codebase before.
|
|
if [ "$rc" -ne 0 ] && [ -n "${FAILLOG:-}" ]; then
|
|
cp -f "$LOG" "$FAILLOG" 2>/dev/null || true
|
|
fi
|
|
rm -f "$LOG"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
PATHS=(
|
|
/etc
|
|
/root
|
|
# /usr/local rather than /usr/local/bin: shanty's live database is bind
|
|
# mounted from /usr/local/shanty, not from anywhere under /home/connor.
|
|
/usr/local
|
|
/var/backups/db
|
|
# Five named podman volumes live here. Four are tiny and one (model-cache)
|
|
# is excluded. Compose bind-mounts almost everything from /home/connor, so
|
|
# this is easy to forget -- and forgetting it silently drops syncthing's
|
|
# and searxng's state from every snapshot.
|
|
/var/lib/containers/storage/volumes
|
|
/home/connor
|
|
)
|
|
|
|
log() { printf '%s restic-backup: %s\n' "$(date -Is)" "$*" | tee -a "$LOG"; }
|
|
|
|
notify() { # notify <priority> <tags> <title> <body>
|
|
local pri=$1 tags=$2 title=$3 body=$4 u p
|
|
[ -r "$NTFY_ENV" ] || return 0
|
|
# PARSED, not sourced. The bot password contains ` and &, so `. $NTFY_ENV`
|
|
# dies with a syntax error -- and it cannot simply be quoted either,
|
|
# because movie_recs_notify reads the same file with a literal split on
|
|
# "=" and would then send the quotes as part of the password.
|
|
u=$(sed -n 's/^NTFY_USER=//p' "$NTFY_ENV" | head -1)
|
|
p=$(sed -n 's/^NTFY_PASS=//p' "$NTFY_ENV" | head -1)
|
|
[ -n "$u" ] && [ -n "$p" ] || return 0
|
|
curl -fsS --max-time 20 \
|
|
-u "$u:$p" \
|
|
-H "Title: $title" -H "Priority: $pri" -H "Tags: $tags" \
|
|
-d "$body" "$NTFY_URL" >/dev/null || true
|
|
}
|
|
|
|
fail() {
|
|
log "FAILED: $1"
|
|
notify urgent "rotating_light" "Backup FAILED on $HOST" \
|
|
"$1"$'\n\n'"$(tail -n 25 "$LOG")"
|
|
exit 1
|
|
}
|
|
|
|
# --- 1. consistent database dumps ------------------------------------------
|
|
log "dumping databases"
|
|
/usr/local/bin/backup-db-dump >>"$LOG" 2>&1 || fail "backup-db-dump failed"
|
|
|
|
# --- 2. snapshot -----------------------------------------------------------
|
|
log "backing up: ${PATHS[*]}"
|
|
rc=0
|
|
nice -n 10 ionice -c2 -n7 restic backup \
|
|
--one-file-system \
|
|
--exclude-file="$EXCLUDES" \
|
|
--exclude-caches \
|
|
--tag "$HOST" \
|
|
--verbose=1 \
|
|
"${PATHS[@]}" >>"$LOG" 2>&1 || rc=$?
|
|
# restic exits 3 when it could not read *some* files but the snapshot was
|
|
# still written. That is worth a warning, not a failure -- a nightly job that
|
|
# hard-fails on one transiently-locked file stops being a backup.
|
|
if [ "$rc" -ne 0 ]; then
|
|
[ "$rc" -eq 3 ] || fail "restic backup exited $rc"
|
|
log "WARN: restic exited 3 (some files unreadable); snapshot was written"
|
|
fi
|
|
|
|
# --- 3. retention ----------------------------------------------------------
|
|
log "forget + prune"
|
|
# --group-by host, NOT the default host+paths. With the default, changing the
|
|
# PATHS list above starts a fresh retention group and the snapshots taken under
|
|
# the old path list are kept forever -- every group gets its own
|
|
# daily/weekly/monthly/yearly allowance. One host per repo, so one group.
|
|
restic forget --prune \
|
|
--group-by host \
|
|
--tag "$HOST" \
|
|
--keep-daily 14 --keep-weekly 8 --keep-monthly 12 --keep-yearly 3 \
|
|
>>"$LOG" 2>&1 || fail "restic forget/prune failed"
|
|
|
|
# --- 4. report -------------------------------------------------------------
|
|
summary=$(grep -E '^(Added to the repository|processed|snapshot [0-9a-f]{8} saved)' "$LOG" | tail -3)
|
|
stats=$(restic stats --mode raw-data latest 2>/dev/null | grep -E 'Total Size' || true)
|
|
log "done"
|
|
notify default "floppy_disk" "Backup OK on $HOST" "${summary:-(no summary)}"$'\n'"$stats"
|