connor
e6644d0616
Replaces the previous repo, which had split into two histories that never met (mainframe on a dead GitLab remote, the laptops on Gitea) with 146 dirty files across three machines and the NAS never enrolled at all. Branch-per-machine is gone. One main, with host differences expressed as small files under hosts/<hostname>/ rather than as branches, so there is nothing to merge. The reconciled zsh layer reduces 15-33 line forks to 1-7 effective lines per host; distro differences (oh-my-zsh prefix, syntax-highlighting path, fd vs fdfind) are probed in common/ instead. Fresh history: the old one carried six plaintext credentials, 45 MB of mail caches, browser caches and vendored binaries. 5,096 tracked files and 144 MB become 462 files and 2.6 MB. The .gitignore is now an allowlist, which is what keeps that true. Root cause of the rot: ~/.local/bin was a symlink to scripts/ with GOPATH inside it, so every go install wrote into version control (2.2 GB on the work laptop). PATH now points at the repo instead of the reverse. Also: Hyprland replaces sway and is sourced in two halves so $browser is defined before use; singleton automations carry ConditionHost= alongside host-layer-only placement; ddns moves from cron to a guarded timer; package manifests and pkg-snapshot/pkg-restore replace the X11-era install_scripts/; networkmanager-dmenu added to system76 (the binding always existed, the package never did).
112 lines
4.0 KiB
Bash
Executable File
112 lines
4.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# ONE-TIME seed of the NAS restic repository.
|
|
#
|
|
# Why this exists rather than just running restic-backup: a single `restic
|
|
# backup` over 520G takes ~12h on this uplink and commits nothing until it
|
|
# finishes. Two attempts were already lost whole -- one to an edit of the
|
|
# running script, one to an unplanned reboot -- because an interrupted run
|
|
# leaves only unreferenced packs that the next run cannot reuse.
|
|
#
|
|
# So: back up in chunks, each of which commits its own snapshot. An
|
|
# interruption now costs one chunk, not the run. Re-running picks up where it
|
|
# left off via the state file, which survives reboots.
|
|
#
|
|
# The chunk list does NOT have to be exhaustive. Once it completes, the normal
|
|
# restic-backup.service run snapshots the full path set and dedupes against
|
|
# everything seeded here, so any file not covered by a chunk is picked up then
|
|
# -- in minutes rather than hours. Afterwards:
|
|
#
|
|
# restic forget --tag seed --group-by '' && restic prune
|
|
#
|
|
# drops the seed snapshots; their data stays, referenced by the real snapshot.
|
|
|
|
set -euo pipefail
|
|
umask 077
|
|
|
|
export RESTIC_CACHE_DIR="$HOME/.cache/restic"
|
|
set -a; . "$HOME/.config/restic/hetzner.env"; set +a
|
|
|
|
EXCLUDES="$HOME/.config/restic/excludes.txt"
|
|
STATE="$HOME/.local/state/restic-seed.done"
|
|
BATCH_BYTES=$(( 15 * 1024 * 1024 * 1024 )) # ~15G per chunk => ~20 min each
|
|
|
|
mkdir -p "$(dirname "$STATE")"; touch "$STATE"
|
|
|
|
log() { printf '%s restic-seed: %s\n' "$(date -Is)" "$*"; }
|
|
|
|
# Backed up whole -- each is small enough to be one chunk.
|
|
WHOLE=(
|
|
/etc
|
|
/home/connor
|
|
/nas/books
|
|
/nas/docs
|
|
/nas/photo/takeout
|
|
/nas/photo/immich/backups
|
|
/nas/audio/podcast
|
|
)
|
|
|
|
# Too big for one chunk; split into batches of their immediate children.
|
|
# admin/ is immich's only library user and holds 185G across 22 year folders;
|
|
# music/ and shanty/ are ~190 artist directories each.
|
|
SPLIT=(
|
|
/nas/photo/immich/library/admin
|
|
/nas/audio/music
|
|
/nas/audio/shanty
|
|
)
|
|
|
|
# run_chunk <path>...
|
|
# Keyed by the md5 of its path list so the state file survives re-ordering.
|
|
run_chunk() {
|
|
local key; key=$(printf '%s\n' "$@" | md5sum | cut -c1-12)
|
|
if grep -qx "$key" "$STATE"; then
|
|
log "skip [$key] $1 ${2:+(+$(($#-1)) more)}"
|
|
return 0
|
|
fi
|
|
log "chunk [$key] $1 ${2:+(+$(($#-1)) more)}"
|
|
# Exit 3 means some files were unreadable but the snapshot was written --
|
|
# still progress, still worth recording.
|
|
local rc=0
|
|
nice -n 10 ionice -c2 -n7 restic backup \
|
|
--one-file-system \
|
|
--exclude-file="$EXCLUDES" \
|
|
--exclude-caches \
|
|
--tag nas --tag seed \
|
|
"$@" || rc=$?
|
|
if [ "$rc" -eq 0 ] || [ "$rc" -eq 3 ]; then
|
|
printf '%s\n' "$key" >> "$STATE"
|
|
# NOT `[ ... ] && log`: under `set -e` a false test makes that the
|
|
# function's exit status and kills the whole seed on the first
|
|
# SUCCESSFUL chunk.
|
|
if [ "$rc" -eq 3 ]; then log " (exit 3: some files unreadable)"; fi
|
|
else
|
|
log " FAILED rc=$rc -- leaving unrecorded so a re-run retries it"
|
|
return "$rc"
|
|
fi
|
|
}
|
|
|
|
for p in "${WHOLE[@]}"; do
|
|
[ -e "$p" ] || { log "missing, skipping: $p"; continue; }
|
|
run_chunk "$p"
|
|
done
|
|
|
|
for root in "${SPLIT[@]}"; do
|
|
[ -d "$root" ] || { log "missing, skipping: $root"; continue; }
|
|
batch=(); size=0
|
|
while IFS= read -r -d '' child; do
|
|
csize=$(du -sb --apparent-size "$child" 2>/dev/null | cut -f1) || csize=0
|
|
# Flush first if adding this child would overflow, so a single
|
|
# oversized child still gets a chunk of its own rather than being
|
|
# merged into a huge one.
|
|
if [ "${#batch[@]}" -gt 0 ] && [ $(( size + csize )) -gt "$BATCH_BYTES" ]; then
|
|
run_chunk "${batch[@]}"
|
|
batch=(); size=0
|
|
fi
|
|
batch+=("$child"); size=$(( size + csize ))
|
|
done < <(find "$root" -mindepth 1 -maxdepth 1 -print0 | sort -z)
|
|
if [ "${#batch[@]}" -gt 0 ]; then run_chunk "${batch[@]}"; fi
|
|
done
|
|
|
|
log "seed complete -- $(wc -l < "$STATE") chunks recorded"
|
|
log "next: systemctl start restic-backup.service (full run, dedupes against these)"
|