533 lines
23 KiB
Bash
Executable File
533 lines
23 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Restore drill and emergency restore tool for the Hetzner restic repos.
|
|
#
|
|
# Purpose: prove that backups can be read and used with nothing but this script,
|
|
# restore-credentials.age and the password written on paper. The plan and reasoning are in
|
|
# ~/guides/mainframe/agent-security-plan.md, phase 1.
|
|
#
|
|
# Stages, in order:
|
|
#
|
|
# restore-drill bundle build the encrypted credentials file (system76)
|
|
# restore-drill cold-start stage 1, list both repos and restore one small file from each
|
|
# restore-drill full-restore stage 2, restore the mainframe snapshot (root, mainframe)
|
|
# restore-drill spot-checks stage 3, compare files and test the database dumps (root, mainframe)
|
|
# restore-drill apps-up stage 4, run copies of three apps on a sealed network (root, mainframe)
|
|
# restore-drill apps-down stage 4, stop those copies
|
|
# restore-drill nas stage 5, restore from the NAS repo and load Immich's dump (root, mainframe)
|
|
# restore-drill cleanup stage 6, delete everything the drill restored (root, mainframe)
|
|
#
|
|
# In a real loss: "cold-start" still shows whether credentials work, TARGET below is where
|
|
# files land, and the time check in full_restore can be deleted. Stages 3 and 4 compare against
|
|
# or run on the live machine, so treat them as examples of how to use the restored files.
|
|
#
|
|
# This file holds no secrets, so it is tracked. Keep a copy on Proton Drive beside the bundle,
|
|
# because the usual copy of this repo is on Gitea, which runs on mainframe.
|
|
|
|
set -euo pipefail
|
|
# Anything this script creates may hold secrets, so it starts private to the owner.
|
|
umask 077
|
|
|
|
# --- Settings that may need changing ---------------------------------------------------------
|
|
|
|
# One Storage Box, reached on port 23, because Hetzner keeps port 22 for something else.
|
|
BOX_HOST=u491382.your-storagebox.de
|
|
BOX_PORT=23
|
|
|
|
# Pinned so the sub-account password is never sent to an impostor. If Hetzner replaces this key,
|
|
# ssh refuses to connect. Check the new one against Hetzner's published fingerprints, then
|
|
# replace this value.
|
|
BOX_HOST_KEY='ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs'
|
|
|
|
# Read-only sub-accounts. They can read the repos but cannot change or delete anything, so a
|
|
# leaked bundle cannot destroy backups. The writable accounts are -sub1 (mainframe) and -sub2 (nas).
|
|
declare -A REPO_USER=([mainframe]=u491382-sub3 [nas]=u491382-sub4)
|
|
|
|
# Hetzner shows a sub-account's base directory as /home, and the repo sits in restic/ under it.
|
|
REPO_PATH=/home/restic
|
|
|
|
# Outside /home/connor, so the nightly backup does not pick up this copy. Not in /tmp, which is
|
|
# a 32 GB tmpfs on mainframe and too small.
|
|
TARGET=/home/restore-test
|
|
|
|
# Small files that exist in each repo, used to prove a restore works without a long download.
|
|
declare -A SAMPLE_FILE=(
|
|
[mainframe]=/home/connor/compose.yml
|
|
[nas]=/nas/docs/caddy_ca_cert.crt
|
|
)
|
|
|
|
# Stage 5 loads this into a scratch database. Immich and its photos are in the NAS repo so that
|
|
# one snapshot captures both halves.
|
|
IMMICH_DUMP=/nas/photo/immich/backups/immich-pgdump.sql.zst
|
|
|
|
# Immich needs its own Postgres build because the dump uses its vector extensions. Must match the
|
|
# image in compose.yml.
|
|
IMMICH_DB_IMAGE=ghcr.io/immich-app/postgres@sha256:41eacbe83eca995561fe43814fd4891e16e39632806253848efaf04d3c8a8b84
|
|
HEDGEDOC_DB_IMAGE=docker.io/library/mariadb:latest
|
|
|
|
# --- Common helpers --------------------------------------------------------------------------
|
|
|
|
# Under sudo, $HOME may be root's. Credentials and results belong to the person running this.
|
|
invoking_user=${SUDO_USER:-$(id -un)}
|
|
invoking_home=$(getent passwd "$invoking_user" | cut -d: -f6)
|
|
DRILL_DIR=$invoking_home/restore-drill
|
|
BUNDLE=$DRILL_DIR/restore-credentials.age
|
|
RESULTS=$DRILL_DIR/results.log
|
|
|
|
WORK=
|
|
SCRATCH_CONTAINERS=()
|
|
|
|
die() { printf 'FAIL: %s\n' "$*" >&2; exit 1; }
|
|
|
|
need() {
|
|
local tool
|
|
for tool in "$@"; do
|
|
command -v "$tool" >/dev/null || die "$tool is not installed"
|
|
done
|
|
}
|
|
|
|
need_root() { [ "$(id -u)" -eq 0 ] || die "run this stage with sudo"; }
|
|
|
|
# Results outlive the cleanup stage, because they are the record that the drill happened. They
|
|
# hold timings and counts only, never file contents or credentials.
|
|
note() {
|
|
printf '%s\n' "$*"
|
|
mkdir -p "$DRILL_DIR"
|
|
printf '%s %s\n' "$(date -Is)" "$*" >>"$RESULTS"
|
|
# A root run must not leave the log unwritable for the laptop user.
|
|
[ "$(id -u)" -ne 0 ] || chown "$invoking_user": "$DRILL_DIR" "$RESULTS"
|
|
}
|
|
|
|
# Runs on every exit, including failures, so a drill leaves no decrypted state or stray containers.
|
|
on_exit() {
|
|
local name
|
|
for name in ${SCRATCH_CONTAINERS[@]+"${SCRATCH_CONTAINERS[@]}"}; do
|
|
podman rm -f -t 2 "$name" >/dev/null 2>&1 || true
|
|
done
|
|
[ -z "$WORK" ] || rm -rf -- "$WORK"
|
|
}
|
|
trap on_exit EXIT
|
|
|
|
# Holds the restic cache and pinned known_hosts. It stays out of /tmp because the cache can be large.
|
|
make_work_dir() {
|
|
mkdir -p "$DRILL_DIR"
|
|
WORK=$(mktemp -d -p "$DRILL_DIR" work.XXXXXX)
|
|
printf '[%s]:%s %s\n' "$BOX_HOST" "$BOX_PORT" "$BOX_HOST_KEY" >"$WORK/known_hosts"
|
|
}
|
|
|
|
# Decrypts into memory only. age asks for the paper password on the terminal.
|
|
declare -A CREDS=()
|
|
load_credentials() {
|
|
need age
|
|
[ -r "$BUNDLE" ] || die "no bundle at $BUNDLE"
|
|
local plain line
|
|
plain=$(age -d "$BUNDLE") || die "could not decrypt the bundle, check the password"
|
|
while IFS= read -r line; do
|
|
# Split on the first = only, because passwords may contain more of them.
|
|
CREDS[${line%%=*}]=${line#*=}
|
|
done <<<"$plain"
|
|
}
|
|
|
|
# Every repo access goes through here, so these rules apply to all of them.
|
|
repo_restic() {
|
|
local repo=$1; shift
|
|
local user=${REPO_USER[$repo]}
|
|
# Passwords only, with keys turned off, so this tests what is in the bundle rather than
|
|
# something this machine happens to have.
|
|
local ssh_cmd="sshpass -e ssh -p $BOX_PORT -l $user"
|
|
ssh_cmd+=" -o PubkeyAuthentication=no -o PreferredAuthentications=password"
|
|
ssh_cmd+=" -o NumberOfPasswordPrompts=1"
|
|
ssh_cmd+=" -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$WORK/known_hosts"
|
|
ssh_cmd+=" $BOX_HOST -s sftp"
|
|
# Secrets travel in the environment because arguments are visible to every user in ps.
|
|
# --no-lock is required since a read-only account cannot write restic's lock files.
|
|
SSHPASS=${CREDS[${repo}_sftp]} RESTIC_PASSWORD=${CREDS[${repo}_restic]} \
|
|
restic -r "sftp:$user@$BOX_HOST:$REPO_PATH" \
|
|
-o sftp.command="$ssh_cmd" \
|
|
--no-lock --cache-dir "$WORK/cache" \
|
|
"$@"
|
|
}
|
|
|
|
# The backup jobs prune around these hours. A restore then could lose data it is still reading,
|
|
# since --no-lock means restic cannot make the prune wait.
|
|
refuse_during_backup_window() {
|
|
local hour
|
|
hour=$(date +%-H)
|
|
if [ "$hour" -ge 2 ] && [ "$hour" -lt 6 ]; then
|
|
die "backups and prunes run around 02:30-03:30. Try after 06:00"
|
|
fi
|
|
}
|
|
|
|
# --- bundle ----------------------------------------------------------------------------------
|
|
|
|
ask_secret() {
|
|
local prompt=$1 value
|
|
# -s keeps it off the screen. A pasted value is fine.
|
|
IFS= read -rs -p "$prompt: " value </dev/tty
|
|
printf '\n' >/dev/tty
|
|
[ -n "$value" ] || die "empty value for: $prompt"
|
|
printf '%s' "$value"
|
|
}
|
|
|
|
cmd_bundle() {
|
|
need age
|
|
# Overwriting could destroy the only working copy, so replacing one is a deliberate act.
|
|
[ ! -e "$BUNDLE" ] || die "$BUNDLE already exists. Move it first if you mean to replace it"
|
|
mkdir -p "$DRILL_DIR"
|
|
|
|
cat <<EOF
|
|
This asks for four values, then for the paper password twice. Nothing is echoed.
|
|
To read the restic passwords, use another terminal:
|
|
ssh mainframe sudo cat /etc/restic/password
|
|
ssh nas cat .config/restic/password
|
|
EOF
|
|
local mf_restic mf_sftp nas_restic nas_sftp
|
|
mf_restic=$(ask_secret "mainframe restic password")
|
|
mf_sftp=$(ask_secret "${REPO_USER[mainframe]} password")
|
|
nas_restic=$(ask_secret "nas restic password")
|
|
nas_sftp=$(ask_secret "${REPO_USER[nas]} password")
|
|
|
|
# printf is a shell builtin, so the values never appear in a process listing. No plaintext
|
|
# file is written at any point.
|
|
printf 'mainframe_restic=%s\nmainframe_sftp=%s\nnas_restic=%s\nnas_sftp=%s\n' \
|
|
"$mf_restic" "$mf_sftp" "$nas_restic" "$nas_sftp" | age -p -o "$BUNDLE"
|
|
|
|
note "bundle written to $BUNDLE"
|
|
echo "Next: run cold-start, which also proves the paper password decrypts this file."
|
|
echo "Then copy $BUNDLE and this script to Proton Drive."
|
|
}
|
|
|
|
# --- Stage 1: cold start ---------------------------------------------------------------------
|
|
|
|
cmd_cold_start() {
|
|
need restic sshpass
|
|
make_work_dir
|
|
load_credentials
|
|
|
|
local repo out sample
|
|
for repo in mainframe nas; do
|
|
echo "--- $repo repo, as ${REPO_USER[$repo]}"
|
|
out=$(repo_restic "$repo" snapshots --latest 1 --json) \
|
|
|| die "$repo repo could not be read. Check the account, password and path"
|
|
# Catches sub3 and sub4 being pointed at each other's directories.
|
|
printf '%s' "$out" | grep -q "\"hostname\":\"$repo\"" \
|
|
|| die "$repo repo has no snapshot from host $repo. Check the sub-account's base directory"
|
|
repo_restic "$repo" snapshots --latest 1 --compact
|
|
|
|
# Listing proves the password. Restoring proves the data comes back over a read-only account.
|
|
sample=${SAMPLE_FILE[$repo]}
|
|
repo_restic "$repo" restore latest --target "$WORK/sample-$repo" \
|
|
--include "$sample" --verify
|
|
[ -s "$WORK/sample-$repo$sample" ] || die "$sample was not restored from the $repo repo"
|
|
note "cold-start: $repo repo listed and $sample restored"
|
|
done
|
|
echo "Stage 1 passed. Copy the bundle and this script to Proton Drive if you have not yet."
|
|
}
|
|
|
|
# --- Stage 2: full restore -------------------------------------------------------------------
|
|
|
|
cmd_full_restore() {
|
|
need_root
|
|
need restic sshpass
|
|
refuse_during_backup_window
|
|
# Never restore over something, since that could hide missing files.
|
|
if [ -e "$TARGET" ] && [ -n "$(ls -A "$TARGET")" ]; then
|
|
die "$TARGET is not empty. Run cleanup first"
|
|
fi
|
|
make_work_dir
|
|
load_credentials
|
|
|
|
# A restore that runs out of disk partway wastes hours. Ten percent covers filesystem overhead.
|
|
local needed free
|
|
needed=$(repo_restic mainframe stats latest --mode restore-size --json \
|
|
| grep -o '"total_size":[0-9]*' | cut -d: -f2)
|
|
free=$(df --output=avail -B1 "$(dirname "$TARGET")" | tail -1)
|
|
[ "$free" -gt $((needed + needed / 10)) ] || die "need about $needed bytes, have $free"
|
|
|
|
# Mode 700 on the top directory protects every secret and setuid file restored beneath it.
|
|
mkdir -p "$TARGET"
|
|
chmod 700 "$TARGET"
|
|
|
|
local started finished
|
|
started=$(date +%s)
|
|
# Low priority so running services are not starved. The subshell keeps that to the restore, and
|
|
# lowers its own priority because wrapping it in a new bash would put credentials in arguments.
|
|
# --verify reads each file back and checks it.
|
|
(
|
|
renice -n 19 -p "$BASHPID" >/dev/null
|
|
ionice -c 3 -p "$BASHPID"
|
|
repo_restic mainframe restore latest --target "$TARGET" --verify
|
|
)
|
|
finished=$(date +%s)
|
|
|
|
# Compare with the snapshot's own listing. A partial restore is the failure this catches.
|
|
local expected actual
|
|
expected=$(repo_restic mainframe ls latest --json | grep -c '"type":"file"' || true)
|
|
actual=$(find "$TARGET" -type f | wc -l)
|
|
[ "$expected" -eq "$actual" ] || die "snapshot lists $expected files, $actual restored"
|
|
|
|
note "full-restore: $actual files, $(du -sh "$TARGET" | cut -f1), $(((finished - started) / 60)) minutes"
|
|
}
|
|
|
|
# --- Stage 3: spot checks --------------------------------------------------------------------
|
|
|
|
# A row total across all tables. Virtual tables can fail to count, and that is not corruption.
|
|
sqlite_row_total() {
|
|
local db=$1 table count total=0
|
|
while IFS= read -r table; do
|
|
count=$(sqlite3 -readonly "$db" "SELECT count(*) FROM \"$table\"" 2>/dev/null || echo 0)
|
|
total=$((total + count))
|
|
done < <(sqlite3 -readonly "$db" "SELECT name FROM sqlite_master WHERE type='table'")
|
|
printf '%s' "$total"
|
|
}
|
|
|
|
cmd_spot_checks() {
|
|
need_root
|
|
need sqlite3 zstd podman
|
|
[ -d "$TARGET" ] || die "$TARGET does not exist. Run full-restore first"
|
|
|
|
# Files that a rebuild would most depend on. Only same, differs or missing is printed, so no
|
|
# contents reach a terminal or a log. "differs" is normal for anything edited since the snapshot.
|
|
local path
|
|
echo "--- critical files against live copies"
|
|
for path in \
|
|
/etc/fstab \
|
|
/etc/restic/excludes.txt \
|
|
/etc/sudoers \
|
|
/home/connor/compose.yml \
|
|
/home/connor/Caddyfile \
|
|
/home/connor/.env \
|
|
/home/connor/.secrets.zsh \
|
|
/home/connor/.ssh/id_ed25519 \
|
|
/home/connor/config/litellm/config.yaml \
|
|
; do
|
|
if [ ! -f "$TARGET$path" ]; then
|
|
note "spot-check: MISSING $path"
|
|
elif cmp -s "$TARGET$path" "$path"; then
|
|
echo "same $path"
|
|
else
|
|
note "spot-check: differs $path"
|
|
fi
|
|
done
|
|
|
|
# These dumps are what a database restore would use. Row totals against the live database show
|
|
# the dump is recent as well as intact, so small differences are expected.
|
|
local spec name live dump check
|
|
echo "--- SQLite dumps"
|
|
for spec in \
|
|
vaultwarden:/home/connor/data/bitwarden/db.sqlite3 \
|
|
gitea:/home/connor/data/gitea/gitea/gitea.db \
|
|
traggo:/home/connor/data/traggo/traggo.db \
|
|
sftpgo:/home/connor/data/sftpgo/sftpgo.db \
|
|
shanty:/usr/local/shanty/shanty.db \
|
|
; do
|
|
name=${spec%%:*}
|
|
live=${spec#*:}
|
|
dump=$TARGET/var/backups/db/$name.sqlite3
|
|
[ -f "$dump" ] || { note "spot-check: MISSING dump for $name"; continue; }
|
|
check=$(sqlite3 -readonly "$dump" 'PRAGMA integrity_check')
|
|
[ "$check" = ok ] || { note "spot-check: $name dump FAILED integrity check"; continue; }
|
|
note "spot-check: $name intact, $(sqlite_row_total "$dump") rows, live has $(sqlite_row_total "$live")"
|
|
done
|
|
|
|
# HedgeDoc is the one database dump on mainframe that is not a SQLite file. Loading it is the only
|
|
# real test. --network none is why an empty root password is safe here.
|
|
echo "--- HedgeDoc dump into a scratch MariaDB"
|
|
local db=drill-mariadb tries=60
|
|
SCRATCH_CONTAINERS+=("$db")
|
|
podman run -d --rm --name "$db" --network none --pull=never \
|
|
-e MARIADB_ALLOW_EMPTY_ROOT_PASSWORD=1 -e MARIADB_DATABASE=hedgedoc \
|
|
"$HEDGEDOC_DB_IMAGE" >/dev/null
|
|
until podman exec "$db" mariadb-admin -u root ping >/dev/null 2>&1; do
|
|
tries=$((tries - 1))
|
|
[ "$tries" -gt 0 ] || die "scratch MariaDB did not start"
|
|
sleep 1
|
|
done
|
|
# The image restarts its server after first-run setup, so wait for that database too.
|
|
tries=60
|
|
until podman exec "$db" mariadb -u root hedgedoc -e 'SELECT 1' >/dev/null 2>&1; do
|
|
tries=$((tries - 1))
|
|
[ "$tries" -gt 0 ] || die "scratch MariaDB did not finish setup"
|
|
sleep 1
|
|
done
|
|
zstd -dc "$TARGET/var/backups/db/hedgedoc.sql.zst" \
|
|
| podman exec -i "$db" mariadb -u root hedgedoc
|
|
note "spot-check: HedgeDoc dump loaded, $(podman exec "$db" mariadb -u root hedgedoc -Ne 'SELECT count(*) FROM Notes') notes"
|
|
}
|
|
|
|
# --- Stage 4: apps ---------------------------------------------------------------------------
|
|
|
|
DRILL_NET=drill-sealed
|
|
|
|
cmd_apps_up() {
|
|
need_root
|
|
need podman curl
|
|
[ -d "$TARGET/home/connor/data" ] || die "$TARGET has no restored data. Run full-restore first"
|
|
|
|
# --internal has no route out. These are live copies of real apps holding real data, and they must
|
|
# not send mail, fire webhooks or reach the real services.
|
|
podman network exists "$DRILL_NET" || podman network create --internal "$DRILL_NET" >/dev/null
|
|
|
|
local home=$TARGET/home/connor
|
|
mkdir -p "$TARGET/drill-empty-music"
|
|
|
|
# --pull=never uses the images already here, because Shanty's is hosted on Gitea, which is on this
|
|
# machine. Containers write only to the restored copy, which cleanup deletes.
|
|
podman run -d --name drill-bitwarden --network "$DRILL_NET" --pull=never \
|
|
-v "$home/data/bitwarden:/data" \
|
|
-e SIGNUPS_ALLOWED=false \
|
|
docker.io/vaultwarden/server:latest-alpine >/dev/null
|
|
podman run -d --name drill-gitea --network "$DRILL_NET" --pull=never \
|
|
-v "$home/data/gitea:/data" \
|
|
-e USER=connor -e USER_UID=1000 -e USER_GID=1000 \
|
|
docker.gitea.com/gitea:latest >/dev/null
|
|
# Music lives in the NAS repo, so Shanty sees an empty library. The test is its database and login.
|
|
podman run -d --name drill-shanty --network "$DRILL_NET" --pull=never \
|
|
-v "$home/config/shanty:/config" \
|
|
-v "$TARGET/usr/local/shanty:/data" \
|
|
-v "$TARGET/drill-empty-music:/music" \
|
|
-e SHANTY_WEB_BIND=0.0.0.0 \
|
|
git.rcjohnstone.com/connor/shanty:latest >/dev/null
|
|
|
|
# Prove the seal before anyone logs in. Reaching 10.0.0.2 would mean reaching Caddy and the real
|
|
# services. If this fails, stop everything rather than continue.
|
|
local dest
|
|
for dest in 1.1.1.1 10.0.0.2; do
|
|
if podman exec drill-bitwarden wget -q -T 3 -O /dev/null "http://$dest/" 2>/dev/null; then
|
|
cmd_apps_down
|
|
die "copies can reach $dest, so the network is not sealed. They have been stopped"
|
|
fi
|
|
done
|
|
|
|
local ip_bw ip_gitea ip_shanty
|
|
ip_bw=$(podman inspect drill-bitwarden --format "{{(index .NetworkSettings.Networks \"$DRILL_NET\").IPAddress}}")
|
|
ip_gitea=$(podman inspect drill-gitea --format "{{(index .NetworkSettings.Networks \"$DRILL_NET\").IPAddress}}")
|
|
ip_shanty=$(podman inspect drill-shanty --format "{{(index .NetworkSettings.Networks \"$DRILL_NET\").IPAddress}}")
|
|
|
|
note "apps-up: copies running on sealed network $DRILL_NET"
|
|
# A tunnel works because the host can reach these addresses. localhost also counts as a secure
|
|
# origin, which Vaultwarden's web vault requires for its cryptography.
|
|
cat <<EOF
|
|
|
|
From your laptop, open a tunnel:
|
|
|
|
ssh -L 18080:$ip_bw:80 -L 18081:$ip_gitea:3000 -L 18082:$ip_shanty:8085 mainframe
|
|
|
|
Then log in and check for recent data:
|
|
|
|
Vaultwarden http://localhost:18080
|
|
Gitea http://localhost:18081
|
|
Shanty http://localhost:18082
|
|
|
|
Use a private browser window, so the copies' cookies do not mix with the real sites.
|
|
Do not change anything you would want to keep. These copies get deleted.
|
|
When finished: sudo restore-drill apps-down
|
|
EOF
|
|
}
|
|
|
|
cmd_apps_down() {
|
|
need_root
|
|
local name
|
|
for name in drill-bitwarden drill-gitea drill-shanty; do
|
|
podman rm -f -t 5 "$name" >/dev/null 2>&1 || true
|
|
done
|
|
podman network rm "$DRILL_NET" >/dev/null 2>&1 || true
|
|
echo "App copies stopped."
|
|
}
|
|
|
|
# --- Stage 5: NAS ----------------------------------------------------------------------------
|
|
|
|
cmd_nas() {
|
|
need_root
|
|
need restic sshpass zstd podman
|
|
refuse_during_backup_window
|
|
make_work_dir
|
|
load_credentials
|
|
|
|
# Runs here rather than on the NAS because this machine has Podman and the Immich database image.
|
|
local dest=$TARGET/nas-sample
|
|
mkdir -p "$dest"
|
|
chmod 700 "$TARGET"
|
|
repo_restic nas restore latest --target "$dest" \
|
|
--include "$IMMICH_DUMP" --include "${SAMPLE_FILE[nas]}" --verify
|
|
[ -s "$dest$IMMICH_DUMP" ] || die "Immich dump was not restored"
|
|
|
|
# A database is no use without its photos, so restore one original as well. The first listed is
|
|
# as good as any, and writing the listing to a file avoids a broken pipe under pipefail.
|
|
repo_restic nas ls latest /nas/photo/immich/library --recursive --json >"$WORK/library.json" || true
|
|
local photo
|
|
photo=$(grep -m1 '"type":"file"' "$WORK/library.json" | sed -E 's/.*"path":"([^"]*)".*/\1/' || true)
|
|
if [ -n "$photo" ]; then
|
|
repo_restic nas restore latest --target "$dest" --include "$photo" --verify
|
|
[ -s "$dest$photo" ] || die "sample photo was not restored"
|
|
note "nas: sample photo restored"
|
|
else
|
|
note "nas: NO photo found under /nas/photo/immich/library"
|
|
fi
|
|
|
|
# --network none is why trust authentication is safe here. The dump recreates its own roles.
|
|
local db=drill-postgres tries=120
|
|
SCRATCH_CONTAINERS+=("$db")
|
|
podman run -d --rm --name "$db" --network none \
|
|
--shm-size 128m \
|
|
-e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_USER=postgres \
|
|
"$IMMICH_DB_IMAGE" >/dev/null
|
|
# Postgres restarts after first-run setup, so ask for a real query instead of pg_isready alone.
|
|
sleep 5
|
|
until podman exec "$db" psql -U postgres -Atc 'SELECT 1' >/dev/null 2>&1; do
|
|
tries=$((tries - 1))
|
|
[ "$tries" -gt 0 ] || die "scratch Postgres did not start"
|
|
sleep 1
|
|
done
|
|
|
|
local started finished assets
|
|
started=$(date +%s)
|
|
# Errors stay on the terminal without stopping the load. pg_dumpall output commonly trips on roles
|
|
# that already exist, so the row count below decides the result.
|
|
zstd -dc "$dest$IMMICH_DUMP" | podman exec -i "$db" psql -q -U postgres -v ON_ERROR_STOP=0 -f - >/dev/null
|
|
finished=$(date +%s)
|
|
assets=$(podman exec "$db" psql -U postgres -d immich -Atc 'SELECT count(*) FROM asset')
|
|
[ "$assets" -gt 0 ] || die "Immich dump loaded with no assets"
|
|
note "nas: Immich dump loaded in $(((finished - started) / 60)) minutes, $assets assets"
|
|
}
|
|
|
|
# --- Stage 6: cleanup ------------------------------------------------------------------------
|
|
|
|
cmd_cleanup() {
|
|
need_root
|
|
cmd_apps_down
|
|
# The restored tree holds every secret on the machine, so this stage is not optional.
|
|
if [ -d "$TARGET" ]; then
|
|
# Guards against an edited TARGET turning this line into something much worse.
|
|
case $TARGET in
|
|
/home/restore-test) rm -rf -- "$TARGET" ;;
|
|
*) die "TARGET changed to $TARGET, so delete it by hand after checking it" ;;
|
|
esac
|
|
fi
|
|
# The laptop and Proton Drive hold the permanent copies. A bundle left on a machine this drill is
|
|
# meant to survive the loss of adds nothing.
|
|
if [ "$(uname -n)" = mainframe ]; then
|
|
rm -f -- "$BUNDLE"
|
|
fi
|
|
rm -rf -- "$DRILL_DIR"/work.*
|
|
note "cleanup: restored data removed"
|
|
echo "Results kept at $RESULTS. Copy them into the plan."
|
|
}
|
|
|
|
# --- Entry point -----------------------------------------------------------------------------
|
|
|
|
case ${1:-} in
|
|
bundle) cmd_bundle ;;
|
|
cold-start) cmd_cold_start ;;
|
|
full-restore) cmd_full_restore ;;
|
|
spot-checks) cmd_spot_checks ;;
|
|
apps-up) cmd_apps_up ;;
|
|
apps-down) cmd_apps_down ;;
|
|
nas) cmd_nas ;;
|
|
cleanup) cmd_cleanup ;;
|
|
*)
|
|
sed -n '/^# Stages/,/^# In a real loss/p' "$0" | sed '$d'
|
|
exit 2
|
|
;;
|
|
esac
|