#!/usr/bin/env bash
#
# Restore drill and emergency restore tool for the Hetzner restic repos.
#
# Purpose: prove that backups can be read and used with nothing but this script,
# restore-credentials.age and the password written on paper. The plan and reasoning are in
# ~/guides/mainframe/agent-security-plan.md, phase 1.
#
# Stages, in order:
#
#   restore-drill bundle        build the encrypted credentials file (system76)
#   restore-drill cold-start    stage 1, list both repos and restore one small file from each
#   restore-drill full-restore  stage 2, restore the mainframe snapshot (root, mainframe)
#   restore-drill spot-checks   stage 3, compare files and test the database dumps (root, mainframe)
#   restore-drill apps-up       stage 4, run copies of three apps on a sealed network (root, mainframe)
#   restore-drill apps-down     stage 4, stop those copies
#   restore-drill nas           stage 5, restore from the NAS repo and load Immich's dump (root, mainframe)
#   restore-drill cleanup       stage 6, delete everything the drill restored (root, mainframe)
#
# In a real loss: "cold-start" still shows whether credentials work, TARGET below is where
# files land, and the time check in full_restore can be deleted. Stages 3 and 4 compare against
# or run on the live machine, so treat them as examples of how to use the restored files.
#
# This file holds no secrets, so it is tracked. Keep a copy on Proton Drive beside the bundle,
# because the usual copy of this repo is on Gitea, which runs on mainframe.

set -euo pipefail
# Anything this script creates may hold secrets, so it starts private to the owner.
umask 077

# --- Settings that may need changing ---------------------------------------------------------

# One Storage Box, reached on port 23, because Hetzner keeps port 22 for something else.
BOX_HOST=u491382.your-storagebox.de
BOX_PORT=23

# Pinned so the sub-account password is never sent to an impostor. If Hetzner replaces this key,
# ssh refuses to connect. Check the new one against Hetzner's published fingerprints, then
# replace this value.
BOX_HOST_KEY='ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs'

# Read-only sub-accounts. They can read the repos but cannot change or delete anything, so a
# leaked bundle cannot destroy backups. The writable accounts are -sub1 (mainframe) and -sub2 (nas).
declare -A REPO_USER=([mainframe]=u491382-sub3 [nas]=u491382-sub4)

# Hetzner shows a sub-account's base directory as /home, and the repo sits in restic/ under it.
REPO_PATH=/home/restic

# Outside /home/connor, so the nightly backup does not pick up this copy. Not in /tmp, which is
# a 32 GB tmpfs on mainframe and too small.
TARGET=/home/restore-test

# Small files that exist in each repo, used to prove a restore works without a long download.
declare -A SAMPLE_FILE=(
    [mainframe]=/home/connor/compose.yml
    [nas]=/nas/docs/caddy_ca_cert.crt
)

# Stage 5 loads this into a scratch database. Immich and its photos are in the NAS repo so that
# one snapshot captures both halves.
IMMICH_DUMP=/nas/photo/immich/backups/immich-pgdump.sql.zst

# Immich needs its own Postgres build because the dump uses its vector extensions. Must match the
# image in compose.yml.
IMMICH_DB_IMAGE=ghcr.io/immich-app/postgres@sha256:41eacbe83eca995561fe43814fd4891e16e39632806253848efaf04d3c8a8b84
HEDGEDOC_DB_IMAGE=docker.io/library/mariadb:latest

# --- Common helpers --------------------------------------------------------------------------

# Under sudo, $HOME may be root's. Credentials and results belong to the person running this.
invoking_user=${SUDO_USER:-$(id -un)}
invoking_home=$(getent passwd "$invoking_user" | cut -d: -f6)
DRILL_DIR=$invoking_home/restore-drill
BUNDLE=$DRILL_DIR/restore-credentials.age
RESULTS=$DRILL_DIR/results.log

WORK=
SCRATCH_CONTAINERS=()

die() { printf 'FAIL: %s\n' "$*" >&2; exit 1; }

need() {
    local tool
    for tool in "$@"; do
        command -v "$tool" >/dev/null || die "$tool is not installed"
    done
}

need_root() { [ "$(id -u)" -eq 0 ] || die "run this stage with sudo"; }

# Results outlive the cleanup stage, because they are the record that the drill happened. They
# hold timings and counts only, never file contents or credentials.
note() {
    printf '%s\n' "$*"
    mkdir -p "$DRILL_DIR"
    printf '%s %s\n' "$(date -Is)" "$*" >>"$RESULTS"
    # A root run must not leave the log unwritable for the laptop user.
    [ "$(id -u)" -ne 0 ] || chown "$invoking_user": "$DRILL_DIR" "$RESULTS"
}

# Runs on every exit, including failures, so a drill leaves no decrypted state or stray containers.
on_exit() {
    local name
    for name in ${SCRATCH_CONTAINERS[@]+"${SCRATCH_CONTAINERS[@]}"}; do
        podman rm -f -t 2 "$name" >/dev/null 2>&1 || true
    done
    [ -z "$WORK" ] || rm -rf -- "$WORK"
}
trap on_exit EXIT

# Holds the restic cache and pinned known_hosts. It stays out of /tmp because the cache can be large.
make_work_dir() {
    mkdir -p "$DRILL_DIR"
    WORK=$(mktemp -d -p "$DRILL_DIR" work.XXXXXX)
    printf '[%s]:%s %s\n' "$BOX_HOST" "$BOX_PORT" "$BOX_HOST_KEY" >"$WORK/known_hosts"
}

# Decrypts into memory only. age asks for the paper password on the terminal.
declare -A CREDS=()
load_credentials() {
    need age
    [ -r "$BUNDLE" ] || die "no bundle at $BUNDLE"
    local plain line
    plain=$(age -d "$BUNDLE") || die "could not decrypt the bundle, check the password"
    while IFS= read -r line; do
        # Split on the first = only, because passwords may contain more of them.
        CREDS[${line%%=*}]=${line#*=}
    done <<<"$plain"
}

# Every repo access goes through here, so these rules apply to all of them.
repo_restic() {
    local repo=$1; shift
    local user=${REPO_USER[$repo]}
    # Passwords only, with keys turned off, so this tests what is in the bundle rather than
    # something this machine happens to have.
    local ssh_cmd="sshpass -e ssh -p $BOX_PORT -l $user"
    ssh_cmd+=" -o PubkeyAuthentication=no -o PreferredAuthentications=password"
    ssh_cmd+=" -o NumberOfPasswordPrompts=1"
    ssh_cmd+=" -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$WORK/known_hosts"
    ssh_cmd+=" $BOX_HOST -s sftp"
    # Secrets travel in the environment because arguments are visible to every user in ps.
    # --no-lock is required since a read-only account cannot write restic's lock files.
    SSHPASS=${CREDS[${repo}_sftp]} RESTIC_PASSWORD=${CREDS[${repo}_restic]} \
        restic -r "sftp:$user@$BOX_HOST:$REPO_PATH" \
        -o sftp.command="$ssh_cmd" \
        --no-lock --cache-dir "$WORK/cache" \
        "$@"
}

# The backup jobs prune around these hours. A restore then could lose data it is still reading,
# since --no-lock means restic cannot make the prune wait.
refuse_during_backup_window() {
    local hour
    hour=$(date +%-H)
    if [ "$hour" -ge 2 ] && [ "$hour" -lt 6 ]; then
        die "backups and prunes run around 02:30-03:30. Try after 06:00"
    fi
}

# --- bundle ----------------------------------------------------------------------------------

ask_secret() {
    local prompt=$1 value
    # -s keeps it off the screen. A pasted value is fine.
    IFS= read -rs -p "$prompt: " value </dev/tty
    printf '\n' >/dev/tty
    [ -n "$value" ] || die "empty value for: $prompt"
    printf '%s' "$value"
}

cmd_bundle() {
    need age
    # Overwriting could destroy the only working copy, so replacing one is a deliberate act.
    [ ! -e "$BUNDLE" ] || die "$BUNDLE already exists. Move it first if you mean to replace it"
    mkdir -p "$DRILL_DIR"

    cat <<EOF
This asks for four values, then for the paper password twice. Nothing is echoed.
To read the restic passwords, use another terminal:
  ssh mainframe sudo cat /etc/restic/password
  ssh nas cat .config/restic/password
EOF
    local mf_restic mf_sftp nas_restic nas_sftp
    mf_restic=$(ask_secret "mainframe restic password")
    mf_sftp=$(ask_secret "${REPO_USER[mainframe]} password")
    nas_restic=$(ask_secret "nas restic password")
    nas_sftp=$(ask_secret "${REPO_USER[nas]} password")

    # printf is a shell builtin, so the values never appear in a process listing. No plaintext
    # file is written at any point.
    printf 'mainframe_restic=%s\nmainframe_sftp=%s\nnas_restic=%s\nnas_sftp=%s\n' \
        "$mf_restic" "$mf_sftp" "$nas_restic" "$nas_sftp" | age -p -o "$BUNDLE"

    note "bundle written to $BUNDLE"
    echo "Next: run cold-start, which also proves the paper password decrypts this file."
    echo "Then copy $BUNDLE and this script to Proton Drive."
}

# --- Stage 1: cold start ---------------------------------------------------------------------

cmd_cold_start() {
    need restic sshpass
    make_work_dir
    load_credentials

    local repo out sample
    for repo in mainframe nas; do
        echo "--- $repo repo, as ${REPO_USER[$repo]}"
        out=$(repo_restic "$repo" snapshots --latest 1 --json) \
            || die "$repo repo could not be read. Check the account, password and path"
        # Catches sub3 and sub4 being pointed at each other's directories.
        printf '%s' "$out" | grep -q "\"hostname\":\"$repo\"" \
            || die "$repo repo has no snapshot from host $repo. Check the sub-account's base directory"
        repo_restic "$repo" snapshots --latest 1 --compact

        # Listing proves the password. Restoring proves the data comes back over a read-only account.
        sample=${SAMPLE_FILE[$repo]}
        repo_restic "$repo" restore latest --target "$WORK/sample-$repo" \
            --include "$sample" --verify
        [ -s "$WORK/sample-$repo$sample" ] || die "$sample was not restored from the $repo repo"
        note "cold-start: $repo repo listed and $sample restored"
    done
    echo "Stage 1 passed. Copy the bundle and this script to Proton Drive if you have not yet."
}

# --- Stage 2: full restore -------------------------------------------------------------------

cmd_full_restore() {
    need_root
    need restic sshpass
    refuse_during_backup_window
    # Never restore over something, since that could hide missing files.
    if [ -e "$TARGET" ] && [ -n "$(ls -A "$TARGET")" ]; then
        die "$TARGET is not empty. Run cleanup first"
    fi
    make_work_dir
    load_credentials

    # A restore that runs out of disk partway wastes hours. Ten percent covers filesystem overhead.
    local needed free
    needed=$(repo_restic mainframe stats latest --mode restore-size --json \
        | grep -o '"total_size":[0-9]*' | cut -d: -f2)
    free=$(df --output=avail -B1 "$(dirname "$TARGET")" | tail -1)
    [ "$free" -gt $((needed + needed / 10)) ] || die "need about $needed bytes, have $free"

    # Mode 700 on the top directory protects every secret and setuid file restored beneath it.
    mkdir -p "$TARGET"
    chmod 700 "$TARGET"

    local started finished
    started=$(date +%s)
    # Low priority so running services are not starved. The subshell keeps that to the restore, and
    # lowers its own priority because wrapping it in a new bash would put credentials in arguments.
    # --verify reads each file back and checks it.
    (
        renice -n 19 -p "$BASHPID" >/dev/null
        ionice -c 3 -p "$BASHPID"
        repo_restic mainframe restore latest --target "$TARGET" --verify
    )
    finished=$(date +%s)

    # Compare with the snapshot's own listing. A partial restore is the failure this catches.
    local expected actual
    expected=$(repo_restic mainframe ls latest --json | grep -c '"type":"file"' || true)
    actual=$(find "$TARGET" -type f | wc -l)
    [ "$expected" -eq "$actual" ] || die "snapshot lists $expected files, $actual restored"

    note "full-restore: $actual files, $(du -sh "$TARGET" | cut -f1), $(((finished - started) / 60)) minutes"
}

# --- Stage 3: spot checks --------------------------------------------------------------------

# A row total across all tables. Virtual tables can fail to count, and that is not corruption.
sqlite_row_total() {
    local db=$1 table count total=0
    while IFS= read -r table; do
        count=$(sqlite3 -readonly "$db" "SELECT count(*) FROM \"$table\"" 2>/dev/null || echo 0)
        total=$((total + count))
    done < <(sqlite3 -readonly "$db" "SELECT name FROM sqlite_master WHERE type='table'")
    printf '%s' "$total"
}

cmd_spot_checks() {
    need_root
    need sqlite3 zstd podman
    [ -d "$TARGET" ] || die "$TARGET does not exist. Run full-restore first"

    # Files that a rebuild would most depend on. Only same, differs or missing is printed, so no
    # contents reach a terminal or a log. "differs" is normal for anything edited since the snapshot.
    local path
    echo "--- critical files against live copies"
    for path in \
        /etc/fstab \
        /etc/restic/excludes.txt \
        /etc/sudoers \
        /home/connor/compose.yml \
        /home/connor/Caddyfile \
        /home/connor/.env \
        /home/connor/.secrets.zsh \
        /home/connor/.ssh/id_ed25519 \
        /home/connor/config/litellm/config.yaml \
    ; do
        if [ ! -f "$TARGET$path" ]; then
            note "spot-check: MISSING $path"
        elif cmp -s "$TARGET$path" "$path"; then
            echo "same     $path"
        else
            note "spot-check: differs $path"
        fi
    done

    # These dumps are what a database restore would use. Row totals against the live database show
    # the dump is recent as well as intact, so small differences are expected.
    local spec name live dump check
    echo "--- SQLite dumps"
    for spec in \
        vaultwarden:/home/connor/data/bitwarden/db.sqlite3 \
        gitea:/home/connor/data/gitea/gitea/gitea.db \
        traggo:/home/connor/data/traggo/traggo.db \
        sftpgo:/home/connor/data/sftpgo/sftpgo.db \
        shanty:/usr/local/shanty/shanty.db \
    ; do
        name=${spec%%:*}
        live=${spec#*:}
        dump=$TARGET/var/backups/db/$name.sqlite3
        [ -f "$dump" ] || { note "spot-check: MISSING dump for $name"; continue; }
        check=$(sqlite3 -readonly "$dump" 'PRAGMA integrity_check')
        [ "$check" = ok ] || { note "spot-check: $name dump FAILED integrity check"; continue; }
        note "spot-check: $name intact, $(sqlite_row_total "$dump") rows, live has $(sqlite_row_total "$live")"
    done

    # HedgeDoc is the one database dump on mainframe that is not a SQLite file. Loading it is the only
    # real test. --network none is why an empty root password is safe here.
    echo "--- HedgeDoc dump into a scratch MariaDB"
    local db=drill-mariadb tries=60
    SCRATCH_CONTAINERS+=("$db")
    podman run -d --rm --name "$db" --network none --pull=never \
        -e MARIADB_ALLOW_EMPTY_ROOT_PASSWORD=1 -e MARIADB_DATABASE=hedgedoc \
        "$HEDGEDOC_DB_IMAGE" >/dev/null
    until podman exec "$db" mariadb-admin -u root ping >/dev/null 2>&1; do
        tries=$((tries - 1))
        [ "$tries" -gt 0 ] || die "scratch MariaDB did not start"
        sleep 1
    done
    # The image restarts its server after first-run setup, so wait for that database too.
    tries=60
    until podman exec "$db" mariadb -u root hedgedoc -e 'SELECT 1' >/dev/null 2>&1; do
        tries=$((tries - 1))
        [ "$tries" -gt 0 ] || die "scratch MariaDB did not finish setup"
        sleep 1
    done
    zstd -dc "$TARGET/var/backups/db/hedgedoc.sql.zst" \
        | podman exec -i "$db" mariadb -u root hedgedoc
    note "spot-check: HedgeDoc dump loaded, $(podman exec "$db" mariadb -u root hedgedoc -Ne 'SELECT count(*) FROM Notes') notes"
}

# --- Stage 4: apps ---------------------------------------------------------------------------

DRILL_NET=drill-sealed

cmd_apps_up() {
    need_root
    need podman curl
    [ -d "$TARGET/home/connor/data" ] || die "$TARGET has no restored data. Run full-restore first"

    # --internal has no route out. These are live copies of real apps holding real data, and they must
    # not send mail, fire webhooks or reach the real services.
    podman network exists "$DRILL_NET" || podman network create --internal "$DRILL_NET" >/dev/null

    local home=$TARGET/home/connor
    mkdir -p "$TARGET/drill-empty-music"

    # --pull=never uses the images already here, because Shanty's is hosted on Gitea, which is on this
    # machine. Containers write only to the restored copy, which cleanup deletes.
    podman run -d --name drill-bitwarden --network "$DRILL_NET" --pull=never \
        -v "$home/data/bitwarden:/data" \
        -e SIGNUPS_ALLOWED=false \
        docker.io/vaultwarden/server:latest-alpine >/dev/null
    podman run -d --name drill-gitea --network "$DRILL_NET" --pull=never \
        -v "$home/data/gitea:/data" \
        -e USER=connor -e USER_UID=1000 -e USER_GID=1000 \
        docker.gitea.com/gitea:latest >/dev/null
    # Music lives in the NAS repo, so Shanty sees an empty library. The test is its database and login.
    podman run -d --name drill-shanty --network "$DRILL_NET" --pull=never \
        -v "$home/config/shanty:/config" \
        -v "$TARGET/usr/local/shanty:/data" \
        -v "$TARGET/drill-empty-music:/music" \
        -e SHANTY_WEB_BIND=0.0.0.0 \
        git.rcjohnstone.com/connor/shanty:latest >/dev/null

    # Prove the seal before anyone logs in. Reaching 10.0.0.2 would mean reaching Caddy and the real
    # services. If this fails, stop everything rather than continue.
    local dest
    for dest in 1.1.1.1 10.0.0.2; do
        if podman exec drill-bitwarden wget -q -T 3 -O /dev/null "http://$dest/" 2>/dev/null; then
            cmd_apps_down
            die "copies can reach $dest, so the network is not sealed. They have been stopped"
        fi
    done

    local ip_bw ip_gitea ip_shanty
    ip_bw=$(podman inspect drill-bitwarden --format "{{(index .NetworkSettings.Networks \"$DRILL_NET\").IPAddress}}")
    ip_gitea=$(podman inspect drill-gitea --format "{{(index .NetworkSettings.Networks \"$DRILL_NET\").IPAddress}}")
    ip_shanty=$(podman inspect drill-shanty --format "{{(index .NetworkSettings.Networks \"$DRILL_NET\").IPAddress}}")

    note "apps-up: copies running on sealed network $DRILL_NET"
    # A tunnel works because the host can reach these addresses. localhost also counts as a secure
    # origin, which Vaultwarden's web vault requires for its cryptography.
    cat <<EOF

From your laptop, open a tunnel:

  ssh -L 18080:$ip_bw:80 -L 18081:$ip_gitea:3000 -L 18082:$ip_shanty:8085 mainframe

Then log in and check for recent data:

  Vaultwarden  http://localhost:18080
  Gitea        http://localhost:18081
  Shanty       http://localhost:18082

Use a private browser window, so the copies' cookies do not mix with the real sites.
Do not change anything you would want to keep. These copies get deleted.
When finished: sudo restore-drill apps-down
EOF
}

cmd_apps_down() {
    need_root
    local name
    for name in drill-bitwarden drill-gitea drill-shanty; do
        podman rm -f -t 5 "$name" >/dev/null 2>&1 || true
    done
    podman network rm "$DRILL_NET" >/dev/null 2>&1 || true
    echo "App copies stopped."
}

# --- Stage 5: NAS ----------------------------------------------------------------------------

cmd_nas() {
    need_root
    need restic sshpass zstd podman
    refuse_during_backup_window
    make_work_dir
    load_credentials

    # Runs here rather than on the NAS because this machine has Podman and the Immich database image.
    local dest=$TARGET/nas-sample
    mkdir -p "$dest"
    chmod 700 "$TARGET"
    repo_restic nas restore latest --target "$dest" \
        --include "$IMMICH_DUMP" --include "${SAMPLE_FILE[nas]}" --verify
    [ -s "$dest$IMMICH_DUMP" ] || die "Immich dump was not restored"

    # A database is no use without its photos, so restore one original as well. The first listed is
    # as good as any, and writing the listing to a file avoids a broken pipe under pipefail.
    repo_restic nas ls latest /nas/photo/immich/library --recursive --json >"$WORK/library.json" || true
    local photo
    photo=$(grep -m1 '"type":"file"' "$WORK/library.json" | sed -E 's/.*"path":"([^"]*)".*/\1/' || true)
    if [ -n "$photo" ]; then
        repo_restic nas restore latest --target "$dest" --include "$photo" --verify
        [ -s "$dest$photo" ] || die "sample photo was not restored"
        note "nas: sample photo restored"
    else
        note "nas: NO photo found under /nas/photo/immich/library"
    fi

    # --network none is why trust authentication is safe here. The dump recreates its own roles.
    local db=drill-postgres tries=120
    SCRATCH_CONTAINERS+=("$db")
    podman run -d --rm --name "$db" --network none \
        --shm-size 128m \
        -e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_USER=postgres \
        "$IMMICH_DB_IMAGE" >/dev/null
    # Postgres restarts after first-run setup, so ask for a real query instead of pg_isready alone.
    sleep 5
    until podman exec "$db" psql -U postgres -Atc 'SELECT 1' >/dev/null 2>&1; do
        tries=$((tries - 1))
        [ "$tries" -gt 0 ] || die "scratch Postgres did not start"
        sleep 1
    done

    local started finished assets
    started=$(date +%s)
    # Errors stay on the terminal without stopping the load. pg_dumpall output commonly trips on roles
    # that already exist, so the row count below decides the result.
    zstd -dc "$dest$IMMICH_DUMP" | podman exec -i "$db" psql -q -U postgres -v ON_ERROR_STOP=0 -f - >/dev/null
    finished=$(date +%s)
    assets=$(podman exec "$db" psql -U postgres -d immich -Atc 'SELECT count(*) FROM asset')
    [ "$assets" -gt 0 ] || die "Immich dump loaded with no assets"
    note "nas: Immich dump loaded in $(((finished - started) / 60)) minutes, $assets assets"
}

# --- Stage 6: cleanup ------------------------------------------------------------------------

cmd_cleanup() {
    need_root
    cmd_apps_down
    # The restored tree holds every secret on the machine, so this stage is not optional.
    if [ -d "$TARGET" ]; then
        # Guards against an edited TARGET turning this line into something much worse.
        case $TARGET in
            /home/restore-test) rm -rf -- "$TARGET" ;;
            *) die "TARGET changed to $TARGET, so delete it by hand after checking it" ;;
        esac
    fi
    # The laptop and Proton Drive hold the permanent copies. A bundle left on a machine this drill is
    # meant to survive the loss of adds nothing.
    if [ "$(uname -n)" = mainframe ]; then
        rm -f -- "$BUNDLE"
    fi
    rm -rf -- "$DRILL_DIR"/work.*
    note "cleanup: restored data removed"
    echo "Results kept at $RESULTS. Copy them into the plan."
}

# --- Entry point -----------------------------------------------------------------------------

case ${1:-} in
    bundle)       cmd_bundle ;;
    cold-start)   cmd_cold_start ;;
    full-restore) cmd_full_restore ;;
    spot-checks)  cmd_spot_checks ;;
    apps-up)      cmd_apps_up ;;
    apps-down)    cmd_apps_down ;;
    nas)          cmd_nas ;;
    cleanup)      cmd_cleanup ;;
    *)
        sed -n '/^# Stages/,/^# In a real loss/p' "$0" | sed '$d'
        exit 2
        ;;
esac