From 621788cde110c5c9f10dd1874d3b4ad12e1a2aa6 Mon Sep 17 00:00:00 2001 From: Connor Johnstone Date: Fri, 6 Nov 2026 16:00:36 -0500 Subject: [PATCH] Many things, but also restic script --- common/claude/CLAUDE.md | 8 +- common/claude/memory/-home-connor/MEMORY.md | 2 + .../-home-connor/agent-security-plan.md | 19 + .../-home-connor/rbw-add-vaultwarden-422.md | 20 + common/opencode/opencode.json | 8 +- common/opencode/plugins/comment-check.ts | 160 ++++++ hosts/mainframe/sbin/restore-drill | 532 ++++++++++++++++++ meta/hosts/mainframe.conf.yaml | 1 + meta/hosts/sctfw004.conf.yaml | 1 + packages/mainframe.aur | 1 + packages/mainframe.pacman | 5 + 11 files changed, 754 insertions(+), 3 deletions(-) create mode 100644 common/claude/memory/-home-connor/agent-security-plan.md create mode 100644 common/claude/memory/-home-connor/rbw-add-vaultwarden-422.md create mode 100644 common/opencode/plugins/comment-check.ts create mode 100755 hosts/mainframe/sbin/restore-drill diff --git a/common/claude/CLAUDE.md b/common/claude/CLAUDE.md index c7de86a..a83bc8b 100644 --- a/common/claude/CLAUDE.md +++ b/common/claude/CLAUDE.md @@ -60,8 +60,12 @@ The five things most likely to waste an hour if you don't know them: before emitting text, or a dropped parameter. Local models do it too. Give reasoning models real headroom before concluding anything. -5. **Zen (`zen/*`) is under a ToS block — make no calls to it**, including - "harmless" ones. Entries stay in config in an access group no key holds. +5. **OpenCode Zen and Z.ai GLM were deleted from the LiteLLM config on + 2026-11-06** — Zen's ToS block was never reversed, and Connor cancelled the + GLM subscription. Don't add `zen/*` or `glm/*` back; the history (including + the z.ai two-endpoint trap) is in `~/guides/llm/robots.md` section 8 and + 14.1b-k, and the pre-removal config is + `~/config/litellm/config.yaml.pre-glm-zen-removal`. 6. **The OpenCode sidebar under-reports cost — badly.** It omits delegated child sessions, where most agent work happens: it showed ~$5.80 against a diff --git a/common/claude/memory/-home-connor/MEMORY.md b/common/claude/memory/-home-connor/MEMORY.md index 961b87e..e583b6a 100644 --- a/common/claude/memory/-home-connor/MEMORY.md +++ b/common/claude/memory/-home-connor/MEMORY.md @@ -22,3 +22,5 @@ - [Discuss before adding content](discuss-before-adding-content.md) — in his notes, record what he said; suggest extras in chat, add only on agreement - [Container storage on /home](container-storage-on-home.md) — bind mounts since 2026-10-29; overlay origin xattrs and conmon KillMode traps - [Caddy 2.11.6 cuts HTTP/1.1 streams at 60 s](caddy-2116-http1-stream-cut.md) — OpenCode ECONNRESET; upgraded to 2.11.7; test --http1.1 vs --http2 first +- [Agent security plan](agent-security-plan.md) — agreed 2026-11-06 in ~/guides/mainframe; read before changing sudo, docker group, launchers or web fetching +- [rbw add 422 on Vaultwarden](rbw-add-vaultwarden-422.md) — rbw 1.15.0 add/edit fail against Vaultwarden 1.37 (missing encryptedFor); reads work; non-tty add reads stdin diff --git a/common/claude/memory/-home-connor/agent-security-plan.md b/common/claude/memory/-home-connor/agent-security-plan.md new file mode 100644 index 0000000..045ac51 --- /dev/null +++ b/common/claude/memory/-home-connor/agent-security-plan.md @@ -0,0 +1,19 @@ +--- +name: agent-security-plan +description: "Agreed plan for sandboxing Claude Code, OpenCode and Hermes, hardening mainframe, and drilling restores; agreed 2026-11-06" +metadata: + node_type: memory + type: project + originSessionId: 91ecb21b-604e-40e1-9420-908de11f9496 + modified: 2026-11-06T18:13:08.400Z +--- + +Plan is in `~/guides/mainframe/agent-security-plan.md`. Read it before changing sudo, the +`docker` group, Home Assistant privileges, `act_runner`, agent launchers, or web fetching. + +**Why:** Connor ranks damage to mainframe and credential theft as his top concerns, and +has never exercised a restore. + +**How to apply:** Check the machine rather than assuming a phase has landed. SSH passphrases +and Hermes write approvals were declined, so don't re-propose them. Rootless Podman is +deferred for reasons recorded in the plan. diff --git a/common/claude/memory/-home-connor/rbw-add-vaultwarden-422.md b/common/claude/memory/-home-connor/rbw-add-vaultwarden-422.md new file mode 100644 index 0000000..9949760 --- /dev/null +++ b/common/claude/memory/-home-connor/rbw-add-vaultwarden-422.md @@ -0,0 +1,20 @@ +--- +name: rbw-add-vaultwarden-422 +description: "rbw 1.15.0 cannot create or edit entries against Vaultwarden 1.37 (HTTP 422, missing encryptedFor); reads still work" +metadata: + node_type: memory + type: project + originSessionId: 08d4907d-b091-423a-8695-bd5ec699d8ca + modified: 2026-11-04T23:14:31.174Z +--- + +As of 2026-11-04, `rbw add` fails with `api request returned error: 422` against +bw.rcjohnstone.com (Vaultwarden 1.37.5). The Vaultwarden log shows +`missing field encryptedFor`. Upstream: doy/rbw issue #385, unmerged PR #386. +`rbw get`, `list` and `sync` are unaffected. + +**Why:** easy to misread as a bad command or a server fault, and nothing is written. + +**How to apply:** check whether rbw has been updated past 1.15.0 before retrying. +With stdin that is not a terminal, `rbw add` reads the password from stdin and +ignores $EDITOR, so an agent shell with an open stdin hangs. Pipe the password in. diff --git a/common/opencode/opencode.json b/common/opencode/opencode.json index 31b7503..728f1f0 100644 --- a/common/opencode/opencode.json +++ b/common/opencode/opencode.json @@ -10,6 +10,12 @@ } }, "permission": { - "plan_enter": "deny" + "plan_enter": "deny", + "skill": { + "*": "deny", + "customize-opencode": "allow", + "rust-crate-release": "allow", + "playlist": "allow" + } } } diff --git a/common/opencode/plugins/comment-check.ts b/common/opencode/plugins/comment-check.ts new file mode 100644 index 0000000..8671efb --- /dev/null +++ b/common/opencode/plugins/comment-check.ts @@ -0,0 +1,160 @@ +// After each file edit, list the comment lines it added and ask the model to +// justify or remove them. Everything runs locally: no binary, no network. +import { readFileSync } from "node:fs" +import { extname, isAbsolute, relative, resolve } from "node:path" + +const EDIT_TOOLS = new Set(["edit", "write", "apply_patch"]) + +const LINE_MARKERS: Record = { + "//": [".rs", ".c", ".h", ".cc", ".cpp", ".cxx", ".hpp", ".hh", ".js", ".jsx", ".mjs", ".cjs", + ".ts", ".tsx", ".mts", ".cts", ".go", ".java", ".kt", ".swift", ".cs", ".scala", ".dart", + ".zig", ".proto", ".fbs", ".glsl", ".wgsl"], + "#": [".py", ".pyi", ".sh", ".bash", ".zsh", ".rb", ".pl", ".r", ".jl", ".nim", ".cmake"], + "--": [".lua", ".sql", ".hs"], + ";": [".el", ".clj", ".lisp", ".scm"], +} + +const MARKER_BY_EXT = new Map() +for (const [marker, exts] of Object.entries(LINE_MARKERS)) { + for (const ext of exts) MARKER_BY_EXT.set(ext, marker) +} + +// Comments that tools or the user's own rules require. +const EXEMPT = [ + /^#!/, + /\bTODO:/, + /\bSAFETY:/, + /SPDX-License-Identifier/, + /-\*-.*-\*-/, + /\b(noqa|type:\s*ignore|pyright:|mypy:|pylint:|ruff:|fmt:\s*(on|off|skip))/, + /\b(eslint|prettier|biome|tslint)-/, + /@ts-(ignore|expect-error|nocheck)/, + /\b(NOLINT|clang-format|shellcheck)\b/, + /^\/\/\s*(go:|\+build)/, + /^\/\/\/\s* i + 1 && close <= i + 3) i = close + continue + } + if (ch === '"' || ch === "'" || ch === "`") { + quote = ch + continue + } + if (!line.startsWith(marker, i)) continue + if (marker === "#" && i > 0 && !/\s/.test(line[i - 1])) continue + if (marker === "//" && i > 0 && line[i - 1] === ":") continue + return i + } + return -1 +} + +function commentText(line: string, marker: string): string | undefined { + const trimmed = line.trim() + if (marker === "//" && (trimmed.startsWith("/*") || (/^\*(\s|\/|$)/.test(trimmed) && !trimmed.endsWith(";")))) { + return trimmed.replace(/^[/*\s]+$/, "") || undefined + } + if (marker === "#" && /^("""|''')/.test(trimmed)) return trimmed + const at = commentStart(line, marker) + if (at < 0) return undefined + const text = line.slice(at).trim() + if (text.replace(/^[^\w]+/, "") === "") return undefined + return text +} + +function readOrEmpty(path: string): string { + try { + return readFileSync(path, "utf8") + } catch { + return "" + } +} + +function targetPaths(tool: string, args: any, base: string): string[] { + const raw: string[] = [] + if (tool === "apply_patch") { + const text: string = args?.patchText ?? args?.patch ?? "" + for (const m of text.matchAll(/^\*\*\* (?:Add|Update) File: (.+)$/gm)) raw.push(m[1].trim()) + for (const m of text.matchAll(/^\*\*\* Move to: (.+)$/gm)) raw.push(m[1].trim()) + } else if (args?.filePath) { + raw.push(args.filePath) + } + return raw.map((p) => (isAbsolute(p) ? p : resolve(base, p))) +} + +function addedComments(before: string, after: string, marker: string): { line: number; text: string }[] { + const remaining = new Map() + for (const l of before.split("\n")) remaining.set(l, (remaining.get(l) ?? 0) + 1) + const found: { line: number; text: string }[] = [] + after.split("\n").forEach((l, i) => { + const left = remaining.get(l) ?? 0 + if (left > 0) { + remaining.set(l, left - 1) + return + } + const text = commentText(l, marker) + if (text && !EXEMPT.some((re) => re.test(text))) found.push({ line: i + 1, text }) + }) + return found +} + +const RULES = `Check each one against the comment rules: +- Keep a comment only where the why is not obvious from the code. +- Remove comments that restate what the code does, narrate the change, describe previous behavior, or explain a departure from an earlier approach. Say those things in the chat instead. +- Prefer one short line to a block. No semicolons or em-dashes in prose comments. +- Deferred work keeps a literal TODO: marker. +Remove the ones that fail with another edit now. If a flagged comment is warranted, leave it and do not mention this check.` + +export const CommentCheck = async ({ directory }: { directory: string }) => { + const pending = new Map>() + + return { + "tool.execute.before": async (input: { tool: string; callID: string }, output: { args: any }) => { + if (!EDIT_TOOLS.has(input.tool)) return + const snapshot = new Map() + for (const p of targetPaths(input.tool, output.args, directory)) { + if (markerFor(p)) snapshot.set(p, readOrEmpty(p)) + } + if (snapshot.size) pending.set(input.callID, snapshot) + }, + + "tool.execute.after": async ( + input: { tool: string; callID: string; args: any }, + output: { output: string }, + ) => { + const snapshot = pending.get(input.callID) + if (!snapshot) return + pending.delete(input.callID) + + const report: string[] = [] + for (const [path, before] of snapshot) { + const marker = markerFor(path)! + for (const c of addedComments(before, readOrEmpty(path), marker)) { + report.push(` ${relative(directory, path)}:${c.line} ${c.text}`) + } + } + if (!report.length) return + + output.output += `\n\n\nThis edit added ${report.length} comment line(s):\n${report.join("\n")}\n\n${RULES}\n` + }, + } +} diff --git a/hosts/mainframe/sbin/restore-drill b/hosts/mainframe/sbin/restore-drill new file mode 100755 index 0000000..0a5fadb --- /dev/null +++ b/hosts/mainframe/sbin/restore-drill @@ -0,0 +1,532 @@ +#!/usr/bin/env bash +# +# Restore drill and emergency restore tool for the Hetzner restic repos. +# +# Purpose: prove that backups can be read and used with nothing but this script, +# restore-credentials.age and the password written on paper. The plan and reasoning are in +# ~/guides/mainframe/agent-security-plan.md, phase 1. +# +# Stages, in order: +# +# restore-drill bundle build the encrypted credentials file (system76) +# restore-drill cold-start stage 1, list both repos and restore one small file from each +# restore-drill full-restore stage 2, restore the mainframe snapshot (root, mainframe) +# restore-drill spot-checks stage 3, compare files and test the database dumps (root, mainframe) +# restore-drill apps-up stage 4, run copies of three apps on a sealed network (root, mainframe) +# restore-drill apps-down stage 4, stop those copies +# restore-drill nas stage 5, restore from the NAS repo and load Immich's dump (root, mainframe) +# restore-drill cleanup stage 6, delete everything the drill restored (root, mainframe) +# +# In a real loss: "cold-start" still shows whether credentials work, TARGET below is where +# files land, and the time check in full_restore can be deleted. Stages 3 and 4 compare against +# or run on the live machine, so treat them as examples of how to use the restored files. +# +# This file holds no secrets, so it is tracked. Keep a copy on Proton Drive beside the bundle, +# because the usual copy of this repo is on Gitea, which runs on mainframe. + +set -euo pipefail +# Anything this script creates may hold secrets, so it starts private to the owner. +umask 077 + +# --- Settings that may need changing --------------------------------------------------------- + +# One Storage Box, reached on port 23, because Hetzner keeps port 22 for something else. +BOX_HOST=u491382.your-storagebox.de +BOX_PORT=23 + +# Pinned so the sub-account password is never sent to an impostor. If Hetzner replaces this key, +# ssh refuses to connect. Check the new one against Hetzner's published fingerprints, then +# replace this value. +BOX_HOST_KEY='ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs' + +# Read-only sub-accounts. They can read the repos but cannot change or delete anything, so a +# leaked bundle cannot destroy backups. The writable accounts are -sub1 (mainframe) and -sub2 (nas). +declare -A REPO_USER=([mainframe]=u491382-sub3 [nas]=u491382-sub4) + +# Hetzner shows a sub-account's base directory as /home, and the repo sits in restic/ under it. +REPO_PATH=/home/restic + +# Outside /home/connor, so the nightly backup does not pick up this copy. Not in /tmp, which is +# a 32 GB tmpfs on mainframe and too small. +TARGET=/home/restore-test + +# Small files that exist in each repo, used to prove a restore works without a long download. +declare -A SAMPLE_FILE=( + [mainframe]=/home/connor/compose.yml + [nas]=/nas/docs/caddy_ca_cert.crt +) + +# Stage 5 loads this into a scratch database. Immich and its photos are in the NAS repo so that +# one snapshot captures both halves. +IMMICH_DUMP=/nas/photo/immich/backups/immich-pgdump.sql.zst + +# Immich needs its own Postgres build because the dump uses its vector extensions. Must match the +# image in compose.yml. +IMMICH_DB_IMAGE=ghcr.io/immich-app/postgres@sha256:41eacbe83eca995561fe43814fd4891e16e39632806253848efaf04d3c8a8b84 +HEDGEDOC_DB_IMAGE=docker.io/library/mariadb:latest + +# --- Common helpers -------------------------------------------------------------------------- + +# Under sudo, $HOME may be root's. Credentials and results belong to the person running this. +invoking_user=${SUDO_USER:-$(id -un)} +invoking_home=$(getent passwd "$invoking_user" | cut -d: -f6) +DRILL_DIR=$invoking_home/restore-drill +BUNDLE=$DRILL_DIR/restore-credentials.age +RESULTS=$DRILL_DIR/results.log + +WORK= +SCRATCH_CONTAINERS=() + +die() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } + +need() { + local tool + for tool in "$@"; do + command -v "$tool" >/dev/null || die "$tool is not installed" + done +} + +need_root() { [ "$(id -u)" -eq 0 ] || die "run this stage with sudo"; } + +# Results outlive the cleanup stage, because they are the record that the drill happened. They +# hold timings and counts only, never file contents or credentials. +note() { + printf '%s\n' "$*" + mkdir -p "$DRILL_DIR" + printf '%s %s\n' "$(date -Is)" "$*" >>"$RESULTS" + # A root run must not leave the log unwritable for the laptop user. + [ "$(id -u)" -ne 0 ] || chown "$invoking_user": "$DRILL_DIR" "$RESULTS" +} + +# Runs on every exit, including failures, so a drill leaves no decrypted state or stray containers. +on_exit() { + local name + for name in ${SCRATCH_CONTAINERS[@]+"${SCRATCH_CONTAINERS[@]}"}; do + podman rm -f -t 2 "$name" >/dev/null 2>&1 || true + done + [ -z "$WORK" ] || rm -rf -- "$WORK" +} +trap on_exit EXIT + +# Holds the restic cache and pinned known_hosts. It stays out of /tmp because the cache can be large. +make_work_dir() { + mkdir -p "$DRILL_DIR" + WORK=$(mktemp -d -p "$DRILL_DIR" work.XXXXXX) + printf '[%s]:%s %s\n' "$BOX_HOST" "$BOX_PORT" "$BOX_HOST_KEY" >"$WORK/known_hosts" +} + +# Decrypts into memory only. age asks for the paper password on the terminal. +declare -A CREDS=() +load_credentials() { + need age + [ -r "$BUNDLE" ] || die "no bundle at $BUNDLE" + local plain line + plain=$(age -d "$BUNDLE") || die "could not decrypt the bundle, check the password" + while IFS= read -r line; do + # Split on the first = only, because passwords may contain more of them. + CREDS[${line%%=*}]=${line#*=} + done <<<"$plain" +} + +# Every repo access goes through here, so these rules apply to all of them. +repo_restic() { + local repo=$1; shift + local user=${REPO_USER[$repo]} + # Passwords only, with keys turned off, so this tests what is in the bundle rather than + # something this machine happens to have. + local ssh_cmd="sshpass -e ssh -p $BOX_PORT -l $user" + ssh_cmd+=" -o PubkeyAuthentication=no -o PreferredAuthentications=password" + ssh_cmd+=" -o NumberOfPasswordPrompts=1" + ssh_cmd+=" -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$WORK/known_hosts" + ssh_cmd+=" $BOX_HOST -s sftp" + # Secrets travel in the environment because arguments are visible to every user in ps. + # --no-lock is required since a read-only account cannot write restic's lock files. + SSHPASS=${CREDS[${repo}_sftp]} RESTIC_PASSWORD=${CREDS[${repo}_restic]} \ + restic -r "sftp:$user@$BOX_HOST:$REPO_PATH" \ + -o sftp.command="$ssh_cmd" \ + --no-lock --cache-dir "$WORK/cache" \ + "$@" +} + +# The backup jobs prune around these hours. A restore then could lose data it is still reading, +# since --no-lock means restic cannot make the prune wait. +refuse_during_backup_window() { + local hour + hour=$(date +%-H) + if [ "$hour" -ge 2 ] && [ "$hour" -lt 6 ]; then + die "backups and prunes run around 02:30-03:30. Try after 06:00" + fi +} + +# --- bundle ---------------------------------------------------------------------------------- + +ask_secret() { + local prompt=$1 value + # -s keeps it off the screen. A pasted value is fine. + IFS= read -rs -p "$prompt: " value /dev/tty + [ -n "$value" ] || die "empty value for: $prompt" + printf '%s' "$value" +} + +cmd_bundle() { + need age + # Overwriting could destroy the only working copy, so replacing one is a deliberate act. + [ ! -e "$BUNDLE" ] || die "$BUNDLE already exists. Move it first if you mean to replace it" + mkdir -p "$DRILL_DIR" + + cat </dev/null + ionice -c 3 -p "$BASHPID" + repo_restic mainframe restore latest --target "$TARGET" --verify + ) + finished=$(date +%s) + + # Compare with the snapshot's own listing. A partial restore is the failure this catches. + local expected actual + expected=$(repo_restic mainframe ls latest --json | grep -c '"type":"file"' || true) + actual=$(find "$TARGET" -type f | wc -l) + [ "$expected" -eq "$actual" ] || die "snapshot lists $expected files, $actual restored" + + note "full-restore: $actual files, $(du -sh "$TARGET" | cut -f1), $(((finished - started) / 60)) minutes" +} + +# --- Stage 3: spot checks -------------------------------------------------------------------- + +# A row total across all tables. Virtual tables can fail to count, and that is not corruption. +sqlite_row_total() { + local db=$1 table count total=0 + while IFS= read -r table; do + count=$(sqlite3 -readonly "$db" "SELECT count(*) FROM \"$table\"" 2>/dev/null || echo 0) + total=$((total + count)) + done < <(sqlite3 -readonly "$db" "SELECT name FROM sqlite_master WHERE type='table'") + printf '%s' "$total" +} + +cmd_spot_checks() { + need_root + need sqlite3 zstd podman + [ -d "$TARGET" ] || die "$TARGET does not exist. Run full-restore first" + + # Files that a rebuild would most depend on. Only same, differs or missing is printed, so no + # contents reach a terminal or a log. "differs" is normal for anything edited since the snapshot. + local path + echo "--- critical files against live copies" + for path in \ + /etc/fstab \ + /etc/restic/excludes.txt \ + /etc/sudoers \ + /home/connor/compose.yml \ + /home/connor/Caddyfile \ + /home/connor/.env \ + /home/connor/.secrets.zsh \ + /home/connor/.ssh/id_ed25519 \ + /home/connor/config/litellm/config.yaml \ + ; do + if [ ! -f "$TARGET$path" ]; then + note "spot-check: MISSING $path" + elif cmp -s "$TARGET$path" "$path"; then + echo "same $path" + else + note "spot-check: differs $path" + fi + done + + # These dumps are what a database restore would use. Row totals against the live database show + # the dump is recent as well as intact, so small differences are expected. + local spec name live dump check + echo "--- SQLite dumps" + for spec in \ + vaultwarden:/home/connor/data/bitwarden/db.sqlite3 \ + gitea:/home/connor/data/gitea/gitea/gitea.db \ + traggo:/home/connor/data/traggo/traggo.db \ + sftpgo:/home/connor/data/sftpgo/sftpgo.db \ + shanty:/usr/local/shanty/shanty.db \ + ; do + name=${spec%%:*} + live=${spec#*:} + dump=$TARGET/var/backups/db/$name.sqlite3 + [ -f "$dump" ] || { note "spot-check: MISSING dump for $name"; continue; } + check=$(sqlite3 -readonly "$dump" 'PRAGMA integrity_check') + [ "$check" = ok ] || { note "spot-check: $name dump FAILED integrity check"; continue; } + note "spot-check: $name intact, $(sqlite_row_total "$dump") rows, live has $(sqlite_row_total "$live")" + done + + # HedgeDoc is the one database dump on mainframe that is not a SQLite file. Loading it is the only + # real test. --network none is why an empty root password is safe here. + echo "--- HedgeDoc dump into a scratch MariaDB" + local db=drill-mariadb tries=60 + SCRATCH_CONTAINERS+=("$db") + podman run -d --rm --name "$db" --network none --pull=never \ + -e MARIADB_ALLOW_EMPTY_ROOT_PASSWORD=1 -e MARIADB_DATABASE=hedgedoc \ + "$HEDGEDOC_DB_IMAGE" >/dev/null + until podman exec "$db" mariadb-admin -u root ping >/dev/null 2>&1; do + tries=$((tries - 1)) + [ "$tries" -gt 0 ] || die "scratch MariaDB did not start" + sleep 1 + done + # The image restarts its server after first-run setup, so wait for that database too. + tries=60 + until podman exec "$db" mariadb -u root hedgedoc -e 'SELECT 1' >/dev/null 2>&1; do + tries=$((tries - 1)) + [ "$tries" -gt 0 ] || die "scratch MariaDB did not finish setup" + sleep 1 + done + zstd -dc "$TARGET/var/backups/db/hedgedoc.sql.zst" \ + | podman exec -i "$db" mariadb -u root hedgedoc + note "spot-check: HedgeDoc dump loaded, $(podman exec "$db" mariadb -u root hedgedoc -Ne 'SELECT count(*) FROM Notes') notes" +} + +# --- Stage 4: apps --------------------------------------------------------------------------- + +DRILL_NET=drill-sealed + +cmd_apps_up() { + need_root + need podman curl + [ -d "$TARGET/home/connor/data" ] || die "$TARGET has no restored data. Run full-restore first" + + # --internal has no route out. These are live copies of real apps holding real data, and they must + # not send mail, fire webhooks or reach the real services. + podman network exists "$DRILL_NET" || podman network create --internal "$DRILL_NET" >/dev/null + + local home=$TARGET/home/connor + mkdir -p "$TARGET/drill-empty-music" + + # --pull=never uses the images already here, because Shanty's is hosted on Gitea, which is on this + # machine. Containers write only to the restored copy, which cleanup deletes. + podman run -d --name drill-bitwarden --network "$DRILL_NET" --pull=never \ + -v "$home/data/bitwarden:/data" \ + -e SIGNUPS_ALLOWED=false \ + docker.io/vaultwarden/server:latest-alpine >/dev/null + podman run -d --name drill-gitea --network "$DRILL_NET" --pull=never \ + -v "$home/data/gitea:/data" \ + -e USER=connor -e USER_UID=1000 -e USER_GID=1000 \ + docker.gitea.com/gitea:latest >/dev/null + # Music lives in the NAS repo, so Shanty sees an empty library. The test is its database and login. + podman run -d --name drill-shanty --network "$DRILL_NET" --pull=never \ + -v "$home/config/shanty:/config" \ + -v "$TARGET/usr/local/shanty:/data" \ + -v "$TARGET/drill-empty-music:/music" \ + -e SHANTY_WEB_BIND=0.0.0.0 \ + git.rcjohnstone.com/connor/shanty:latest >/dev/null + + # Prove the seal before anyone logs in. Reaching 10.0.0.2 would mean reaching Caddy and the real + # services. If this fails, stop everything rather than continue. + local dest + for dest in 1.1.1.1 10.0.0.2; do + if podman exec drill-bitwarden wget -q -T 3 -O /dev/null "http://$dest/" 2>/dev/null; then + cmd_apps_down + die "copies can reach $dest, so the network is not sealed. They have been stopped" + fi + done + + local ip_bw ip_gitea ip_shanty + ip_bw=$(podman inspect drill-bitwarden --format "{{(index .NetworkSettings.Networks \"$DRILL_NET\").IPAddress}}") + ip_gitea=$(podman inspect drill-gitea --format "{{(index .NetworkSettings.Networks \"$DRILL_NET\").IPAddress}}") + ip_shanty=$(podman inspect drill-shanty --format "{{(index .NetworkSettings.Networks \"$DRILL_NET\").IPAddress}}") + + note "apps-up: copies running on sealed network $DRILL_NET" + # A tunnel works because the host can reach these addresses. localhost also counts as a secure + # origin, which Vaultwarden's web vault requires for its cryptography. + cat </dev/null 2>&1 || true + done + podman network rm "$DRILL_NET" >/dev/null 2>&1 || true + echo "App copies stopped." +} + +# --- Stage 5: NAS ---------------------------------------------------------------------------- + +cmd_nas() { + need_root + need restic sshpass zstd podman + refuse_during_backup_window + make_work_dir + load_credentials + + # Runs here rather than on the NAS because this machine has Podman and the Immich database image. + local dest=$TARGET/nas-sample + mkdir -p "$dest" + chmod 700 "$TARGET" + repo_restic nas restore latest --target "$dest" \ + --include "$IMMICH_DUMP" --include "${SAMPLE_FILE[nas]}" --verify + [ -s "$dest$IMMICH_DUMP" ] || die "Immich dump was not restored" + + # A database is no use without its photos, so restore one original as well. The first listed is + # as good as any, and writing the listing to a file avoids a broken pipe under pipefail. + repo_restic nas ls latest /nas/photo/immich/library --recursive --json >"$WORK/library.json" || true + local photo + photo=$(grep -m1 '"type":"file"' "$WORK/library.json" | sed -E 's/.*"path":"([^"]*)".*/\1/' || true) + if [ -n "$photo" ]; then + repo_restic nas restore latest --target "$dest" --include "$photo" --verify + [ -s "$dest$photo" ] || die "sample photo was not restored" + note "nas: sample photo restored" + else + note "nas: NO photo found under /nas/photo/immich/library" + fi + + # --network none is why trust authentication is safe here. The dump recreates its own roles. + local db=drill-postgres tries=120 + SCRATCH_CONTAINERS+=("$db") + podman run -d --rm --name "$db" --network none \ + --shm-size 128m \ + -e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_USER=postgres \ + "$IMMICH_DB_IMAGE" >/dev/null + # Postgres restarts after first-run setup, so ask for a real query instead of pg_isready alone. + sleep 5 + until podman exec "$db" psql -U postgres -Atc 'SELECT 1' >/dev/null 2>&1; do + tries=$((tries - 1)) + [ "$tries" -gt 0 ] || die "scratch Postgres did not start" + sleep 1 + done + + local started finished assets + started=$(date +%s) + # Errors stay on the terminal without stopping the load. pg_dumpall output commonly trips on roles + # that already exist, so the row count below decides the result. + zstd -dc "$dest$IMMICH_DUMP" | podman exec -i "$db" psql -q -U postgres -v ON_ERROR_STOP=0 -f - >/dev/null + finished=$(date +%s) + assets=$(podman exec "$db" psql -U postgres -d immich -Atc 'SELECT count(*) FROM asset') + [ "$assets" -gt 0 ] || die "Immich dump loaded with no assets" + note "nas: Immich dump loaded in $(((finished - started) / 60)) minutes, $assets assets" +} + +# --- Stage 6: cleanup ------------------------------------------------------------------------ + +cmd_cleanup() { + need_root + cmd_apps_down + # The restored tree holds every secret on the machine, so this stage is not optional. + if [ -d "$TARGET" ]; then + # Guards against an edited TARGET turning this line into something much worse. + case $TARGET in + /home/restore-test) rm -rf -- "$TARGET" ;; + *) die "TARGET changed to $TARGET, so delete it by hand after checking it" ;; + esac + fi + # The laptop and Proton Drive hold the permanent copies. A bundle left on a machine this drill is + # meant to survive the loss of adds nothing. + if [ "$(uname -n)" = mainframe ]; then + rm -f -- "$BUNDLE" + fi + rm -rf -- "$DRILL_DIR"/work.* + note "cleanup: restored data removed" + echo "Results kept at $RESULTS. Copy them into the plan." +} + +# --- Entry point ----------------------------------------------------------------------------- + +case ${1:-} in + bundle) cmd_bundle ;; + cold-start) cmd_cold_start ;; + full-restore) cmd_full_restore ;; + spot-checks) cmd_spot_checks ;; + apps-up) cmd_apps_up ;; + apps-down) cmd_apps_down ;; + nas) cmd_nas ;; + cleanup) cmd_cleanup ;; + *) + sed -n '/^# Stages/,/^# In a real loss/p' "$0" | sed '$d' + exit 2 + ;; +esac diff --git a/meta/hosts/mainframe.conf.yaml b/meta/hosts/mainframe.conf.yaml index a316d55..2c6e5ad 100644 --- a/meta/hosts/mainframe.conf.yaml +++ b/meta/hosts/mainframe.conf.yaml @@ -25,6 +25,7 @@ ~/.config/opencode/work.json: common/opencode/work.json ~/.config/opencode/local.json: common/opencode/local.json ~/.config/opencode/tui.json: common/opencode/tui.json + ~/.config/opencode/plugins/comment-check.ts: common/opencode/plugins/comment-check.ts ~/.config/opencode/agents/architect.md: common/opencode/agents/architect.md ~/.config/opencode/agents/brainstorm.md: common/opencode/agents/brainstorm.md ~/.config/opencode/agents/dev.md: common/opencode/agents/dev.md diff --git a/meta/hosts/sctfw004.conf.yaml b/meta/hosts/sctfw004.conf.yaml index 7a49c6a..b336024 100644 --- a/meta/hosts/sctfw004.conf.yaml +++ b/meta/hosts/sctfw004.conf.yaml @@ -51,6 +51,7 @@ ~/.config/opencode/work.json: common/opencode/work.json ~/.config/opencode/local.json: common/opencode/local.json ~/.config/opencode/tui.json: common/opencode/tui.json + ~/.config/opencode/plugins/comment-check.ts: common/opencode/plugins/comment-check.ts ~/.config/opencode/agents/architect.md: common/opencode/agents/architect.md ~/.config/opencode/agents/brainstorm.md: common/opencode/agents/brainstorm.md ~/.config/opencode/agents/dev.md: common/opencode/agents/dev.md diff --git a/packages/mainframe.aur b/packages/mainframe.aur index 02c9a59..facabca 100644 --- a/packages/mainframe.aur +++ b/packages/mainframe.aur @@ -1,6 +1,7 @@ act-runner-bin bashmount claude-code +gitea-mcp llama.cpp-cuda nvidia-patch-git oh-my-zsh-git diff --git a/packages/mainframe.pacman b/packages/mainframe.pacman index 8ee6cda..01e7330 100644 --- a/packages/mainframe.pacman +++ b/packages/mainframe.pacman @@ -14,6 +14,7 @@ dhcpcd difftastic dmidecode docker +doctl dosfstools dust dysk @@ -126,6 +127,7 @@ nvtop openbox openssh orca +packer pacman-contrib pandoc-cli papers @@ -169,7 +171,10 @@ ttf-nerd-fonts-symbols udisks2 unzip uv +vulkan-headers +vulkan-swrast waybar +waypipe wayvnc wdisplays wget