Many things, but also restic script
This commit is contained in:
connor committed
1 parent
b4bc5e1031
commit
621788cde1
11 files changed
+754
-3
No files matched your search
@@ -60,8 +60,12 @@ The five things most likely to waste an hour if you don't know them:
|
||||
before emitting text, or a dropped parameter. Local models do it too. Give
|
||||
reasoning models real headroom before concluding anything.
|
||||
|
||||
5. **Zen (`zen/*`) is under a ToS block — make no calls to it**, including
|
||||
"harmless" ones. Entries stay in config in an access group no key holds.
|
||||
5. **OpenCode Zen and Z.ai GLM were deleted from the LiteLLM config on
|
||||
2026-11-06** — Zen's ToS block was never reversed, and Connor cancelled the
|
||||
GLM subscription. Don't add `zen/*` or `glm/*` back; the history (including
|
||||
the z.ai two-endpoint trap) is in `~/guides/llm/robots.md` section 8 and
|
||||
14.1b-k, and the pre-removal config is
|
||||
`~/config/litellm/config.yaml.pre-glm-zen-removal`.
|
||||
|
||||
6. **The OpenCode sidebar under-reports cost — badly.** It omits delegated child
|
||||
sessions, where most agent work happens: it showed ~$5.80 against a
|
||||
|
||||
@@ -22,3 +22,5 @@
|
||||
- [Discuss before adding content](discuss-before-adding-content.md) — in his notes, record what he said; suggest extras in chat, add only on agreement
|
||||
- [Container storage on /home](container-storage-on-home.md) — bind mounts since 2026-10-29; overlay origin xattrs and conmon KillMode traps
|
||||
- [Caddy 2.11.6 cuts HTTP/1.1 streams at 60 s](caddy-2116-http1-stream-cut.md) — OpenCode ECONNRESET; upgraded to 2.11.7; test --http1.1 vs --http2 first
|
||||
- [Agent security plan](agent-security-plan.md) — agreed 2026-11-06 in ~/guides/mainframe; read before changing sudo, docker group, launchers or web fetching
|
||||
- [rbw add 422 on Vaultwarden](rbw-add-vaultwarden-422.md) — rbw 1.15.0 add/edit fail against Vaultwarden 1.37 (missing encryptedFor); reads work; non-tty add reads stdin
|
||||
@@ -0,0 +1,19 @@
|
||||
---
|
||||
name: agent-security-plan
|
||||
description: "Agreed plan for sandboxing Claude Code, OpenCode and Hermes, hardening mainframe, and drilling restores; agreed 2026-11-06"
|
||||
metadata:
|
||||
node_type: memory
|
||||
type: project
|
||||
originSessionId: 91ecb21b-604e-40e1-9420-908de11f9496
|
||||
modified: 2026-11-06T18:13:08.400Z
|
||||
---
|
||||
|
||||
Plan is in `~/guides/mainframe/agent-security-plan.md`. Read it before changing sudo, the
|
||||
`docker` group, Home Assistant privileges, `act_runner`, agent launchers, or web fetching.
|
||||
|
||||
**Why:** Connor ranks damage to mainframe and credential theft as his top concerns, and
|
||||
has never exercised a restore.
|
||||
|
||||
**How to apply:** Check the machine rather than assuming a phase has landed. SSH passphrases
|
||||
and Hermes write approvals were declined, so don't re-propose them. Rootless Podman is
|
||||
deferred for reasons recorded in the plan.
|
||||
@@ -0,0 +1,20 @@
|
||||
---
|
||||
name: rbw-add-vaultwarden-422
|
||||
description: "rbw 1.15.0 cannot create or edit entries against Vaultwarden 1.37 (HTTP 422, missing encryptedFor); reads still work"
|
||||
metadata:
|
||||
node_type: memory
|
||||
type: project
|
||||
originSessionId: 08d4907d-b091-423a-8695-bd5ec699d8ca
|
||||
modified: 2026-11-04T23:14:31.174Z
|
||||
---
|
||||
|
||||
As of 2026-11-04, `rbw add` fails with `api request returned error: 422` against
|
||||
bw.rcjohnstone.com (Vaultwarden 1.37.5). The Vaultwarden log shows
|
||||
`missing field encryptedFor`. Upstream: doy/rbw issue #385, unmerged PR #386.
|
||||
`rbw get`, `list` and `sync` are unaffected.
|
||||
|
||||
**Why:** easy to misread as a bad command or a server fault, and nothing is written.
|
||||
|
||||
**How to apply:** check whether rbw has been updated past 1.15.0 before retrying.
|
||||
With stdin that is not a terminal, `rbw add` reads the password from stdin and
|
||||
ignores $EDITOR, so an agent shell with an open stdin hangs. Pipe the password in.
|
||||
@@ -10,6 +10,12 @@
|
||||
}
|
||||
},
|
||||
"permission": {
|
||||
"plan_enter": "deny"
|
||||
"plan_enter": "deny",
|
||||
"skill": {
|
||||
"*": "deny",
|
||||
"customize-opencode": "allow",
|
||||
"rust-crate-release": "allow",
|
||||
"playlist": "allow"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
// After each file edit, list the comment lines it added and ask the model to
|
||||
// justify or remove them. Everything runs locally: no binary, no network.
|
||||
import { readFileSync } from "node:fs"
|
||||
import { extname, isAbsolute, relative, resolve } from "node:path"
|
||||
|
||||
const EDIT_TOOLS = new Set(["edit", "write", "apply_patch"])
|
||||
|
||||
const LINE_MARKERS: Record<string, string[]> = {
|
||||
"//": [".rs", ".c", ".h", ".cc", ".cpp", ".cxx", ".hpp", ".hh", ".js", ".jsx", ".mjs", ".cjs",
|
||||
".ts", ".tsx", ".mts", ".cts", ".go", ".java", ".kt", ".swift", ".cs", ".scala", ".dart",
|
||||
".zig", ".proto", ".fbs", ".glsl", ".wgsl"],
|
||||
"#": [".py", ".pyi", ".sh", ".bash", ".zsh", ".rb", ".pl", ".r", ".jl", ".nim", ".cmake"],
|
||||
"--": [".lua", ".sql", ".hs"],
|
||||
";": [".el", ".clj", ".lisp", ".scm"],
|
||||
}
|
||||
|
||||
const MARKER_BY_EXT = new Map<string, string>()
|
||||
for (const [marker, exts] of Object.entries(LINE_MARKERS)) {
|
||||
for (const ext of exts) MARKER_BY_EXT.set(ext, marker)
|
||||
}
|
||||
|
||||
// Comments that tools or the user's own rules require.
|
||||
const EXEMPT = [
|
||||
/^#!/,
|
||||
/\bTODO:/,
|
||||
/\bSAFETY:/,
|
||||
/SPDX-License-Identifier/,
|
||||
/-\*-.*-\*-/,
|
||||
/\b(noqa|type:\s*ignore|pyright:|mypy:|pylint:|ruff:|fmt:\s*(on|off|skip))/,
|
||||
/\b(eslint|prettier|biome|tslint)-/,
|
||||
/@ts-(ignore|expect-error|nocheck)/,
|
||||
/\b(NOLINT|clang-format|shellcheck)\b/,
|
||||
/^\/\/\s*(go:|\+build)/,
|
||||
/^\/\/\/\s*</,
|
||||
]
|
||||
|
||||
function markerFor(path: string): string | undefined {
|
||||
const name = path.split("/").pop() ?? ""
|
||||
if (name === "CMakeLists.txt") return "#"
|
||||
return MARKER_BY_EXT.get(extname(path).toLowerCase())
|
||||
}
|
||||
|
||||
// Index where a comment starts on this line, ignoring markers inside quotes.
|
||||
function commentStart(line: string, marker: string): number {
|
||||
let quote = ""
|
||||
for (let i = 0; i < line.length; i++) {
|
||||
const ch = line[i]
|
||||
if (quote) {
|
||||
if (ch === "\\") i++
|
||||
else if (ch === quote) quote = ""
|
||||
continue
|
||||
}
|
||||
if (ch === "'" && marker === "//") {
|
||||
// Only a char literal opens a quote here, so a Rust lifetime does not.
|
||||
const close = line.indexOf("'", i + 1)
|
||||
if (close > i + 1 && close <= i + 3) i = close
|
||||
continue
|
||||
}
|
||||
if (ch === '"' || ch === "'" || ch === "`") {
|
||||
quote = ch
|
||||
continue
|
||||
}
|
||||
if (!line.startsWith(marker, i)) continue
|
||||
if (marker === "#" && i > 0 && !/\s/.test(line[i - 1])) continue
|
||||
if (marker === "//" && i > 0 && line[i - 1] === ":") continue
|
||||
return i
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
function commentText(line: string, marker: string): string | undefined {
|
||||
const trimmed = line.trim()
|
||||
if (marker === "//" && (trimmed.startsWith("/*") || (/^\*(\s|\/|$)/.test(trimmed) && !trimmed.endsWith(";")))) {
|
||||
return trimmed.replace(/^[/*\s]+$/, "") || undefined
|
||||
}
|
||||
if (marker === "#" && /^("""|''')/.test(trimmed)) return trimmed
|
||||
const at = commentStart(line, marker)
|
||||
if (at < 0) return undefined
|
||||
const text = line.slice(at).trim()
|
||||
if (text.replace(/^[^\w]+/, "") === "") return undefined
|
||||
return text
|
||||
}
|
||||
|
||||
function readOrEmpty(path: string): string {
|
||||
try {
|
||||
return readFileSync(path, "utf8")
|
||||
} catch {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
function targetPaths(tool: string, args: any, base: string): string[] {
|
||||
const raw: string[] = []
|
||||
if (tool === "apply_patch") {
|
||||
const text: string = args?.patchText ?? args?.patch ?? ""
|
||||
for (const m of text.matchAll(/^\*\*\* (?:Add|Update) File: (.+)$/gm)) raw.push(m[1].trim())
|
||||
for (const m of text.matchAll(/^\*\*\* Move to: (.+)$/gm)) raw.push(m[1].trim())
|
||||
} else if (args?.filePath) {
|
||||
raw.push(args.filePath)
|
||||
}
|
||||
return raw.map((p) => (isAbsolute(p) ? p : resolve(base, p)))
|
||||
}
|
||||
|
||||
function addedComments(before: string, after: string, marker: string): { line: number; text: string }[] {
|
||||
const remaining = new Map<string, number>()
|
||||
for (const l of before.split("\n")) remaining.set(l, (remaining.get(l) ?? 0) + 1)
|
||||
const found: { line: number; text: string }[] = []
|
||||
after.split("\n").forEach((l, i) => {
|
||||
const left = remaining.get(l) ?? 0
|
||||
if (left > 0) {
|
||||
remaining.set(l, left - 1)
|
||||
return
|
||||
}
|
||||
const text = commentText(l, marker)
|
||||
if (text && !EXEMPT.some((re) => re.test(text))) found.push({ line: i + 1, text })
|
||||
})
|
||||
return found
|
||||
}
|
||||
|
||||
const RULES = `Check each one against the comment rules:
|
||||
- Keep a comment only where the why is not obvious from the code.
|
||||
- Remove comments that restate what the code does, narrate the change, describe previous behavior, or explain a departure from an earlier approach. Say those things in the chat instead.
|
||||
- Prefer one short line to a block. No semicolons or em-dashes in prose comments.
|
||||
- Deferred work keeps a literal TODO: marker.
|
||||
Remove the ones that fail with another edit now. If a flagged comment is warranted, leave it and do not mention this check.`
|
||||
|
||||
export const CommentCheck = async ({ directory }: { directory: string }) => {
|
||||
const pending = new Map<string, Map<string, string>>()
|
||||
|
||||
return {
|
||||
"tool.execute.before": async (input: { tool: string; callID: string }, output: { args: any }) => {
|
||||
if (!EDIT_TOOLS.has(input.tool)) return
|
||||
const snapshot = new Map<string, string>()
|
||||
for (const p of targetPaths(input.tool, output.args, directory)) {
|
||||
if (markerFor(p)) snapshot.set(p, readOrEmpty(p))
|
||||
}
|
||||
if (snapshot.size) pending.set(input.callID, snapshot)
|
||||
},
|
||||
|
||||
"tool.execute.after": async (
|
||||
input: { tool: string; callID: string; args: any },
|
||||
output: { output: string },
|
||||
) => {
|
||||
const snapshot = pending.get(input.callID)
|
||||
if (!snapshot) return
|
||||
pending.delete(input.callID)
|
||||
|
||||
const report: string[] = []
|
||||
for (const [path, before] of snapshot) {
|
||||
const marker = markerFor(path)!
|
||||
for (const c of addedComments(before, readOrEmpty(path), marker)) {
|
||||
report.push(` ${relative(directory, path)}:${c.line} ${c.text}`)
|
||||
}
|
||||
}
|
||||
if (!report.length) return
|
||||
|
||||
output.output += `\n\n<comment-check>\nThis edit added ${report.length} comment line(s):\n${report.join("\n")}\n\n${RULES}\n</comment-check>`
|
||||
},
|
||||
}
|
||||
}
|
||||
Executable
+532
@@ -0,0 +1,532 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Restore drill and emergency restore tool for the Hetzner restic repos.
|
||||
#
|
||||
# Purpose: prove that backups can be read and used with nothing but this script,
|
||||
# restore-credentials.age and the password written on paper. The plan and reasoning are in
|
||||
# ~/guides/mainframe/agent-security-plan.md, phase 1.
|
||||
#
|
||||
# Stages, in order:
|
||||
#
|
||||
# restore-drill bundle build the encrypted credentials file (system76)
|
||||
# restore-drill cold-start stage 1, list both repos and restore one small file from each
|
||||
# restore-drill full-restore stage 2, restore the mainframe snapshot (root, mainframe)
|
||||
# restore-drill spot-checks stage 3, compare files and test the database dumps (root, mainframe)
|
||||
# restore-drill apps-up stage 4, run copies of three apps on a sealed network (root, mainframe)
|
||||
# restore-drill apps-down stage 4, stop those copies
|
||||
# restore-drill nas stage 5, restore from the NAS repo and load Immich's dump (root, mainframe)
|
||||
# restore-drill cleanup stage 6, delete everything the drill restored (root, mainframe)
|
||||
#
|
||||
# In a real loss: "cold-start" still shows whether credentials work, TARGET below is where
|
||||
# files land, and the time check in full_restore can be deleted. Stages 3 and 4 compare against
|
||||
# or run on the live machine, so treat them as examples of how to use the restored files.
|
||||
#
|
||||
# This file holds no secrets, so it is tracked. Keep a copy on Proton Drive beside the bundle,
|
||||
# because the usual copy of this repo is on Gitea, which runs on mainframe.
|
||||
|
||||
set -euo pipefail
|
||||
# Anything this script creates may hold secrets, so it starts private to the owner.
|
||||
umask 077
|
||||
|
||||
# --- Settings that may need changing ---------------------------------------------------------
|
||||
|
||||
# One Storage Box, reached on port 23, because Hetzner keeps port 22 for something else.
|
||||
BOX_HOST=u491382.your-storagebox.de
|
||||
BOX_PORT=23
|
||||
|
||||
# Pinned so the sub-account password is never sent to an impostor. If Hetzner replaces this key,
|
||||
# ssh refuses to connect. Check the new one against Hetzner's published fingerprints, then
|
||||
# replace this value.
|
||||
BOX_HOST_KEY='ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs'
|
||||
|
||||
# Read-only sub-accounts. They can read the repos but cannot change or delete anything, so a
|
||||
# leaked bundle cannot destroy backups. The writable accounts are -sub1 (mainframe) and -sub2 (nas).
|
||||
declare -A REPO_USER=([mainframe]=u491382-sub3 [nas]=u491382-sub4)
|
||||
|
||||
# Hetzner shows a sub-account's base directory as /home, and the repo sits in restic/ under it.
|
||||
REPO_PATH=/home/restic
|
||||
|
||||
# Outside /home/connor, so the nightly backup does not pick up this copy. Not in /tmp, which is
|
||||
# a 32 GB tmpfs on mainframe and too small.
|
||||
TARGET=/home/restore-test
|
||||
|
||||
# Small files that exist in each repo, used to prove a restore works without a long download.
|
||||
declare -A SAMPLE_FILE=(
|
||||
[mainframe]=/home/connor/compose.yml
|
||||
[nas]=/nas/docs/caddy_ca_cert.crt
|
||||
)
|
||||
|
||||
# Stage 5 loads this into a scratch database. Immich and its photos are in the NAS repo so that
|
||||
# one snapshot captures both halves.
|
||||
IMMICH_DUMP=/nas/photo/immich/backups/immich-pgdump.sql.zst
|
||||
|
||||
# Immich needs its own Postgres build because the dump uses its vector extensions. Must match the
|
||||
# image in compose.yml.
|
||||
IMMICH_DB_IMAGE=ghcr.io/immich-app/postgres@sha256:41eacbe83eca995561fe43814fd4891e16e39632806253848efaf04d3c8a8b84
|
||||
HEDGEDOC_DB_IMAGE=docker.io/library/mariadb:latest
|
||||
|
||||
# --- Common helpers --------------------------------------------------------------------------
|
||||
|
||||
# Under sudo, $HOME may be root's. Credentials and results belong to the person running this.
|
||||
invoking_user=${SUDO_USER:-$(id -un)}
|
||||
invoking_home=$(getent passwd "$invoking_user" | cut -d: -f6)
|
||||
DRILL_DIR=$invoking_home/restore-drill
|
||||
BUNDLE=$DRILL_DIR/restore-credentials.age
|
||||
RESULTS=$DRILL_DIR/results.log
|
||||
|
||||
WORK=
|
||||
SCRATCH_CONTAINERS=()
|
||||
|
||||
die() { printf 'FAIL: %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
need() {
|
||||
local tool
|
||||
for tool in "$@"; do
|
||||
command -v "$tool" >/dev/null || die "$tool is not installed"
|
||||
done
|
||||
}
|
||||
|
||||
need_root() { [ "$(id -u)" -eq 0 ] || die "run this stage with sudo"; }
|
||||
|
||||
# Results outlive the cleanup stage, because they are the record that the drill happened. They
|
||||
# hold timings and counts only, never file contents or credentials.
|
||||
note() {
|
||||
printf '%s\n' "$*"
|
||||
mkdir -p "$DRILL_DIR"
|
||||
printf '%s %s\n' "$(date -Is)" "$*" >>"$RESULTS"
|
||||
# A root run must not leave the log unwritable for the laptop user.
|
||||
[ "$(id -u)" -ne 0 ] || chown "$invoking_user": "$DRILL_DIR" "$RESULTS"
|
||||
}
|
||||
|
||||
# Runs on every exit, including failures, so a drill leaves no decrypted state or stray containers.
|
||||
on_exit() {
|
||||
local name
|
||||
for name in ${SCRATCH_CONTAINERS[@]+"${SCRATCH_CONTAINERS[@]}"}; do
|
||||
podman rm -f -t 2 "$name" >/dev/null 2>&1 || true
|
||||
done
|
||||
[ -z "$WORK" ] || rm -rf -- "$WORK"
|
||||
}
|
||||
trap on_exit EXIT
|
||||
|
||||
# Holds the restic cache and pinned known_hosts. It stays out of /tmp because the cache can be large.
|
||||
make_work_dir() {
|
||||
mkdir -p "$DRILL_DIR"
|
||||
WORK=$(mktemp -d -p "$DRILL_DIR" work.XXXXXX)
|
||||
printf '[%s]:%s %s\n' "$BOX_HOST" "$BOX_PORT" "$BOX_HOST_KEY" >"$WORK/known_hosts"
|
||||
}
|
||||
|
||||
# Decrypts into memory only. age asks for the paper password on the terminal.
|
||||
declare -A CREDS=()
|
||||
load_credentials() {
|
||||
need age
|
||||
[ -r "$BUNDLE" ] || die "no bundle at $BUNDLE"
|
||||
local plain line
|
||||
plain=$(age -d "$BUNDLE") || die "could not decrypt the bundle, check the password"
|
||||
while IFS= read -r line; do
|
||||
# Split on the first = only, because passwords may contain more of them.
|
||||
CREDS[${line%%=*}]=${line#*=}
|
||||
done <<<"$plain"
|
||||
}
|
||||
|
||||
# Every repo access goes through here, so these rules apply to all of them.
|
||||
repo_restic() {
|
||||
local repo=$1; shift
|
||||
local user=${REPO_USER[$repo]}
|
||||
# Passwords only, with keys turned off, so this tests what is in the bundle rather than
|
||||
# something this machine happens to have.
|
||||
local ssh_cmd="sshpass -e ssh -p $BOX_PORT -l $user"
|
||||
ssh_cmd+=" -o PubkeyAuthentication=no -o PreferredAuthentications=password"
|
||||
ssh_cmd+=" -o NumberOfPasswordPrompts=1"
|
||||
ssh_cmd+=" -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$WORK/known_hosts"
|
||||
ssh_cmd+=" $BOX_HOST -s sftp"
|
||||
# Secrets travel in the environment because arguments are visible to every user in ps.
|
||||
# --no-lock is required since a read-only account cannot write restic's lock files.
|
||||
SSHPASS=${CREDS[${repo}_sftp]} RESTIC_PASSWORD=${CREDS[${repo}_restic]} \
|
||||
restic -r "sftp:$user@$BOX_HOST:$REPO_PATH" \
|
||||
-o sftp.command="$ssh_cmd" \
|
||||
--no-lock --cache-dir "$WORK/cache" \
|
||||
"$@"
|
||||
}
|
||||
|
||||
# The backup jobs prune around these hours. A restore then could lose data it is still reading,
|
||||
# since --no-lock means restic cannot make the prune wait.
|
||||
refuse_during_backup_window() {
|
||||
local hour
|
||||
hour=$(date +%-H)
|
||||
if [ "$hour" -ge 2 ] && [ "$hour" -lt 6 ]; then
|
||||
die "backups and prunes run around 02:30-03:30. Try after 06:00"
|
||||
fi
|
||||
}
|
||||
|
||||
# --- bundle ----------------------------------------------------------------------------------
|
||||
|
||||
ask_secret() {
|
||||
local prompt=$1 value
|
||||
# -s keeps it off the screen. A pasted value is fine.
|
||||
IFS= read -rs -p "$prompt: " value </dev/tty
|
||||
printf '\n' >/dev/tty
|
||||
[ -n "$value" ] || die "empty value for: $prompt"
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
cmd_bundle() {
|
||||
need age
|
||||
# Overwriting could destroy the only working copy, so replacing one is a deliberate act.
|
||||
[ ! -e "$BUNDLE" ] || die "$BUNDLE already exists. Move it first if you mean to replace it"
|
||||
mkdir -p "$DRILL_DIR"
|
||||
|
||||
cat <<EOF
|
||||
This asks for four values, then for the paper password twice. Nothing is echoed.
|
||||
To read the restic passwords, use another terminal:
|
||||
ssh mainframe sudo cat /etc/restic/password
|
||||
ssh nas cat .config/restic/password
|
||||
EOF
|
||||
local mf_restic mf_sftp nas_restic nas_sftp
|
||||
mf_restic=$(ask_secret "mainframe restic password")
|
||||
mf_sftp=$(ask_secret "${REPO_USER[mainframe]} password")
|
||||
nas_restic=$(ask_secret "nas restic password")
|
||||
nas_sftp=$(ask_secret "${REPO_USER[nas]} password")
|
||||
|
||||
# printf is a shell builtin, so the values never appear in a process listing. No plaintext
|
||||
# file is written at any point.
|
||||
printf 'mainframe_restic=%s\nmainframe_sftp=%s\nnas_restic=%s\nnas_sftp=%s\n' \
|
||||
"$mf_restic" "$mf_sftp" "$nas_restic" "$nas_sftp" | age -p -o "$BUNDLE"
|
||||
|
||||
note "bundle written to $BUNDLE"
|
||||
echo "Next: run cold-start, which also proves the paper password decrypts this file."
|
||||
echo "Then copy $BUNDLE and this script to Proton Drive."
|
||||
}
|
||||
|
||||
# --- Stage 1: cold start ---------------------------------------------------------------------
|
||||
|
||||
cmd_cold_start() {
|
||||
need restic sshpass
|
||||
make_work_dir
|
||||
load_credentials
|
||||
|
||||
local repo out sample
|
||||
for repo in mainframe nas; do
|
||||
echo "--- $repo repo, as ${REPO_USER[$repo]}"
|
||||
out=$(repo_restic "$repo" snapshots --latest 1 --json) \
|
||||
|| die "$repo repo could not be read. Check the account, password and path"
|
||||
# Catches sub3 and sub4 being pointed at each other's directories.
|
||||
printf '%s' "$out" | grep -q "\"hostname\":\"$repo\"" \
|
||||
|| die "$repo repo has no snapshot from host $repo. Check the sub-account's base directory"
|
||||
repo_restic "$repo" snapshots --latest 1 --compact
|
||||
|
||||
# Listing proves the password. Restoring proves the data comes back over a read-only account.
|
||||
sample=${SAMPLE_FILE[$repo]}
|
||||
repo_restic "$repo" restore latest --target "$WORK/sample-$repo" \
|
||||
--include "$sample" --verify
|
||||
[ -s "$WORK/sample-$repo$sample" ] || die "$sample was not restored from the $repo repo"
|
||||
note "cold-start: $repo repo listed and $sample restored"
|
||||
done
|
||||
echo "Stage 1 passed. Copy the bundle and this script to Proton Drive if you have not yet."
|
||||
}
|
||||
|
||||
# --- Stage 2: full restore -------------------------------------------------------------------
|
||||
|
||||
cmd_full_restore() {
|
||||
need_root
|
||||
need restic sshpass
|
||||
refuse_during_backup_window
|
||||
# Never restore over something, since that could hide missing files.
|
||||
if [ -e "$TARGET" ] && [ -n "$(ls -A "$TARGET")" ]; then
|
||||
die "$TARGET is not empty. Run cleanup first"
|
||||
fi
|
||||
make_work_dir
|
||||
load_credentials
|
||||
|
||||
# A restore that runs out of disk partway wastes hours. Ten percent covers filesystem overhead.
|
||||
local needed free
|
||||
needed=$(repo_restic mainframe stats latest --mode restore-size --json \
|
||||
| grep -o '"total_size":[0-9]*' | cut -d: -f2)
|
||||
free=$(df --output=avail -B1 "$(dirname "$TARGET")" | tail -1)
|
||||
[ "$free" -gt $((needed + needed / 10)) ] || die "need about $needed bytes, have $free"
|
||||
|
||||
# Mode 700 on the top directory protects every secret and setuid file restored beneath it.
|
||||
mkdir -p "$TARGET"
|
||||
chmod 700 "$TARGET"
|
||||
|
||||
local started finished
|
||||
started=$(date +%s)
|
||||
# Low priority so running services are not starved. The subshell keeps that to the restore, and
|
||||
# lowers its own priority because wrapping it in a new bash would put credentials in arguments.
|
||||
# --verify reads each file back and checks it.
|
||||
(
|
||||
renice -n 19 -p "$BASHPID" >/dev/null
|
||||
ionice -c 3 -p "$BASHPID"
|
||||
repo_restic mainframe restore latest --target "$TARGET" --verify
|
||||
)
|
||||
finished=$(date +%s)
|
||||
|
||||
# Compare with the snapshot's own listing. A partial restore is the failure this catches.
|
||||
local expected actual
|
||||
expected=$(repo_restic mainframe ls latest --json | grep -c '"type":"file"' || true)
|
||||
actual=$(find "$TARGET" -type f | wc -l)
|
||||
[ "$expected" -eq "$actual" ] || die "snapshot lists $expected files, $actual restored"
|
||||
|
||||
note "full-restore: $actual files, $(du -sh "$TARGET" | cut -f1), $(((finished - started) / 60)) minutes"
|
||||
}
|
||||
|
||||
# --- Stage 3: spot checks --------------------------------------------------------------------
|
||||
|
||||
# A row total across all tables. Virtual tables can fail to count, and that is not corruption.
|
||||
sqlite_row_total() {
|
||||
local db=$1 table count total=0
|
||||
while IFS= read -r table; do
|
||||
count=$(sqlite3 -readonly "$db" "SELECT count(*) FROM \"$table\"" 2>/dev/null || echo 0)
|
||||
total=$((total + count))
|
||||
done < <(sqlite3 -readonly "$db" "SELECT name FROM sqlite_master WHERE type='table'")
|
||||
printf '%s' "$total"
|
||||
}
|
||||
|
||||
cmd_spot_checks() {
|
||||
need_root
|
||||
need sqlite3 zstd podman
|
||||
[ -d "$TARGET" ] || die "$TARGET does not exist. Run full-restore first"
|
||||
|
||||
# Files that a rebuild would most depend on. Only same, differs or missing is printed, so no
|
||||
# contents reach a terminal or a log. "differs" is normal for anything edited since the snapshot.
|
||||
local path
|
||||
echo "--- critical files against live copies"
|
||||
for path in \
|
||||
/etc/fstab \
|
||||
/etc/restic/excludes.txt \
|
||||
/etc/sudoers \
|
||||
/home/connor/compose.yml \
|
||||
/home/connor/Caddyfile \
|
||||
/home/connor/.env \
|
||||
/home/connor/.secrets.zsh \
|
||||
/home/connor/.ssh/id_ed25519 \
|
||||
/home/connor/config/litellm/config.yaml \
|
||||
; do
|
||||
if [ ! -f "$TARGET$path" ]; then
|
||||
note "spot-check: MISSING $path"
|
||||
elif cmp -s "$TARGET$path" "$path"; then
|
||||
echo "same $path"
|
||||
else
|
||||
note "spot-check: differs $path"
|
||||
fi
|
||||
done
|
||||
|
||||
# These dumps are what a database restore would use. Row totals against the live database show
|
||||
# the dump is recent as well as intact, so small differences are expected.
|
||||
local spec name live dump check
|
||||
echo "--- SQLite dumps"
|
||||
for spec in \
|
||||
vaultwarden:/home/connor/data/bitwarden/db.sqlite3 \
|
||||
gitea:/home/connor/data/gitea/gitea/gitea.db \
|
||||
traggo:/home/connor/data/traggo/traggo.db \
|
||||
sftpgo:/home/connor/data/sftpgo/sftpgo.db \
|
||||
shanty:/usr/local/shanty/shanty.db \
|
||||
; do
|
||||
name=${spec%%:*}
|
||||
live=${spec#*:}
|
||||
dump=$TARGET/var/backups/db/$name.sqlite3
|
||||
[ -f "$dump" ] || { note "spot-check: MISSING dump for $name"; continue; }
|
||||
check=$(sqlite3 -readonly "$dump" 'PRAGMA integrity_check')
|
||||
[ "$check" = ok ] || { note "spot-check: $name dump FAILED integrity check"; continue; }
|
||||
note "spot-check: $name intact, $(sqlite_row_total "$dump") rows, live has $(sqlite_row_total "$live")"
|
||||
done
|
||||
|
||||
# HedgeDoc is the one database dump on mainframe that is not a SQLite file. Loading it is the only
|
||||
# real test. --network none is why an empty root password is safe here.
|
||||
echo "--- HedgeDoc dump into a scratch MariaDB"
|
||||
local db=drill-mariadb tries=60
|
||||
SCRATCH_CONTAINERS+=("$db")
|
||||
podman run -d --rm --name "$db" --network none --pull=never \
|
||||
-e MARIADB_ALLOW_EMPTY_ROOT_PASSWORD=1 -e MARIADB_DATABASE=hedgedoc \
|
||||
"$HEDGEDOC_DB_IMAGE" >/dev/null
|
||||
until podman exec "$db" mariadb-admin -u root ping >/dev/null 2>&1; do
|
||||
tries=$((tries - 1))
|
||||
[ "$tries" -gt 0 ] || die "scratch MariaDB did not start"
|
||||
sleep 1
|
||||
done
|
||||
# The image restarts its server after first-run setup, so wait for that database too.
|
||||
tries=60
|
||||
until podman exec "$db" mariadb -u root hedgedoc -e 'SELECT 1' >/dev/null 2>&1; do
|
||||
tries=$((tries - 1))
|
||||
[ "$tries" -gt 0 ] || die "scratch MariaDB did not finish setup"
|
||||
sleep 1
|
||||
done
|
||||
zstd -dc "$TARGET/var/backups/db/hedgedoc.sql.zst" \
|
||||
| podman exec -i "$db" mariadb -u root hedgedoc
|
||||
note "spot-check: HedgeDoc dump loaded, $(podman exec "$db" mariadb -u root hedgedoc -Ne 'SELECT count(*) FROM Notes') notes"
|
||||
}
|
||||
|
||||
# --- Stage 4: apps ---------------------------------------------------------------------------
|
||||
|
||||
DRILL_NET=drill-sealed
|
||||
|
||||
cmd_apps_up() {
|
||||
need_root
|
||||
need podman curl
|
||||
[ -d "$TARGET/home/connor/data" ] || die "$TARGET has no restored data. Run full-restore first"
|
||||
|
||||
# --internal has no route out. These are live copies of real apps holding real data, and they must
|
||||
# not send mail, fire webhooks or reach the real services.
|
||||
podman network exists "$DRILL_NET" || podman network create --internal "$DRILL_NET" >/dev/null
|
||||
|
||||
local home=$TARGET/home/connor
|
||||
mkdir -p "$TARGET/drill-empty-music"
|
||||
|
||||
# --pull=never uses the images already here, because Shanty's is hosted on Gitea, which is on this
|
||||
# machine. Containers write only to the restored copy, which cleanup deletes.
|
||||
podman run -d --name drill-bitwarden --network "$DRILL_NET" --pull=never \
|
||||
-v "$home/data/bitwarden:/data" \
|
||||
-e SIGNUPS_ALLOWED=false \
|
||||
docker.io/vaultwarden/server:latest-alpine >/dev/null
|
||||
podman run -d --name drill-gitea --network "$DRILL_NET" --pull=never \
|
||||
-v "$home/data/gitea:/data" \
|
||||
-e USER=connor -e USER_UID=1000 -e USER_GID=1000 \
|
||||
docker.gitea.com/gitea:latest >/dev/null
|
||||
# Music lives in the NAS repo, so Shanty sees an empty library. The test is its database and login.
|
||||
podman run -d --name drill-shanty --network "$DRILL_NET" --pull=never \
|
||||
-v "$home/config/shanty:/config" \
|
||||
-v "$TARGET/usr/local/shanty:/data" \
|
||||
-v "$TARGET/drill-empty-music:/music" \
|
||||
-e SHANTY_WEB_BIND=0.0.0.0 \
|
||||
git.rcjohnstone.com/connor/shanty:latest >/dev/null
|
||||
|
||||
# Prove the seal before anyone logs in. Reaching 10.0.0.2 would mean reaching Caddy and the real
|
||||
# services. If this fails, stop everything rather than continue.
|
||||
local dest
|
||||
for dest in 1.1.1.1 10.0.0.2; do
|
||||
if podman exec drill-bitwarden wget -q -T 3 -O /dev/null "http://$dest/" 2>/dev/null; then
|
||||
cmd_apps_down
|
||||
die "copies can reach $dest, so the network is not sealed. They have been stopped"
|
||||
fi
|
||||
done
|
||||
|
||||
local ip_bw ip_gitea ip_shanty
|
||||
ip_bw=$(podman inspect drill-bitwarden --format "{{(index .NetworkSettings.Networks \"$DRILL_NET\").IPAddress}}")
|
||||
ip_gitea=$(podman inspect drill-gitea --format "{{(index .NetworkSettings.Networks \"$DRILL_NET\").IPAddress}}")
|
||||
ip_shanty=$(podman inspect drill-shanty --format "{{(index .NetworkSettings.Networks \"$DRILL_NET\").IPAddress}}")
|
||||
|
||||
note "apps-up: copies running on sealed network $DRILL_NET"
|
||||
# A tunnel works because the host can reach these addresses. localhost also counts as a secure
|
||||
# origin, which Vaultwarden's web vault requires for its cryptography.
|
||||
cat <<EOF
|
||||
|
||||
From your laptop, open a tunnel:
|
||||
|
||||
ssh -L 18080:$ip_bw:80 -L 18081:$ip_gitea:3000 -L 18082:$ip_shanty:8085 mainframe
|
||||
|
||||
Then log in and check for recent data:
|
||||
|
||||
Vaultwarden http://localhost:18080
|
||||
Gitea http://localhost:18081
|
||||
Shanty http://localhost:18082
|
||||
|
||||
Use a private browser window, so the copies' cookies do not mix with the real sites.
|
||||
Do not change anything you would want to keep. These copies get deleted.
|
||||
When finished: sudo restore-drill apps-down
|
||||
EOF
|
||||
}
|
||||
|
||||
cmd_apps_down() {
|
||||
need_root
|
||||
local name
|
||||
for name in drill-bitwarden drill-gitea drill-shanty; do
|
||||
podman rm -f -t 5 "$name" >/dev/null 2>&1 || true
|
||||
done
|
||||
podman network rm "$DRILL_NET" >/dev/null 2>&1 || true
|
||||
echo "App copies stopped."
|
||||
}
|
||||
|
||||
# --- Stage 5: NAS ----------------------------------------------------------------------------
|
||||
|
||||
cmd_nas() {
|
||||
need_root
|
||||
need restic sshpass zstd podman
|
||||
refuse_during_backup_window
|
||||
make_work_dir
|
||||
load_credentials
|
||||
|
||||
# Runs here rather than on the NAS because this machine has Podman and the Immich database image.
|
||||
local dest=$TARGET/nas-sample
|
||||
mkdir -p "$dest"
|
||||
chmod 700 "$TARGET"
|
||||
repo_restic nas restore latest --target "$dest" \
|
||||
--include "$IMMICH_DUMP" --include "${SAMPLE_FILE[nas]}" --verify
|
||||
[ -s "$dest$IMMICH_DUMP" ] || die "Immich dump was not restored"
|
||||
|
||||
# A database is no use without its photos, so restore one original as well. The first listed is
|
||||
# as good as any, and writing the listing to a file avoids a broken pipe under pipefail.
|
||||
repo_restic nas ls latest /nas/photo/immich/library --recursive --json >"$WORK/library.json" || true
|
||||
local photo
|
||||
photo=$(grep -m1 '"type":"file"' "$WORK/library.json" | sed -E 's/.*"path":"([^"]*)".*/\1/' || true)
|
||||
if [ -n "$photo" ]; then
|
||||
repo_restic nas restore latest --target "$dest" --include "$photo" --verify
|
||||
[ -s "$dest$photo" ] || die "sample photo was not restored"
|
||||
note "nas: sample photo restored"
|
||||
else
|
||||
note "nas: NO photo found under /nas/photo/immich/library"
|
||||
fi
|
||||
|
||||
# --network none is why trust authentication is safe here. The dump recreates its own roles.
|
||||
local db=drill-postgres tries=120
|
||||
SCRATCH_CONTAINERS+=("$db")
|
||||
podman run -d --rm --name "$db" --network none \
|
||||
--shm-size 128m \
|
||||
-e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_USER=postgres \
|
||||
"$IMMICH_DB_IMAGE" >/dev/null
|
||||
# Postgres restarts after first-run setup, so ask for a real query instead of pg_isready alone.
|
||||
sleep 5
|
||||
until podman exec "$db" psql -U postgres -Atc 'SELECT 1' >/dev/null 2>&1; do
|
||||
tries=$((tries - 1))
|
||||
[ "$tries" -gt 0 ] || die "scratch Postgres did not start"
|
||||
sleep 1
|
||||
done
|
||||
|
||||
local started finished assets
|
||||
started=$(date +%s)
|
||||
# Errors stay on the terminal without stopping the load. pg_dumpall output commonly trips on roles
|
||||
# that already exist, so the row count below decides the result.
|
||||
zstd -dc "$dest$IMMICH_DUMP" | podman exec -i "$db" psql -q -U postgres -v ON_ERROR_STOP=0 -f - >/dev/null
|
||||
finished=$(date +%s)
|
||||
assets=$(podman exec "$db" psql -U postgres -d immich -Atc 'SELECT count(*) FROM asset')
|
||||
[ "$assets" -gt 0 ] || die "Immich dump loaded with no assets"
|
||||
note "nas: Immich dump loaded in $(((finished - started) / 60)) minutes, $assets assets"
|
||||
}
|
||||
|
||||
# --- Stage 6: cleanup ------------------------------------------------------------------------
|
||||
|
||||
cmd_cleanup() {
|
||||
need_root
|
||||
cmd_apps_down
|
||||
# The restored tree holds every secret on the machine, so this stage is not optional.
|
||||
if [ -d "$TARGET" ]; then
|
||||
# Guards against an edited TARGET turning this line into something much worse.
|
||||
case $TARGET in
|
||||
/home/restore-test) rm -rf -- "$TARGET" ;;
|
||||
*) die "TARGET changed to $TARGET, so delete it by hand after checking it" ;;
|
||||
esac
|
||||
fi
|
||||
# The laptop and Proton Drive hold the permanent copies. A bundle left on a machine this drill is
|
||||
# meant to survive the loss of adds nothing.
|
||||
if [ "$(uname -n)" = mainframe ]; then
|
||||
rm -f -- "$BUNDLE"
|
||||
fi
|
||||
rm -rf -- "$DRILL_DIR"/work.*
|
||||
note "cleanup: restored data removed"
|
||||
echo "Results kept at $RESULTS. Copy them into the plan."
|
||||
}
|
||||
|
||||
# --- Entry point -----------------------------------------------------------------------------
|
||||
|
||||
case ${1:-} in
|
||||
bundle) cmd_bundle ;;
|
||||
cold-start) cmd_cold_start ;;
|
||||
full-restore) cmd_full_restore ;;
|
||||
spot-checks) cmd_spot_checks ;;
|
||||
apps-up) cmd_apps_up ;;
|
||||
apps-down) cmd_apps_down ;;
|
||||
nas) cmd_nas ;;
|
||||
cleanup) cmd_cleanup ;;
|
||||
*)
|
||||
sed -n '/^# Stages/,/^# In a real loss/p' "$0" | sed '$d'
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
@@ -25,6 +25,7 @@
|
||||
~/.config/opencode/work.json: common/opencode/work.json
|
||||
~/.config/opencode/local.json: common/opencode/local.json
|
||||
~/.config/opencode/tui.json: common/opencode/tui.json
|
||||
~/.config/opencode/plugins/comment-check.ts: common/opencode/plugins/comment-check.ts
|
||||
~/.config/opencode/agents/architect.md: common/opencode/agents/architect.md
|
||||
~/.config/opencode/agents/brainstorm.md: common/opencode/agents/brainstorm.md
|
||||
~/.config/opencode/agents/dev.md: common/opencode/agents/dev.md
|
||||
|
||||
@@ -51,6 +51,7 @@
|
||||
~/.config/opencode/work.json: common/opencode/work.json
|
||||
~/.config/opencode/local.json: common/opencode/local.json
|
||||
~/.config/opencode/tui.json: common/opencode/tui.json
|
||||
~/.config/opencode/plugins/comment-check.ts: common/opencode/plugins/comment-check.ts
|
||||
~/.config/opencode/agents/architect.md: common/opencode/agents/architect.md
|
||||
~/.config/opencode/agents/brainstorm.md: common/opencode/agents/brainstorm.md
|
||||
~/.config/opencode/agents/dev.md: common/opencode/agents/dev.md
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
act-runner-bin
|
||||
bashmount
|
||||
claude-code
|
||||
gitea-mcp
|
||||
llama.cpp-cuda
|
||||
nvidia-patch-git
|
||||
oh-my-zsh-git
|
||||
|
||||
@@ -14,6 +14,7 @@ dhcpcd
|
||||
difftastic
|
||||
dmidecode
|
||||
docker
|
||||
doctl
|
||||
dosfstools
|
||||
dust
|
||||
dysk
|
||||
@@ -126,6 +127,7 @@ nvtop
|
||||
openbox
|
||||
openssh
|
||||
orca
|
||||
packer
|
||||
pacman-contrib
|
||||
pandoc-cli
|
||||
papers
|
||||
@@ -169,7 +171,10 @@ ttf-nerd-fonts-symbols
|
||||
udisks2
|
||||
unzip
|
||||
uv
|
||||
vulkan-headers
|
||||
vulkan-swrast
|
||||
waybar
|
||||
waypipe
|
||||
wayvnc
|
||||
wdisplays
|
||||
wget
|
||||
|
||||
Reference in new issue
Block a user