From fecda07e5cf699c404c5a14b42deb990374ffbb0 Mon Sep 17 00:00:00 2001
From: Connor Johnstone <c@rcjohnstone.com>
Date: Thu, 17 Sep 2026 12:34:59 -0400
Subject: [PATCH] backup-db-dump: force the zstd writes, follow sftpgo's db

Two independent reasons the nightly backup was not producing a snapshot.

zstd refuses to overwrite its -o target. A dump that fails midway therefore
wedges the job permanently rather than just losing one night: pg_dumpall dies
(the immich db is not up yet right after a boot), zstd still writes its empty
frame, pipefail aborts before the mv, and the orphaned .tmp then makes every
later run fail instantly on "already exists; not overwritten". Three nights
were lost that way from 2026-09-15. The sqlite branch already rm -f'd its tmp;
these two did not. -f closes it.

sftpgo moved its state onto the /var/lib/sftpgo bind mount, so the old
/home/connor/docs/sftpgo.db only tripped the "db missing, skipping" WARN --
sftpgo went unbacked-up from 2026-09-09 while the run still reported success.
Left as a WARN deliberately: a decommissioned service should not break the
nightly run. Read the WARNs.
---
 hosts/mainframe/sbin/backup-db-dump | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/hosts/mainframe/sbin/backup-db-dump b/hosts/mainframe/sbin/backup-db-dump
index 14f6b29..d472b79 100755
--- a/hosts/mainframe/sbin/backup-db-dump
+++ b/hosts/mainframe/sbin/backup-db-dump
@@ -20,8 +20,24 @@
 #    the container -- so the file is left exactly as compose expects and read
 #    with a plain parser instead.
 #
+#  * The sqlite paths are all under /home/connor/data/<svc>/. sftpgo's was
+#    /home/connor/docs/sftpgo.db until the service moved its state onto the
+#    /var/lib/sftpgo bind mount; the old path then just tripped the "db
+#    missing, skipping" WARN, so sftpgo silently went unbacked-up from
+#    2026-09-09 while the job still reported success. A skip is a WARN by
+#    design -- check the WARNs, they do not fail the run.
+#
 #  * Every file is written to .tmp and renamed, so a concurrent backup never
 #    snapshots a truncated dump.
+#
+#  * The zstd calls pass -f. Without it a single failed run wedges the job
+#    PERMANENTLY: pg_dumpall fails (e.g. the db is not up yet right after a
+#    boot), zstd still writes its empty frame, pipefail aborts before the mv,
+#    and the orphaned .tmp then makes every later run die at once on
+#    "zstd: ... already exists; not overwritten". That is what happened from
+#    2026-09-15 to 2026-09-17 -- three nights with no backup, and the ntfy
+#    failure text named the dump, not the stale file. The sqlite branch below
+#    already rm -f'd its .tmp; these two did not.
 
 set -euo pipefail
 umask 077
@@ -47,14 +63,14 @@ else
 fi
 log "immich postgres -> $tgt"
 podman exec connor_immich_db_1 sh -c "pg_dumpall -U '$IMMICH_DB_USERNAME'" \
-    | zstd -q -o "$tgt/immich-pgdump.sql.zst.tmp"
+    | zstd -q -f -o "$tgt/immich-pgdump.sql.zst.tmp"
 mv -f "$tgt/immich-pgdump.sql.zst.tmp" "$tgt/immich-pgdump.sql.zst"
 
 # --- MariaDB (HedgeDoc) ----------------------------------------------------
 log "hedgedoc mariadb"
 podman exec connor_hedgedocdb_1 sh -c \
     "mariadb-dump --single-transaction -u root -p'$HEDGEDOC_DB_ROOT_PASSWORD' hedgedoc" \
-    | zstd -q -o "$DEST/hedgedoc.sql.zst.tmp"
+    | zstd -q -f -o "$DEST/hedgedoc.sql.zst.tmp"
 mv -f "$DEST/hedgedoc.sql.zst.tmp" "$DEST/hedgedoc.sql.zst"
 
 # --- SQLite ----------------------------------------------------------------
@@ -62,7 +78,7 @@ for spec in \
     "vaultwarden:/home/connor/data/bitwarden/db.sqlite3" \
     "gitea:/home/connor/data/gitea/gitea/gitea.db" \
     "traggo:/home/connor/data/traggo/traggo.db" \
-    "sftpgo:/home/connor/docs/sftpgo.db" \
+    "sftpgo:/home/connor/data/sftpgo/sftpgo.db" \
     "shanty:/usr/local/shanty/shanty.db" \
 ; do
     name=${spec%%:*}; path=${spec#*:}