diff --git a/hosts/mainframe/sbin/backup-db-dump b/hosts/mainframe/sbin/backup-db-dump index 14f6b29..d472b79 100755 --- a/hosts/mainframe/sbin/backup-db-dump +++ b/hosts/mainframe/sbin/backup-db-dump @@ -20,8 +20,24 @@ # the container -- so the file is left exactly as compose expects and read # with a plain parser instead. # +# * The sqlite paths are all under /home/connor/data//. sftpgo's was +# /home/connor/docs/sftpgo.db until the service moved its state onto the +# /var/lib/sftpgo bind mount; the old path then just tripped the "db +# missing, skipping" WARN, so sftpgo silently went unbacked-up from +# 2026-09-09 while the job still reported success. A skip is a WARN by +# design -- check the WARNs, they do not fail the run. +# # * Every file is written to .tmp and renamed, so a concurrent backup never # snapshots a truncated dump. +# +# * The zstd calls pass -f. Without it a single failed run wedges the job +# PERMANENTLY: pg_dumpall fails (e.g. the db is not up yet right after a +# boot), zstd still writes its empty frame, pipefail aborts before the mv, +# and the orphaned .tmp then makes every later run die at once on +# "zstd: ... already exists; not overwritten". That is what happened from +# 2026-09-15 to 2026-09-17 -- three nights with no backup, and the ntfy +# failure text named the dump, not the stale file. The sqlite branch below +# already rm -f'd its .tmp; these two did not. set -euo pipefail umask 077 @@ -47,14 +63,14 @@ else fi log "immich postgres -> $tgt" podman exec connor_immich_db_1 sh -c "pg_dumpall -U '$IMMICH_DB_USERNAME'" \ - | zstd -q -o "$tgt/immich-pgdump.sql.zst.tmp" + | zstd -q -f -o "$tgt/immich-pgdump.sql.zst.tmp" mv -f "$tgt/immich-pgdump.sql.zst.tmp" "$tgt/immich-pgdump.sql.zst" # --- MariaDB (HedgeDoc) ---------------------------------------------------- log "hedgedoc mariadb" podman exec connor_hedgedocdb_1 sh -c \ "mariadb-dump --single-transaction -u root -p'$HEDGEDOC_DB_ROOT_PASSWORD' hedgedoc" \ - | zstd -q -o "$DEST/hedgedoc.sql.zst.tmp" + | zstd -q -f -o "$DEST/hedgedoc.sql.zst.tmp" mv -f "$DEST/hedgedoc.sql.zst.tmp" "$DEST/hedgedoc.sql.zst" # --- SQLite ---------------------------------------------------------------- @@ -62,7 +78,7 @@ for spec in \ "vaultwarden:/home/connor/data/bitwarden/db.sqlite3" \ "gitea:/home/connor/data/gitea/gitea/gitea.db" \ "traggo:/home/connor/data/traggo/traggo.db" \ - "sftpgo:/home/connor/docs/sftpgo.db" \ + "sftpgo:/home/connor/data/sftpgo/sftpgo.db" \ "shanty:/usr/local/shanty/shanty.db" \ ; do name=${spec%%:*}; path=${spec#*:}