#!/usr/bin/env bash # OnFailure= handler for the restic units. Invoked by systemd with the failed # unit's name as $1. # # Why this exists: restic-backup notifies from inside the script, but # restic-check calls /usr/bin/restic directly and restic-seed had no notify at # all. Both could therefore fail completely silently -- and on 2026-08-21 the # seed did exactly that, dying on a dropped SFTP connection and leaving a stale # lock that then failed the Sunday check, with nothing said for three days. # "Silence is the alarm" only works if every unit can actually raise one. set -uo pipefail UNIT=${1:-unknown.service} NTFY_URL=https://ntfy.rcjohnstone.com/backup for f in /etc/ntfy/publish.env /home/connor/.config/ntfy/publish.env; do [ -r "$f" ] && { NTFY_ENV=$f; break; } done [ -n "${NTFY_ENV:-}" ] || exit 0 # PARSED, not sourced -- the bot password contains ` and &, so sourcing dies # with a syntax error. Same reason restic-backup's notify() uses sed. u=$(sed -n 's/^NTFY_USER=//p' "$NTFY_ENV" | head -1) p=$(sed -n 's/^NTFY_PASS=//p' "$NTFY_ENV" | head -1) [ -n "$u" ] && [ -n "$p" ] || exit 0 HOST=$(uname -n); HOST=${HOST%%.*} RESULT=$(systemctl show -p Result --value "$UNIT" 2>/dev/null) BODY=$(printf '%s failed on %s (Result=%s)\n\n%s\n' \ "$UNIT" "$HOST" "$RESULT" \ "$(journalctl -u "$UNIT" -n 20 --no-pager -o cat 2>/dev/null | tail -c 1200)") curl -fsS --max-time 20 -u "$u:$p" \ -H "Title: $HOST: $UNIT FAILED" -H "Priority: urgent" -H "Tags: rotating_light" \ -d "$BODY" "$NTFY_URL" >/dev/null || true