Rebuild dotfiles as one branch with per-host layers
Replaces the previous repo, which had split into two histories that never met (mainframe on a dead GitLab remote, the laptops on Gitea) with 146 dirty files across three machines and the NAS never enrolled at all. Branch-per-machine is gone. One main, with host differences expressed as small files under hosts/<hostname>/ rather than as branches, so there is nothing to merge. The reconciled zsh layer reduces 15-33 line forks to 1-7 effective lines per host; distro differences (oh-my-zsh prefix, syntax-highlighting path, fd vs fdfind) are probed in common/ instead. Fresh history: the old one carried six plaintext credentials, 45 MB of mail caches, browser caches and vendored binaries. 5,096 tracked files and 144 MB become 462 files and 2.6 MB. The .gitignore is now an allowlist, which is what keeps that true. Root cause of the rot: ~/.local/bin was a symlink to scripts/ with GOPATH inside it, so every go install wrote into version control (2.2 GB on the work laptop). PATH now points at the repo instead of the reverse. Also: Hyprland replaces sway and is sourced in two halves so $browser is defined before use; singleton automations carry ConditionHost= alongside host-layer-only placement; ddns moves from cron to a guarded timer; package manifests and pkg-snapshot/pkg-restore replace the X11-era install_scripts/; networkmanager-dmenu added to system76 (the binding always existed, the package never did).
This commit is contained in:
Executable
+443
@@ -0,0 +1,443 @@
|
||||
#!/usr/bin/python3
|
||||
"""Archive stale unread mail out of the Proton inbox. Never deletes.
|
||||
|
||||
The inbox is meant to hold new mail plus read mail kept on purpose. Unread mail
|
||||
accumulates faster than it gets triaged, so anything still unread after a few
|
||||
days is moved to Archive.
|
||||
|
||||
Deliberately has no LLM in it. Of ~1200 inbox messages sampled, 81% carry a
|
||||
machine-readable bulk marker, and the ~19% residual is transactional rather
|
||||
than personal. Since transactional mail is archived on the same rule, every
|
||||
decision reduces to a flag test, a set membership test or a string match --
|
||||
deterministic, explainable in the log, and not improved by a model.
|
||||
|
||||
Safety, in order of importance:
|
||||
* nothing is ever deleted; the only operation is IMAP MOVE to Archive
|
||||
* BODY.PEEK everywhere, so nothing is ever marked read as a side effect
|
||||
* INBOX is the only mailbox ever opened read-write
|
||||
* every archived message is recorded with its Message-ID and the rule that
|
||||
fired, and any run can be undone
|
||||
* a message with no Message-ID is left alone, because it could not be undone
|
||||
* two interlocks abort the run if the mailbox does not look like itself
|
||||
"""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import email
|
||||
import email.policy
|
||||
import email.utils
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import tomllib
|
||||
|
||||
# Shared with rent_utilities. sys.path rather than PYTHONPATH so this works
|
||||
# when run by hand as well as from the unit.
|
||||
sys.path.insert(0, os.path.expanduser("~/.local/lib/pymail"))
|
||||
from protonimap import ( # noqa: E402
|
||||
Fatal, imap_connect, imap_date, notify_failure, push, _quote)
|
||||
|
||||
CONFIG = os.path.expanduser("~/.config/inbox-tidy/tidy.toml")
|
||||
LOG = os.path.expanduser("~/.local/share/inbox-tidy/archived.jsonl")
|
||||
|
||||
HEADERS = ("FROM SUBJECT DATE MESSAGE-ID LIST-UNSUBSCRIBE LIST-ID PRECEDENCE "
|
||||
"AUTO-SUBMITTED FEEDBACK-ID X-CAMPAIGNID")
|
||||
BULK_HEADERS = ("List-Unsubscribe", "List-Id", "Feedback-ID", "X-CampaignID")
|
||||
|
||||
# Skip reasons, in evaluation order. Also the keys in the run summary.
|
||||
KEPT_FLAGGED = "flagged-or-answered"
|
||||
KEPT_SENDER = "protected-sender"
|
||||
KEPT_CORRESPONDENT = "known-correspondent"
|
||||
KEPT_SUBJECT = "protected-subject"
|
||||
KEPT_NO_MSGID = "no-message-id"
|
||||
ARCHIVE = "archive"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# helpers
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def load_config(path=None):
|
||||
path = path or CONFIG
|
||||
try:
|
||||
with open(path, "rb") as fh:
|
||||
return tomllib.load(fh)
|
||||
except (OSError, tomllib.TOMLDecodeError) as exc:
|
||||
raise Fatal("Cannot read %s: %s" % (path, exc))
|
||||
|
||||
|
||||
def addr_of(header_value):
|
||||
"""Bare lowercase address out of a From/To header, or ''."""
|
||||
if not header_value:
|
||||
return ""
|
||||
pairs = email.utils.getaddresses([str(header_value)])
|
||||
return pairs[0][1].lower().strip() if pairs and pairs[0][1] else ""
|
||||
|
||||
|
||||
def domain_of(addr):
|
||||
return addr.rsplit("@", 1)[-1] if "@" in addr else ""
|
||||
|
||||
|
||||
def sender_protected(addr, entries):
|
||||
"""An entry containing @ matches the address; otherwise the domain.
|
||||
|
||||
Domain entries match subdomains too, so `chase.com` also covers
|
||||
`fraudalert.chase.com` -- which is the case that actually matters.
|
||||
"""
|
||||
dom = domain_of(addr)
|
||||
for e in entries:
|
||||
e = e.lower().strip()
|
||||
if not e:
|
||||
continue
|
||||
if "@" in e:
|
||||
if addr == e:
|
||||
return True
|
||||
elif dom == e or dom.endswith("." + e):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def fetch_headers(conn, ids, extra="FLAGS"):
|
||||
"""[(uid, flags, email.Message)] for a list of sequence ids, batched.
|
||||
|
||||
BODY.PEEK, never BODY: fetching a body with BODY would set \\Seen on mail
|
||||
this job exists to leave unread.
|
||||
"""
|
||||
out = []
|
||||
for i in range(0, len(ids), 200):
|
||||
chunk = b",".join(ids[i:i + 200]).decode()
|
||||
typ, data = conn.fetch(
|
||||
chunk, "(UID %s BODY.PEEK[HEADER.FIELDS (%s)])" % (extra, HEADERS))
|
||||
if typ != "OK":
|
||||
continue
|
||||
for item in data:
|
||||
if not isinstance(item, tuple):
|
||||
continue
|
||||
meta = item[0].decode(errors="replace")
|
||||
m_uid = re.search(r"UID (\d+)", meta)
|
||||
m_fl = re.search(r"FLAGS \(([^)]*)\)", meta)
|
||||
if not m_uid:
|
||||
continue
|
||||
msg = email.message_from_bytes(item[1], policy=email.policy.default)
|
||||
out.append((m_uid.group(1), (m_fl.group(1) if m_fl else ""), msg))
|
||||
return out
|
||||
|
||||
|
||||
def correspondents(conn, cfg):
|
||||
"""Addresses this account has actually written to, from the Sent folder.
|
||||
|
||||
A behavioural allowlist: someone you have emailed is someone whose mail
|
||||
should not be swept up. Cheap to derive and needs no maintenance.
|
||||
"""
|
||||
box = cfg["imap"].get("sent_mailbox", "Sent")
|
||||
typ, _ = conn.select(_quote(box), readonly=True)
|
||||
if typ != "OK":
|
||||
raise Fatal("Cannot open the Sent mailbox %r to build the "
|
||||
"correspondent allowlist." % box)
|
||||
typ, data = conn.search(None, "ALL")
|
||||
ids = data[0].split() if typ == "OK" else []
|
||||
ids = ids[-cfg["imap"].get("sent_scan_limit", 3000):]
|
||||
found = set()
|
||||
for i in range(0, len(ids), 200):
|
||||
chunk = b",".join(ids[i:i + 200]).decode()
|
||||
typ, data = conn.fetch(
|
||||
chunk, "(BODY.PEEK[HEADER.FIELDS (TO CC BCC)])")
|
||||
if typ != "OK":
|
||||
continue
|
||||
for item in data:
|
||||
if not isinstance(item, tuple):
|
||||
continue
|
||||
h = email.message_from_bytes(item[1], policy=email.policy.default)
|
||||
for field in ("To", "Cc", "Bcc"):
|
||||
for _, a in email.utils.getaddresses(h.get_all(field, [])):
|
||||
if a and "@" in a:
|
||||
found.add(a.lower().strip())
|
||||
return found
|
||||
|
||||
|
||||
def decide(msg, flags, cfg, corr):
|
||||
"""-> (outcome, detail). Guards first; anything uncertain is kept."""
|
||||
if "\\Flagged" in flags or "\\Answered" in flags:
|
||||
return KEPT_FLAGGED, flags.strip()
|
||||
|
||||
frm = addr_of(msg.get("From"))
|
||||
if sender_protected(frm, cfg.get("protect_senders", [])):
|
||||
return KEPT_SENDER, frm
|
||||
if frm and frm in corr:
|
||||
return KEPT_CORRESPONDENT, frm
|
||||
|
||||
subject = str(msg.get("Subject", ""))
|
||||
for pat in cfg.get("protect_subjects", []):
|
||||
if re.search(pat, subject, re.I):
|
||||
return KEPT_SUBJECT, pat
|
||||
|
||||
# Undo works by Message-ID. Without one the move could not be reversed,
|
||||
# so it is not made -- a message stuck in the inbox is a far smaller
|
||||
# problem than one that cannot be brought back.
|
||||
if not str(msg.get("Message-ID", "")).strip():
|
||||
return KEPT_NO_MSGID, ""
|
||||
|
||||
return ARCHIVE, ""
|
||||
|
||||
|
||||
def bulk_markers(msg):
|
||||
return [h for h in BULK_HEADERS if msg.get(h)] + (
|
||||
["Precedence"] if (msg.get("Precedence") or "").lower().strip()
|
||||
in ("bulk", "list", "junk") else [])
|
||||
|
||||
|
||||
def log_records(records):
|
||||
with open(LOG, "a") as fh:
|
||||
for r in records:
|
||||
fh.write(json.dumps(r, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def read_log():
|
||||
try:
|
||||
with open(LOG) as fh:
|
||||
return [json.loads(l) for l in fh if l.strip()]
|
||||
except FileNotFoundError:
|
||||
return []
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# modes
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def gather(conn, cfg, args):
|
||||
"""-> (decisions, corr) where decisions is [(uid, msg, outcome, detail)]."""
|
||||
corr = correspondents(conn, cfg)
|
||||
minc = cfg.get("min_correspondents", 25)
|
||||
if len(corr) < minc and not args.check:
|
||||
raise Fatal(
|
||||
"The correspondent allowlist has only %d address(es); %d are "
|
||||
"required.\n\n"
|
||||
"That guard is derived from the Sent mailbox, so a nearly empty "
|
||||
"Sent means the guard is silently doing nothing -- mail from "
|
||||
"people you actually write to would be archived like any "
|
||||
"circular. Refusing to run.\n\n"
|
||||
"If Bridge is still doing its first sync, wait for it to finish "
|
||||
"and try again:\n"
|
||||
" sudo podman logs --tail 5 connor_protonmail-bridge_1\n"
|
||||
"Lower `min_correspondents` in %s only if this account genuinely "
|
||||
"sends very little mail." % (len(corr), minc, CONFIG))
|
||||
|
||||
conn.select("INBOX", readonly=True)
|
||||
cutoff = dt.date.today() - dt.timedelta(days=cfg.get("age_days", 3))
|
||||
typ, data = conn.search(None, '(UNSEEN BEFORE "%s")' % imap_date(cutoff))
|
||||
if typ != "OK":
|
||||
raise Fatal("IMAP search of INBOX failed: %s" % typ)
|
||||
ids = data[0].split()
|
||||
|
||||
cap = cfg.get("expected_unread_max", 900)
|
||||
if len(ids) > cap and not args.check:
|
||||
raise Fatal(
|
||||
"%d messages in INBOX are unread and older than %d days, which is "
|
||||
"above the sanity limit of %d.\n\n"
|
||||
"This limit exists because Bridge's \\Seen flags did not initially "
|
||||
"match the Proton UI -- it reported 1249 unread where the UI "
|
||||
"showed 646. Archiving on wrong flags would move READ mail, which "
|
||||
"is exactly the mail the inbox is meant to keep.\n\n"
|
||||
"Compare `inbox_tidy --check` against the unread count in the "
|
||||
"Proton web UI. If they agree, this really is the backlog and "
|
||||
"`expected_unread_max` in %s should be raised. If they disagree, "
|
||||
"do not run this." % (len(ids), cfg.get("age_days", 3), cap, CONFIG))
|
||||
|
||||
decisions = []
|
||||
for uid, flags, msg in fetch_headers(conn, ids):
|
||||
outcome, detail = decide(msg, flags, cfg, corr)
|
||||
decisions.append((uid, msg, outcome, detail))
|
||||
return decisions, corr
|
||||
|
||||
|
||||
def do_check(conn, cfg):
|
||||
print("Mailbox state (compare INBOX unseen against the Proton web UI):\n")
|
||||
for box in ("INBOX", "Archive", cfg["imap"].get("sent_mailbox", "Sent")):
|
||||
typ, st = conn.status(_quote(box), "(MESSAGES UNSEEN)")
|
||||
print(" %-10s %s" % (box, st[0].decode() if typ == "OK" else "?"))
|
||||
corr = correspondents(conn, cfg)
|
||||
conn.select("INBOX", readonly=True)
|
||||
cutoff = dt.date.today() - dt.timedelta(days=cfg.get("age_days", 3))
|
||||
typ, data = conn.search(None, '(UNSEEN BEFORE "%s")' % imap_date(cutoff))
|
||||
n = len(data[0].split()) if typ == "OK" else 0
|
||||
print("\n correspondents from Sent : %d (minimum %d)"
|
||||
% (len(corr), cfg.get("min_correspondents", 25)))
|
||||
print(" candidates (unread >%dd) : %d (sanity limit %d)"
|
||||
% (cfg.get("age_days", 3), n, cfg.get("expected_unread_max", 900)))
|
||||
print(" enabled : %s" % cfg.get("enabled", False))
|
||||
print("\nProceed only if INBOX unseen matches the web UI and the "
|
||||
"correspondent count looks real.")
|
||||
return 0
|
||||
|
||||
|
||||
def do_run(conn, cfg, args):
|
||||
decisions, corr = gather(conn, cfg, args)
|
||||
to_move = [(u, m) for u, m, o, _ in decisions if o == ARCHIVE]
|
||||
limit = args.max if args.max is not None else cfg.get("max_per_run", 250)
|
||||
batch, held = to_move[:limit], to_move[limit:]
|
||||
|
||||
counts = {}
|
||||
for _, _, outcome, _ in decisions:
|
||||
counts[outcome] = counts.get(outcome, 0) + 1
|
||||
|
||||
if args.dry_run:
|
||||
for uid, msg, outcome, detail in decisions:
|
||||
print("%-20s %-34s %s" % (
|
||||
outcome, addr_of(msg.get("From"))[:34],
|
||||
str(msg.get("Subject", ""))[:60]) +
|
||||
((" [%s]" % detail) if detail and outcome != ARCHIVE else ""))
|
||||
print("\n%s" % summarise(counts, len(batch), len(held), None))
|
||||
return 0
|
||||
|
||||
if not cfg.get("enabled", False):
|
||||
raise Fatal(
|
||||
"`enabled` is false in %s.\n\nThe job is installed but not armed. "
|
||||
"Review `inbox_tidy --check` and `--dry-run` first, then set "
|
||||
"enabled = true." % CONFIG)
|
||||
|
||||
run_id = time.strftime("%Y%m%dT%H%M%S")
|
||||
moved = []
|
||||
if batch:
|
||||
conn.select("INBOX", readonly=False) # the ONLY read-write select
|
||||
for i in range(0, len(batch), 100):
|
||||
part = batch[i:i + 100]
|
||||
typ, _ = conn.uid("MOVE", ",".join(u for u, _ in part), '"Archive"')
|
||||
if typ != "OK":
|
||||
raise Fatal(
|
||||
"IMAP MOVE failed after %d message(s) of this run.\n\n"
|
||||
"Anything already moved is in Archive and is recorded in "
|
||||
"%s under run %s, so `inbox_tidy --undo %s` will bring it "
|
||||
"back." % (len(moved), LOG, run_id, run_id))
|
||||
for uid, msg in part:
|
||||
moved.append({
|
||||
"run": run_id,
|
||||
"at": dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
"action": "archive",
|
||||
"uid": uid,
|
||||
"message_id": str(msg.get("Message-ID", "")).strip(),
|
||||
"from": addr_of(msg.get("From")),
|
||||
"subject": str(msg.get("Subject", ""))[:200],
|
||||
"date": str(msg.get("Date", "")),
|
||||
"bulk": bulk_markers(msg),
|
||||
"src": "INBOX", "dst": "Archive",
|
||||
})
|
||||
log_records(moved[-len(part):])
|
||||
|
||||
body = summarise(counts, len(moved), len(held), run_id)
|
||||
push(cfg, "Inbox tidy: %d archived" % len(moved), body, "broom")
|
||||
print(body)
|
||||
return 0
|
||||
|
||||
|
||||
def summarise(counts, n_moved, n_held, run_id):
|
||||
lines = ["**%d archived**" % n_moved]
|
||||
if n_held:
|
||||
lines.append("%d over the per-run cap, left for the next run." % n_held)
|
||||
lines.append("")
|
||||
lines.append("Kept:")
|
||||
for k in (KEPT_FLAGGED, KEPT_CORRESPONDENT, KEPT_SENDER, KEPT_SUBJECT,
|
||||
KEPT_NO_MSGID):
|
||||
if counts.get(k):
|
||||
lines.append(" %-22s %d" % (k, counts[k]))
|
||||
if not any(counts.get(k) for k in (KEPT_FLAGGED, KEPT_CORRESPONDENT,
|
||||
KEPT_SENDER, KEPT_SUBJECT,
|
||||
KEPT_NO_MSGID)):
|
||||
lines.append(" (nothing was held back by a guard)")
|
||||
if run_id:
|
||||
lines += ["", "Undo: `inbox_tidy --undo %s`" % run_id]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def do_undo(conn, cfg, run_id):
|
||||
recs = [r for r in read_log()
|
||||
if r.get("action") == "archive" and r.get("run") == run_id]
|
||||
if not recs:
|
||||
raise Fatal("No archive records for run %r in %s." % (run_id, LOG))
|
||||
|
||||
conn.select('"Archive"', readonly=False)
|
||||
restored, missing = [], []
|
||||
for r in recs:
|
||||
mid = r.get("message_id", "")
|
||||
if not mid:
|
||||
missing.append(r)
|
||||
continue
|
||||
typ, data = conn.search(None, '(HEADER MESSAGE-ID "%s")' % mid)
|
||||
ids = data[0].split() if typ == "OK" else []
|
||||
if not ids:
|
||||
missing.append(r)
|
||||
continue
|
||||
typ, d2 = conn.fetch(b",".join(ids).decode(), "(UID)")
|
||||
uids = [m.group(1) for m in
|
||||
(re.search(r"UID (\d+)", x.decode(errors="replace"))
|
||||
for x in d2 if isinstance(x, bytes)) if m]
|
||||
if not uids:
|
||||
missing.append(r)
|
||||
continue
|
||||
typ, _ = conn.uid("MOVE", ",".join(uids), "INBOX")
|
||||
(restored if typ == "OK" else missing).append(r)
|
||||
|
||||
log_records([{
|
||||
"run": run_id, "at": dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
"action": "undo", "restored": len(restored), "missing": len(missing),
|
||||
}])
|
||||
body = ("**Undo of run %s**\n\n%d restored to INBOX.\n%d not found in "
|
||||
"Archive (moved or deleted by hand since)."
|
||||
% (run_id, len(restored), len(missing)))
|
||||
print(body)
|
||||
push(cfg, "Inbox tidy: undo %s" % run_id, body, "leftwards_arrow_with_hook")
|
||||
return 0 if not missing else 1
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--config", metavar="PATH",
|
||||
help="use an alternative config (for testing a rule change "
|
||||
"without touching the live one)")
|
||||
ap.add_argument("--check", action="store_true",
|
||||
help="report mailbox state and the guard inputs; change nothing")
|
||||
ap.add_argument("--dry-run", action="store_true",
|
||||
help="print the decision for every candidate; move nothing")
|
||||
ap.add_argument("--max", type=int, help="override the per-run cap")
|
||||
ap.add_argument("--undo", metavar="RUN_ID", help="restore a run from Archive")
|
||||
ap.add_argument("--undo-last", action="store_true",
|
||||
help="restore the most recent archive run")
|
||||
args = ap.parse_args()
|
||||
|
||||
try:
|
||||
cfg = load_config(args.config)
|
||||
except Fatal as exc:
|
||||
print("inbox_tidy: %s" % exc, file=sys.stderr)
|
||||
return 2
|
||||
|
||||
conn = None
|
||||
try:
|
||||
conn = imap_connect(cfg["imap"])
|
||||
if args.undo or args.undo_last:
|
||||
run_id = args.undo
|
||||
if args.undo_last:
|
||||
runs = [r["run"] for r in read_log() if r.get("action") == "archive"]
|
||||
if not runs:
|
||||
raise Fatal("Nothing in %s to undo." % LOG)
|
||||
run_id = runs[-1]
|
||||
return do_undo(conn, cfg, run_id)
|
||||
if args.check:
|
||||
return do_check(conn, cfg)
|
||||
return do_run(conn, cfg, args)
|
||||
except Fatal as exc:
|
||||
body = "**Inbox tidy could not run.**\n\n%s" % exc
|
||||
if args.dry_run or args.check:
|
||||
print(body, file=sys.stderr)
|
||||
else:
|
||||
notify_failure(cfg, "Inbox tidy: FAILED", body)
|
||||
return 1
|
||||
finally:
|
||||
if conn is not None:
|
||||
try:
|
||||
conn.logout()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user