#!/usr/bin/python3 """Archive stale unread mail out of the Proton inbox. Never deletes. The inbox is meant to hold new mail plus read mail kept on purpose. Unread mail accumulates faster than it gets triaged, so anything still unread after a few days is moved to Archive. Deliberately has no LLM in it. Of ~1200 inbox messages sampled, 81% carry a machine-readable bulk marker, and the ~19% residual is transactional rather than personal. Since transactional mail is archived on the same rule, every decision reduces to a flag test, a set membership test or a string match -- deterministic, explainable in the log, and not improved by a model. Safety, in order of importance: * nothing is ever deleted; the only operation is IMAP MOVE to Archive * BODY.PEEK everywhere, so nothing is ever marked read as a side effect * INBOX is the only mailbox ever opened read-write * every archived message is recorded with its Message-ID and the rule that fired, and any run can be undone * a message with no Message-ID is left alone, because it could not be undone * two interlocks abort the run if the mailbox does not look like itself """ import argparse import datetime as dt import email import email.policy import email.utils import json import os import re import sys import time import tomllib # Shared with rent_utilities. sys.path rather than PYTHONPATH so this works # when run by hand as well as from the unit. sys.path.insert(0, os.path.expanduser("~/.local/lib/pymail")) from protonimap import ( # noqa: E402 Fatal, imap_connect, imap_date, notify_failure, push, _quote) CONFIG = os.path.expanduser("~/.config/inbox-tidy/tidy.toml") LOG = os.path.expanduser("~/.local/share/inbox-tidy/archived.jsonl") HEADERS = ("FROM SUBJECT DATE MESSAGE-ID LIST-UNSUBSCRIBE LIST-ID PRECEDENCE " "AUTO-SUBMITTED FEEDBACK-ID X-CAMPAIGNID") BULK_HEADERS = ("List-Unsubscribe", "List-Id", "Feedback-ID", "X-CampaignID") # Skip reasons, in evaluation order. Also the keys in the run summary. KEPT_FLAGGED = "flagged-or-answered" KEPT_SENDER = "protected-sender" KEPT_CORRESPONDENT = "known-correspondent" KEPT_SUBJECT = "protected-subject" KEPT_NO_MSGID = "no-message-id" ARCHIVE = "archive" # -------------------------------------------------------------------------- # helpers # -------------------------------------------------------------------------- def load_config(path=None): path = path or CONFIG try: with open(path, "rb") as fh: return tomllib.load(fh) except (OSError, tomllib.TOMLDecodeError) as exc: raise Fatal("Cannot read %s: %s" % (path, exc)) def addr_of(header_value): """Bare lowercase address out of a From/To header, or ''.""" if not header_value: return "" pairs = email.utils.getaddresses([str(header_value)]) return pairs[0][1].lower().strip() if pairs and pairs[0][1] else "" def domain_of(addr): return addr.rsplit("@", 1)[-1] if "@" in addr else "" def sender_protected(addr, entries): """An entry containing @ matches the address; otherwise the domain. Domain entries match subdomains too, so `chase.com` also covers `fraudalert.chase.com` -- which is the case that actually matters. """ dom = domain_of(addr) for e in entries: e = e.lower().strip() if not e: continue if "@" in e: if addr == e: return True elif dom == e or dom.endswith("." + e): return True return False def fetch_headers(conn, ids, extra="FLAGS"): """[(uid, flags, email.Message)] for a list of sequence ids, batched. BODY.PEEK, never BODY: fetching a body with BODY would set \\Seen on mail this job exists to leave unread. """ out = [] for i in range(0, len(ids), 200): chunk = b",".join(ids[i:i + 200]).decode() typ, data = conn.fetch( chunk, "(UID %s BODY.PEEK[HEADER.FIELDS (%s)])" % (extra, HEADERS)) if typ != "OK": continue for item in data: if not isinstance(item, tuple): continue meta = item[0].decode(errors="replace") m_uid = re.search(r"UID (\d+)", meta) m_fl = re.search(r"FLAGS \(([^)]*)\)", meta) if not m_uid: continue msg = email.message_from_bytes(item[1], policy=email.policy.default) out.append((m_uid.group(1), (m_fl.group(1) if m_fl else ""), msg)) return out def correspondents(conn, cfg): """Addresses this account has actually written to, from the Sent folder. A behavioural allowlist: someone you have emailed is someone whose mail should not be swept up. Cheap to derive and needs no maintenance. """ box = cfg["imap"].get("sent_mailbox", "Sent") typ, _ = conn.select(_quote(box), readonly=True) if typ != "OK": raise Fatal("Cannot open the Sent mailbox %r to build the " "correspondent allowlist." % box) typ, data = conn.search(None, "ALL") ids = data[0].split() if typ == "OK" else [] ids = ids[-cfg["imap"].get("sent_scan_limit", 3000):] found = set() for i in range(0, len(ids), 200): chunk = b",".join(ids[i:i + 200]).decode() typ, data = conn.fetch( chunk, "(BODY.PEEK[HEADER.FIELDS (TO CC BCC)])") if typ != "OK": continue for item in data: if not isinstance(item, tuple): continue h = email.message_from_bytes(item[1], policy=email.policy.default) for field in ("To", "Cc", "Bcc"): for _, a in email.utils.getaddresses(h.get_all(field, [])): if a and "@" in a: found.add(a.lower().strip()) return found def decide(msg, flags, cfg, corr): """-> (outcome, detail). Guards first; anything uncertain is kept.""" if "\\Flagged" in flags or "\\Answered" in flags: return KEPT_FLAGGED, flags.strip() frm = addr_of(msg.get("From")) if sender_protected(frm, cfg.get("protect_senders", [])): return KEPT_SENDER, frm if frm and frm in corr: return KEPT_CORRESPONDENT, frm subject = str(msg.get("Subject", "")) for pat in cfg.get("protect_subjects", []): if re.search(pat, subject, re.I): return KEPT_SUBJECT, pat # Undo works by Message-ID. Without one the move could not be reversed, # so it is not made -- a message stuck in the inbox is a far smaller # problem than one that cannot be brought back. if not str(msg.get("Message-ID", "")).strip(): return KEPT_NO_MSGID, "" return ARCHIVE, "" def bulk_markers(msg): return [h for h in BULK_HEADERS if msg.get(h)] + ( ["Precedence"] if (msg.get("Precedence") or "").lower().strip() in ("bulk", "list", "junk") else []) def log_records(records): with open(LOG, "a") as fh: for r in records: fh.write(json.dumps(r, sort_keys=True) + "\n") def read_log(): try: with open(LOG) as fh: return [json.loads(l) for l in fh if l.strip()] except FileNotFoundError: return [] # -------------------------------------------------------------------------- # modes # -------------------------------------------------------------------------- def gather(conn, cfg, args): """-> (decisions, corr) where decisions is [(uid, msg, outcome, detail)].""" corr = correspondents(conn, cfg) minc = cfg.get("min_correspondents", 25) if len(corr) < minc and not args.check: raise Fatal( "The correspondent allowlist has only %d address(es); %d are " "required.\n\n" "That guard is derived from the Sent mailbox, so a nearly empty " "Sent means the guard is silently doing nothing -- mail from " "people you actually write to would be archived like any " "circular. Refusing to run.\n\n" "If Bridge is still doing its first sync, wait for it to finish " "and try again:\n" " sudo podman logs --tail 5 connor_protonmail-bridge_1\n" "Lower `min_correspondents` in %s only if this account genuinely " "sends very little mail." % (len(corr), minc, CONFIG)) conn.select("INBOX", readonly=True) cutoff = dt.date.today() - dt.timedelta(days=cfg.get("age_days", 3)) typ, data = conn.search(None, '(UNSEEN BEFORE "%s")' % imap_date(cutoff)) if typ != "OK": raise Fatal("IMAP search of INBOX failed: %s" % typ) ids = data[0].split() cap = cfg.get("expected_unread_max", 900) if len(ids) > cap and not args.check: raise Fatal( "%d messages in INBOX are unread and older than %d days, which is " "above the sanity limit of %d.\n\n" "This limit exists because Bridge's \\Seen flags did not initially " "match the Proton UI -- it reported 1249 unread where the UI " "showed 646. Archiving on wrong flags would move READ mail, which " "is exactly the mail the inbox is meant to keep.\n\n" "Compare `inbox_tidy --check` against the unread count in the " "Proton web UI. If they agree, this really is the backlog and " "`expected_unread_max` in %s should be raised. If they disagree, " "do not run this." % (len(ids), cfg.get("age_days", 3), cap, CONFIG)) decisions = [] for uid, flags, msg in fetch_headers(conn, ids): outcome, detail = decide(msg, flags, cfg, corr) decisions.append((uid, msg, outcome, detail)) return decisions, corr def do_check(conn, cfg): print("Mailbox state (compare INBOX unseen against the Proton web UI):\n") for box in ("INBOX", "Archive", cfg["imap"].get("sent_mailbox", "Sent")): typ, st = conn.status(_quote(box), "(MESSAGES UNSEEN)") print(" %-10s %s" % (box, st[0].decode() if typ == "OK" else "?")) corr = correspondents(conn, cfg) conn.select("INBOX", readonly=True) cutoff = dt.date.today() - dt.timedelta(days=cfg.get("age_days", 3)) typ, data = conn.search(None, '(UNSEEN BEFORE "%s")' % imap_date(cutoff)) n = len(data[0].split()) if typ == "OK" else 0 print("\n correspondents from Sent : %d (minimum %d)" % (len(corr), cfg.get("min_correspondents", 25))) print(" candidates (unread >%dd) : %d (sanity limit %d)" % (cfg.get("age_days", 3), n, cfg.get("expected_unread_max", 900))) print(" enabled : %s" % cfg.get("enabled", False)) print("\nProceed only if INBOX unseen matches the web UI and the " "correspondent count looks real.") return 0 def do_run(conn, cfg, args): decisions, corr = gather(conn, cfg, args) to_move = [(u, m) for u, m, o, _ in decisions if o == ARCHIVE] limit = args.max if args.max is not None else cfg.get("max_per_run", 250) batch, held = to_move[:limit], to_move[limit:] counts = {} for _, _, outcome, _ in decisions: counts[outcome] = counts.get(outcome, 0) + 1 if args.dry_run: for uid, msg, outcome, detail in decisions: print("%-20s %-34s %s" % ( outcome, addr_of(msg.get("From"))[:34], str(msg.get("Subject", ""))[:60]) + ((" [%s]" % detail) if detail and outcome != ARCHIVE else "")) print("\n%s" % summarise(counts, len(batch), len(held), None)) return 0 if not cfg.get("enabled", False): raise Fatal( "`enabled` is false in %s.\n\nThe job is installed but not armed. " "Review `inbox_tidy --check` and `--dry-run` first, then set " "enabled = true." % CONFIG) run_id = time.strftime("%Y%m%dT%H%M%S") moved = [] if batch: conn.select("INBOX", readonly=False) # the ONLY read-write select for i in range(0, len(batch), 100): part = batch[i:i + 100] typ, _ = conn.uid("MOVE", ",".join(u for u, _ in part), '"Archive"') if typ != "OK": raise Fatal( "IMAP MOVE failed after %d message(s) of this run.\n\n" "Anything already moved is in Archive and is recorded in " "%s under run %s, so `inbox_tidy --undo %s` will bring it " "back." % (len(moved), LOG, run_id, run_id)) for uid, msg in part: moved.append({ "run": run_id, "at": dt.datetime.now(dt.timezone.utc).isoformat(), "action": "archive", "uid": uid, "message_id": str(msg.get("Message-ID", "")).strip(), "from": addr_of(msg.get("From")), "subject": str(msg.get("Subject", ""))[:200], "date": str(msg.get("Date", "")), "bulk": bulk_markers(msg), "src": "INBOX", "dst": "Archive", }) log_records(moved[-len(part):]) body = summarise(counts, len(moved), len(held), run_id) push(cfg, "Inbox tidy: %d archived" % len(moved), body, "broom") print(body) return 0 def summarise(counts, n_moved, n_held, run_id): lines = ["**%d archived**" % n_moved] if n_held: lines.append("%d over the per-run cap, left for the next run." % n_held) lines.append("") lines.append("Kept:") for k in (KEPT_FLAGGED, KEPT_CORRESPONDENT, KEPT_SENDER, KEPT_SUBJECT, KEPT_NO_MSGID): if counts.get(k): lines.append(" %-22s %d" % (k, counts[k])) if not any(counts.get(k) for k in (KEPT_FLAGGED, KEPT_CORRESPONDENT, KEPT_SENDER, KEPT_SUBJECT, KEPT_NO_MSGID)): lines.append(" (nothing was held back by a guard)") if run_id: lines += ["", "Undo: `inbox_tidy --undo %s`" % run_id] return "\n".join(lines) def do_undo(conn, cfg, run_id): recs = [r for r in read_log() if r.get("action") == "archive" and r.get("run") == run_id] if not recs: raise Fatal("No archive records for run %r in %s." % (run_id, LOG)) conn.select('"Archive"', readonly=False) restored, missing = [], [] for r in recs: mid = r.get("message_id", "") if not mid: missing.append(r) continue typ, data = conn.search(None, '(HEADER MESSAGE-ID "%s")' % mid) ids = data[0].split() if typ == "OK" else [] if not ids: missing.append(r) continue typ, d2 = conn.fetch(b",".join(ids).decode(), "(UID)") uids = [m.group(1) for m in (re.search(r"UID (\d+)", x.decode(errors="replace")) for x in d2 if isinstance(x, bytes)) if m] if not uids: missing.append(r) continue typ, _ = conn.uid("MOVE", ",".join(uids), "INBOX") (restored if typ == "OK" else missing).append(r) log_records([{ "run": run_id, "at": dt.datetime.now(dt.timezone.utc).isoformat(), "action": "undo", "restored": len(restored), "missing": len(missing), }]) body = ("**Undo of run %s**\n\n%d restored to INBOX.\n%d not found in " "Archive (moved or deleted by hand since)." % (run_id, len(restored), len(missing))) print(body) push(cfg, "Inbox tidy: undo %s" % run_id, body, "leftwards_arrow_with_hook") return 0 if not missing else 1 def main(): ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--config", metavar="PATH", help="use an alternative config (for testing a rule change " "without touching the live one)") ap.add_argument("--check", action="store_true", help="report mailbox state and the guard inputs; change nothing") ap.add_argument("--dry-run", action="store_true", help="print the decision for every candidate; move nothing") ap.add_argument("--max", type=int, help="override the per-run cap") ap.add_argument("--undo", metavar="RUN_ID", help="restore a run from Archive") ap.add_argument("--undo-last", action="store_true", help="restore the most recent archive run") args = ap.parse_args() try: cfg = load_config(args.config) except Fatal as exc: print("inbox_tidy: %s" % exc, file=sys.stderr) return 2 conn = None try: conn = imap_connect(cfg["imap"]) if args.undo or args.undo_last: run_id = args.undo if args.undo_last: runs = [r["run"] for r in read_log() if r.get("action") == "archive"] if not runs: raise Fatal("Nothing in %s to undo." % LOG) run_id = runs[-1] return do_undo(conn, cfg, run_id) if args.check: return do_check(conn, cfg) return do_run(conn, cfg, args) except Fatal as exc: body = "**Inbox tidy could not run.**\n\n%s" % exc if args.dry_run or args.check: print(body, file=sys.stderr) else: notify_failure(cfg, "Inbox tidy: FAILED", body) return 1 finally: if conn is not None: try: conn.logout() except Exception: pass if __name__ == "__main__": sys.exit(main())