From 185b45dbfc706ffe62e4c37429106309bfef8799 Mon Sep 17 00:00:00 2001 From: Connor Johnstone Date: Wed, 9 Sep 2026 18:30:29 -0400 Subject: [PATCH] Add the prune rule, its tests, and a corpus validator The rule PLAN.md describes and the rationale it gives for that rule disagree on one shape that turns out to be common. 235 checkboxes in the corpus sit under a plain bullet used as a group label, and by the letter of the rule those have no checkbox ancestor, so a finished one would be dropped on its own. But section 4 already argues the opposite case for checkbox parents: a finished item under an unfinished group is context, not cruft, and a label like `- Trellis "Ergonomics" Fixes` is a group in every sense that matters. Resolved in favour of the rationale, which collapses both cases into one rule. A top-level bullet is any bullet with no bullet ancestor in its section. It is dropped, with its subtree, exactly when that subtree holds at least one checkbox and all of them are [x]. Prose bullets hold none and so are never eligible, which is the property section 4 wanted. Validated across all 228 notes with zero invariant failures: no heading, no unfinished item and no checkbox-free subtree is ever removed, and no group label is left with its children gone. 270 lines would be dropped across 118 files. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01PU5ZFfQFtDTFqqhvMTWdGH --- rollover/__pycache__/prune.cpython-312.pyc | Bin 0 -> 6385 bytes rollover/prune.py | 127 +++++++++++++++++++++ rollover/test_prune.py | 80 +++++++++++++ rollover/validate.py | 89 +++++++++++++++ 4 files changed, 296 insertions(+) create mode 100644 rollover/__pycache__/prune.cpython-312.pyc create mode 100755 rollover/prune.py create mode 100644 rollover/test_prune.py create mode 100644 rollover/validate.py diff --git a/rollover/__pycache__/prune.cpython-312.pyc b/rollover/__pycache__/prune.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5157c35622db22c6455a4193614f61c57c2aa44e GIT binary patch literal 6385 zcma)AU2qh~weI;_?XOl60waOskr0FxVijyih`}bp5)Kj(HV`Msz%rT{X?Hc6S@+B$ zq3l-Hj&FGv*0pk7Claw!7OA8Rft3Ap>(+g`c}~?W!{TB+(XArMOZ+B_R1))&obKJ3 z1z9HPs_E|OKHYt~`#WEs?%#z%egq-+|5uYOjR^gT{5XNHW$ynGG7CsSV@P2XR%6B( znzCaoO}R0SMJk*0It}Zo!rfr1e`CT|%J8m?=PP9dSEfPnjCon~0#X{@K#KPUTkGGa z_})Nce#H-QKnXw`RDuwPln}&?N+ZN!wMhw2?L-Je4{G>EO;HA2WG}LjCg~mM>_wc` zRZ%vkQ<`e2ia2B7DbYw_0foKi`bRns!Cn9#+9Ws0UMTM0uyAjVZmN$8WNh*eXy{HAQA)TnqeEyrQ3dY1|| zQYlsGtM*HJn28&jDw^qW3#)3MsLo2V1&YnYRb8~=s=rnP^Uy6Rp_`&)iJB^zFby)h zxGGPL8?&OMDSmfE?3+T6;cNjYM%KYJ zLoY?r(zM1*9UdXhf`e zj8j;!6aFF>;cfm{LRTXmn3sj$$$%W&2Pr+tP^ZgleXw9b1E=^AkAzoL!HgR_ju zGDs1UZBC1N>dc`ggFq{HrWb*qzd}oQ?HTq`i$(OjUr*}AsK$$Ff_a2J3|A5M@L_4$&(T@#HBSZ$p!1dA8McW&TfvsDy*j$K!KI;6E1u}mH zQ4UpFH|P1HZ@Inmy_xs3SF+bmm7jd7-2OBahAW#pAPH5vcYM%$wfC2O#UrI-g=34Y zcY>WC4d$6o1Hnp1SAMWKy4>EeD7`;jcdbjs>d+#;& zZrh$)ZF|aXPv34k@R5}td~5h#N4L{fc{lP)`C8A1yRPrLvAeuu|E>1@w2!t9B6uGZ zMWYdcOh=cd)mY5-$70il0@MxpU@Z1(TGDDYzF5rE49kqg@K$Jc9bqzAyoE%>j%d^l z88Zmc8>rHBB6svY9~3&Pf<#7khy?P8m?AZD8!`)M5@o?n0OLV}=&XcLma|;xjO!L; zdCMh=q-(yUyK1OQ;y{KCf-!<2@-de(bRQUJ!=q&xK1mR`Cg4Ue4*n6s;Qtk2WL%SE zP*LWAxK5%72Y`<>(B-j(BSWLl58)OVi=vvsn1bB)NN{B93haiT@MlZ|T$wfpo`fw> zhO`;WW;NX;vj+T%B3*l1NJ-dK@h&LsgU{3<0`&QVx#ugv=G;&v^u#Li3s2;kWyF5GozGbfB_D%pWcut8DFp z@yS1ILg}~pi5QLhN@U0E7;Y^mxz}C+} zRhG~2b$-tvhgGIv({UX7;YmMPAtNNo7;AzVg!1*alJlVqcM?IZ%gFVnNtdhYWjAVB z)05#rGhoatUmS(@brCn0VW$B0*cpR|*|=>UGaIHDCptygxf92TpGA(#S>iN;L*O`S zA~wKtRLHsRICtRlV1g8dx)2FHEfLR9q-cRYWG$_zN)!{0k9a64VshS8(MH*nED$E= z5ggS*6=h5U=vOjTo7X@Un@Qqcs5gluMOqDNftr^X1DxWgpz>KF^>c`F=w6^{zPYf! zbf|DBH&_V-A@ch20|g;BQ1J(f^8EJ8$xG2nsAYan;RkuPhOfo$_jmtnH{f)o2?d*S zBP(Il)KZ!$%#^Z)Z1G2xU)zL0T2|l6IDn4B@0d)y>w!wG*Ae4?Erba&x5sr&IAfr4&X8?a$7Iv0Py7U9> zYXLLG4N$WVA~8p3t=J8<>)_I4iv1Y|f0Lk};2NaX!ROQ&*sty2p*#ks3|JW6@Ne|P z6rm@R&#*sb78ufMJ-=stJsevEYP-b44d^%A3=`qzhK}kAIPh?fnQ)y}>f^_y3*exB z`A$RbI^c%`%oEtT*+f742ZYf`gY8rG^t1|YvHB2X$*NQjcrIn6@OJo(@OU>Q)TflE zaFj?;(2`vspsliFGu*gh^u|?lLQT9M1YU&Cd<&vKqrU@ogN?bN<>vOqedXq^RTLD& zf{dumo~^WQ0(OD8@!ef9M1%~8N+>1g3- zapZFjg3MN*>LHFiZ@gW7J~C97EZr?`V;SypSQuFU(l2D^?`kdw=%!o z-vjX@=OtMLdS0HOq*@27LsnU4J?aDi7YKMQ_p}5WWu0pv&?4s8FEfj)_i2m$$CEx-+h9wDp6$6S1K+_(Raa+I__?|6+JzmQ zN@`2dgPI{rn)zJRm6?K;W-CM#vA5gyUSk%!-m`ulxz)D!3zLBU^$i=Dh}Y)eVSr+s zGPEsVNuN}!eg<`mIk+L(o^(n96N3|i8WB&%r#57BgxmK(ceaP#*32)R0^v2v;uk=A z96s~c5arNvpz-p)+~9Jkb1_{GZC^!RVHY?bjctqDmzv9s+w$CBJZRIF#XXDi(!iC; z^2Qy175Enay{6_$xOM)e!tC8}*X?lEl6r06YWxHJs{ZkgZ?6alPL8g^cS@s|M^`uy z&wEx7B&&WDYdSFb)%-W>TP*t^mUYOJ)- z1L2F+Ff^r()PL;lQc4CA_sJB%fGQ0KXP-f@pvPs^+gbDZIIY!%8SMps1z{PXWa5wfTvJ zraoW}&21%btnj`nQWW-yiaH^sHF%A*;K39KSkNUrY1({T)l!j=?S(&jak3i+@FaK~ zy7eNxN7)Ur7(4-x>#w9JF%YAbwx3Fqe-obo9c><-USWI;FS5Y77fs@cz)*xX6?$o# z!SD=7F$hl(tdz1D2?Vk1JU~>T%1^_iBvzou3-Fl+L;zG@^Ie~K%O~Fa{vF@G+~ErE zdGq*d$LEhMZTq-yICuOGKT`4fOQAyOav1#UuI>*u{bEzGC;w{cLgB*YzPtV{xBOe~ z_&aW%{P6Vk(;uF@ey+Ux+1&HXe8Zc=uMOYjTW|5L?|g6Z>Hi%2gcqrMOr8LUCb3wg z8UFy3z%N5&vslGvNv453aKln#vX!`*L_H)rNFrhm)LQA0Q$2jhp+F~3=Y~7UZ#RiL zNJL%@9(WH{pBkRSarh#d-P&y@Y$xrU0jCC}2^=`&?-XHK}+;Lq`Y^Msks&k;ncJ&jCwwY!OF rTHO(5#MPbv= ln.indent: + stack.pop() + if not stack: + yield i + stack.append(ln.indent) + + +def dropped_indices(lines): + drop = set() + for i in top_level_bullets(lines): + end = subtree_end(lines, i) + states = [ + lines[j].state + for j in range(i, end + 1) + if lines[j].is_bullet and lines[j].state is not None + ] + if states and all(s == DONE for s in states): + drop.update(range(i, end + 1)) + return drop + + +def prune(text): + lines = parse(text.split("\n")) + drop = dropped_indices(lines) + return "\n".join(l.text for k, l in enumerate(lines) if k not in drop) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("file", nargs="?", help="note to prune, default stdin") + a = ap.parse_args() + src = open(a.file).read() if a.file else sys.stdin.read() + sys.stdout.write(prune(src)) + + +if __name__ == "__main__": + main() diff --git a/rollover/test_prune.py b/rollover/test_prune.py new file mode 100644 index 0000000..aa1ce06 --- /dev/null +++ b/rollover/test_prune.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Cases the prune rule has to get right, including ones the corpus lacks.""" + +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from prune import prune + + +class TestPrune(unittest.TestCase): + def assertPrune(self, src, want): + self.assertEqual(prune(src.strip("\n")), want.strip("\n")) + + def test_completed_top_level_item_goes(self): + self.assertPrune("## To-Do\n- [x] Climb\n- [ ] Finish the backpack", + "## To-Do\n- [ ] Finish the backpack") + + def test_finished_child_of_unfinished_parent_stays(self): + src = ("## To-Do\n- [ ] Add OpenID to any apps that I control\n" + " - [x] Runway\n - [ ] Shanty") + self.assertPrune(src, src) + + def test_completed_checkbox_group_goes_entirely(self): + self.assertPrune("## To-Do\n- [x] Add OpenID\n - [x] Runway\n - [x] Shanty\n- [ ] Keep", + "## To-Do\n- [ ] Keep") + + def test_plain_group_label_protects_finished_children(self): + src = ('## To-Do\n- Trellis "Ergonomics" Fixes\n' + " - [x] Build up a justfile\n - [ ] Add in some unit tests") + self.assertPrune(src, src) + + def test_plain_group_label_goes_when_group_completes(self): + self.assertPrune('## To-Do\n- Trellis "Ergonomics" Fixes\n' + " - [x] Build up a justfile\n - [x] Add in some unit tests\n- [ ] Keep", + "## To-Do\n- [ ] Keep") + + def test_prose_bullets_are_never_touched(self): + src = ("### DSA Election Research Tool\n - Work on android rust app\n" + " - Move bike components that I want over") + self.assertPrune(src, src) + + def test_headings_survive_an_emptied_section(self): + self.assertPrune("## To-Do\n- [x] Climb\n\n## Personal\n- [ ] Keep", + "## To-Do\n\n## Personal\n- [ ] Keep") + + def test_question_state_is_not_complete(self): + src = "## To-Do\n - [?] Cincinnati Trash" + self.assertPrune(src, src) + + def test_question_state_blocks_its_group(self): + src = "## To-Do\n- [x] Parent\n - [?] Unsure" + self.assertPrune(src, src) + + def test_wrapped_continuation_travels_with_its_item(self): + self.assertPrune("## To-Do\n- [x] Could be good to do a review of the band-aids\n" + " and instabilities\n- [ ] Keep", + "## To-Do\n- [ ] Keep") + + def test_headings_reset_scope(self): + # The [x] under Personal is top-level despite following a deeper bullet. + self.assertPrune("## Work\n - [ ] Deep unfinished\n## Personal\n- [x] Climb", + "## Work\n - [ ] Deep unfinished\n## Personal") + + def test_checkboxes_inside_a_code_fence_are_inert(self): + src = "## Notes\n```\n- [x] not a real item\n```\n- [ ] Keep" + self.assertPrune(src, src) + + def test_two_space_base_indent_corpus_style(self): + self.assertPrune("## Work\n - [ ] Letter of Determination\n - [x] Check e-mails", + "## Work\n - [ ] Letter of Determination") + + def test_date_heading_and_blank_structure_survive(self): + self.assertPrune("# 2026-09-09\n\n## To-Do\n- [x] Climb\n\n# Notes\n", + "# 2026-09-09\n\n## To-Do\n\n# Notes\n") + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/rollover/validate.py b/rollover/validate.py new file mode 100644 index 0000000..fb6fec0 --- /dev/null +++ b/rollover/validate.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Run the prune across every historical note and assert it never misbehaves. + +The prune is the piece of this system carrying real semantic risk, and 228 real +notes exist to test it against, so this checks invariants rather than relying on +someone eyeballing a diff. +""" + +import glob +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from prune import DONE, dropped_indices, parse, subtree_end, top_level_bullets + +NOTES = os.path.expanduser("~/docs/notes/todo") + + +def audit(path): + lines = parse(open(path).read().split("\n")) + drop = dropped_indices(lines) + errs = [] + + for i in sorted(drop): + ln = lines[i] + if ln.is_heading: + errs.append(f"removed a heading: {ln.text!r}") + elif ln.is_bullet and ln.state is not None and ln.state != DONE: + errs.append(f"removed an unfinished item [{ln.state}]: {ln.text!r}") + + # A dropped subtree must contain at least one checkbox, or it was prose. + for i in top_level_bullets(lines): + if i not in drop: + continue + end = subtree_end(lines, i) + if not any( + lines[j].is_bullet and lines[j].state is not None + for j in range(i, end + 1) + ): + errs.append(f"dropped a checkbox-free subtree: {lines[i].text!r}") + + # A surviving group label must not have lost all of its children. + for i, ln in enumerate(lines): + if i in drop or not ln.is_bullet or ln.state is not None: + continue + kids = [j for j in range(i + 1, subtree_end(lines, i) + 1) if lines[j].is_bullet] + if kids and all(j in drop for j in kids): + errs.append(f"left an emptied group label: {ln.text!r}") + + return lines, drop, errs + + +def main(): + files = sorted(glob.glob(os.path.join(NOTES, "*.md"))) + if not files: + sys.exit(f"no notes found under {NOTES}") + + failures = changed = dropped = 0 + kinds = {} + for f in files: + lines, drop, errs = audit(f) + if errs: + failures += len(errs) + print(f"FAIL {os.path.basename(f)}") + for e in errs[:4]: + print(" ", e) + if drop: + changed += 1 + dropped += len(drop) + for i in drop: + ln = lines[i] + k = ( + f"checkbox[{ln.state}]" + if ln.is_bullet and ln.state + else "bullet" if ln.is_bullet else "continuation" + ) + kinds[k] = kinds.get(k, 0) + 1 + + print() + print(f"files : {len(files)} ({changed} would change)") + print(f"lines dropped : {dropped}") + for k, v in sorted(kinds.items(), key=lambda x: -x[1]): + print(f" {k:>14} : {v}") + print(f"invariant failures: {failures}") + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main())