Add the prune rule, its tests, and a corpus validator
The rule PLAN.md describes and the rationale it gives for that rule disagree on one shape that turns out to be common. 235 checkboxes in the corpus sit under a plain bullet used as a group label, and by the letter of the rule those have no checkbox ancestor, so a finished one would be dropped on its own. But section 4 already argues the opposite case for checkbox parents: a finished item under an unfinished group is context, not cruft, and a label like `- Trellis "Ergonomics" Fixes` is a group in every sense that matters. Resolved in favour of the rationale, which collapses both cases into one rule. A top-level bullet is any bullet with no bullet ancestor in its section. It is dropped, with its subtree, exactly when that subtree holds at least one checkbox and all of them are [x]. Prose bullets hold none and so are never eligible, which is the property section 4 wanted. Validated across all 228 notes with zero invariant failures: no heading, no unfinished item and no checkbox-free subtree is ever removed, and no group label is left with its children gone. 270 lines would be dropped across 118 files. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PU5ZFfQFtDTFqqhvMTWdGH
This commit is contained in:
Connor JohnstoneandClaude Opus 5 committed
1 parent
727813b5e9
commit
185b45dbfc
4 files changed
+296
No files matched your search
Binary file not shown.
Executable
+127
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prune completed work out of a daily note.
|
||||
|
||||
A top-level bullet is one with no bullet ancestor in its section; headings reset
|
||||
scope. Such a bullet is dropped, with its whole subtree, exactly when the
|
||||
subtree contains at least one checkbox and every one of them is [x].
|
||||
|
||||
That single rule covers both shapes that occur in practice. `- [x] Climb` is a
|
||||
subtree of one and goes. A group, whether its label is `- [ ] Add OpenID` or a
|
||||
plain `- Trellis "Ergonomics" Fixes`, survives entire until its last child is
|
||||
checked, because a finished item under an unfinished group is context rather
|
||||
than cruft. Prose bullets hold no checkboxes and so are never eligible at all.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
|
||||
BULLET = re.compile(r"^(\s*)([-*+])\s+(?:\[(.)\]\s+)?(.*)$")
|
||||
HEADING = re.compile(r"^#{1,6}\s")
|
||||
FENCE = re.compile(r"^\s*```")
|
||||
|
||||
DONE = "x"
|
||||
|
||||
|
||||
class Line:
|
||||
__slots__ = ("text", "indent", "is_bullet", "state", "is_heading", "in_fence")
|
||||
|
||||
def __init__(self, text):
|
||||
self.text = text
|
||||
self.is_heading = bool(HEADING.match(text))
|
||||
self.in_fence = False
|
||||
m = BULLET.match(text) if text.strip() else None
|
||||
if m:
|
||||
self.is_bullet = True
|
||||
self.indent = len(m.group(1))
|
||||
self.state = (m.group(3) or "").lower() or None
|
||||
else:
|
||||
self.is_bullet = False
|
||||
self.indent = len(text) - len(text.lstrip())
|
||||
self.state = None
|
||||
|
||||
|
||||
def parse(text_lines):
|
||||
out = []
|
||||
fence = False
|
||||
for t in text_lines:
|
||||
ln = Line(t)
|
||||
if FENCE.match(t):
|
||||
fence = not fence
|
||||
ln.in_fence = True
|
||||
elif fence:
|
||||
ln.in_fence = True
|
||||
if ln.in_fence:
|
||||
ln.is_bullet = False
|
||||
ln.is_heading = False
|
||||
out.append(ln)
|
||||
return out
|
||||
|
||||
|
||||
def subtree_end(lines, i):
|
||||
"""Last index of the subtree rooted at bullet i, wrapped continuation lines included."""
|
||||
base = lines[i].indent
|
||||
last = i
|
||||
j = i + 1
|
||||
while j < len(lines):
|
||||
ln = lines[j]
|
||||
if ln.in_fence or ln.is_heading:
|
||||
break
|
||||
if not ln.text.strip():
|
||||
j += 1
|
||||
continue
|
||||
if ln.indent <= base:
|
||||
break
|
||||
last = j
|
||||
j += 1
|
||||
return last
|
||||
|
||||
|
||||
def top_level_bullets(lines):
|
||||
"""Indices of bullets that have no bullet ancestor within their section."""
|
||||
stack = []
|
||||
for i, ln in enumerate(lines):
|
||||
if ln.in_fence:
|
||||
continue
|
||||
if ln.is_heading:
|
||||
stack = []
|
||||
continue
|
||||
if not ln.is_bullet:
|
||||
continue
|
||||
while stack and stack[-1] >= ln.indent:
|
||||
stack.pop()
|
||||
if not stack:
|
||||
yield i
|
||||
stack.append(ln.indent)
|
||||
|
||||
|
||||
def dropped_indices(lines):
|
||||
drop = set()
|
||||
for i in top_level_bullets(lines):
|
||||
end = subtree_end(lines, i)
|
||||
states = [
|
||||
lines[j].state
|
||||
for j in range(i, end + 1)
|
||||
if lines[j].is_bullet and lines[j].state is not None
|
||||
]
|
||||
if states and all(s == DONE for s in states):
|
||||
drop.update(range(i, end + 1))
|
||||
return drop
|
||||
|
||||
|
||||
def prune(text):
|
||||
lines = parse(text.split("\n"))
|
||||
drop = dropped_indices(lines)
|
||||
return "\n".join(l.text for k, l in enumerate(lines) if k not in drop)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("file", nargs="?", help="note to prune, default stdin")
|
||||
a = ap.parse_args()
|
||||
src = open(a.file).read() if a.file else sys.stdin.read()
|
||||
sys.stdout.write(prune(src))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Cases the prune rule has to get right, including ones the corpus lacks."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from prune import prune
|
||||
|
||||
|
||||
class TestPrune(unittest.TestCase):
|
||||
def assertPrune(self, src, want):
|
||||
self.assertEqual(prune(src.strip("\n")), want.strip("\n"))
|
||||
|
||||
def test_completed_top_level_item_goes(self):
|
||||
self.assertPrune("## To-Do\n- [x] Climb\n- [ ] Finish the backpack",
|
||||
"## To-Do\n- [ ] Finish the backpack")
|
||||
|
||||
def test_finished_child_of_unfinished_parent_stays(self):
|
||||
src = ("## To-Do\n- [ ] Add OpenID to any apps that I control\n"
|
||||
" - [x] Runway\n - [ ] Shanty")
|
||||
self.assertPrune(src, src)
|
||||
|
||||
def test_completed_checkbox_group_goes_entirely(self):
|
||||
self.assertPrune("## To-Do\n- [x] Add OpenID\n - [x] Runway\n - [x] Shanty\n- [ ] Keep",
|
||||
"## To-Do\n- [ ] Keep")
|
||||
|
||||
def test_plain_group_label_protects_finished_children(self):
|
||||
src = ('## To-Do\n- Trellis "Ergonomics" Fixes\n'
|
||||
" - [x] Build up a justfile\n - [ ] Add in some unit tests")
|
||||
self.assertPrune(src, src)
|
||||
|
||||
def test_plain_group_label_goes_when_group_completes(self):
|
||||
self.assertPrune('## To-Do\n- Trellis "Ergonomics" Fixes\n'
|
||||
" - [x] Build up a justfile\n - [x] Add in some unit tests\n- [ ] Keep",
|
||||
"## To-Do\n- [ ] Keep")
|
||||
|
||||
def test_prose_bullets_are_never_touched(self):
|
||||
src = ("### DSA Election Research Tool\n - Work on android rust app\n"
|
||||
" - Move bike components that I want over")
|
||||
self.assertPrune(src, src)
|
||||
|
||||
def test_headings_survive_an_emptied_section(self):
|
||||
self.assertPrune("## To-Do\n- [x] Climb\n\n## Personal\n- [ ] Keep",
|
||||
"## To-Do\n\n## Personal\n- [ ] Keep")
|
||||
|
||||
def test_question_state_is_not_complete(self):
|
||||
src = "## To-Do\n - [?] Cincinnati Trash"
|
||||
self.assertPrune(src, src)
|
||||
|
||||
def test_question_state_blocks_its_group(self):
|
||||
src = "## To-Do\n- [x] Parent\n - [?] Unsure"
|
||||
self.assertPrune(src, src)
|
||||
|
||||
def test_wrapped_continuation_travels_with_its_item(self):
|
||||
self.assertPrune("## To-Do\n- [x] Could be good to do a review of the band-aids\n"
|
||||
" and instabilities\n- [ ] Keep",
|
||||
"## To-Do\n- [ ] Keep")
|
||||
|
||||
def test_headings_reset_scope(self):
|
||||
# The [x] under Personal is top-level despite following a deeper bullet.
|
||||
self.assertPrune("## Work\n - [ ] Deep unfinished\n## Personal\n- [x] Climb",
|
||||
"## Work\n - [ ] Deep unfinished\n## Personal")
|
||||
|
||||
def test_checkboxes_inside_a_code_fence_are_inert(self):
|
||||
src = "## Notes\n```\n- [x] not a real item\n```\n- [ ] Keep"
|
||||
self.assertPrune(src, src)
|
||||
|
||||
def test_two_space_base_indent_corpus_style(self):
|
||||
self.assertPrune("## Work\n - [ ] Letter of Determination\n - [x] Check e-mails",
|
||||
"## Work\n - [ ] Letter of Determination")
|
||||
|
||||
def test_date_heading_and_blank_structure_survive(self):
|
||||
self.assertPrune("# 2026-09-09\n\n## To-Do\n- [x] Climb\n\n# Notes\n",
|
||||
"# 2026-09-09\n\n## To-Do\n\n# Notes\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the prune across every historical note and assert it never misbehaves.
|
||||
|
||||
The prune is the piece of this system carrying real semantic risk, and 228 real
|
||||
notes exist to test it against, so this checks invariants rather than relying on
|
||||
someone eyeballing a diff.
|
||||
"""
|
||||
|
||||
import glob
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from prune import DONE, dropped_indices, parse, subtree_end, top_level_bullets
|
||||
|
||||
NOTES = os.path.expanduser("~/docs/notes/todo")
|
||||
|
||||
|
||||
def audit(path):
|
||||
lines = parse(open(path).read().split("\n"))
|
||||
drop = dropped_indices(lines)
|
||||
errs = []
|
||||
|
||||
for i in sorted(drop):
|
||||
ln = lines[i]
|
||||
if ln.is_heading:
|
||||
errs.append(f"removed a heading: {ln.text!r}")
|
||||
elif ln.is_bullet and ln.state is not None and ln.state != DONE:
|
||||
errs.append(f"removed an unfinished item [{ln.state}]: {ln.text!r}")
|
||||
|
||||
# A dropped subtree must contain at least one checkbox, or it was prose.
|
||||
for i in top_level_bullets(lines):
|
||||
if i not in drop:
|
||||
continue
|
||||
end = subtree_end(lines, i)
|
||||
if not any(
|
||||
lines[j].is_bullet and lines[j].state is not None
|
||||
for j in range(i, end + 1)
|
||||
):
|
||||
errs.append(f"dropped a checkbox-free subtree: {lines[i].text!r}")
|
||||
|
||||
# A surviving group label must not have lost all of its children.
|
||||
for i, ln in enumerate(lines):
|
||||
if i in drop or not ln.is_bullet or ln.state is not None:
|
||||
continue
|
||||
kids = [j for j in range(i + 1, subtree_end(lines, i) + 1) if lines[j].is_bullet]
|
||||
if kids and all(j in drop for j in kids):
|
||||
errs.append(f"left an emptied group label: {ln.text!r}")
|
||||
|
||||
return lines, drop, errs
|
||||
|
||||
|
||||
def main():
|
||||
files = sorted(glob.glob(os.path.join(NOTES, "*.md")))
|
||||
if not files:
|
||||
sys.exit(f"no notes found under {NOTES}")
|
||||
|
||||
failures = changed = dropped = 0
|
||||
kinds = {}
|
||||
for f in files:
|
||||
lines, drop, errs = audit(f)
|
||||
if errs:
|
||||
failures += len(errs)
|
||||
print(f"FAIL {os.path.basename(f)}")
|
||||
for e in errs[:4]:
|
||||
print(" ", e)
|
||||
if drop:
|
||||
changed += 1
|
||||
dropped += len(drop)
|
||||
for i in drop:
|
||||
ln = lines[i]
|
||||
k = (
|
||||
f"checkbox[{ln.state}]"
|
||||
if ln.is_bullet and ln.state
|
||||
else "bullet" if ln.is_bullet else "continuation"
|
||||
)
|
||||
kinds[k] = kinds.get(k, 0) + 1
|
||||
|
||||
print()
|
||||
print(f"files : {len(files)} ({changed} would change)")
|
||||
print(f"lines dropped : {dropped}")
|
||||
for k, v in sorted(kinds.items(), key=lambda x: -x[1]):
|
||||
print(f" {k:>14} : {v}")
|
||||
print(f"invariant failures: {failures}")
|
||||
return 1 if failures else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user