Add tomd tool

Author: vinniefalco

paperworks/lib/inventory.py

@@ -128,7 +128,7 @@ def scan_markdown_dirs(watch_dirs):
     Returns dict keyed by normalized doc_number.
     """
     papers = {}
-    for entry in watch_dirs:
+    for dir_idx, entry in enumerate(watch_dirs, 1):
         if not entry.get("enabled", True):
             continue
         dirpath = Path(entry["path"])
@@ -167,6 +167,7 @@ def scan_markdown_dirs(watch_dirs):
                 "brutal_summary": brutal,
                 "md_path": str(md_path),
                 "md_mtime": md_path.stat().st_mtime,
+                "folder_idx": dir_idx,
             }
     return papers
 
@@ -323,6 +324,7 @@ def build_inventory(watch_dirs, output_dir, remote_papers=None):
             "stale_pdf": stale_pdf,
             "stale_remote_meta": stale_remote_meta,
             "warnings": warnings,
+            "folder_idx": md.get("folder_idx") if md else None,
         }
 
     # Group by base, keep only latest revision, attach prior revisions

paperworks/lib/templates/index.html

@@ -224,6 +224,7 @@
     <div class="table-wrap" id="table-wrap">
       <table>
         <thead><tr>
+          <th style="width:24px;text-align:center;cursor:pointer" onclick="toggleFolderSort()" title="Sort by folder">#</th>
           <th style="width:95px">Document</th>
           <th style="width:30px"></th>
           <th style="width:34px"></th>
@@ -345,7 +346,7 @@ <h2>Activity Log</h2>
 <div id="toast"></div>
 
 <script>
-let _papers=[], _filter='', _workingSet=new Set(), _dirtySet=new Set(), _queueDepth=0, _expandedIdx=null;
+let _papers=[], _filter='', _workingSet=new Set(), _dirtySet=new Set(), _queueDepth=0, _expandedIdx=null, _folderSort=false;
 
 // -- Tabs --
 function showTab(name) {
@@ -401,15 +402,18 @@ <h2>Activity Log</h2>
 
 // -- Papers --
 
+function toggleFolderSort(){_folderSort=!_folderSort;renderPapers();}
 function renderPapers() {
   const tbody=document.getElementById('tbody'), empty=document.getElementById('empty'), tw=document.getElementById('table-wrap');
   const ft=_filter.toLowerCase();
   const indexed=_papers.map((p,i)=>({p,i})).filter(({p})=>!ft||[p.doc_number,p.title,p.audience].filter(Boolean).join(' ').toLowerCase().includes(ft));
+  if(_folderSort) indexed.sort((a,b)=>{const fa=a.p.folder_idx||999,fb=b.p.folder_idx||999;return fa!==fb?fa-fb:0;});
   if(!indexed.length){tbody.innerHTML='';empty.style.display='block';tw.style.display='none';return;}
   empty.style.display='none';tw.style.display='block';
   tbody.innerHTML=indexed.map(({p,i:idx})=>{
     const dirty=p.doc_number&&p.pdf_path&&_dirtySet.has(p.doc_number)?'<span class="dirty-dot"></span>':'';
     return `<tr onclick="toggleDetail(${idx})" data-idx="${idx}">
+      <td class="mono muted" style="width:24px;text-align:center;font-size:10px">${esc(''+(p.folder_idx||''))}</td>
       <td class="mono" style="padding-right:2px">${esc(p.doc_number)||'-'}${dirty}</td>
       <td style="width:30px;text-align:center;padding-left:0;padding-right:4px">${p.md_path?`<a class="file-btn file-btn-md" href="/api/file?path=${encodeURIComponent(p.md_path)}" target="_blank" onclick="event.stopPropagation()" title="Open Markdown">MD</a>`:''}</td>
       <td style="width:34px;text-align:center;padding-left:0;padding-right:4px">${p.pdf_path?`<a class="file-btn file-btn-pdf" href="/api/file?path=${encodeURIComponent(p.pdf_path)}" target="_blank" onclick="event.stopPropagation()" title="Open PDF">PDF</a>`:''}</td>
@@ -444,7 +448,7 @@ <h2>Activity Log</h2>
   let acts='';
   if(p.md_path) acts+=`<button class="btn btn-blue${wk}" onclick="event.stopPropagation();renderSingle(${idx})"${dis}>RENDER</button>`;
   if(p.pdf_path&&p.remote) acts+=`<button class="btn btn-blue${wk}" onclick="event.stopPropagation();submitUpload(${idx})"${dis}>UPLOAD</button>`;
-  d.innerHTML=`<td colspan="6"><div class="detail-content"><div class="detail-grid">
+  d.innerHTML=`<td colspan="7"><div class="detail-content"><div class="detail-grid">
     <span class="dl">Document</span><span class="dv">${esc(p.doc_number)||'-'}</span>
     <span class="dl">Title</span><span class="dv">${esc(p.title)||'-'}</span>
     <span class="dl">Authors</span><span class="dv">${esc(p.authors)||'-'}</span>

tomd/.gitignore

@@ -0,0 +1,3 @@
+**/__pycache__/
+**/.pytest_cache/
+*.pyc

tomd/CLAUDE.md

@@ -0,0 +1,138 @@
+# tomd - Agent Rules
+
+## What This Is
+
+tomd is a hybrid PDF-to-Markdown converter. It uses deterministic text extraction and multi-signal classification to produce Markdown, with optional LLM resolution for ambiguous sections. HTML conversion is planned as a second converter.
+
+## Architecture
+
+Pipeline execution order:
+
+1. Per-page: dual extract (MuPDF + spatial) + edge items + link collection
+2. Close document
+3. Readability check (early exit if garbage)
+4. Header/footer detection and stripping (both paths)
+5. Text cleanup: NBSP, whitespace, dehyphenation, cross-page join (both paths)
+6. Span normalization: snap bold/italic boundaries to word edges (both paths)
+7. Table detection from MuPDF block positions; exclude table regions from spatial
+8. Dual-path comparison -> Sections (confident or uncertain per page)
+9. Merge table sections into position
+10. Structure: metadata extraction, heading/list/paragraph classification, position-based list detection, paragraph merging, code block detection, language label stripping, nesting validation
+11. TOC stripping (fuzzy match against headings)
+12. Emit .md + optional .prompts.md
+
+## Multi-Signal Confidence (Critical)
+
+Never classify based on a single signal. Every structural decision (heading, paragraph, list, code, table) must consider all available signals and produce a confidence level.
+
+Available signals and their reliability:
+- **Section numbering** (highest) - dotted decimal numbers give unambiguous depth
+- **Font size** (high) - relative to the most common (body) size
+- **Font weight/style** (medium) - bold/italic flags from font metadata
+- **Known section names** (high for WG21) - `Abstract`, `References`, `Wording`, etc.
+- **Line geometry** (medium) - length, indentation, vertical gaps
+- **Dual-path agreement** (high) - MuPDF and spatial rules agree on boundaries
+
+When signals agree, confidence is high. When they disagree, flag for LLM review. Never silently pick one signal over another - the disagreement is the data.
+
+## Preserve All Metadata
+
+Never discard information from the PDF during extraction. Text is the primary output, but font size, font name, font flags, coordinates, and page boundaries are preserved as annotations. Downstream phases use this metadata for confidence scoring and LLM prompt context.
+
+Discard nothing. Use everything.
+
+## Dual Extraction Path
+
+Every PDF page is processed through two independent extraction paths:
+1. **MuPDF path** - `page.get_text("dict")` for MuPDF's block/line/span grouping
+2. **Spatial path** - `page.get_text("rawdict")` with four coordinate rules:
+   - Horizontal close -> same word
+   - Horizontal far -> word break
+   - Vertical close + left reset -> line continuation (same paragraph)
+   - Vertical far -> paragraph break
+
+Both produce the same intermediate format. Agreement = confident. Disagreement = uncertain. Never skip one path. The comparison is the confidence mechanism.
+
+When paths disagree: MuPDF version goes in the output (it's more battle-tested). Both versions go in the LLM prompt for reconciliation. The prompt must require all data verbatim - the LLM fixes structure, never content.
+
+## Heading Rules
+
+- Heading level is derived from section numbering depth: `2.1.3` = depth 3 = `####` (depth + 1 because `#` is reserved for the document title)
+- Font size provides an independent heading level estimate by ranking sizes larger than body
+- All signals are evaluated; confidence depends on agreement count
+- Nesting must be validated: no heading may skip more than one level deeper than its predecessor
+- When signals conflict, section number wins if present; font-size ranking wins otherwise at lower confidence
+- Known unnumbered sections (`Abstract`, `Revision History`, `References`, `Acknowledgements`, `Motivation`, `Wording`, `Proposed Wording`, `Design Decisions`) are top-level (`##`)
+- Title is the first non-metadata text block before any numbered section, must have font size larger than body
+
+## Honest Output
+
+The tool must never silently produce bad Markdown.
+
+- If a region is uncertain, emit the MuPDF version in the output marked with `<!-- tomd:uncertain:L{start}-L{end} -->`
+- The companion prompts file includes BOTH extraction versions, surrounding context, and all raw metadata
+- LLM prompts must require verbatim data preservation - the LLM fixes structure, never content
+- If no prompts file is needed (zero uncertain regions), don't write one.
+- High-confidence output should look like a human wrote the Markdown - proper heading nesting, unwrapped paragraph lines, correct list formatting, blank lines between blocks
+
+## Markdown Quality
+
+The output Markdown must be clean and readable:
+- Paragraphs are single unwrapped lines (no hard wraps from PDF line breaks)
+- One blank line between all block elements (paragraphs, headings, lists, code blocks)
+- Headings use ATX style (`##` not underlines)
+- Lists use the marker from the source when detectable (`-`, `*`, `1.`)
+- No trailing whitespace on lines
+- No redundant blank lines (max one between blocks)
+- Dehyphenate broken words across lines (`imple-` + `mentation` -> `implementation`)
+- Join paragraphs that span page breaks (no terminal punctuation + lowercase continuation = same paragraph)
+- Hyperlinks become `[text](url)` - only http, https, mailto schemes
+- WG21 metadata block becomes YAML front matter
+- Collapse multiple spaces, replace non-breaking spaces, normalize whitespace
+
+## LLM Integration (v2)
+
+Auto-resolution via `--llm` flag is deferred to v2. For v1, the tool produces a companion `.prompts.md` file that the user feeds to any LLM manually. The prompts file is plain Markdown - usable by any LLM, any interface.
+
+## File Map
+
+- `main.py` - CLI entry point. Argparse, glob expansion, output path logic, main(). No conversion logic.
+- `lib/__init__.py` - Package marker for shared library.
+- `lib/similarity.py` - Dual-algorithm string similarity (SequenceMatcher + Jaccard). Per-algorithm thresholds, 200-char circuit breaker. Format-agnostic.
+- `lib/toc.py` - Table of Contents detection. Matches section texts against known headings using fuzzy similarity. Bridges small gaps. Format-agnostic - no dependency on PDF types.
+- `lib/pdf/__init__.py` - Exports `convert_pdf()`. Orchestrates the full pipeline in order.
+- `lib/pdf/types.py` - Data classes (`Block`, `Span`, `Line`, `Section`, `PageEdgeItem`), enums (`Confidence`, `SectionKind`), named constants, precompiled regex, `is_readable()`.
+- `lib/pdf/extract.py` - Dual extraction: `extract_mupdf()` (dict API) and `extract_spatial()` (rawdict + four spatial rules). Link collection and attachment. Calls `classify_monospace` during span construction.
+- `lib/pdf/mono.py` - Triple-signal monospace detection. Font name decomposition (strip modifiers, split camelCase, check keywords), glyph width uniformity, glyph spacing uniformity.
+- `lib/pdf/cleanup.py` - Header/footer detection (top-3/bottom-3 edge items), repeating strip, span whitespace (NBSP, multi-space on non-mono), dehyphenation, cross-page join, zero-width char strip.
+- `lib/pdf/spans.py` - Span normalization. Snaps bold/italic style boundaries to word edges. Monospace exempt.
+- `lib/pdf/table.py` - Table detection from MuPDF block/line positions. Detects columnar layout (x-gap between lines), extracts as high-confidence TABLE sections, excludes table regions from spatial path.
+- `lib/pdf/structure.py` - Dual-path comparison, metadata extraction, heading intelligence (multi-signal), position-based list detection (x-coordinates), paragraph merging, code block detection, language label detection, nesting validation.
+- `lib/pdf/emit.py` - Markdown generation (headings, paragraphs, code blocks, tables, nested lists) with span-level formatting (inline code, bold, italic, links). Prompts file generation for uncertain regions.
+
+## Header/Footer Stripping
+
+Before dual extraction, scan all pages for repeating content at page edges.
+
+- For each page, capture the top 3 and bottom 3 text items by y-coordinate
+- Compare across pages: same text at same y on 50%+ of pages = repeating = strip
+- Page numbers: same y, content is a bare number or "Page N" or "N of M" = strip
+- Running doc numbers: same y, content matches document number pattern = strip
+- Strip these items from page data before extraction runs. They are not content.
+
+## Text Cleanup Rules
+
+- **Dehyphenation**: line ends with `-`, next line starts lowercase -> join word, remove hyphen. Skip known compound prefixes (`self-`, `non-`, `well-`, `cross-`).
+- **Cross-page join**: last block on page N has no terminal punctuation, first block on page N+1 starts lowercase -> same paragraph, join.
+- **Link extraction**: collected during Phase 2 via `page.get_links()`, matched to text by bounding rect -> `[text](url)`. Only http/https/mailto.
+- **Whitespace**: collapse runs, replace non-breaking spaces, strip trailing.
+- **WG21 metadata**: Document Number / Date / Reply-to / Audience at top of page 1 -> YAML front matter.
+
+## tomd-Specific Extensions
+
+These extend general rules in the root CLAUDE.md with project-specific instances.
+
+- `fitz.open()` must always be paired with `doc.close()` in a `finally` block. Never rely on garbage collection.
+- Font metadata thresholds (what counts as "larger than body," "horizontal close," etc.) must be named constants, not magic numbers scattered in code.
+- The four spatial rules are the foundation. Changes to their thresholds affect everything downstream. Test thoroughly.
+- Regex patterns for section numbers, known section names, list markers, and metadata fields must be precompiled at module level and defined in one place.