De-duplicate repeated content sections (0.9.19)

Sparse or hand-drawn posters can make the model loop and emit the same
section several times (e.g. three identical untitled "than you might
think!" sections on the Turing Way sketchnote poster), and the raw-text
recovery step could re-add an already-present fragment. Post-processing
now collapses sections with identical normalized content to one entry,
keeping the first occurrence and preferring a titled copy. Distinct
sections are unaffected.

Author: jimnoneill

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.9.19] - 2026-06-10
+
+### Fixed
+
+- **Duplicate content sections are now de-duplicated.** On sparse or hand-drawn posters the model can loop and emit the same section several times (observed as three identical untitled "than you might think!" sections on the Turing Way sketchnote poster), and the raw-text recovery step could re-add a fragment that already appeared. Post-processing now collapses sections with identical normalized content (case- and whitespace-insensitive) to a single entry, keeping the first occurrence and preferring a copy that carries a title. Distinct sections are unaffected.
+
 ## [0.9.18] - 2026-06-10
 
 ### Fixed

poster2json/extract.py

@@ -2349,6 +2349,26 @@ def _postprocess_json(
                         "sectionContent": blob,
                     })
 
+            # Drop duplicate sections. The model can repeat a section verbatim
+            # (a generation loop, common on sparse or hand-drawn posters), and
+            # the raw-text recovery above can re-add a fragment that already
+            # appears several times. De-dupe by normalized content, keeping the
+            # first occurrence and preferring a copy that carries a title.
+            deduped = []
+            seen_by_content = {}
+            for sec in cleaned_sections:
+                ckey = " ".join(sec.get("sectionContent", "").lower().split())
+                if not ckey:
+                    continue
+                if ckey in seen_by_content:
+                    idx = seen_by_content[ckey]
+                    if not deduped[idx].get("sectionTitle") and sec.get("sectionTitle"):
+                        deduped[idx] = sec
+                    continue
+                seen_by_content[ckey] = len(deduped)
+                deduped.append(sec)
+            cleaned_sections = deduped
+
             result["content"]["sections"] = cleaned_sections
 
     # Build sections from raw text when LLM produced none

pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 
 name = "poster2json"
-version = "0.9.18"
+version = "0.9.19"
 description = "Convert scientific posters (PDF/images) to structured JSON metadata using Large Language Models"
 
 packages = [{ include = "poster2json" }]

tests/test_normalize.py

@@ -478,6 +478,47 @@ def test_creator_nametype_personal_vs_organizational():
     assert out["creators"][1]["nameType"] == "Organizational"
 
 
+def test_postprocess_dedupes_repeated_sections():
+    from poster2json.extract import _postprocess_json
+
+    # The model looped and emitted the same fragment as three untitled sections
+    # (the TTW sketchnote-poster case): they must collapse to one.
+    data = {"content": {"sections": [
+        {"sectionTitle": "", "sectionContent": "than you might think!"},
+        {"sectionTitle": "", "sectionContent": "than you might think!"},
+        {"sectionTitle": "", "sectionContent": "than you might think!"},
+    ]}}
+    out = _postprocess_json(data, raw_text="")
+    secs = out["content"]["sections"]
+    assert len(secs) == 1
+    assert secs[0]["sectionContent"] == "than you might think!"
+
+
+def test_postprocess_dedup_prefers_titled_section():
+    from poster2json.extract import _postprocess_json
+
+    # Same content appears untitled then titled; keep the titled copy.
+    data = {"content": {"sections": [
+        {"sectionContent": "shared body text content goes here"},
+        {"sectionTitle": "Collaboration", "sectionContent": "shared body text content goes here"},
+    ]}}
+    out = _postprocess_json(data, raw_text="")
+    secs = out["content"]["sections"]
+    assert len(secs) == 1
+    assert secs[0].get("sectionTitle") == "Collaboration"
+
+
+def test_postprocess_keeps_distinct_sections():
+    from poster2json.extract import _postprocess_json
+
+    data = {"content": {"sections": [
+        {"sectionTitle": "Intro", "sectionContent": "first distinct section body"},
+        {"sectionTitle": "Methods", "sectionContent": "second distinct section body"},
+    ]}}
+    out = _postprocess_json(data, raw_text="")
+    assert len(out["content"]["sections"]) == 2
+
+
 def test_orcid_enrichment_uses_affiliation_resolver():
     from poster2json.orcid import enrich_creators_orcid