finalize output - remove duplicated paragraphs

Author: lfoppiano

Readme.md

@@ -30,6 +30,7 @@ A simple, efficient Python client for [GROBID](https://github.com/kermitt2/grobi
 - **Command Line & Library**: Use as a standalone CLI tool or import into your Python projects
 - **Coordinate Extraction**: Optional PDF coordinate extraction for precise element positioning
 - **Sentence Segmentation**: Layout-aware sentence segmentation capabilities
+- **JSON Output**: Convert TEI XML output to structured JSON format with CORD-19-like structure
 
 ## 📋 Prerequisites
 
@@ -40,6 +41,7 @@ A simple, efficient Python client for [GROBID](https://github.com/kermitt2/grobi
   - Default server: `http://localhost:8070`
   - Online demo: https://lfoppiano-grobid.hf.space (usage limits apply), more details [here](https://grobid.readthedocs.io/en/latest/getting_started/#using-grobid-from-the-cloud).
 
+
 > [!IMPORTANT]
 > GROBID supports Windows only through Docker containers. See the [Docker documentation](https://grobid.readthedocs.io/en/latest/Grobid-docker/) for details.
 
@@ -131,6 +133,7 @@ grobid_client [OPTIONS] SERVICE
 | `--teiCoordinates` | Add PDF coordinates to XML |
 | `--segmentSentences` | Segment sentences with coordinates |
 | `--flavor` | Processing flavor for fulltext extraction |
+| `--json` | Convert TEI output to JSON format |
 
 #### Examples
 
@@ -141,11 +144,14 @@ grobid_client --input ~/documents --output ~/results processFulltextDocument
 # High concurrency with coordinates
 grobid_client --input ~/pdfs --output ~/tei --n 20 --teiCoordinates processFulltextDocument
 
+# Process with JSON output
+grobid_client --input ~/pdfs --output ~/results --json processFulltextDocument
+
 # Process citations with custom server
 grobid_client --server https://grobid.example.com --input ~/citations.txt processCitationList
 
-# Force reprocessing with sentence segmentation
-grobid_client --input ~/docs --force --segmentSentences processFulltextDocument
+# Force reprocessing with sentence segmentation and JSON output
+grobid_client --input ~/docs --force --segmentSentences --json processFulltextDocument
 ```
 
 ### Python Library
@@ -188,6 +194,14 @@ client.process(
     segmentSentences=True
 )
 
+# Process with JSON output
+client.process(
+    service="processFulltextDocument",
+    input_path="/path/to/pdfs",
+    output_path="/path/to/output",
+    json_output=True
+)
+
 # Process citation lists
 client.process(
     service="processCitationList",
@@ -234,6 +248,87 @@ Extracts complete document structure including headers, body text, figures, tabl
 grobid_client --input pdfs/ --output results/ processFulltextDocument
 ```
 
+### JSON Output Format
+
+When using the `--json` flag, the client converts TEI XML output to a structured JSON format similar to CORD-19. This provides:
+
+- **Structured Bibliography**: Title, authors, DOI, publication date, journal information
+- **Body Text**: Paragraphs and sentences with metadata and reference annotations
+- **Figures and Tables**: Structured JSON format for tables with headers, rows, and metadata
+- **Reference Information**: In-text citations with offsets and targets
+
+#### JSON Structure
+
+```json
+{
+  "level": "paragraph",
+  "biblio": {
+    "title": "Document Title",
+    "authors": ["Author 1", "Author 2"],
+    "doi": "10.1000/example",
+    "publication_date": "2023-01-01",
+    "journal": "Journal Name",
+    "abstract": [...]
+  },
+  "body_text": [
+    {
+      "id": "p_12345",
+      "text": "Paragraph text with citations [1].",
+      "head_section": "Introduction",
+      "refs": [
+        {
+          "type": "bibr",
+          "target": "b1",
+          "text": "[1]",
+          "offset_start": 25,
+          "offset_end": 28
+        }
+      ]
+    }
+  ],
+  "figures_and_tables": [
+    {
+      "id": "table_1",
+      "type": "table",
+      "label": "Table 1",
+      "head": "Sample Data",
+      "content": {
+        "headers": ["Header 1", "Header 2"],
+        "rows": [["Value 1", "Value 2"]],
+        "metadata": {
+          "row_count": 1,
+          "column_count": 2,
+          "has_headers": true
+        }
+      }
+    }
+  ]
+}
+```
+
+#### Usage Examples
+
+```bash
+# Generate both TEI and JSON outputs
+grobid_client --input pdfs/ --output results/ --json processFulltextDocument
+
+# JSON output with coordinates and sentence segmentation
+grobid_client --input pdfs/ --output results/ --json --teiCoordinates --segmentSentences processFulltextDocument
+```
+
+```python
+# Python library usage
+client.process(
+    service="processFulltextDocument",
+    input_path="/path/to/pdfs",
+    output_path="/path/to/output",
+    json_output=True
+)
+```
+
+> [!NOTE]
+> When using `--json`, the `--force` flag only checks for existing TEI files. If a TEI file is rewritten (due to `--force`), the corresponding JSON file is automatically rewritten as well.
+
 ### Header Document Processing
 Extracts only document metadata (title, authors, abstract, etc.).
 

grobid_client/format/TEI2LossyJSON.py

@@ -181,7 +181,7 @@ def convert_tei_file(self, tei_file: Union[Path, BinaryIO], stream: bool = False
                         head = item.head
                         label = item.label
                         if item.has_attr("type") and item.attrs["type"] == "table":
-                            content = xml_table_to_markdown(item.table) if item.table else None
+                            json_content = xml_table_to_json(item.table) if item.table else None
                             note = item.note
                             figures_and_tables.append(
                                 {
@@ -190,7 +190,7 @@ def convert_tei_file(self, tei_file: Union[Path, BinaryIO], stream: bool = False
                                     "head": head.text if head else "",
                                     "type": "table",
                                     "desc": desc.text if desc else "",
-                                    "content": content,
+                                    "content": json_content,
                                     "note": note.text if note else "",
                                     "coords": [
                                         box_to_dict(coord.split(","))
@@ -228,35 +228,46 @@ def _iter_passages_from_soup(self, soup: BeautifulSoup, passage_level: str) -> I
 
     def _iter_passages_from_soup_for_text(self, text_node: Tag, passage_level: str) -> Iterator[Dict[str, Union[str, Dict[str, str]]]]:
         head_paragraph = None
-        div_nodes = text_node.find_all("div")
-        for id_div, div in enumerate(div_nodes):
-            head = div.find("head")
-            p_nodes = div.find_all("p")
-            head_section = None
-
-            if head:
-                if len(p_nodes) == 0:
-                    head_paragraph = head.text
+        
+        # Process body and back sections, but only their direct child divs
+        for section in text_node.find_all(['body', 'back']):
+            # Only get direct child divs of this section
+            div_nodes = [child for child in section.children if hasattr(child, 'name') and child.name == "div"]
+            
+            for id_div, div in enumerate(div_nodes):
+                head = div.find("head")
+                p_nodes = div.find_all("p")
+                head_section = None
+
+                if head:
+                    if len(p_nodes) == 0:
+                        head_paragraph = head.text
+                    else:
+                        head_section = head.text
                 else:
-                    head_section = head.text
-
-            for id_p, p in enumerate(p_nodes):
-                paragraph_id = get_random_id(prefix="p_")
-                if passage_level == "sentence":
-                    for id_s, sentence in enumerate(p.find_all("s")):
-                        struct = get_formatted_passage(head_paragraph, head_section, paragraph_id, sentence)
+                    # If no head element, try to use the type attribute as head_section
+                    div_type = div.get("type")
+                    if div_type and len(p_nodes) > 0:
+                        head_section = div_type.title()  # Capitalize first letter
+
+                for id_p, p in enumerate(p_nodes):
+                    paragraph_id = get_random_id(prefix="p_")
+                    
+                    if passage_level == "sentence":
+                        for id_s, sentence in enumerate(p.find_all("s")):
+                            struct = get_formatted_passage(head_paragraph, head_section, paragraph_id, sentence)
+                            if self.validate_refs:
+                                for ref in struct['refs']:
+                                    assert ref['offset_start'] < ref['offset_end']
+                                    assert struct['text'][ref['offset_start']:ref['offset_end']] == ref['text']
+                            yield struct
+                    else:
+                        struct = get_formatted_passage(head_paragraph, head_section, paragraph_id, p)
                         if self.validate_refs:
                             for ref in struct['refs']:
                                 assert ref['offset_start'] < ref['offset_end']
                                 assert struct['text'][ref['offset_start']:ref['offset_end']] == ref['text']
                         yield struct
-                else:
-                    struct = get_formatted_passage(head_paragraph, head_section, paragraph_id, p)
-                    if self.validate_refs:
-                        for ref in struct['refs']:
-                            assert ref['offset_start'] < ref['offset_end']
-                            assert struct['text'][ref['offset_start']:ref['offset_end']] == ref['text']
-                    yield struct
 
     def process_directory(self, directory: Union[str, Path], pattern: str = "*.tei.xml", parallel: bool = True, workers: int = None) -> Iterator[Dict]:
         """Process a directory of TEI files and yield converted documents.
@@ -289,7 +300,7 @@ def _convert_file_worker(path: str):
     from bs4 import BeautifulSoup
     import dateparser
     # Reuse existing top-level helpers from this module by importing here
-    from grobid_client.format.TEI2LossyJSON import box_to_dict, get_random_id, get_formatted_passage, get_refs_with_offsets, xml_table_to_markdown
+    from grobid_client.format.TEI2LossyJSON import box_to_dict, get_random_id, get_formatted_passage, get_refs_with_offsets, xml_table_to_markdown, xml_table_to_json
     content = open(path, 'r').read()
     soup = BeautifulSoup(content, 'xml')
     converter = TEI2LossyJSONConverter()
@@ -377,6 +388,56 @@ def xml_table_to_markdown(table_element):
     return "\n".join(markdown_lines) if markdown_lines else None
 
 
+def xml_table_to_json(table_element):
+    """Convert XML table to JSON format."""
+    if not table_element:
+        return None
+    
+    table_data = {
+        "headers": [],
+        "rows": [],
+        "metadata": {}
+    }
+    
+    # Check if table has a header row (thead)
+    thead = table_element.find("thead")
+    if thead:
+        header_row = thead.find("row")
+        if header_row:
+            for cell in header_row.find_all("cell"):
+                cell_text = cell.get_text().strip()
+                table_data["headers"].append(cell_text)
+    
+    # Process table body rows
+    tbody = table_element.find("tbody")
+    if tbody:
+        rows = tbody.find_all("row")
+    else:
+        # If no tbody, get all rows
+        rows = table_element.find_all("row")
+        # Skip first row if we already processed it as header
+        if thead and rows:
+            rows = rows[1:]
+    
+    for row in rows:
+        row_data = []
+        for cell in row.find_all("cell"):
+            cell_text = cell.get_text().strip()
+            row_data.append(cell_text)
+        
+        if row_data:
+            table_data["rows"].append(row_data)
+    
+    # Add metadata
+    table_data["metadata"] = {
+        "row_count": len(table_data["rows"]),
+        "column_count": len(table_data["headers"]) if table_data["headers"] else (len(table_data["rows"][0]) if table_data["rows"] else 0),
+        "has_headers": len(table_data["headers"]) > 0
+    }
+    
+    return table_data if table_data["rows"] else None
+
+
 # Backwards compatible top-level function that uses the class
 def convert_tei_file(tei_file: Union[Path, BinaryIO], stream: bool = False):
     converter = TEI2LossyJSONConverter()

grobid_client/grobid_client.py

@@ -506,6 +506,7 @@ def process_batch(
 
                             if json_data:
                                 json_filename = filename.replace('.grobid.tei.xml', '.json')
+                                # Always write JSON file when TEI is written (respects --force behavior)
                                 with open(json_filename, 'w', encoding='utf8') as json_file:
                                     json.dump(json_data, json_file, indent=2, ensure_ascii=False)
                                 self.logger.debug(f"Successfully wrote JSON file: {json_filename}")