feat(doc): add stable elements[] output for unified element consumption

Phase 1: Add `elements[]` output - a unified indexed array combining
blocks, tables, and pages into a single stable machine-readable format.

- Add StructuredElementKind enum (Block, Table, Page)
- Add StructuredElement struct with index, kind, label, content, page,
  source, location, attributes, and structured_payload
- Add elements field to ParsedDocument
- Add build_elements() method to ParsedDocument
- Add extract_elements_for_output() for JSON serialization
- Include elements in parse result metadata

Author: RoyLin

core/src/doc/mod.rs

@@ -11,7 +11,7 @@ pub(crate) mod registry;
 pub use model::{
     DocumentBlock, DocumentBlockKind, DocumentBlockLocation, DocumentConfidence,
     DocumentExtractionMetadata, DocumentMetadata, DocumentProvenance, ExtractedDocument,
-    ParsedDocument,
+    ParsedDocument, StructuredElement, StructuredElementKind,
 };
 pub use parser::DocumentParser;
 pub use registry::DocumentParserRegistry;

core/src/doc/model.rs

@@ -242,6 +242,126 @@ impl PageInfo {
     }
 }
 
+/// Unified element kinds for structured element output.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub enum StructuredElementKind {
+    /// A document block (paragraph, heading, etc.)
+    Block,
+    /// A table element
+    Table,
+    /// A page element
+    Page,
+}
+
+impl StructuredElementKind {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            Self::Block => "block",
+            Self::Table => "table",
+            Self::Page => "page",
+        }
+    }
+}
+
+/// Unified element representation for stable machine-readable element output.
+/// Combines blocks, tables, and pages into a single indexed array for downstream consumption.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct StructuredElement {
+    /// Element index in the flattened elements array.
+    pub index: usize,
+    /// Element type for discrimination.
+    pub kind: StructuredElementKind,
+    /// Element label/name if available.
+    pub label: Option<String>,
+    /// Element content or summary.
+    pub content: String,
+    /// Page number if available.
+    pub page: Option<usize>,
+    /// Source/location reference.
+    pub source: Option<String>,
+    /// Location metadata.
+    pub location: Option<DocumentBlockLocation>,
+    /// Extended attributes for kind-specific data.
+    pub attributes: BTreeMap<String, String>,
+    /// Structured payload for tables and other structured content.
+    pub structured_payload: Option<String>,
+}
+
+impl StructuredElement {
+    /// Create a new structured element from a document block.
+    pub fn from_block(block: &DocumentBlock, index: usize) -> Self {
+        Self {
+            index,
+            kind: StructuredElementKind::Block,
+            label: block.label.clone(),
+            content: block.content.clone(),
+            page: block.location.as_ref().and_then(|l| l.page),
+            source: block.location.as_ref().and_then(|l| l.source.clone()),
+            location: block.location.clone(),
+            attributes: block.attributes.clone(),
+            structured_payload: block.structured_payload.clone(),
+        }
+    }
+
+    /// Create a new structured element from a structured table.
+    pub fn from_table(table: &StructuredTable, index: usize) -> Self {
+        let content = if table.rows.is_empty() {
+            String::new()
+        } else {
+            table
+                .rows
+                .iter()
+                .map(|row| row.join("\t"))
+                .collect::<Vec<_>>()
+                .join("\n")
+        };
+
+        let mut attributes = BTreeMap::new();
+        attributes.insert("row_count".to_string(), table.row_count.to_string());
+        attributes.insert("column_count".to_string(), table.column_count.to_string());
+
+        Self {
+            index,
+            kind: StructuredElementKind::Table,
+            label: table.label.clone(),
+            content,
+            page: table.page,
+            source: table.source.clone(),
+            location: table.location.clone(),
+            attributes,
+            structured_payload: None,
+        }
+    }
+
+    /// Create a new structured element from a page info.
+    pub fn from_page(page: &PageInfo, index: usize) -> Self {
+        Self {
+            index,
+            kind: StructuredElementKind::Page,
+            label: None,
+            content: page.preview.clone().unwrap_or_default(),
+            page: Some(page.page),
+            source: page.source.clone(),
+            location: None,
+            attributes: {
+                let mut attrs = BTreeMap::new();
+                attrs.insert("block_count".to_string(), page.block_count.to_string());
+                if page.continued_from_previous_page {
+                    attrs.insert(
+                        "continued_from_previous_page".to_string(),
+                        "true".to_string(),
+                    );
+                }
+                if page.continued_to_next_page {
+                    attrs.insert("continued_to_next_page".to_string(), "true".to_string());
+                }
+                attrs
+            },
+            structured_payload: None,
+        }
+    }
+}
+
 #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
 pub struct ParsedDocument {
     pub title: Option<String>,
@@ -251,6 +371,8 @@ pub struct ParsedDocument {
     pub tables: Vec<StructuredTable>,
     /// Page-level information for documents with page structure.
     pub pages: Vec<PageInfo>,
+    /// Unified elements array combining blocks, tables, and pages.
+    pub elements: Vec<StructuredElement>,
 }
 
 impl ParsedDocument {
@@ -269,6 +391,7 @@ impl ParsedDocument {
             metadata: None,
             tables: Vec::new(),
             pages: Vec::new(),
+            elements: Vec::new(),
         }
     }
 
@@ -290,6 +413,33 @@ impl ParsedDocument {
         self.blocks.len()
     }
 
+    /// Build the unified elements array from blocks, tables, and pages.
+    /// Elements are ordered: blocks first, then tables, then pages.
+    pub fn build_elements(&mut self) {
+        let mut elements = Vec::new();
+        let mut index = 0;
+
+        // Add blocks
+        for block in &self.blocks {
+            elements.push(StructuredElement::from_block(block, index));
+            index += 1;
+        }
+
+        // Add tables
+        for table in &self.tables {
+            elements.push(StructuredElement::from_table(table, index));
+            index += 1;
+        }
+
+        // Add pages
+        for page in &self.pages {
+            elements.push(StructuredElement::from_page(page, index));
+            index += 1;
+        }
+
+        self.elements = elements;
+    }
+
     pub fn non_empty_block_count(&self) -> usize {
         self.blocks
             .iter()

core/src/document/consume.rs

@@ -659,6 +659,7 @@ mod tests {
             llm_answer: Some("Key finding"),
             tables: &[],
             pages: &[],
+            elements: &[],
         };
 
         let built = build_parse_result(&input);
@@ -728,6 +729,7 @@ mod tests {
             llm_answer: None,
             tables: &[],
             pages: &[],
+            elements: &[],
         };
 
         let output = build_parse_tool_output(&input);

core/src/document/consume_parse.rs

@@ -201,6 +201,7 @@ pub(crate) struct PreparedParseDocument {
     pub structured_payloads: Vec<serde_json::Value>,
     pub tables: Vec<serde_json::Value>,
     pub pages: Vec<serde_json::Value>,
+    pub elements: Vec<serde_json::Value>,
     pub structural_summary: String,
     pub document_runtime: Option<serde_json::Value>,
     pub document_quality: Option<crate::document_pipeline::DocumentQualityReport>,
@@ -414,6 +415,7 @@ pub(crate) fn prepare_parse_document_from_path(
     let structured_payloads = summarize_structured_payloads(&raw_document, block_kind_label);
     let tables = extract_tables_for_output(&raw_document);
     let pages = summarize_pages_for_parse_report(&raw_document);
+    let elements = extract_elements_for_output(&raw_document);
     let structural_summary = build_structural_summary(
         &raw_document,
         crate::document_service_types::structural_summary_style_for_strategy_label(&strategy_label),
@@ -455,6 +457,7 @@ pub(crate) fn prepare_parse_document_from_path(
         structured_payloads,
         tables,
         pages,
+        elements,
         structural_summary,
         document_runtime,
         document_quality,
@@ -1242,6 +1245,7 @@ pub(crate) struct ParseResultInput<'a> {
     pub structured_payloads: &'a [serde_json::Value],
     pub tables: &'a [serde_json::Value],
     pub pages: &'a [serde_json::Value],
+    pub elements: &'a [serde_json::Value],
     pub max_chars: usize,
     pub structural_summary: &'a str,
     pub llm_answer: Option<&'a str>,
@@ -1315,6 +1319,7 @@ pub(crate) fn build_parse_result(input: &ParseResultInput<'_>) -> BuiltParseResu
         "structured_payloads": input.structured_payloads,
         "tables": input.tables,
         "pages": input.pages,
+        "elements": input.elements,
         "max_chars": input.max_chars,
         "query_aware_selection": input.query_used,
     });
@@ -1356,6 +1361,7 @@ pub(crate) fn build_parse_tool_output_from_prepared(
         structured_payloads: &prepared.structured_payloads,
         tables: &prepared.tables,
         pages: &prepared.pages,
+        elements: &prepared.elements,
         max_chars,
         structural_summary: &prepared.structural_summary,
         llm_answer,
@@ -1508,6 +1514,109 @@ fn extract_tables_for_output(doc: &ParsedDocument) -> Vec<serde_json::Value> {
         .collect()
 }
 
+/// Extract unified elements for stable machine-readable `elements[]` output.
+/// Combines blocks, tables, and pages into a single indexed array.
+fn extract_elements_for_output(doc: &ParsedDocument) -> Vec<serde_json::Value> {
+    use crate::doc::StructuredElementKind;
+
+    let mut elements = Vec::new();
+    let mut index = 0;
+
+    // Add blocks as elements
+    for block in &doc.blocks {
+        let location_display = block
+            .location
+            .as_ref()
+            .map(|loc| crate::document_render::format_block_location(loc))
+            .unwrap_or_default();
+
+        elements.push(json!({
+            "index": index,
+            "kind": "block",
+            "kind_detail": match block.kind {
+                DocumentBlockKind::Paragraph => "paragraph",
+                DocumentBlockKind::Heading => "heading",
+                DocumentBlockKind::Table => "table",
+                DocumentBlockKind::Section => "section",
+                DocumentBlockKind::Metadata => "metadata",
+                DocumentBlockKind::Slide => "slide",
+                DocumentBlockKind::EmailHeader => "email_header",
+                DocumentBlockKind::Code => "code",
+                DocumentBlockKind::Raw => "raw",
+            },
+            "label": block.label,
+            "content": block.content,
+            "page": block.location.as_ref().and_then(|l| l.page),
+            "source": block.location.as_ref().and_then(|l| l.source.clone()),
+            "location": block.location.as_ref().map(|location: &DocumentBlockLocation| json!({
+                "source": location.source,
+                "page": location.page,
+                "ordinal": location.ordinal,
+                "continued_from_previous_page": location.continued_from_previous_page,
+                "continued_to_next_page": location.continued_to_next_page,
+                "display": location_display,
+            })),
+            "attributes": block.attributes,
+            "structured_payload": block.structured_payload,
+        }));
+        index += 1;
+    }
+
+    // Add tables as elements
+    for table in &doc.tables {
+        elements.push(json!({
+            "index": index,
+            "kind": "table",
+            "kind_detail": "table",
+            "label": table.label,
+            "content": if table.rows.is_empty() {
+                String::new()
+            } else {
+                table.rows.iter().map(|row| row.join("\t")).collect::<Vec<_>>().join("\n")
+            },
+            "page": table.page,
+            "source": table.source,
+            "location": table.location.as_ref().map(|location: &DocumentBlockLocation| json!({
+                "source": location.source,
+                "page": location.page,
+                "ordinal": location.ordinal,
+                "continued_from_previous_page": location.continued_from_previous_page,
+                "continued_to_next_page": location.continued_to_next_page,
+                "display": crate::document_render::format_block_location(location),
+            })),
+            "attributes": {
+                "row_count": table.row_count,
+                "column_count": table.column_count,
+            },
+            "structured_payload": null,
+        }));
+        index += 1;
+    }
+
+    // Add pages as elements
+    for page in &doc.pages {
+        elements.push(json!({
+            "index": index,
+            "kind": "page",
+            "kind_detail": "page",
+            "label": null,
+            "content": page.preview.clone().unwrap_or_default(),
+            "page": page.page,
+            "source": page.source,
+            "location": null,
+            "attributes": {
+                "block_count": page.block_count,
+                "continued_from_previous_page": page.continued_from_previous_page,
+                "continued_to_next_page": page.continued_to_next_page,
+            },
+            "structured_payload": null,
+        }));
+        index += 1;
+    }
+
+    elements
+}
+
 fn summarize_pages_for_parse_report(doc: &ParsedDocument) -> Vec<serde_json::Value> {
     use std::collections::BTreeMap;
 

core/src/document/engine_tests.rs

@@ -128,6 +128,7 @@ fn build_parse_tool_output_preserves_strategy_metadata() {
             llm_answer: None,
             tables: &[],
             pages: &[],
+            elements: &[],
         },
     );
     assert!(output.success);
@@ -218,6 +219,7 @@ fn build_parse_tool_output_surfaces_structured_payloads() {
             llm_answer: None,
             tables: &[],
             pages: &[],
+            elements: &[],
         },
     );