core: record item source spans in CellItem

src/matcher.rs

@@ -358,6 +358,7 @@ fn process_atomic(
     col: usize,
     input_text: &str,
     item_index: usize,
+    span: (usize, usize),
     sem: &mut SemanticsCore,
 ) -> Result<(), SemErr> {
     if atomic.idd == Idd::Skip {
@@ -376,6 +377,7 @@ fn process_atomic(
         row,
         col,
         ty,
+        span,
     });
     for action_spec in &atomic.actions {
         let action = instantiate_action(ItemId::Cell(item_id), action_spec, sem)?;
@@ -384,19 +386,33 @@ fn process_atomic(
     Ok(())
 }
 
+/// Non-empty `java_trim`med parts of a literal split as
+/// (original part position, trimmed part, byte span in the original cell
+/// text); `base` is the offset of `text` within that cell text.
+fn split_with_spans(delim: &str, text: &str, base: usize) -> Vec<(usize, String, (usize, usize))> {
+    let mut out = Vec::new();
+    let mut start = 0usize;
+    for (i, part) in split_literal(delim, text).into_iter().enumerate() {
+        let trimmed = java_trim(&part);
+        if !trimmed.is_empty() {
+            let lead = part.len() - part.trim_start_matches(|c: char| c <= ' ').len();
+            let from = base + start + lead;
+            out.push((i, trimmed.to_string(), (from, from + trimmed.len())));
+        }
+        start += part.len() + delim.len();
+    }
+    out
+}
+
 fn process_delimited(
     d: &DelimitedSpec,
     row: usize,
     col: usize,
     text: &str,
     sem: &mut SemanticsCore,
 ) -> Result<(), SemErr> {
-    let parts = split_literal(&d.delimiter, text);
-    for (i, part) in parts.iter().enumerate() {
-        let part = java_trim(part);
-        if !part.is_empty() {
-            process_atomic(&d.atom, row, col, part, i, sem)?;
-        }
+    for (i, part, span) in split_with_spans(&d.delimiter, text, 0) {
+        process_atomic(&d.atom, row, col, &part, i, span, sem)?;
     }
     Ok(())
 }
@@ -444,16 +460,13 @@ fn process_compound(
         let substring = &text[pos..end_pos];
         match &seg.spec {
             ContentSpec::Atomic(a) => {
-                process_atomic(a, row, col, substring, item_index, sem)?;
+                process_atomic(a, row, col, substring, item_index, (pos, end_pos), sem)?;
                 item_index += 1;
             }
             ContentSpec::Delimited(d) => {
-                for part in split_literal(&d.delimiter, substring) {
-                    let trimmed = java_trim(&part);
-                    if !trimmed.is_empty() {
-                        process_atomic(&d.atom, row, col, trimmed, item_index, sem)?;
-                        item_index += 1;
-                    }
+                for (_, part, span) in split_with_spans(&d.delimiter, substring, pos) {
+                    process_atomic(&d.atom, row, col, &part, item_index, span, sem)?;
+                    item_index += 1;
                 }
             }
             _ => unreachable!("compound segment restricted to atomic/delimited"),
@@ -476,7 +489,8 @@ fn process_content_spec(
     match cs {
         ContentSpec::Atomic(a) => {
             let text = syntax.cell(row, col).text.clone();
-            process_atomic(a, row, col, &text, 0, sem)
+            let span = (0, text.len());
+            process_atomic(a, row, col, &text, 0, span, sem)
         }
         ContentSpec::Delimited(d) => {
             let text = syntax.cell(row, col).text.clone();

src/py.rs

@@ -85,6 +85,7 @@ pub fn call_item_filter(
             ty: it.ty,
             row: it.row,
             col: it.col,
+            span: it.span,
             table: table.as_ref().map(|t| t.clone_ref(py)),
         };
         let res = func
@@ -635,6 +636,7 @@ pub struct PyCellDerivedItem {
     pub ty: sp::ItemType,
     pub row: usize,
     pub col: usize,
+    pub span: (usize, usize),
     pub table: Option<Py<PyTableSyntax>>,
 }
 
@@ -644,6 +646,12 @@ impl PyCellDerivedItem {
     fn str(&self) -> String {
         self.s.clone()
     }
+    /// Byte range of the item's source segment within the raw cell text
+    /// (before extractors).
+    #[getter]
+    fn span(&self) -> (usize, usize) {
+        self.span
+    }
     #[getter]
     fn tags(&self) -> Vec<String> {
         self.tags.clone()
@@ -2610,6 +2618,7 @@ impl PyTableSemantics {
                 ty: it.ty,
                 row: it.row,
                 col: it.col,
+                span: it.span,
                 table: Some(self.table.clone_ref(py)),
             })
             .collect()

src/semantics.rs

@@ -21,6 +21,10 @@ pub struct CellItem {
     pub row: usize,
     pub col: usize,
     pub ty: ItemType,
+    /// Byte range of the item's source segment within the raw cell text
+    /// (before extractors): atomic content spans the whole text, delimited
+    /// and compound content spans the segment the item was derived from.
+    pub span: (usize, usize),
 }
 
 /// Context-derived item.

src/tests.rs

@@ -112,6 +112,25 @@ fn end_to_end_match_and_interpret() {
     let sem = match_atp(&pattern, &mut syntax, Vec::new())
         .unwrap()
         .expect("pattern must match");
+
+    // Item spans: the compound cell "0 Jan" (row 1, col 1) yields two VAL
+    // items whose spans point at the segments of the raw cell text.
+    let compound: Vec<_> = sem
+        .cell_items
+        .iter()
+        .filter(|it| it.row == 1 && it.col == 1)
+        .collect();
+    assert_eq!(compound.len(), 2);
+    assert_eq!((compound[0].s.as_str(), compound[0].span), ("0", (0, 1)));
+    assert_eq!((compound[1].s.as_str(), compound[1].span), ("Jan", (2, 5)));
+    // Atomic cells span their whole text.
+    let atomic = sem
+        .cell_items
+        .iter()
+        .find(|it| it.row == 1 && it.col == 0)
+        .unwrap();
+    assert_eq!(atomic.span, (0, "IKT".len()));
+
     let rs = interpret(&InterpreterCfg::default(), &syntax, &sem, None).unwrap();
 
     assert_eq!(rs.schema.attributes, vec!["ND", "AIRLINE", "AIRPORT", "MON"]);