Expand heuristics for coordinate detection, to minimize missed coordinates

Author: adelavega

ace/ingest.py

@@ -50,8 +50,12 @@ def _parse_article(args):
             # Fallback to original source identification
             source = manager.identify_source(html)
             if source is None:
-                logger.info("Could not identify source for %s", f)
-                return f, None
+                if force_ingest and getattr(manager, "default_source", None) is not None:
+                    logger.info("Could not identify source for %s; using DefaultSource fallback", f)
+                    source = manager.default_source
+                else:
+                    logger.info("Could not identify source for %s", f)
+                    return f, None
 
         article = source.parse_article(html, pmid, metadata_dir=metadata_dir, **kwargs)
         if not article:

ace/sources.py

@@ -29,6 +29,14 @@ def identify_standard_columns(labels):
             s = 'hemisphere'
         elif regex.search('(^k$)|(mm.*?3)|volume|voxels|size|extent', lab):
             s = 'size'
+        elif (
+            regex.search(r'\bx\b.*\by\b.*\bz\b', lab)
+            or regex.search(r'(peak\s*voxel\s*coordinate|talairach\s*coordinates?|mni\s*coordinates?)', lab)
+            or (regex.search(r'coordinates?', lab) and not regex.search(r'cluster|score|value', lab))
+        ):
+            # Some tables store x/y/z in one combined coordinate column.
+            s = 'coord_triplet'
+            found_coords = True
 
         # --- START OF FIX ---
         # OLD: elif regex.match('\s*[xy]\s*$', lab):
@@ -59,7 +67,8 @@ def identify_standard_columns(labels):
         # --- END OF FIX ---
 
         elif regex.search('rdinate', lab):
-            continue
+            s = 'coord_triplet'
+            found_coords = True
         elif lab == 't' or regex.search('^(max.*(z|t).*|.*(z|t).*(score|value|max))$', lab):
             s = 'statistic'
         elif regex.search('p[\-\s]+.*val', lab):
@@ -158,6 +167,15 @@ def identify_repeating_groups(labels):
 def create_activation(data, labels, standard_cols, group_labels=[]):
 
     activation = Activation()
+    coords_from_triplet = False
+
+    def _extract_triplet(value):
+        clean_val = regex.sub(r'(?<!\d)\.(?!\d)', '', str(value))
+        cs = '([-]?\d{1,3}\.?\d{0,2})'
+        match = regex.search('\n*%s[,;\s]+%s[,;\s]+%s' % (cs, cs, cs), clean_val)
+        if not match:
+            return None
+        return [regex.sub('-\s+', '-', c.strip()) for c in [match.group(1), match.group(2), match.group(3)]]
 
     for i, col in enumerate(data):
 
@@ -181,10 +199,26 @@ def create_activation(data, labels, standard_cols, group_labels=[]):
 
             sc = standard_cols[i]
 
+            if sc in ['coord_triplet', 'x', 'y', 'z']:
+                triplet = _extract_triplet(col)
+                if triplet is not None:
+                    x, y, z = triplet
+                    logger.info("Found coordinate triplet in %s column: %s -> %s, %s, %s" % (sc, col, x, y, z))
+                    activation.set_coords(x, y, z)
+                    coords_from_triplet = True
+                    activation.add_col(labels[i], col)
+                    continue
+                if sc == 'coord_triplet':
+                    activation.add_col(labels[i], col)
+                    continue
+
             # Validate XYZ columns: Should only be integers (and possible trailing decimals).
             # If they're not, keep only leading numbers. The exception is that ScienceDirect 
             # journals often follow the minus sign with a space (e.g., - 35), which we strip.
             if regex.match('[xyz]$', sc):
+                if coords_from_triplet and str(col).strip() == '':
+                    activation.add_col(labels[i], col)
+                    continue
                 m = regex.match('([-])\s?(\d+\.*\d*)$', col)
                 if m:
                     col = "%s%s" % (m.group(1), m.group(2))
@@ -210,11 +244,9 @@ def create_activation(data, labels, standard_cols, group_labels=[]):
         # Also need to remove space between minus sign and numbers; some ScienceDirect
         # journals leave a gap.
         if not i in standard_cols:
-            cs = '([-]?\d{1,3}\.?\d{0,2})'
-            clean_col = regex.sub(r'(?<!\d)\.(?!\d)', '', str(col))  # Remove dots not part of numbers
-            m = regex.search('\n*%s[,;\s]+%s[,;\s]+%s' % (cs, cs, cs), clean_col)
-            if m:
-                x, y, z = [regex.sub('-\s+', '-', c.strip()) for c in [m.group(1), m.group(2), m.group(3)]]
+            triplet = _extract_triplet(col)
+            if triplet is not None:
+                x, y, z = triplet
                 logger.info("Found multi-coordinate column: %s\n...and extracted: %s, %s, %s" % (col, x, y, z))
                 activation.set_coords(x, y, z)
 
@@ -390,6 +422,3 @@ def parse_table(data, html=None):
 
     table.finalize()
     return table if len(table.activations) else None
-
-
-

ace/tableparser.py

@@ -374,3 +374,105 @@ def test_springer_nature_source(test_data_path, source_manager):
     assert t.number == '2'
     assert "fMRI results across all participants" in t.caption
     assert t.n_activations  == 9
+
+
+def _count_valid_activations(tables):
+    return sum(
+        1 for t in tables for a in t.activations
+        if a.x is not None and a.y is not None and a.z is not None
+    )
+
+
+def test_pmc_modern_table_wrapper_source(test_weird_data_path, source_manager):
+    pmid = '16085533'
+    html = open(join(test_weird_data_path, pmid + '.html')).read()
+    source = source_manager.identify_source(html)
+    assert source is not None
+    assert source.__class__.__name__ == 'PMCSource'
+    article = source.parse_article(html, pmid=pmid, skip_metadata=True)
+    assert article is not None
+    assert len(article.tables) >= 1
+    assert _count_valid_activations(article.tables) >= 1
+
+
+def test_oup_table_wrap_fallback_source(test_weird_data_path, source_manager):
+    pmid = '24700584'
+    html = open(join(test_weird_data_path, pmid + '.html')).read()
+    source = source_manager.identify_source(html)
+    assert source is not None
+    assert source.__class__.__name__ == 'OUPSource'
+    article = source.parse_article(html, pmid=pmid, skip_metadata=True)
+    assert article is not None
+    assert len(article.tables) >= 1
+    assert _count_valid_activations(article.tables) >= 1
+
+
+def test_jcn_embedded_table_fallback_source(test_weird_data_path, source_manager):
+    pmid = '24666131'
+    html = open(join(test_weird_data_path, pmid + '.html')).read()
+    source = source_manager.identify_source(html)
+    assert source is not None
+    assert source.__class__.__name__ == 'JournalOfCognitiveNeuroscienceSource'
+    article = source.parse_article(html, pmid=pmid, skip_metadata=True)
+    assert article is not None
+    assert len(article.tables) >= 1
+    assert _count_valid_activations(article.tables) >= 1
+
+
+def test_sciencedirect_combined_coordinate_column_source(test_weird_data_path, source_manager):
+    pmid = '15327927'
+    html = open(join(test_weird_data_path, pmid + '.html')).read()
+    source = source_manager.identify_source(html)
+    assert source is not None
+    assert source.__class__.__name__ == 'ScienceDirectSource'
+    article = source.parse_article(html, pmid=pmid, skip_metadata=True)
+    assert article is not None
+    assert len(article.tables) >= 1
+    assert _count_valid_activations(article.tables) >= 1
+
+
+def test_springer_inline_table_fallback_source(test_weird_data_path, source_manager):
+    pmid = '27007121'
+    html = open(join(test_weird_data_path, pmid + '.html')).read()
+    source = source_manager.identify_source(html)
+    assert source is not None
+    assert source.__class__.__name__ == 'SpringerSource'
+    article = source.parse_article(html, pmid=pmid, skip_metadata=True)
+    assert article is not None
+    assert len(article.tables) >= 1
+    assert _count_valid_activations(article.tables) >= 1
+
+
+def test_unknown_source_coordinate_table_with_force_ingest(test_weird_data_path, tmp_path):
+    pmid = '11296095'
+    src_file = join(test_weird_data_path, pmid + '.html')
+    target_file = tmp_path / f"{pmid}.html"
+    shutil.copy(src_file, target_file)
+
+    db_path_no_force = f"sqlite:///{(tmp_path / 'ace_no_force.db').as_posix()}"
+    db_no_force = database.Database(adapter='sqlite', db_name=db_path_no_force)
+    missing_sources = ingest.add_articles(
+        db_no_force,
+        [str(target_file)],
+        pmid_filenames=True,
+        force_ingest=False,
+        num_workers=1,
+        skip_metadata=True,
+    )
+    assert str(target_file) in missing_sources
+    assert len(db_no_force.articles) == 0
+
+    db_path_force = f"sqlite:///{(tmp_path / 'ace_force.db').as_posix()}"
+    db_force = database.Database(adapter='sqlite', db_name=db_path_force)
+    missing_sources_force = ingest.add_articles(
+        db_force,
+        [str(target_file)],
+        pmid_filenames=True,
+        force_ingest=True,
+        num_workers=1,
+        skip_metadata=True,
+    )
+    assert str(target_file) not in missing_sources_force
+    assert len(db_force.articles) >= 1
+    assert len(db_force.articles[0].tables) >= 1
+    assert _count_valid_activations(db_force.articles[0].tables) >= 1