Merge pull request #575 from Knowledge-Graph-Hub/fix/metpo-drop-deprecated-terms

Drop owl:deprecated ontology terms before KG ingest

Author: realmarcin

kg_microbe/transform_utils/ontologies/ontologies_transform.py

@@ -207,6 +207,14 @@ def parse(self, name: str, data_file: Optional[Path], source: str) -> None:
         # entries so the load proceeds.
         self._sanitize_obograph_synonyms(Path(data_file))
 
+        # Drop owl:deprecated (obsolete) classes so retired terms do not enter
+        # the KG as orphan nodes. METPO, for example, retains ~1,200 deprecated
+        # classes from an ID-scheme migration alongside its ~370 active terms;
+        # ingesting all of them inflates the graph with disconnected obsolete
+        # nodes. Removal happens before the KGX load so neither the nodes nor
+        # any edges touching them reach the output.
+        self._drop_deprecated_terms(Path(data_file))
+
         transform(
             inputs=[data_file],
             input_format="obojson",
@@ -262,6 +270,71 @@ def _sanitize_obograph_synonyms(self, json_path: Path) -> None:
             with open(json_path, "w", encoding="utf-8") as f:
                 json.dump(data, f)
 
+    @staticmethod
+    def _is_deprecated_node(node: dict) -> bool:
+        """
+        Return True if an obograph node is flagged ``owl:deprecated true``.
+
+        Obograph JSON encodes deprecation either as a top-level
+        ``meta.deprecated`` boolean or as a ``meta.basicPropertyValues`` entry
+        whose predicate is ``owl#deprecated`` with the literal value ``"true"``.
+        Both forms are checked.
+        """
+        meta = node.get("meta") or {}
+        if meta.get("deprecated") is True:
+            return True
+        for bpv in meta.get("basicPropertyValues", []) or []:
+            pred = str(bpv.get("pred", ""))
+            if pred.endswith("owl#deprecated") and str(bpv.get("val", "")).lower() == "true":
+                return True
+        return False
+
+    def _drop_deprecated_terms(self, json_path: Path) -> None:
+        """
+        Remove ``owl:deprecated`` (obsolete) classes from an obograph JSON in place.
+
+        Retired terms are dropped before the KGX load so they never become KG
+        nodes, and any edges referencing a dropped term are removed too so no
+        dangling edges remain. Active terms are untouched. The file is only
+        rewritten when something is actually removed.
+        """
+        if not json_path.is_file() or json_path.suffix != ".json":
+            return
+        try:
+            with open(json_path, "r", encoding="utf-8") as f:
+                data = json.load(f)
+        except (OSError, json.JSONDecodeError):
+            return  # let downstream KGX raise the more informative error
+
+        dropped_nodes = 0
+        dropped_edges = 0
+        for graph in data.get("graphs", []) or []:
+            deprecated_ids = {
+                node["id"]
+                for node in graph.get("nodes", []) or []
+                if node.get("id") and self._is_deprecated_node(node)
+            }
+            if not deprecated_ids:
+                continue
+            kept_nodes = [n for n in graph.get("nodes", []) or [] if n.get("id") not in deprecated_ids]
+            dropped_nodes += len(graph.get("nodes", []) or []) - len(kept_nodes)
+            graph["nodes"] = kept_nodes
+
+            edges = graph.get("edges", []) or []
+            kept_edges = [
+                e for e in edges if e.get("sub") not in deprecated_ids and e.get("obj") not in deprecated_ids
+            ]
+            dropped_edges += len(edges) - len(kept_edges)
+            graph["edges"] = kept_edges
+
+        if dropped_nodes:
+            print(
+                f"  Dropped {dropped_nodes} deprecated (owl:deprecated) terms "
+                f"and {dropped_edges} edges touching them from {json_path.name}"
+            )
+            with open(json_path, "w", encoding="utf-8") as f:
+                json.dump(data, f)
+
     def _add_kgx_metadata_to_edges(self, edges_file_path: Path):
         """
         Add knowledge_level and agent_type columns to ontology edge files.

tests/test_ontologies_deprecated_filter.py

@@ -0,0 +1,107 @@
+"""Tests for the owl:deprecated term filter in the ontologies transform."""
+
+import json
+from pathlib import Path
+from unittest import TestCase
+
+from kg_microbe.transform_utils.ontologies.ontologies_transform import OntologiesTransform
+
+
+def _make_obograph():
+    """Build a tiny obograph with one active node, two deprecated nodes, and edges."""
+    return {
+        "graphs": [
+            {
+                "id": "https://w3id.org/metpo/test.json",
+                "nodes": [
+                    {"id": "https://w3id.org/metpo/1000001", "lbl": "active term", "type": "CLASS"},
+                    {
+                        "id": "https://w3id.org/metpo/0000001",
+                        "lbl": "obsolete term (basicPropertyValues)",
+                        "type": "CLASS",
+                        "meta": {
+                            "basicPropertyValues": [
+                                {"pred": "http://www.w3.org/2002/07/owl#deprecated", "val": "true"}
+                            ]
+                        },
+                    },
+                    {
+                        "id": "https://w3id.org/metpo/0000002",
+                        "lbl": "obsolete term (meta.deprecated)",
+                        "type": "CLASS",
+                        "meta": {"deprecated": True},
+                    },
+                ],
+                "edges": [
+                    # edge between two active-ish nodes (subject active, object active) -> kept
+                    {
+                        "sub": "https://w3id.org/metpo/1000001",
+                        "pred": "is_a",
+                        "obj": "https://w3id.org/metpo/1000001",
+                    },
+                    # edge touching a deprecated node -> dropped
+                    {
+                        "sub": "https://w3id.org/metpo/1000001",
+                        "pred": "is_a",
+                        "obj": "https://w3id.org/metpo/0000001",
+                    },
+                ],
+            }
+        ]
+    }
+
+
+class TestDeprecatedFilter(TestCase):
+
+    """Test owl:deprecated term removal from obograph JSON before KGX load."""
+
+    def setUp(self):
+        """Instantiate the transform without running __init__ side effects."""
+        # The filter methods only rely on self._is_deprecated_node; bypass the
+        # base Transform.__init__ (which sets up source dirs) to keep the test
+        # isolated and side-effect free.
+        self.transform = OntologiesTransform.__new__(OntologiesTransform)
+
+    def test_is_deprecated_node_detects_both_encodings(self):
+        """Both basicPropertyValues and meta.deprecated encodings are detected."""
+        graph = _make_obograph()["graphs"][0]
+        active, dep_bpv, dep_meta = graph["nodes"]
+        self.assertFalse(self.transform._is_deprecated_node(active))
+        self.assertTrue(self.transform._is_deprecated_node(dep_bpv))
+        self.assertTrue(self.transform._is_deprecated_node(dep_meta))
+
+    def test_drop_deprecated_terms_removes_nodes_and_dangling_edges(self):
+        """Deprecated nodes and any edges touching them are removed in place."""
+        path = Path(self.tmp_json())
+        self.transform._drop_deprecated_terms(path)
+
+        result = json.loads(path.read_text())
+        graph = result["graphs"][0]
+        node_ids = {n["id"] for n in graph["nodes"]}
+
+        # Only the active term survives.
+        self.assertEqual(node_ids, {"https://w3id.org/metpo/1000001"})
+        # No node flagged deprecated remains.
+        self.assertFalse(any(self.transform._is_deprecated_node(n) for n in graph["nodes"]))
+        # The edge touching a deprecated node is gone; the active-only edge stays.
+        self.assertEqual(len(graph["edges"]), 1)
+        self.assertEqual(graph["edges"][0]["obj"], "https://w3id.org/metpo/1000001")
+
+    def test_active_only_graph_is_untouched(self):
+        """A graph with no deprecated terms is left unchanged (no rewrite)."""
+        data = {"graphs": [{"nodes": [{"id": "https://w3id.org/metpo/1000001", "lbl": "x"}], "edges": []}]}
+        path = Path(self.tmp_json(data))
+        before = path.read_text()
+        self.transform._drop_deprecated_terms(path)
+        self.assertEqual(path.read_text(), before)
+
+    def tmp_json(self, data=None):
+        """Write an obograph dict to a temp file and return its path."""
+        import tempfile
+
+        if data is None:
+            data = _make_obograph()
+        fd = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False)
+        json.dump(data, fd)
+        fd.close()
+        return fd.name