Externalize OLS implementation (#270)

* Externalize OLS implementation

This PR switches out interaction with the EBI's ontology lookup service using the `ols-client` package, which enables interaction with alternative OLS instances like the TIB Hannover one. Probably of interest to @StroemPhi

* Add intermediate class skips

* Update test_bioportal.py

* Update basic_ontology_interface.py

* Update poetry.lock

* Update main.yaml

* Update main.yaml

Author: cthoyt

poetry.lock

@@ -30,6 +30,7 @@ ontoportal-client = "0.0.3"
 curies = "^0.1.5"
 bioregistry = "^0.5.64"
 prefixmaps = "^0.1.2"
+ols-client = "^0.1.1"
 
 [tool.poetry.dev-dependencies]
 pytest = "^5.2"

pyproject.toml

@@ -6,7 +6,11 @@
 
 from oaklib.implementations.funowl.funowl_implementation import FunOwlImplementation
 from oaklib.implementations.gilda import GildaImplementation
-from oaklib.implementations.ols.ols_implementation import OlsImplementation
+from oaklib.implementations.ols import (
+    BaseOlsImplementation,
+    OlsImplementation,
+    TIBOlsImplementation,
+)
 from oaklib.implementations.ontobee.ontobee_implementation import OntobeeImplementation
 from oaklib.implementations.ontoportal.agroportal_implementation import (
     AgroPortalImplementation,
@@ -20,6 +24,9 @@
 from oaklib.implementations.ontoportal.matportal_implementation import (
     MatPortalImplementation,
 )
+from oaklib.implementations.ontoportal.ontoportal_implementation_base import (
+    OntoPortalImplementationBase,
+)
 from oaklib.implementations.pronto.pronto_implementation import ProntoImplementation
 from oaklib.implementations.simpleobo.simple_obo_implementation import (
     SimpleOboImplementation,
@@ -43,6 +50,7 @@
     "EcoPortalImplementation",
     "MatPortalImplementation",
     "OlsImplementation",
+    "TIBOlsImplementation",
     "OntobeeImplementation",
     "ProntoImplementation",
     "SimpleOboImplementation",
@@ -64,6 +72,10 @@
 implementation_resolver: ClassResolver[OntologyInterface] = ClassResolver.from_subclasses(
     OntologyInterface,
     suffix="Implementation",
+    skip={
+        OntoPortalImplementationBase,
+        BaseOlsImplementation,
+    },
 )
 implementation_resolver.synonyms.update(
     {

src/oaklib/implementations/__init__.py

@@ -1,3 +1,13 @@
-from oaklib.datamodels.search import SearchConfiguration
+from .constants import SEARCH_CONFIG
+from .ols_implementation import (
+    BaseOlsImplementation,
+    OlsImplementation,
+    TIBOlsImplementation,
+)
 
-SEARCH_CONFIG = SearchConfiguration()
+__all__ = [
+    "SEARCH_CONFIG",
+    "BaseOlsImplementation",
+    "OlsImplementation",
+    "TIBOlsImplementation",
+]

src/oaklib/implementations/ols/__init__.py

@@ -0,0 +1,7 @@
+from oaklib.datamodels.search import SearchConfiguration
+
+__all__ = [
+    "SEARCH_CONFIG",
+]
+
+SEARCH_CONFIG = SearchConfiguration()

src/oaklib/implementations/ols/constants.py

@@ -1,25 +1,33 @@
-import logging
-import urllib
 from collections import ChainMap
 from dataclasses import dataclass, field
-from typing import Any, Dict, Iterable, Iterator, List, Tuple, Union
+from typing import Any, ClassVar, Dict, Iterable, Iterator, List, Tuple, Union
 
 import requests
+from ols_client import Client, EBIClient, TIBClient
 from sssom_schema import Mapping
 
 from oaklib.datamodels import oxo
 from oaklib.datamodels.oxo import ScopeEnum
 from oaklib.datamodels.search import SearchConfiguration, SearchProperty
 from oaklib.datamodels.text_annotator import TextAnnotation
 from oaklib.datamodels.vocabulary import IS_A, SEMAPV
-from oaklib.implementations.ols import SEARCH_CONFIG
-from oaklib.implementations.ols.oxo_utils import load_oxo_payload
 from oaklib.interfaces.basic_ontology_interface import PREFIX_MAP
 from oaklib.interfaces.mapping_provider_interface import MappingProviderInterface
 from oaklib.interfaces.search_interface import SearchInterface
 from oaklib.interfaces.text_annotator_interface import TextAnnotatorInterface
 from oaklib.types import CURIE, PRED_CURIE
 
+from .constants import SEARCH_CONFIG
+from .oxo_utils import load_oxo_payload
+
+__all__ = [
+    # Abstract classes
+    "BaseOlsImplementation",
+    # Concrete classes
+    "OlsImplementation",
+    "TIBOlsImplementation",
+]
+
 ANNOTATION = Dict[str, Any]
 SEARCH_ROWS = 50
 
@@ -32,19 +40,20 @@
 
 
 @dataclass
-class OlsImplementation(TextAnnotatorInterface, SearchInterface, MappingProviderInterface):
+class BaseOlsImplementation(TextAnnotatorInterface, SearchInterface, MappingProviderInterface):
     """
     Implementation over OLS and OxO APIs
     """
 
-    ols_api_key: str = None
+    ols_client_class: ClassVar[type[Client]]
     label_cache: Dict[CURIE, str] = field(default_factory=lambda: {})
     base_url = "https://www.ebi.ac.uk/spot/oxo/api/mappings"
-    ols_base_url = "https://www.ebi.ac.uk/ols/api"
     _prefix_map: Dict[str, str] = field(default_factory=lambda: {})
     focus_ontology: str = None
+    client: Client = field(init=False)
 
     def __post_init__(self):
+        self.client = self.ols_client_class()
         if self.focus_ontology is None:
             if self.resource:
                 self.focus_ontology = self.resource.slug
@@ -71,30 +80,20 @@ def annotate_text(self, text: str) -> Iterator[TextAnnotation]:
     def ancestors(
         self, start_curies: Union[CURIE, List[CURIE]], predicates: List[PRED_CURIE] = None
     ) -> Iterable[CURIE]:
-        query = "hierarchicalAncestors"
+        func = self.client.iter_hierarchical_ancestors
         if predicates:
             if predicates == [IS_A]:
-                query = "ancestors"
+                func = self.client.iter_ancestors
             elif IS_A not in predicates:
                 raise NotImplementedError(f"OLS always include {IS_A}, you selected: {predicates}")
         if not isinstance(start_curies, list):
             start_curies = [start_curies]
         ancs = set()
         ontology = self.focus_ontology
         for curie in start_curies:
-            term_id = self.curie_to_uri(curie)
-            # must be double encoded https://www.ebi.ac.uk/ols/docs/api
-            term_id_quoted = urllib.parse.quote(term_id, safe="")
-            term_id_quoted = urllib.parse.quote(term_id_quoted, safe="")
-            url = f"{self.ols_base_url}/ontologies/{ontology}/terms/{term_id_quoted}/{query}"
-            logging.debug(f"URL={url}")
-            result = requests.get(url)
-            obj = result.json()
-            if result.status_code == 200 and "_embedded" in obj:
-                ancs.update([x["obo_id"] for x in obj["_embedded"]["terms"]])
-            else:
-                logging.debug(f"No ancestors for {url} (maybe ontology not indexed in OLS?)")
-                ancs = []
+            iri = self.curie_to_uri(curie)
+            records = func(ontology=ontology, iri=iri)
+            ancs.update(record["obo_id"] for record in records)
         return list(ancs)
 
     # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -119,7 +118,6 @@ def basic_search(
             query_fields.update(["description"])
 
         params = {
-            "q": search_term,
             "type": "class",
             "local": "true",
             "fieldList": "iri,label",
@@ -134,19 +132,10 @@ def basic_search(
         if self.focus_ontology:
             params["ontology"] = self.focus_ontology.lower()
 
-        finished = False
-        while not finished:
-            response = requests.get(f"{self.ols_base_url}/search", params=params)
-            logging.debug(f"URL={response.url}")
-            body = response.json()
-            params["start"] += params["rows"]
-            if params["start"] > body["response"]["numFound"]:
-                finished = True
-            for doc in body["response"]["docs"]:
-                curie = self.uri_to_curie(doc["iri"])
-                label = doc["label"]
-                self.label_cache[curie] = label
-                yield curie
+        for record in self.client.search(search_term, params=params):
+            curie = self.uri_to_curie(record["iri"])
+            self.label_cache[curie] = record["label"]
+            yield curie
 
     # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
     # Implements: MappingsInterface
@@ -199,3 +188,15 @@ def convert_payload(self, container: oxo.Container) -> Iterator[Mapping]:
     #                 msdoc.mapping_set.mappings.append(m)
     #                 n += 1
     #     return n
+
+
+class OlsImplementation(BaseOlsImplementation):
+    """Implementation for the EBI OLS instance."""
+
+    ols_client_class = EBIClient
+
+
+class TIBOlsImplementation(BaseOlsImplementation):
+    """Implementation for the TIB Hannover OLS instance."""
+
+    ols_client_class = TIBClient

src/oaklib/implementations/ols/ols_implementation.py

@@ -31,6 +31,7 @@
 MISSING_PREFIX_MAP = dict(
     EFO="http://www.ebi.ac.uk/efo/EFO_",
     SCTID="http://snomed.info/id/",
+    ORPHANET="http://www.orpha.net/ORDO/Orphanet_",
 )
 
 

src/oaklib/interfaces/basic_ontology_interface.py

@@ -11,7 +11,6 @@
 from tests import CELLULAR_COMPONENT, CYTOPLASM, DIGIT, VACUOLE
 
 
-@unittest.skip("TODO: use mock tests, this fails whenever OLS is down")
 class TestOlsImplementation(unittest.TestCase):
     def setUp(self) -> None:
         oi = OlsImplementation(OntologyResource("go"))
@@ -39,7 +38,7 @@ def test_ancestors(self):
 
     def test_basic_search(self):
         self.oi.focus_ontology = None
-        results = list(itertools.islice(self.oi.basic_search("epilepsy"), 20))
+        results = list(self.oi.basic_search("epilepsy"))
         self.assertIn("MONDO:0005027", results)
 
     def test_focus_ontology_search(self):