Docs/update readme and examples

README.md

@@ -1,7 +1,5 @@
 # datastew
 
-
-
 [![DOI](https://zenodo.org/badge/822570156.svg)](https://doi.org/10.5281/zenodo.16871713) ![tests](https://github.com/SCAI-BIO/datastew/actions/workflows/tests.yml/badge.svg) ![GitHub Release](https://img.shields.io/github/v/release/SCAI-BIO/datastew)
 
 Datastew is a python library for intelligent data harmonization using Large Language Model (LLM) vector embeddings.
@@ -54,22 +52,31 @@ df = map_dictionary_to_dictionary(source, target, vectorizer=vectorizer)
 A simple example how to initialize an in memory database and compute a similarity mapping is shown in
 [datastew/scripts/mapping_db_example.py](datastew/scripts/mapping_db_example.py):
 
-1) Initialize the repository and embedding model:
+1.  Initialize the repository and embedding model:
 
     ```python
     from datastew.embedding import Vectorizer
-    from datastew.repository import WeaviateRepository
-    from datastew.repository.model import Terminology, Concept, Mapping
+    from datastew.repository import PostgreSQLRepository
+    from datastew.repository.model import Concept, Mapping, MappingResult, Terminology
+
+    POSTGRES_USER = "user"
+    POSTGRES_PASSWORD = "password"
+    POSTGRES_HOST = "localhost"
+    POSTGRES_PORT = "5432"
+    POSTGRES_DB = "testdb"
 
-    repository = WeaviateRepository(mode='remote', path='localhost', port=8080)
+    connection_string = f"postgresql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}"
+
+    # You can use an OpenAI model if you have an API key:
+    # vectorizer = Vectorizer("text-embedding-3-small", key="your_openai_api_key")
     vectorizer = Vectorizer()
-    # vectorizer = Vectorizer("text-embedding-ada-002", key="your_key") # Use this line for higher accuracy if you have an OpenAI API key
+    repository = PostgreSQLRepository(connection_string, vectorizer=vectorizer)
     ```
 
-2) Create a baseline of data to map to in the initialized repository. Text gets attached to any unique concept of an
-existing or custom vocabulary or terminology namespace in the form of a mapping object containing the text, embedding,
-and the name of sentence embedder used. Multiple Mapping objects with textually different but semantically equal
-descriptions can point to the same Concept.
+2.  Create a baseline of data to map to in the initialized repository. Text gets attached to any unique concept of an
+    existing or custom vocabulary or terminology namespace in the form of a mapping object containing the text, embedding,
+    and the name of sentence embedder used. Multiple Mapping objects with textually different but semantically equal
+    descriptions can point to the same Concept.
 
     ```python
     terminology = Terminology("snomed CT", "SNOMED")
@@ -85,17 +92,22 @@ descriptions can point to the same Concept.
     repository.store_all([terminology, concept1, mapping1, concept2, mapping2])
     ```
 
-3) Retrieve the closest mappings and their similarities for a given text:
+3.  Retrieve the closest mappings and their similarities for a given text:
 
-```python
-text_to_map = "Sugar sickness" # Semantically similar to "Diabetes mellitus (disorder)"
-embedding = vectorizer.get_embedding(text_to_map)
-
-results = repository.get_closest_mappings(embedding, similarities=True, limit=2)
+    ```python
+    query_text = "Sugar sickness"  # semantically similar to "Diabetes mellitus (disorder)"
+    embedding = vectorizer.get_embedding(query_text)
+    results = repository.get_closest_mappings(embedding, similarities=True, limit=2)
+
+    print(f'Query: "{query_text}"\n')
+    for r in results:
+        # If similarities=True, repo returns MappingResult; else Mapping.
+        if isinstance(r, MappingResult):
+            print(r)
+        else:
+            print(str(r))
 
-for result in results:
-    print(result)
-```
+    ```
 
 output:
 

datastew/process/ols.py

@@ -83,11 +83,11 @@ def __process_page(self, page: int) -> Tuple[List[Concept], List[Mapping]]:
             logging.error(f"Failed to fetch concepts and descriptions from OLS for page {page}: {str(e)}")
             return [], []
 
-    def process_to_weaviate(self, repository: BaseRepository):
+    def process_to_repository(self, repository: BaseRepository):
         """
-        Fetches concepts and descriptions from the OLS API and stores them in a Weaviate repository.
+        Fetches concepts and descriptions from the OLS API and stores them in a repository.
 
-        :param repository: The Weaviate repository to store the concepts and mappings.
+        :param repository: The repository to store the concepts and mappings.
 
         :return: None
         """
@@ -102,7 +102,7 @@ def process_to_weaviate(self, repository: BaseRepository):
                 repository.store(mappings[idx])
             self.current_page += 1
 
-    def process_to_weaviate_json(self, dest_path: str):
+    def process_to_json(self, dest_path: str):
         """
         Fetches concepts and descriptions from the OLS API and stores them in a JSON file.
         """

datastew/scripts/export_json_from_weaviate.py

@@ -1,61 +1,74 @@
-from datastew import Concept, Mapping, Terminology
+"""
+Example: Export mappings from an in-memory Weaviate repository to JSONL
+
+This script demonstrates how to:
+1) Initialize an in-memory WeaviateRepository (no external service needed).
+2) Create a small SNOMED CT terminology with several concepts.
+3) Generate embeddings and store Mapping objects.
+4) Export the repository contents to JSONL for downstream use.
+
+Run:
+    python examples/export_weaviate_jsonl.py
+"""
+
 from datastew.embedding import Vectorizer
 from datastew.process.jsonl_adapter import WeaviateJsonlConverter
 from datastew.repository import WeaviateRepository
+from datastew.repository.model import Concept, Mapping, Terminology
 
-repository = WeaviateRepository(mode="memory", path="localhost", port=8080)
-
+# --------------------------------------------------------------------
+# 1) Repository (in-memory) and converter
+# --------------------------------------------------------------------
+repo = WeaviateRepository(mode="memory", path="localhost", port=8080)
 converter = WeaviateJsonlConverter(dest_dir="export")
 
-# Compute and store some exemplary Mappings
+# --------------------------------------------------------------------
+# 2) Vectorizer (use default or specify a model/API key if desired)
+# --------------------------------------------------------------------
+# Example for OpenAI:
+# vectorizer = Vectorizer("text-embedding-3-small", key="your_openai_api_key")
 vectorizer = Vectorizer()
 
+# --------------------------------------------------------------------
+# 3) Define terminology, concepts, and mappings
+# --------------------------------------------------------------------
 terminology = Terminology("snomed CT", "SNOMED")
 
-text1 = "Diabetes mellitus (disorder)"
-concept1 = Concept(terminology, text1, "Concept ID: 11893007")
-mapping1 = Mapping(concept1, text1, vectorizer.get_embedding(text1), vectorizer.model_name)
-
-text2 = "Hypertension (disorder)"
-concept2 = Concept(terminology, text2, "Concept ID: 73211009")
-mapping2 = Mapping(concept2, text2, vectorizer.get_embedding(text2), vectorizer.model_name)
-
-text3 = "Asthma"
-concept3 = Concept(terminology, text3, "Concept ID: 195967001")
-mapping3 = Mapping(concept3, text3, vectorizer.get_embedding(text3), vectorizer.model_name)
-
-text4 = "Heart attack"
-concept4 = Concept(terminology, text4, "Concept ID: 22298006")
-mapping4 = Mapping(concept4, text4, vectorizer.get_embedding(text4), vectorizer.model_name)
-
-text5 = "Common cold"
-concept5 = Concept(terminology, text5, "Concept ID: 13260007")
-mapping5 = Mapping(concept5, text5, vectorizer.get_embedding(text5), vectorizer.model_name)
-
-text6 = "Stroke"
-concept6 = Concept(terminology, text6, "Concept ID: 422504002")
-mapping6 = Mapping(concept6, text6, vectorizer.get_embedding(text6), vectorizer.model_name)
-
-text7 = "Migraine"
-concept7 = Concept(terminology, text7, "Concept ID: 386098009")
-mapping7 = Mapping(concept7, text7, vectorizer.get_embedding(text7), vectorizer.model_name)
-
-text8 = "Influenza"
-concept8 = Concept(terminology, text8, "Concept ID: 57386000")
-mapping8 = Mapping(concept8, text8, vectorizer.get_embedding(text8), vectorizer.model_name)
-
-text9 = "Osteoarthritis"
-concept9 = Concept(terminology, text9, "Concept ID: 399206004")
-mapping9 = Mapping(concept9, text9, vectorizer.get_embedding(text9), vectorizer.model_name)
-
-text10 = "Depression"
-concept10 = Concept(terminology, text10, "Concept ID: 386584008")
-mapping10 = Mapping(concept10, text10, vectorizer.get_embedding(text10), vectorizer.model_name)
-
-repository.store_all([terminology, concept1, mapping1, concept2, mapping2, concept3, mapping3, concept4, mapping4,
-                      concept5, mapping5, concept6, mapping6, concept7, mapping7, concept8, mapping8,
-                      concept9, mapping9, concept10, mapping10])
-
-converter.from_repository(repository)
-
-repository.close()
+# Concept ID -> Preferred label
+concepts_dict = {
+    "11893007": "Diabetes mellitus (disorder)",
+    "73211009": "Hypertension (disorder)",
+    "195967001": "Asthma",
+    "22298006": "Heart attack",
+    "13260007": "Common cold",
+    "422504002": "Stroke",
+    "386098009": "Migraine",
+    "57386000": "Influenza",
+    "399206004": "Osteoarthritis",
+    "386584008": "Depression",
+}
+
+concepts = []
+mappings = []
+
+for cid, label in concepts_dict.items():
+    concept = Concept(terminology, pref_label=label, concept_identifier=cid)
+    embedding = vectorizer.get_embedding(label)
+    mapping = Mapping(
+        concept=concept,
+        text=label,
+        embedding=embedding,
+        sentence_embedder=vectorizer.model_name,
+    )
+    concepts.append(concept)
+    mappings.append(mapping)
+
+# --------------------------------------------------------------------
+# 4) Store data and export to JSONL
+# --------------------------------------------------------------------
+try:
+    repo.store_all([terminology, *concepts, *mappings])
+    converter.from_repository(repo)
+    print("Export complete. JSONL files written to 'export/'.")
+finally:
+    repo.shut_down()

datastew/scripts/fill_db_example.py

@@ -1,54 +1,87 @@
-from datastew.embedding import Vectorizer
-from datastew.repository import WeaviateRepository
-from datastew.repository.model import Concept, Mapping, Terminology
+"""
+Example: Store a small SNOMED CT terminology in PostgreSQL using datastew
 
-# This script demonstrates how to fill a vector database with example data
+This example shows how to:
+1- Start a local PostgreSQL database (via Docker)
+2- Initialize datastew with a vectorizer and repository
+3- Insert a small set of medical concepts and text embeddings
 
-repository = WeaviateRepository(mode="memory", path="localhost", port=8080)
-vectorizer = Vectorizer()
+---
 
-terminology = Terminology("snomed CT", "SNOMED")
+Quick Start
 
-text1 = "Diabetes mellitus (disorder)"
-concept1 = Concept(terminology, text1, "Concept ID: 11893007")
-mapping1 = Mapping(concept1, text1, vectorizer.get_embedding(text1), vectorizer.model_name)
+# 1. Run PostgreSQL locally (in a new terminal):
+docker run -d \
+  --name datastew-postgres \
+  -e POSTGRES_USER=user \
+  -e POSTGRES_PASSWORD=password \
+  -e POSTGRES_DB=testdb \
+  -p 5432:5432 \
+  postgres:15
 
-text2 = "Hypertension (disorder)"
-concept2 = Concept(terminology, text2, "Concept ID: 73211009")
-mapping2 = Mapping(concept2, text2, vectorizer.get_embedding(text2), vectorizer.model_name)
+# 2. Run this script:
+python examples/store_snomed_baseline.py
 
-text3 = "Asthma"
-concept3 = Concept(terminology, text3, "Concept ID: 195967001")
-mapping3 = Mapping(concept3, text3, vectorizer.get_embedding(text3), vectorizer.model_name)
+# 3. (Optional) Retrieve embeddings later using:
+python examples/get_closest_mappings.py
+"""
 
-text4 = "Heart attack"
-concept4 = Concept(terminology, text4, "Concept ID: 22298006")
-mapping4 = Mapping(concept4, text4, vectorizer.get_embedding(text4), vectorizer.model_name)
+from datastew.embedding import Vectorizer
+from datastew.repository import PostgreSQLRepository
+from datastew.repository.model import Concept, Mapping, Terminology
+
+# --------------------------------------------------------------------
+# 1) Connect to PostgreSQL
+# --------------------------------------------------------------------
+POSTGRES_USER = "user"
+POSTGRES_PASSWORD = "password"
+POSTGRES_HOST = "localhost"
+POSTGRES_PORT = "5432"
+POSTGRES_DB = "testdb"
 
-text5 = "Common cold"
-concept5 = Concept(terminology, text5, "Concept ID: 13260007")
-mapping5 = Mapping(concept5, text5, vectorizer.get_embedding(text5), vectorizer.model_name)
+connection_string = f"postgresql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}"
 
-text6 = "Stroke"
-concept6 = Concept(terminology, text6, "Concept ID: 422504002")
-mapping6 = Mapping(concept6, text6, vectorizer.get_embedding(text6), vectorizer.model_name)
+# Use OpenAI embeddings if you have an API key for higher-quality results:
+# vectorizer = Vectorizer("text-embedding-3-small", key="your_openai_api_key")
+vectorizer = Vectorizer()
+repository = PostgreSQLRepository(connection_string, vectorizer=vectorizer)
 
-text7 = "Migraine"
-concept7 = Concept(terminology, text7, "Concept ID: 386098009")
-mapping7 = Mapping(concept7, text7, vectorizer.get_embedding(text7), vectorizer.model_name)
+# --------------------------------------------------------------------
+# 2) Define a terminology namespace
+# --------------------------------------------------------------------
+terminology = Terminology("snomed CT", "SNOMED")
 
-text8 = "Influenza"
-concept8 = Concept(terminology, text8, "Concept ID: 57386000")
-mapping8 = Mapping(concept8, text8, vectorizer.get_embedding(text8), vectorizer.model_name)
+# --------------------------------------------------------------------
+# 3) Define example medical concepts
+# --------------------------------------------------------------------
+concept_texts = {
+    "11893007": "Diabetes mellitus (disorder)",
+    "73211009": "Hypertension (disorder)",
+    "195967001": "Asthma",
+    "22298006": "Heart attack",
+    "13260007": "Common cold",
+    "422504002": "Stroke",
+    "386098009": "Migraine",
+    "57386000": "Influenza",
+    "399206004": "Osteoarthritis",
+    "386584008": "Depression",
+}
 
-text9 = "Osteoarthritis"
-concept9 = Concept(terminology, text9, "Concept ID: 399206004")
-mapping9 = Mapping(concept9, text9, vectorizer.get_embedding(text9), vectorizer.model_name)
+# --------------------------------------------------------------------
+# 4) Convert to datastew concepts + mappings
+# --------------------------------------------------------------------
+concepts, mappings = [], []
+for concept_id, label in concept_texts.items():
+    concept = Concept(terminology, label, concept_id)
+    embedding = vectorizer.get_embedding(label)
+    mapping = Mapping(concept, label, embedding, vectorizer.model_name)
+    concepts.append(concept)
+    mappings.append(mapping)
 
-text10 = "Depression"
-concept10 = Concept(terminology, text10, "Concept ID: 386584008")
-mapping10 = Mapping(concept10, text10, vectorizer.get_embedding(text10), vectorizer.model_name)
+# --------------------------------------------------------------------
+# 5) Store everything in the repository
+# --------------------------------------------------------------------
+repository.store_all([terminology, *concepts, *mappings])
 
-repository.store_all([terminology, concept1, mapping1, concept2, mapping2, concept3, mapping3, concept4, mapping4,
-                      concept5, mapping5, concept6, mapping6, concept7, mapping7, concept8, mapping8,
-                      concept9, mapping9, concept10, mapping10])
+print(f"Stored {len(concepts)} concepts and mappings in PostgreSQL.")
+print("You can now query them using `repository.get_closest_mappings()`.")