From 9e34e501be1a303cef4c58c6341970704dd3e450 Mon Sep 17 00:00:00 2001 From: Mehmet Can Ay Date: Mon, 10 Nov 2025 12:25:32 +0100 Subject: [PATCH 1/2] docs: update and improve usage examples --- README.md | 52 +++++--- datastew/scripts/export_json_from_weaviate.py | 115 ++++++++++-------- datastew/scripts/fill_db_example.py | 113 +++++++++++------ datastew/scripts/mapping_db_example.py | 75 +++++++++--- datastew/scripts/mapping_excel_example.py | 44 ++++++- datastew/scripts/ohdsi_to_jsonl.py | 29 ++++- datastew/scripts/ols_snomed_retrieval.py | 55 +++++++-- datastew/scripts/tsne_visualization.py | 33 ++++- 8 files changed, 375 insertions(+), 141 deletions(-) diff --git a/README.md b/README.md index 6ccff7e..fb72b7d 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ # datastew - - [![DOI](https://zenodo.org/badge/822570156.svg)](https://doi.org/10.5281/zenodo.16871713) ![tests](https://github.com/SCAI-BIO/datastew/actions/workflows/tests.yml/badge.svg) ![GitHub Release](https://img.shields.io/github/v/release/SCAI-BIO/datastew) Datastew is a python library for intelligent data harmonization using Large Language Model (LLM) vector embeddings. @@ -54,22 +52,31 @@ df = map_dictionary_to_dictionary(source, target, vectorizer=vectorizer) A simple example how to initialize an in memory database and compute a similarity mapping is shown in [datastew/scripts/mapping_db_example.py](datastew/scripts/mapping_db_example.py): -1) Initialize the repository and embedding model: +1. Initialize the repository and embedding model: ```python from datastew.embedding import Vectorizer - from datastew.repository import WeaviateRepository - from datastew.repository.model import Terminology, Concept, Mapping + from datastew.repository import PostgreSQLRepository + from datastew.repository.model import Concept, Mapping, MappingResult, Terminology + + POSTGRES_USER = "user" + POSTGRES_PASSWORD = "password" + POSTGRES_HOST = "localhost" + POSTGRES_PORT = "5432" + POSTGRES_DB = "testdb" - repository = WeaviateRepository(mode='remote', path='localhost', port=8080) + connection_string = f"postgresql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}" + + # You can use an OpenAI model if you have an API key: + # vectorizer = Vectorizer("text-embedding-3-small", key="your_openai_api_key") vectorizer = Vectorizer() - # vectorizer = Vectorizer("text-embedding-ada-002", key="your_key") # Use this line for higher accuracy if you have an OpenAI API key + repository = PostgreSQLRepository(connection_string, vectorizer=vectorizer) ``` -2) Create a baseline of data to map to in the initialized repository. Text gets attached to any unique concept of an -existing or custom vocabulary or terminology namespace in the form of a mapping object containing the text, embedding, -and the name of sentence embedder used. Multiple Mapping objects with textually different but semantically equal -descriptions can point to the same Concept. +2. Create a baseline of data to map to in the initialized repository. Text gets attached to any unique concept of an + existing or custom vocabulary or terminology namespace in the form of a mapping object containing the text, embedding, + and the name of sentence embedder used. Multiple Mapping objects with textually different but semantically equal + descriptions can point to the same Concept. ```python terminology = Terminology("snomed CT", "SNOMED") @@ -85,17 +92,22 @@ descriptions can point to the same Concept. repository.store_all([terminology, concept1, mapping1, concept2, mapping2]) ``` -3) Retrieve the closest mappings and their similarities for a given text: +3. Retrieve the closest mappings and their similarities for a given text: -```python -text_to_map = "Sugar sickness" # Semantically similar to "Diabetes mellitus (disorder)" -embedding = vectorizer.get_embedding(text_to_map) - -results = repository.get_closest_mappings(embedding, similarities=True, limit=2) + ```python + query_text = "Sugar sickness" # semantically similar to "Diabetes mellitus (disorder)" + embedding = vectorizer.get_embedding(query_text) + results = repository.get_closest_mappings(embedding, similarities=True, limit=2) + + print(f'Query: "{query_text}"\n') + for r in results: + # If similarities=True, repo returns MappingResult; else Mapping. + if isinstance(r, MappingResult): + print(r) + else: + print(str(r)) -for result in results: - print(result) -``` + ``` output: diff --git a/datastew/scripts/export_json_from_weaviate.py b/datastew/scripts/export_json_from_weaviate.py index c88cc1d..cce4344 100644 --- a/datastew/scripts/export_json_from_weaviate.py +++ b/datastew/scripts/export_json_from_weaviate.py @@ -1,61 +1,74 @@ -from datastew import Concept, Mapping, Terminology +""" +Example: Export mappings from an in-memory Weaviate repository to JSONL + +This script demonstrates how to: +1) Initialize an in-memory WeaviateRepository (no external service needed). +2) Create a small SNOMED CT terminology with several concepts. +3) Generate embeddings and store Mapping objects. +4) Export the repository contents to JSONL for downstream use. + +Run: + python examples/export_weaviate_jsonl.py +""" + from datastew.embedding import Vectorizer from datastew.process.jsonl_adapter import WeaviateJsonlConverter from datastew.repository import WeaviateRepository +from datastew.repository.model import Concept, Mapping, Terminology -repository = WeaviateRepository(mode="memory", path="localhost", port=8080) - +# -------------------------------------------------------------------- +# 1) Repository (in-memory) and converter +# -------------------------------------------------------------------- +repo = WeaviateRepository(mode="memory", path="localhost", port=8080) converter = WeaviateJsonlConverter(dest_dir="export") -# Compute and store some exemplary Mappings +# -------------------------------------------------------------------- +# 2) Vectorizer (use default or specify a model/API key if desired) +# -------------------------------------------------------------------- +# Example for OpenAI: +# vectorizer = Vectorizer("text-embedding-3-small", key="your_openai_api_key") vectorizer = Vectorizer() +# -------------------------------------------------------------------- +# 3) Define terminology, concepts, and mappings +# -------------------------------------------------------------------- terminology = Terminology("snomed CT", "SNOMED") -text1 = "Diabetes mellitus (disorder)" -concept1 = Concept(terminology, text1, "Concept ID: 11893007") -mapping1 = Mapping(concept1, text1, vectorizer.get_embedding(text1), vectorizer.model_name) - -text2 = "Hypertension (disorder)" -concept2 = Concept(terminology, text2, "Concept ID: 73211009") -mapping2 = Mapping(concept2, text2, vectorizer.get_embedding(text2), vectorizer.model_name) - -text3 = "Asthma" -concept3 = Concept(terminology, text3, "Concept ID: 195967001") -mapping3 = Mapping(concept3, text3, vectorizer.get_embedding(text3), vectorizer.model_name) - -text4 = "Heart attack" -concept4 = Concept(terminology, text4, "Concept ID: 22298006") -mapping4 = Mapping(concept4, text4, vectorizer.get_embedding(text4), vectorizer.model_name) - -text5 = "Common cold" -concept5 = Concept(terminology, text5, "Concept ID: 13260007") -mapping5 = Mapping(concept5, text5, vectorizer.get_embedding(text5), vectorizer.model_name) - -text6 = "Stroke" -concept6 = Concept(terminology, text6, "Concept ID: 422504002") -mapping6 = Mapping(concept6, text6, vectorizer.get_embedding(text6), vectorizer.model_name) - -text7 = "Migraine" -concept7 = Concept(terminology, text7, "Concept ID: 386098009") -mapping7 = Mapping(concept7, text7, vectorizer.get_embedding(text7), vectorizer.model_name) - -text8 = "Influenza" -concept8 = Concept(terminology, text8, "Concept ID: 57386000") -mapping8 = Mapping(concept8, text8, vectorizer.get_embedding(text8), vectorizer.model_name) - -text9 = "Osteoarthritis" -concept9 = Concept(terminology, text9, "Concept ID: 399206004") -mapping9 = Mapping(concept9, text9, vectorizer.get_embedding(text9), vectorizer.model_name) - -text10 = "Depression" -concept10 = Concept(terminology, text10, "Concept ID: 386584008") -mapping10 = Mapping(concept10, text10, vectorizer.get_embedding(text10), vectorizer.model_name) - -repository.store_all([terminology, concept1, mapping1, concept2, mapping2, concept3, mapping3, concept4, mapping4, - concept5, mapping5, concept6, mapping6, concept7, mapping7, concept8, mapping8, - concept9, mapping9, concept10, mapping10]) - -converter.from_repository(repository) - -repository.close() +# Concept ID -> Preferred label +concepts_dict = { + "11893007": "Diabetes mellitus (disorder)", + "73211009": "Hypertension (disorder)", + "195967001": "Asthma", + "22298006": "Heart attack", + "13260007": "Common cold", + "422504002": "Stroke", + "386098009": "Migraine", + "57386000": "Influenza", + "399206004": "Osteoarthritis", + "386584008": "Depression", +} + +concepts = [] +mappings = [] + +for cid, label in concepts_dict.items(): + concept = Concept(terminology, pref_label=label, concept_identifier=cid) + embedding = vectorizer.get_embedding(label) + mapping = Mapping( + concept=concept, + text=label, + embedding=embedding, + sentence_embedder=vectorizer.model_name, + ) + concepts.append(concept) + mappings.append(mapping) + +# -------------------------------------------------------------------- +# 4) Store data and export to JSONL +# -------------------------------------------------------------------- +try: + repo.store_all([terminology, *concepts, *mappings]) + converter.from_repository(repo) + print("Export complete. JSONL files written to 'export/'.") +finally: + repo.shut_down() diff --git a/datastew/scripts/fill_db_example.py b/datastew/scripts/fill_db_example.py index de6a693..a79a2f0 100644 --- a/datastew/scripts/fill_db_example.py +++ b/datastew/scripts/fill_db_example.py @@ -1,54 +1,87 @@ -from datastew.embedding import Vectorizer -from datastew.repository import WeaviateRepository -from datastew.repository.model import Concept, Mapping, Terminology +""" +Example: Store a small SNOMED CT terminology in PostgreSQL using datastew -# This script demonstrates how to fill a vector database with example data +This example shows how to: +1- Start a local PostgreSQL database (via Docker) +2- Initialize datastew with a vectorizer and repository +3- Insert a small set of medical concepts and text embeddings -repository = WeaviateRepository(mode="memory", path="localhost", port=8080) -vectorizer = Vectorizer() +--- -terminology = Terminology("snomed CT", "SNOMED") +Quick Start -text1 = "Diabetes mellitus (disorder)" -concept1 = Concept(terminology, text1, "Concept ID: 11893007") -mapping1 = Mapping(concept1, text1, vectorizer.get_embedding(text1), vectorizer.model_name) +# 1. Run PostgreSQL locally (in a new terminal): +docker run -d \ + --name datastew-postgres \ + -e POSTGRES_USER=user \ + -e POSTGRES_PASSWORD=password \ + -e POSTGRES_DB=testdb \ + -p 5432:5432 \ + postgres:15 -text2 = "Hypertension (disorder)" -concept2 = Concept(terminology, text2, "Concept ID: 73211009") -mapping2 = Mapping(concept2, text2, vectorizer.get_embedding(text2), vectorizer.model_name) +# 2. Run this script: +python examples/store_snomed_baseline.py -text3 = "Asthma" -concept3 = Concept(terminology, text3, "Concept ID: 195967001") -mapping3 = Mapping(concept3, text3, vectorizer.get_embedding(text3), vectorizer.model_name) +# 3. (Optional) Retrieve embeddings later using: +python examples/get_closest_mappings.py +""" -text4 = "Heart attack" -concept4 = Concept(terminology, text4, "Concept ID: 22298006") -mapping4 = Mapping(concept4, text4, vectorizer.get_embedding(text4), vectorizer.model_name) +from datastew.embedding import Vectorizer +from datastew.repository import PostgreSQLRepository +from datastew.repository.model import Concept, Mapping, Terminology + +# -------------------------------------------------------------------- +# 1) Connect to PostgreSQL +# -------------------------------------------------------------------- +POSTGRES_USER = "user" +POSTGRES_PASSWORD = "password" +POSTGRES_HOST = "localhost" +POSTGRES_PORT = "5432" +POSTGRES_DB = "testdb" -text5 = "Common cold" -concept5 = Concept(terminology, text5, "Concept ID: 13260007") -mapping5 = Mapping(concept5, text5, vectorizer.get_embedding(text5), vectorizer.model_name) +connection_string = f"postgresql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}" -text6 = "Stroke" -concept6 = Concept(terminology, text6, "Concept ID: 422504002") -mapping6 = Mapping(concept6, text6, vectorizer.get_embedding(text6), vectorizer.model_name) +# Use OpenAI embeddings if you have an API key for higher-quality results: +# vectorizer = Vectorizer("text-embedding-3-small", key="your_openai_api_key") +vectorizer = Vectorizer() +repository = PostgreSQLRepository(connection_string, vectorizer=vectorizer) -text7 = "Migraine" -concept7 = Concept(terminology, text7, "Concept ID: 386098009") -mapping7 = Mapping(concept7, text7, vectorizer.get_embedding(text7), vectorizer.model_name) +# -------------------------------------------------------------------- +# 2) Define a terminology namespace +# -------------------------------------------------------------------- +terminology = Terminology("snomed CT", "SNOMED") -text8 = "Influenza" -concept8 = Concept(terminology, text8, "Concept ID: 57386000") -mapping8 = Mapping(concept8, text8, vectorizer.get_embedding(text8), vectorizer.model_name) +# -------------------------------------------------------------------- +# 3) Define example medical concepts +# -------------------------------------------------------------------- +concept_texts = { + "11893007": "Diabetes mellitus (disorder)", + "73211009": "Hypertension (disorder)", + "195967001": "Asthma", + "22298006": "Heart attack", + "13260007": "Common cold", + "422504002": "Stroke", + "386098009": "Migraine", + "57386000": "Influenza", + "399206004": "Osteoarthritis", + "386584008": "Depression", +} -text9 = "Osteoarthritis" -concept9 = Concept(terminology, text9, "Concept ID: 399206004") -mapping9 = Mapping(concept9, text9, vectorizer.get_embedding(text9), vectorizer.model_name) +# -------------------------------------------------------------------- +# 4) Convert to datastew concepts + mappings +# -------------------------------------------------------------------- +concepts, mappings = [], [] +for concept_id, label in concept_texts.items(): + concept = Concept(terminology, label, concept_id) + embedding = vectorizer.get_embedding(label) + mapping = Mapping(concept, label, embedding, vectorizer.model_name) + concepts.append(concept) + mappings.append(mapping) -text10 = "Depression" -concept10 = Concept(terminology, text10, "Concept ID: 386584008") -mapping10 = Mapping(concept10, text10, vectorizer.get_embedding(text10), vectorizer.model_name) +# -------------------------------------------------------------------- +# 5) Store everything in the repository +# -------------------------------------------------------------------- +repository.store_all([terminology, *concepts, *mappings]) -repository.store_all([terminology, concept1, mapping1, concept2, mapping2, concept3, mapping3, concept4, mapping4, - concept5, mapping5, concept6, mapping6, concept7, mapping7, concept8, mapping8, - concept9, mapping9, concept10, mapping10]) +print(f"Stored {len(concepts)} concepts and mappings in PostgreSQL.") +print("You can now query them using `repository.get_closest_mappings()`.") diff --git a/datastew/scripts/mapping_db_example.py b/datastew/scripts/mapping_db_example.py index e2f76f5..caa7277 100644 --- a/datastew/scripts/mapping_db_example.py +++ b/datastew/scripts/mapping_db_example.py @@ -1,20 +1,51 @@ +""" +Example: Retrieve the closest text embeddings and their similarities. + +This script shows how to: +1. Start a local PostgreSQL instance via Docker. +2. Store two simple SNOMED CT concepts in the database. +3. Retrieve the closest mappings for an input phrase. + +--- + +Quick start + +# Start PostgreSQL via Docker: +docker run -d \ + --name datastew-postgres \ + -e POSTGRES_USER=user \ + -e POSTGRES_PASSWORD=password \ + -e POSTGRES_DB=testdb \ + -p 5432:5432 \ + postgres:15 + +# Run this script: +python examples/get_closest_mappings.py +""" + from datastew.embedding import Vectorizer -from datastew.repository import WeaviateRepository -from datastew.repository.model import Concept, Mapping, Terminology +from datastew.repository import PostgreSQLRepository +from datastew.repository.model import Concept, Mapping, MappingResult, Terminology -# This script demonstrates how to retrieve the closest text embeddings and their similarities for a given text +# -------------------------------------------------------------------- +# 1) Connect to PostgreSQL +# -------------------------------------------------------------------- +POSTGRES_USER = "user" +POSTGRES_PASSWORD = "password" +POSTGRES_HOST = "localhost" +POSTGRES_PORT = "5432" +POSTGRES_DB = "testdb" -# 1) Initialize the repository and embedding model +connection_string = f"postgresql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}" -repository = WeaviateRepository(mode="remote", path="localhost", port=8080) +# You can use an OpenAI model if you have an API key: +# vectorizer = Vectorizer("text-embedding-3-small", key="your_openai_api_key") vectorizer = Vectorizer() -# vectorizer = Vectorizer("text-embedding-ada-002", key="your_key") # Use this line for higher accuracy if you have an OpenAI API key - -# 2) Create a baseline of data to map to in the initialized repository. Text gets attached to any unique concept of an -# existing or custom vocabulary or terminology namespace in the form of a mapping object containing the text, embedding, -# and the name of sentence embedder used. Multiple Mapping objects with textually different but semantically equal -# descriptions can point to the same Concept. +repository = PostgreSQLRepository(connection_string, vectorizer=vectorizer) +# -------------------------------------------------------------------- +# 2) Add a small SNOMED CT baseline +# -------------------------------------------------------------------- terminology = Terminology("snomed CT", "SNOMED") text1 = "Diabetes mellitus (disorder)" @@ -27,12 +58,20 @@ repository.store_all([terminology, concept1, mapping1, concept2, mapping2]) -# 3) Retrieve the closest mappings and their similarities for a given text - -text_to_map = "Sugar sickness" # Semantically similar to "Diabetes mellitus (disorder)" -embedding = vectorizer.get_embedding(text_to_map) +# -------------------------------------------------------------------- +# 3) Find closest mappings for a new phrase +# -------------------------------------------------------------------- +query_text = "Sugar sickness" # semantically similar to "Diabetes mellitus (disorder)" +embedding = vectorizer.get_embedding(query_text) results = repository.get_closest_mappings(embedding, similarities=True, limit=2) -# 4) print the mappings and their similarities -for result in results: - print(result) +# -------------------------------------------------------------------- +# 4) Display the results +# -------------------------------------------------------------------- +print(f'Query: "{query_text}"\n') +for r in results: + # If similarities=True, repo returns MappingResult; else Mapping. + if isinstance(r, MappingResult): + print(r) + else: + print(str(r)) diff --git a/datastew/scripts/mapping_excel_example.py b/datastew/scripts/mapping_excel_example.py index da4b781..4a63ca9 100644 --- a/datastew/scripts/mapping_excel_example.py +++ b/datastew/scripts/mapping_excel_example.py @@ -1,9 +1,49 @@ +""" +Example: Map variables between two data dictionaries using datastew + +This script shows how to automatically map variables from a *source* +data dictionary to a *target* data dictionary based on semantic similarity +between their variable names and descriptions. + +--- + +Quick Start + +# 1. Prepare two Excel files: +# source.xlsx — contains your original study variable names +# target.xlsx — contains standardized variable names you want to map to +# Each file must include at least: +# - a column with variable names (e.g., "var") +# - a column with variable descriptions (e.g., "desc") + +# 2. Run the script: +python examples/map_data_dictionaries.py + +# 3. The output will be written to: +# result.xlsx — containing matched variables and similarity scores +""" + from datastew.process.mapping import map_dictionary_to_dictionary from datastew.process.parsing import DataDictionarySource -# Variable and description refer to the corresponding column names in your excel sheet +# -------------------------------------------------------------------- +# 1) Load source and target data dictionaries +# -------------------------------------------------------------------- +# Each DataDictionarySource represents one Excel sheet containing variable names and descriptions. +# Adjust the column names below ("var" and "desc") to match your Excel file. source = DataDictionarySource("source.xlxs", variable_field="var", description_field="desc") target = DataDictionarySource("target.xlxs", variable_field="var", description_field="desc") +# -------------------------------------------------------------------- +# 2) Perform automated mapping +# -------------------------------------------------------------------- +# This uses LLM-based embeddings under the hood to identify semantically similar variables between the two dictionaries. df = map_dictionary_to_dictionary(source, target) -df.to_excel("result.xlxs") + +# -------------------------------------------------------------------- +# 3) Save results +# -------------------------------------------------------------------- +# The result DataFrame includes the top matches and similarity scores. +output_path = "result.xlsx" +df.to_excel(output_path, index=False) +print(f"Mapped {len(df)} variables. Results saved to '{output_path}'.") diff --git a/datastew/scripts/ohdsi_to_jsonl.py b/datastew/scripts/ohdsi_to_jsonl.py index ded50d4..cb6fa50 100644 --- a/datastew/scripts/ohdsi_to_jsonl.py +++ b/datastew/scripts/ohdsi_to_jsonl.py @@ -1,5 +1,30 @@ +""" +Example: Convert OHDSI vocabulary files to JSONL format for Weaviate import + +This script demonstrates how to use the datastew WeaviateJsonlConverter to +transform an OHDSI-style vocabulary file (e.g., CONCEPT.csv) into a JSONL file +that can be directly imported into a Weaviate vector database. + +Steps: +1. Initialize the converter with an output directory. +2. Convert the OHDSI CONCEPT.csv file to JSONL format. +3. The resulting JSONL files will be saved in the specified output directory. +""" + from datastew.process.jsonl_adapter import WeaviateJsonlConverter -jsonl_converter = WeaviateJsonlConverter("resources/results") +# -------------------------------------------------------------------- +# 1) Initialize the converter +# -------------------------------------------------------------------- +# The output directory will contain the generated JSONL files. +output_directory = "resources/results" +jsonl_converter = WeaviateJsonlConverter(output_directory) + +# -------------------------------------------------------------------- +# 2) Convert the OHDSI CONCEPT.csv file +# -------------------------------------------------------------------- +# The input file should follow the standard OHDSI vocabulary structure. +input_file = "resources/CONCEPT.csv" +jsonl_converter.from_ohdsi(input_file) -jsonl_converter.from_ohdsi("resources/CONCEPT.csv") \ No newline at end of file +print(f"Conversion complete. JSONL files written to: {output_directory}") diff --git a/datastew/scripts/ols_snomed_retrieval.py b/datastew/scripts/ols_snomed_retrieval.py index c0200d0..7b2a550 100644 --- a/datastew/scripts/ols_snomed_retrieval.py +++ b/datastew/scripts/ols_snomed_retrieval.py @@ -1,11 +1,52 @@ +""" +Example: Import a terminology from the Ontology Lookup Service (OLS) into PostgreSQL + +This example demonstrates how to: +1. Connect to a PostgreSQL database using the datastew repository. +2. Initialize an embedding model for generating vector embeddings. +3. Import a terminology (for example, SNOMED CT) from the OLS service + directly into the PostgreSQL database. + +Before running this example: +- Ensure a PostgreSQL instance is running locally (for example, via Docker): + + docker run -d \ + --name datastew-postgres \ + -e POSTGRES_USER=user \ + -e POSTGRES_PASSWORD=password \ + -e POSTGRES_DB=testdb \ + -p 5432:5432 \ + postgres:15 +""" + from datastew.embedding import Vectorizer from datastew.process.ols import OLSTerminologyImportTask -from datastew.repository import WeaviateRepository +from datastew.repository import PostgreSQLRepository + +# -------------------------------------------------------------------- +# 1) Connect to PostgreSQL +# -------------------------------------------------------------------- +POSTGRES_USER = "user" +POSTGRES_PASSWORD = "password" +POSTGRES_HOST = "localhost" +POSTGRES_PORT = "5432" +POSTGRES_DB = "testdb" + +connection_string = ( + f"postgresql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@" f"{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}" +) +vectorizer = Vectorizer() +repository = PostgreSQLRepository(connection_string, vectorizer) + +# -------------------------------------------------------------------- +# 3) Import terminology from OLS +# -------------------------------------------------------------------- +# The first argument is the embedding model. +# The second argument is the OLS display name of the terminology. +# The third argument is the short identifier used internally. +task = OLSTerminologyImportTask(vectorizer, "SNOMED CT", "snomed") -# Use a local running weaviate instance on localhost:8080 -repository = WeaviateRepository(mode='remote', path='localhost', port=8080) -embedding_model = Vectorizer() +# This method fetches, embeds, and uploads the terminology to PostgreSQL. +task.process_to_repository(repository) -task = OLSTerminologyImportTask(embedding_model, "SNOMED CT", "snomed") -task.process_to_weaviate(repository) -print("done") \ No newline at end of file +print("Terminology import completed successfully.") diff --git a/datastew/scripts/tsne_visualization.py b/datastew/scripts/tsne_visualization.py index e2cb55b..0d8cc89 100644 --- a/datastew/scripts/tsne_visualization.py +++ b/datastew/scripts/tsne_visualization.py @@ -1,10 +1,41 @@ +""" +Example: Visualize the semantic similarity between two data dictionaries + +This script demonstrates how to: +1. Load two data dictionaries from Excel files. +2. Generate text embeddings for each variable description. +3. Visualize their semantic relationships in a shared 2D embedding space. + +The resulting plot highlights how conceptually similar variables +from both data dictionaries cluster together. + +Before running this example: +- Ensure both Excel files contain columns for variable names and descriptions. +- The column names should match the ones you specify in `variable_field` + and `description_field`. +""" + from datastew.embedding import Vectorizer from datastew.process.parsing import DataDictionarySource from datastew.visualisation import plot_embeddings -# Variable and description refer to the corresponding column names in your excel sheet +# -------------------------------------------------------------------- +# 1) Load data dictionaries +# -------------------------------------------------------------------- +# Replace these filenames and column names with your own. data_dictionary_source_1 = DataDictionarySource("source1.xlsx", variable_field="var", description_field="desc") data_dictionary_source_2 = DataDictionarySource("source2.xlsx", variable_field="var", description_field="desc") +# -------------------------------------------------------------------- +# 2) Initialize the embedding model +# -------------------------------------------------------------------- +# You can also specify a model name or API key if desired: +# vectorizer = Vectorizer("text-embedding-3-small", key="your_openai_api_key") vectorizer = Vectorizer() + +# -------------------------------------------------------------------- +# 3) Generate embeddings and visualize +# -------------------------------------------------------------------- +# The resulting visualization displays variables from both sources +# positioned based on semantic similarity. plot_embeddings([data_dictionary_source_1, data_dictionary_source_2], vectorizer=vectorizer) From 889b3a3fd3afc23eb2629c2f8444e44be150ce42 Mon Sep 17 00:00:00 2001 From: Mehmet Can Ay Date: Mon, 10 Nov 2025 12:26:01 +0100 Subject: [PATCH 2/2] refactor(ols): generalize function names to all supported repositories --- datastew/process/ols.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datastew/process/ols.py b/datastew/process/ols.py index 8b5b9b3..ba625b7 100644 --- a/datastew/process/ols.py +++ b/datastew/process/ols.py @@ -83,11 +83,11 @@ def __process_page(self, page: int) -> Tuple[List[Concept], List[Mapping]]: logging.error(f"Failed to fetch concepts and descriptions from OLS for page {page}: {str(e)}") return [], [] - def process_to_weaviate(self, repository: BaseRepository): + def process_to_repository(self, repository: BaseRepository): """ - Fetches concepts and descriptions from the OLS API and stores them in a Weaviate repository. + Fetches concepts and descriptions from the OLS API and stores them in a repository. - :param repository: The Weaviate repository to store the concepts and mappings. + :param repository: The repository to store the concepts and mappings. :return: None """ @@ -102,7 +102,7 @@ def process_to_weaviate(self, repository: BaseRepository): repository.store(mappings[idx]) self.current_page += 1 - def process_to_weaviate_json(self, dest_path: str): + def process_to_json(self, dest_path: str): """ Fetches concepts and descriptions from the OLS API and stores them in a JSON file. """