From a7c40a615b1085a4ec7fff563759780c9b483241 Mon Sep 17 00:00:00 2001 From: Mehmet Can Ay Date: Mon, 10 Nov 2025 13:11:04 +0100 Subject: [PATCH 1/4] fix: resolve duplicate key errors by using merge() for SQL persistence --- datastew/repository/postgresql.py | 31 ++----------------------------- datastew/repository/sqllite.py | 31 ++----------------------------- 2 files changed, 4 insertions(+), 58 deletions(-) diff --git a/datastew/repository/postgresql.py b/datastew/repository/postgresql.py index d71e353..0ee0cc5 100644 --- a/datastew/repository/postgresql.py +++ b/datastew/repository/postgresql.py @@ -2,7 +2,7 @@ import logging from typing import Any, Dict, List, Literal, Optional, Sequence, Union -from sqlalchemy import create_engine, func, inspect, text +from sqlalchemy import create_engine, func, text from sqlalchemy.orm import joinedload, sessionmaker from datastew.embedding import Vectorizer @@ -49,10 +49,7 @@ def store(self, model_object_instance: Union[Terminology, Concept, Mapping]): :raises ObjectStorageError: If the object cannot be stored (e.g., due to DB errors). """ try: - if self._is_duplicate(model_object_instance): - return - - self.session.add(model_object_instance) + self.session.merge(model_object_instance) self.session.commit() except Exception as e: self.session.rollback() @@ -308,27 +305,3 @@ def _validate_required_fields( def _initialize_pgvector(self): with self.engine.begin() as conn: conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector")) - - def _is_duplicate(self, model_object_instance: Union[Terminology, Concept, Mapping]) -> bool: - """Checks whether an object with the same primary key already exists. - - :param model_object_instance: SQLAlchemy model instance. - :return: True if a duplicate exists, False otherwise. - """ - cls = type(model_object_instance) - pk_attrs = inspect(cls).primary_key - - if len(pk_attrs) != 1: - logger.warning( - f"Duplicate check only supports single-column primary keys. Skipping check for {cls.__name__}" - ) - return False - - pk_attr = pk_attrs[0].name - pk_value = getattr(model_object_instance, pk_attr) - existing = self.session.get(cls, pk_value) - - if existing: - logger.info(f"Skipped storing existing {cls.__name__} with {pk_attr}={pk_value}") - return True - return False diff --git a/datastew/repository/sqllite.py b/datastew/repository/sqllite.py index 5bb779f..28c8bd8 100644 --- a/datastew/repository/sqllite.py +++ b/datastew/repository/sqllite.py @@ -3,7 +3,7 @@ from typing import Any, Dict, List, Literal, Optional, Union import numpy as np -from sqlalchemy import create_engine, func, inspect +from sqlalchemy import create_engine, func from sqlalchemy.orm import joinedload, sessionmaker from sqlalchemy.pool import StaticPool @@ -52,10 +52,7 @@ def store(self, model_object_instance: Union[Terminology, Concept, Mapping]): :raises ObjectStorageError: If the object cannot be stored (e.g., due to DB errors). """ try: - if self._is_duplicate(model_object_instance): - return - - self.session.add(model_object_instance) + self.session.merge(model_object_instance) self.session.commit() except Exception as e: self.session.rollback() @@ -321,27 +318,3 @@ def _validate_required_fields( for key in required_keys: if key not in data: raise ValueError(f"Missing required field '{key}' for {object_type}") - - def _is_duplicate(self, model_object_instance: Union[Terminology, Concept, Mapping]) -> bool: - """Checks whether an object with the same primary key already exists. - - :param model_object_instance: SQLAlchemy model instance. - :return: True if a duplicate exists, False otherwise. - """ - cls = type(model_object_instance) - pk_attrs = inspect(cls).primary_key - - if len(pk_attrs) != 1: - logger.warning( - f"Duplicate check only supports single-column primary keys. Skipping check for {cls.__name__}" - ) - return False - - pk_attr = pk_attrs[0].name - pk_value = getattr(model_object_instance, pk_attr) - existing = self.session.get(cls, pk_value) - - if existing: - logger.info(f"Skipped storing existing {cls.__name__} with {pk_attr}={pk_value}") - return True - return False From 59577dc6f8f44a65b295da94fba55f364f310070 Mon Sep 17 00:00:00 2001 From: Mehmet Can Ay Date: Mon, 10 Nov 2025 13:11:49 +0100 Subject: [PATCH 2/4] fix: update OpenAI SDK usage to match new API syntax --- datastew/embedding.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/datastew/embedding.py b/datastew/embedding.py index 6a18990..3f411d4 100644 --- a/datastew/embedding.py +++ b/datastew/embedding.py @@ -128,8 +128,8 @@ def get_embedding(self, text: str) -> Sequence[float]: # Request from OpenAI API try: - response = openai.Embedding.create(input=[text], model=self.model_name) - embedding = response["data"][0]["embedding"] + response = openai.embeddings.create(input=[text], model=self.model_name) + embedding = response.data[0].embedding self.add_to_cache(text, embedding) return embedding except Exception as e: @@ -150,8 +150,8 @@ def get_embeddings(self, messages: List[str]) -> Sequence[Sequence[float]]: if uncached_messages: try: - response = openai.Embedding.create(model=self.model_name, input=uncached_messages) - new_embeddings = [item["embedding"] for item in response["data"]] + response = openai.embeddings.create(model=self.model_name, input=uncached_messages) + new_embeddings = [item.embedding for item in response.data] for idx, embedding in zip(uncached_indices, new_embeddings): self.add_to_cache(sanitized_messages[idx], embedding) embeddings[idx] = embedding @@ -162,8 +162,8 @@ def get_embeddings(self, messages: List[str]) -> Sequence[Sequence[float]]: return [emb for emb in embeddings if emb is not None] try: - response = openai.Embedding.create(model=self.model_name, input=sanitized_messages) - embeddings = [item["embedding"] for item in response["data"]] + response = openai.embeddings.create(model=self.model_name, input=sanitized_messages) + embeddings = [item.embedding for item in response.data] return embeddings except Exception as e: logging.error(f"Failed processing messages: {e}") From ea65110b478866639a26658ab72307d50b9d7de0 Mon Sep 17 00:00:00 2001 From: Mehmet Can Ay Date: Mon, 10 Nov 2025 13:12:27 +0100 Subject: [PATCH 3/4] docs: correct postgresql image name and Vectorizer parameter name --- datastew/scripts/export_json_from_weaviate.py | 2 +- datastew/scripts/fill_db_example.py | 4 ++-- datastew/scripts/mapping_db_example.py | 4 ++-- datastew/scripts/ols_snomed_retrieval.py | 2 +- datastew/scripts/tsne_visualization.py | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/datastew/scripts/export_json_from_weaviate.py b/datastew/scripts/export_json_from_weaviate.py index cce4344..afbd1e3 100644 --- a/datastew/scripts/export_json_from_weaviate.py +++ b/datastew/scripts/export_json_from_weaviate.py @@ -26,7 +26,7 @@ # 2) Vectorizer (use default or specify a model/API key if desired) # -------------------------------------------------------------------- # Example for OpenAI: -# vectorizer = Vectorizer("text-embedding-3-small", key="your_openai_api_key") +# vectorizer = Vectorizer("text-embedding-3-small", api_key="your_openai_api_key") vectorizer = Vectorizer() # -------------------------------------------------------------------- diff --git a/datastew/scripts/fill_db_example.py b/datastew/scripts/fill_db_example.py index a79a2f0..da73518 100644 --- a/datastew/scripts/fill_db_example.py +++ b/datastew/scripts/fill_db_example.py @@ -17,7 +17,7 @@ -e POSTGRES_PASSWORD=password \ -e POSTGRES_DB=testdb \ -p 5432:5432 \ - postgres:15 + pgvector/pgvector:pg17 # 2. Run this script: python examples/store_snomed_baseline.py @@ -42,7 +42,7 @@ connection_string = f"postgresql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}" # Use OpenAI embeddings if you have an API key for higher-quality results: -# vectorizer = Vectorizer("text-embedding-3-small", key="your_openai_api_key") +# vectorizer = Vectorizer("text-embedding-3-small", api_key="your_openai_api_key") vectorizer = Vectorizer() repository = PostgreSQLRepository(connection_string, vectorizer=vectorizer) diff --git a/datastew/scripts/mapping_db_example.py b/datastew/scripts/mapping_db_example.py index caa7277..81e07f5 100644 --- a/datastew/scripts/mapping_db_example.py +++ b/datastew/scripts/mapping_db_example.py @@ -17,7 +17,7 @@ -e POSTGRES_PASSWORD=password \ -e POSTGRES_DB=testdb \ -p 5432:5432 \ - postgres:15 + pgvector/pgvector:pg17 # Run this script: python examples/get_closest_mappings.py @@ -39,7 +39,7 @@ connection_string = f"postgresql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}" # You can use an OpenAI model if you have an API key: -# vectorizer = Vectorizer("text-embedding-3-small", key="your_openai_api_key") +# vectorizer = Vectorizer("text-embedding-3-small", api_key="your_openai_api_key") vectorizer = Vectorizer() repository = PostgreSQLRepository(connection_string, vectorizer=vectorizer) diff --git a/datastew/scripts/ols_snomed_retrieval.py b/datastew/scripts/ols_snomed_retrieval.py index 7b2a550..0a64f26 100644 --- a/datastew/scripts/ols_snomed_retrieval.py +++ b/datastew/scripts/ols_snomed_retrieval.py @@ -16,7 +16,7 @@ -e POSTGRES_PASSWORD=password \ -e POSTGRES_DB=testdb \ -p 5432:5432 \ - postgres:15 + pgvector/pgvector:pg17 """ from datastew.embedding import Vectorizer diff --git a/datastew/scripts/tsne_visualization.py b/datastew/scripts/tsne_visualization.py index 0d8cc89..672eebb 100644 --- a/datastew/scripts/tsne_visualization.py +++ b/datastew/scripts/tsne_visualization.py @@ -30,7 +30,7 @@ # 2) Initialize the embedding model # -------------------------------------------------------------------- # You can also specify a model name or API key if desired: -# vectorizer = Vectorizer("text-embedding-3-small", key="your_openai_api_key") +# vectorizer = Vectorizer("text-embedding-3-small", api_key="your_openai_api_key") vectorizer = Vectorizer() # -------------------------------------------------------------------- From 5f90f152a261552e327bc7891b8b41f8cf1e5e35 Mon Sep 17 00:00:00 2001 From: Mehmet Can Ay Date: Mon, 10 Nov 2025 13:16:34 +0100 Subject: [PATCH 4/4] docs: correct Vectorizer parameter name --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fb72b7d..d793414 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ A simple example how to initialize an in memory database and compute a similarit connection_string = f"postgresql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}" # You can use an OpenAI model if you have an API key: - # vectorizer = Vectorizer("text-embedding-3-small", key="your_openai_api_key") + # vectorizer = Vectorizer("text-embedding-3-small", api_key="your_openai_api_key") vectorizer = Vectorizer() repository = PostgreSQLRepository(connection_string, vectorizer=vectorizer) ```