Fix/update OpenAI sdk syntax

README.md

@@ -68,7 +68,7 @@ A simple example how to initialize an in memory database and compute a similarit
     connection_string = f"postgresql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}"
 
     # You can use an OpenAI model if you have an API key:
-    # vectorizer = Vectorizer("text-embedding-3-small", key="your_openai_api_key")
+    # vectorizer = Vectorizer("text-embedding-3-small", api_key="your_openai_api_key")
     vectorizer = Vectorizer()
     repository = PostgreSQLRepository(connection_string, vectorizer=vectorizer)
     ```

datastew/embedding.py

@@ -128,8 +128,8 @@ def get_embedding(self, text: str) -> Sequence[float]:
 
         # Request from OpenAI API
         try:
-            response = openai.Embedding.create(input=[text], model=self.model_name)
-            embedding = response["data"][0]["embedding"]
+            response = openai.embeddings.create(input=[text], model=self.model_name)
+            embedding = response.data[0].embedding
             self.add_to_cache(text, embedding)
             return embedding
         except Exception as e:
@@ -150,8 +150,8 @@ def get_embeddings(self, messages: List[str]) -> Sequence[Sequence[float]]:
 
             if uncached_messages:
                 try:
-                    response = openai.Embedding.create(model=self.model_name, input=uncached_messages)
-                    new_embeddings = [item["embedding"] for item in response["data"]]
+                    response = openai.embeddings.create(model=self.model_name, input=uncached_messages)
+                    new_embeddings = [item.embedding for item in response.data]
                     for idx, embedding in zip(uncached_indices, new_embeddings):
                         self.add_to_cache(sanitized_messages[idx], embedding)
                         embeddings[idx] = embedding
@@ -162,8 +162,8 @@ def get_embeddings(self, messages: List[str]) -> Sequence[Sequence[float]]:
             return [emb for emb in embeddings if emb is not None]
 
         try:
-            response = openai.Embedding.create(model=self.model_name, input=sanitized_messages)
-            embeddings = [item["embedding"] for item in response["data"]]
+            response = openai.embeddings.create(model=self.model_name, input=sanitized_messages)
+            embeddings = [item.embedding for item in response.data]
             return embeddings
         except Exception as e:
             logging.error(f"Failed processing messages: {e}")

datastew/repository/postgresql.py

@@ -2,7 +2,7 @@
 import logging
 from typing import Any, Dict, List, Literal, Optional, Sequence, Union
 
-from sqlalchemy import create_engine, func, inspect, text
+from sqlalchemy import create_engine, func, text
 from sqlalchemy.orm import joinedload, sessionmaker
 
 from datastew.embedding import Vectorizer
@@ -49,10 +49,7 @@ def store(self, model_object_instance: Union[Terminology, Concept, Mapping]):
         :raises ObjectStorageError: If the object cannot be stored (e.g., due to DB errors).
         """
         try:
-            if self._is_duplicate(model_object_instance):
-                return
-
-            self.session.add(model_object_instance)
+            self.session.merge(model_object_instance)
             self.session.commit()
         except Exception as e:
             self.session.rollback()
@@ -308,27 +305,3 @@ def _validate_required_fields(
     def _initialize_pgvector(self):
         with self.engine.begin() as conn:
             conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
-
-    def _is_duplicate(self, model_object_instance: Union[Terminology, Concept, Mapping]) -> bool:
-        """Checks whether an object with the same primary key already exists.
-
-        :param model_object_instance: SQLAlchemy model instance.
-        :return: True if a duplicate exists, False otherwise.
-        """
-        cls = type(model_object_instance)
-        pk_attrs = inspect(cls).primary_key
-
-        if len(pk_attrs) != 1:
-            logger.warning(
-                f"Duplicate check only supports single-column primary keys. Skipping check for {cls.__name__}"
-            )
-            return False
-
-        pk_attr = pk_attrs[0].name
-        pk_value = getattr(model_object_instance, pk_attr)
-        existing = self.session.get(cls, pk_value)
-
-        if existing:
-            logger.info(f"Skipped storing existing {cls.__name__} with {pk_attr}={pk_value}")
-            return True
-        return False

datastew/repository/sqllite.py

@@ -3,7 +3,7 @@
 from typing import Any, Dict, List, Literal, Optional, Union
 
 import numpy as np
-from sqlalchemy import create_engine, func, inspect
+from sqlalchemy import create_engine, func
 from sqlalchemy.orm import joinedload, sessionmaker
 from sqlalchemy.pool import StaticPool
 
@@ -52,10 +52,7 @@ def store(self, model_object_instance: Union[Terminology, Concept, Mapping]):
         :raises ObjectStorageError: If the object cannot be stored (e.g., due to DB errors).
         """
         try:
-            if self._is_duplicate(model_object_instance):
-                return
-
-            self.session.add(model_object_instance)
+            self.session.merge(model_object_instance)
             self.session.commit()
         except Exception as e:
             self.session.rollback()
@@ -321,27 +318,3 @@ def _validate_required_fields(
         for key in required_keys:
             if key not in data:
                 raise ValueError(f"Missing required field '{key}' for {object_type}")
-
-    def _is_duplicate(self, model_object_instance: Union[Terminology, Concept, Mapping]) -> bool:
-        """Checks whether an object with the same primary key already exists.
-
-        :param model_object_instance: SQLAlchemy model instance.
-        :return: True if a duplicate exists, False otherwise.
-        """
-        cls = type(model_object_instance)
-        pk_attrs = inspect(cls).primary_key
-
-        if len(pk_attrs) != 1:
-            logger.warning(
-                f"Duplicate check only supports single-column primary keys. Skipping check for {cls.__name__}"
-            )
-            return False
-
-        pk_attr = pk_attrs[0].name
-        pk_value = getattr(model_object_instance, pk_attr)
-        existing = self.session.get(cls, pk_value)
-
-        if existing:
-            logger.info(f"Skipped storing existing {cls.__name__} with {pk_attr}={pk_value}")
-            return True
-        return False

datastew/scripts/export_json_from_weaviate.py

@@ -26,7 +26,7 @@
 # 2) Vectorizer (use default or specify a model/API key if desired)
 # --------------------------------------------------------------------
 # Example for OpenAI:
-# vectorizer = Vectorizer("text-embedding-3-small", key="your_openai_api_key")
+# vectorizer = Vectorizer("text-embedding-3-small", api_key="your_openai_api_key")
 vectorizer = Vectorizer()
 
 # --------------------------------------------------------------------

datastew/scripts/fill_db_example.py

@@ -17,7 +17,7 @@
   -e POSTGRES_PASSWORD=password \
   -e POSTGRES_DB=testdb \
   -p 5432:5432 \
-  postgres:15
+  pgvector/pgvector:pg17
 
 # 2. Run this script:
 python examples/store_snomed_baseline.py
@@ -42,7 +42,7 @@
 connection_string = f"postgresql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}"
 
 # Use OpenAI embeddings if you have an API key for higher-quality results:
-# vectorizer = Vectorizer("text-embedding-3-small", key="your_openai_api_key")
+# vectorizer = Vectorizer("text-embedding-3-small", api_key="your_openai_api_key")
 vectorizer = Vectorizer()
 repository = PostgreSQLRepository(connection_string, vectorizer=vectorizer)
 

datastew/scripts/mapping_db_example.py

@@ -17,7 +17,7 @@
   -e POSTGRES_PASSWORD=password \
   -e POSTGRES_DB=testdb \
   -p 5432:5432 \
-  postgres:15
+  pgvector/pgvector:pg17
 
 # Run this script:
 python examples/get_closest_mappings.py
@@ -39,7 +39,7 @@
 connection_string = f"postgresql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}"
 
 # You can use an OpenAI model if you have an API key:
-# vectorizer = Vectorizer("text-embedding-3-small", key="your_openai_api_key")
+# vectorizer = Vectorizer("text-embedding-3-small", api_key="your_openai_api_key")
 vectorizer = Vectorizer()
 repository = PostgreSQLRepository(connection_string, vectorizer=vectorizer)
 

datastew/scripts/ols_snomed_retrieval.py

@@ -16,7 +16,7 @@
       -e POSTGRES_PASSWORD=password \
       -e POSTGRES_DB=testdb \
       -p 5432:5432 \
-      postgres:15
+      pgvector/pgvector:pg17
 """
 
 from datastew.embedding import Vectorizer

datastew/scripts/tsne_visualization.py

@@ -30,7 +30,7 @@
 # 2) Initialize the embedding model
 # --------------------------------------------------------------------
 # You can also specify a model name or API key if desired:
-# vectorizer = Vectorizer("text-embedding-3-small", key="your_openai_api_key")
+# vectorizer = Vectorizer("text-embedding-3-small", api_key="your_openai_api_key")
 vectorizer = Vectorizer()
 
 # --------------------------------------------------------------------