diff --git a/README.md b/README.md index 37f96b8..816a0e1 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,8 @@ csv, tsv or excel file. An example how to match two separate variable descriptio [datastew/scripts/mapping_excel_example.py](datastew/scripts/mapping_excel_example.py): ```python -from datastew.process.parsing import DataDictionarySource -from datastew.process.mapping import map_dictionary_to_dictionary +from datastew.io.source import DataDictionarySource +from datastew.harmonization import map_dictionary_to_dictionary # Variable and description refer to the corresponding column names in your excel sheet source = DataDictionarySource("source.xlxs", variable_field="var", description_field="desc") @@ -39,7 +39,7 @@ function: ```python from datastew.embedding import Vectorizer -from datastew.process.mapping import map_dictionary_to_dictionary +from datastew.harmonization import map_dictionary_to_dictionary vectorizer = Vectorizer("text-embedding-ada-002", key="your_api_key") df = map_dictionary_to_dictionary(source, target, vectorizer=vectorizer) @@ -135,7 +135,7 @@ language models. An example how to generate a t-sne plot is shown in ```python from datastew.embedding import Vectorizer -from datastew.process.parsing import DataDictionarySource +from datastew.io.source import DataDictionarySource from datastew.visualisation import plot_embeddings # Variable and description refer to the corresponding column names in your excel sheet diff --git a/datastew/__init__.py b/datastew/__init__.py index 831a8a2..f83997d 100644 --- a/datastew/__init__.py +++ b/datastew/__init__.py @@ -1,8 +1,9 @@ from ._version import __version__ from .embedding import Vectorizer - -# Importing submodules to expose their attributes if needed -from .process import jsonl_adapter, mapping, ols, parsing +from .harmonization import mapping +from .integrations import ols +from .io import source +from .io.adapters import jsonl from .repository import model, postgresql from .visualisation import ( bar_chart_average_acc_two_distributions, @@ -13,15 +14,15 @@ __all__ = [ "__version__", - "jsonl_adapter", - "mapping", - "ols", - "parsing", - "model", - "postgresql", "Vectorizer", "bar_chart_average_acc_two_distributions", "enrichment_plot", "get_plot_for_current_database_state", + "jsonl", + "mapping", + "model", + "ols", + "source", "plot_embeddings", + "postgresql", ] diff --git a/datastew/harmonization/__init__.py b/datastew/harmonization/__init__.py new file mode 100644 index 0000000..17dfff1 --- /dev/null +++ b/datastew/harmonization/__init__.py @@ -0,0 +1,3 @@ +from .mapping import map_dictionary_to_dictionary + +__all__ = ["map_dictionary_to_dictionary"] diff --git a/datastew/process/mapping.py b/datastew/harmonization/mapping.py similarity index 98% rename from datastew/process/mapping.py rename to datastew/harmonization/mapping.py index dcc31e8..1ac4916 100644 --- a/datastew/process/mapping.py +++ b/datastew/harmonization/mapping.py @@ -3,7 +3,7 @@ from sklearn.metrics.pairwise import cosine_similarity from datastew.embedding import Vectorizer -from datastew.process.parsing import DataDictionarySource +from datastew.io.source import DataDictionarySource def map_dictionary_to_dictionary( diff --git a/datastew/integrations/__init__.py b/datastew/integrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/datastew/integrations/ols/__init__.py b/datastew/integrations/ols/__init__.py new file mode 100644 index 0000000..cfaf430 --- /dev/null +++ b/datastew/integrations/ols/__init__.py @@ -0,0 +1,3 @@ +from .client import OlsClient + +__all__ = ["OlsClient"] diff --git a/datastew/process/ols.py b/datastew/integrations/ols/client.py similarity index 99% rename from datastew/process/ols.py rename to datastew/integrations/ols/client.py index e901442..a7ea28a 100644 --- a/datastew/process/ols.py +++ b/datastew/integrations/ols/client.py @@ -10,7 +10,7 @@ from datastew.repository.model import Concept, Mapping, Terminology -class OLSTerminologyImportTask: +class OlsClient: def __init__( self, diff --git a/datastew/io/__init__.py b/datastew/io/__init__.py new file mode 100644 index 0000000..5cd6744 --- /dev/null +++ b/datastew/io/__init__.py @@ -0,0 +1,4 @@ +from .importer import Importer +from .source import DataDictionarySource + +__all__ = ["Importer", "DataDictionarySource"] diff --git a/datastew/io/adapters/__init__.py b/datastew/io/adapters/__init__.py new file mode 100644 index 0000000..2d87c47 --- /dev/null +++ b/datastew/io/adapters/__init__.py @@ -0,0 +1,3 @@ +from .jsonl import JsonlAdapter + +__all__ = ["JsonlAdapter"] diff --git a/datastew/process/jsonl_adapter.py b/datastew/io/adapters/jsonl.py similarity index 99% rename from datastew/process/jsonl_adapter.py rename to datastew/io/adapters/jsonl.py index f7755f8..2c763ef 100644 --- a/datastew/process/jsonl_adapter.py +++ b/datastew/io/adapters/jsonl.py @@ -11,7 +11,7 @@ from datastew.repository.model import Concept, Mapping, Terminology -class SQLJsonlConverter: +class JsonlAdapter: def __init__(self, dest_dir: str, buffer_size: int = 1000): """Initialize the converter. diff --git a/datastew/process/importer.py b/datastew/io/importer.py similarity index 99% rename from datastew/process/importer.py rename to datastew/io/importer.py index 4a5014a..7be5bb4 100644 --- a/datastew/process/importer.py +++ b/datastew/io/importer.py @@ -5,14 +5,14 @@ from sqlalchemy import text -from datastew.process.parsing import DataDictionarySource +from datastew.io.source import DataDictionarySource from datastew.repository import PostgreSQLRepository from datastew.repository.model import Concept, Mapping, Terminology logger = logging.getLogger(__name__) -class PostgreSQLImporter: +class Importer: def __init__(self, repository: PostgreSQLRepository): self.repository = repository self.engine = repository.engine diff --git a/datastew/process/parsing.py b/datastew/io/source.py similarity index 100% rename from datastew/process/parsing.py rename to datastew/io/source.py diff --git a/datastew/process/__init__.py b/datastew/process/__init__.py deleted file mode 100644 index ba6095f..0000000 --- a/datastew/process/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from .mapping import map_dictionary_to_dictionary -from .parsing import DataDictionarySource - -__all__ = [ - "map_dictionary_to_dictionary", - "DataDictionarySource" -] \ No newline at end of file diff --git a/datastew/scripts/mapping_excel_example.py b/datastew/scripts/mapping_excel_example.py index 4a63ca9..efa2b69 100644 --- a/datastew/scripts/mapping_excel_example.py +++ b/datastew/scripts/mapping_excel_example.py @@ -23,8 +23,8 @@ # result.xlsx — containing matched variables and similarity scores """ -from datastew.process.mapping import map_dictionary_to_dictionary -from datastew.process.parsing import DataDictionarySource +from datastew.harmonization import map_dictionary_to_dictionary +from datastew.io.source import DataDictionarySource # -------------------------------------------------------------------- # 1) Load source and target data dictionaries diff --git a/datastew/scripts/ohdsi_to_jsonl.py b/datastew/scripts/ohdsi_to_jsonl.py index b6f4436..f27c7f9 100644 --- a/datastew/scripts/ohdsi_to_jsonl.py +++ b/datastew/scripts/ohdsi_to_jsonl.py @@ -11,14 +11,14 @@ 3. The resulting JSONL files will be saved in the specified output directory. """ -from datastew.process.jsonl_adapter import SQLJsonlConverter +from datastew.io.adapters import JsonlAdapter # -------------------------------------------------------------------- # 1) Initialize the converter # -------------------------------------------------------------------- # The output directory will contain the generated JSONL files. output_directory = "resources/results" -jsonl_converter = SQLJsonlConverter(output_directory) +jsonl_converter = JsonlAdapter(output_directory) # -------------------------------------------------------------------- # 2) Convert the OHDSI CONCEPT.csv file diff --git a/datastew/scripts/ols_snomed_retrieval.py b/datastew/scripts/ols_snomed_retrieval.py index 0a64f26..049d80d 100644 --- a/datastew/scripts/ols_snomed_retrieval.py +++ b/datastew/scripts/ols_snomed_retrieval.py @@ -20,7 +20,7 @@ """ from datastew.embedding import Vectorizer -from datastew.process.ols import OLSTerminologyImportTask +from datastew.integrations.ols import OlsClient from datastew.repository import PostgreSQLRepository # -------------------------------------------------------------------- @@ -44,7 +44,7 @@ # The first argument is the embedding model. # The second argument is the OLS display name of the terminology. # The third argument is the short identifier used internally. -task = OLSTerminologyImportTask(vectorizer, "SNOMED CT", "snomed") +task = OlsClient(vectorizer, "SNOMED CT", "snomed") # This method fetches, embeds, and uploads the terminology to PostgreSQL. task.process_to_repository(repository) diff --git a/datastew/scripts/tsne_visualization.py b/datastew/scripts/tsne_visualization.py index 672eebb..492d258 100644 --- a/datastew/scripts/tsne_visualization.py +++ b/datastew/scripts/tsne_visualization.py @@ -16,7 +16,7 @@ """ from datastew.embedding import Vectorizer -from datastew.process.parsing import DataDictionarySource +from datastew.io.source import DataDictionarySource from datastew.visualisation import plot_embeddings # -------------------------------------------------------------------- diff --git a/datastew/visualisation.py b/datastew/visualisation.py index e41e12a..073624c 100644 --- a/datastew/visualisation.py +++ b/datastew/visualisation.py @@ -9,7 +9,7 @@ from sklearn.manifold import TSNE from datastew.embedding import Vectorizer -from datastew.process.parsing import DataDictionarySource +from datastew.io.source import DataDictionarySource from datastew.repository import PostgreSQLRepository diff --git a/tests/test_harmonization.py b/tests/test_harmonization.py new file mode 100644 index 0000000..fb21fb9 --- /dev/null +++ b/tests/test_harmonization.py @@ -0,0 +1,22 @@ +import os +import unittest + +from datastew.harmonization import map_dictionary_to_dictionary +from datastew.io.source import DataDictionarySource + + +class TestHarmonization(unittest.TestCase): + + TEST_DIR_PATH = os.path.dirname(os.path.realpath(__file__)) + + data_dictionary_source = DataDictionarySource( + os.path.join(TEST_DIR_PATH, "resources", "test_data_dict.csv"), "VAR_1", "DESC" + ) + + data_dictionary_target = DataDictionarySource( + os.path.join(TEST_DIR_PATH, "resources", "test_data_dict.xlsx"), "VAR_1", "DESC" + ) + + def test_map_dictionary_to_dictionary(self): + df = map_dictionary_to_dictionary(self.data_dictionary_source, self.data_dictionary_target, limit=2) + print(df) diff --git a/tests/test_postgresql_import.py b/tests/test_importer.py similarity index 96% rename from tests/test_postgresql_import.py rename to tests/test_importer.py index 0884e15..82415d2 100644 --- a/tests/test_postgresql_import.py +++ b/tests/test_importer.py @@ -6,19 +6,19 @@ from typing import Any, Literal from unittest import TestCase -from datastew.process.importer import PostgreSQLImporter -from datastew.process.parsing import DataDictionarySource +from datastew.io.importer import Importer +from datastew.io.source import DataDictionarySource from datastew.repository import PostgreSQLRepository -class TestPostgreSQLImporter(TestCase): +class TestImporter(TestCase): def setUp(self) -> None: POSTGRES_TEST_URL = os.getenv("TEST_POSTGRES_URI", "postgresql://testuser:testpass@localhost/testdb") self.TEST_DIR_PATH = os.path.dirname(os.path.realpath(__file__)) self.repository = PostgreSQLRepository(POSTGRES_TEST_URL) self.repository.clear_all() - self.importer = PostgreSQLImporter(self.repository) + self.importer = Importer(self.repository) self.temp_dir = tempfile.mkdtemp() # Sample data for JSONL files diff --git a/tests/test_json_adapter.py b/tests/test_jsonl_adapter.py similarity index 96% rename from tests/test_json_adapter.py rename to tests/test_jsonl_adapter.py index 20e7ee5..1bb9f02 100644 --- a/tests/test_json_adapter.py +++ b/tests/test_jsonl_adapter.py @@ -7,7 +7,7 @@ import pandas as pd from datastew.embedding import Vectorizer -from datastew.process.jsonl_adapter import SQLJsonlConverter +from datastew.io.adapters import JsonlAdapter class MockVectorizer(Vectorizer): @@ -17,7 +17,7 @@ def get_embeddings(self, texts): return [[0.1, 0.2, 0.3] for _ in texts] -class TestSQLJsonlConverter(unittest.TestCase): +class TestJsonlAdapter(unittest.TestCase): """Unit tests for SQLJsonlConverter""" def setUp(self): @@ -49,7 +49,7 @@ def setUp(self): data.to_csv(self.mock_concept_file, sep="\t", index=False) # Initialize SQLJsonlConverter - self.converter = SQLJsonlConverter(dest_dir=self.temp_dir) + self.converter = JsonlAdapter(dest_dir=self.temp_dir) # Inject Mock Embedding Model self.mock_vectorizer_model = MockVectorizer() diff --git a/tests/test_mapping.py b/tests/test_mapping.py deleted file mode 100644 index abba363..0000000 --- a/tests/test_mapping.py +++ /dev/null @@ -1,19 +0,0 @@ -import os -import unittest - -from datastew.process.mapping import map_dictionary_to_dictionary -from datastew.process.parsing import DataDictionarySource - - -class TestEmbedding(unittest.TestCase): - - TEST_DIR_PATH = os.path.dirname(os.path.realpath(__file__)) - - data_dictionary_source = DataDictionarySource(os.path.join(TEST_DIR_PATH, "resources", "test_data_dict.csv"), - "VAR_1", "DESC") - - data_dictionary_target = DataDictionarySource(os.path.join(TEST_DIR_PATH, "resources", "test_data_dict.xlsx"), "VAR_1", "DESC") - - def test_map_dictionary_to_dictionary(self): - df = map_dictionary_to_dictionary(self.data_dictionary_source, self.data_dictionary_target, limit=2) - print(df) \ No newline at end of file diff --git a/tests/test_postgresql_repository.py b/tests/test_postgresql_repository.py index d0dd052..c7e18d1 100644 --- a/tests/test_postgresql_repository.py +++ b/tests/test_postgresql_repository.py @@ -2,7 +2,7 @@ import unittest from datastew.embedding import Vectorizer -from datastew.process.jsonl_adapter import SQLJsonlConverter +from datastew.io.adapters import JsonlAdapter from datastew.repository import PostgreSQLRepository from datastew.repository.model import MappingResult @@ -37,7 +37,7 @@ def setUpClass(cls): cls.repo_args = (cls.POSTGRES_TEST_URL, cls.vectorizer1) cls.repository = PostgreSQLRepository(*cls.repo_args) - cls.jsonl_converter = SQLJsonlConverter(dest_dir="test_export") + cls.jsonl_converter = JsonlAdapter(dest_dir="test_export") @classmethod def tearDownClass(cls): diff --git a/tests/test_parser.py b/tests/test_source.py similarity index 94% rename from tests/test_parser.py rename to tests/test_source.py index 883a9cd..b0d01db 100644 --- a/tests/test_parser.py +++ b/tests/test_source.py @@ -5,10 +5,10 @@ import numpy as np import pandas as pd -from datastew.process.parsing import DataDictionarySource, EmbeddingSource, MappingSource +from datastew.io.source import DataDictionarySource, EmbeddingSource, MappingSource -class TestParsing(TestCase): +class TestSource(TestCase): TEST_DIR = os.path.dirname(os.path.realpath(__file__)) def setUp(self): @@ -40,7 +40,7 @@ def test_parse_data_dict_excel(self): self.assertIn("variable", df.columns) self.assertIn("description", df.columns) - @patch("datastew.process.parsing.Vectorizer") + @patch("datastew.io.source.Vectorizer") def test_get_embeddings(self, mock_vectorizer_class): mock_vectorizer = Mock() mock_vectorizer.get_embeddings.return_value = [[0.1 * 5]] * 11 diff --git a/tests/test_version.py b/tests/test_version.py index d261e01..35b248c 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -4,7 +4,7 @@ import datastew -class Test(TestCase): +class TestVersion(TestCase): def test_canonical_version(self): version = datastew.__version__ is_canonical = ( diff --git a/tests/test_visualisation.py b/tests/test_visualisation.py index 7f58297..7c17a10 100644 --- a/tests/test_visualisation.py +++ b/tests/test_visualisation.py @@ -4,11 +4,11 @@ import numpy as np import pandas as pd -from datastew.process.parsing import DataDictionarySource, MappingSource +from datastew.io.source import DataDictionarySource, MappingSource from datastew.visualisation import bar_chart_average_acc_two_distributions, enrichment_plot, plot_embeddings -class Test(TestCase): +class TestVisualization(TestCase): TEST_DIR_PATH = os.path.dirname(os.path.realpath(__file__)) mapping_source = MappingSource(os.path.join(TEST_DIR_PATH, "resources", "test_mapping.xlsx"), "VAR_1", "ID_1")