refactor: split process module into multiple modules

README.md

@@ -19,8 +19,8 @@ csv, tsv or excel file. An example how to match two separate variable descriptio
 [datastew/scripts/mapping_excel_example.py](datastew/scripts/mapping_excel_example.py):
 
 ```python
-from datastew.process.parsing import DataDictionarySource
-from datastew.process.mapping import map_dictionary_to_dictionary
+from datastew.io.source import DataDictionarySource
+from datastew.harmonization import map_dictionary_to_dictionary
 
 # Variable and description refer to the corresponding column names in your excel sheet
 source = DataDictionarySource("source.xlxs", variable_field="var", description_field="desc")
@@ -39,7 +39,7 @@ function:
 
 ```python
 from datastew.embedding import Vectorizer
-from datastew.process.mapping import map_dictionary_to_dictionary
+from datastew.harmonization import map_dictionary_to_dictionary
 
 vectorizer = Vectorizer("text-embedding-ada-002", key="your_api_key")
 df = map_dictionary_to_dictionary(source, target, vectorizer=vectorizer)
@@ -135,7 +135,7 @@ language models. An example how to generate a t-sne plot is shown in
 
 ```python
 from datastew.embedding import Vectorizer
-from datastew.process.parsing import DataDictionarySource
+from datastew.io.source import DataDictionarySource
 from datastew.visualisation import plot_embeddings
 
 # Variable and description refer to the corresponding column names in your excel sheet

datastew/__init__.py

@@ -1,8 +1,9 @@
 from ._version import __version__
 from .embedding import Vectorizer
-
-# Importing submodules to expose their attributes if needed
-from .process import jsonl_adapter, mapping, ols, parsing
+from .harmonization import mapping
+from .integrations import ols
+from .io import source
+from .io.adapters import jsonl
 from .repository import model, postgresql
 from .visualisation import (
     bar_chart_average_acc_two_distributions,
@@ -13,15 +14,15 @@
 
 __all__ = [
     "__version__",
-    "jsonl_adapter",
-    "mapping",
-    "ols",
-    "parsing",
-    "model",
-    "postgresql",
     "Vectorizer",
     "bar_chart_average_acc_two_distributions",
     "enrichment_plot",
     "get_plot_for_current_database_state",
+    "jsonl",
+    "mapping",
+    "model",
+    "ols",
+    "source",
     "plot_embeddings",
+    "postgresql",
 ]

datastew/harmonization/__init__.py

@@ -0,0 +1,3 @@
+from .mapping import map_dictionary_to_dictionary
+
+__all__ = ["map_dictionary_to_dictionary"]

datastew/process/mapping.py → datastew/harmonization/mapping.py

@@ -3,7 +3,7 @@
 from sklearn.metrics.pairwise import cosine_similarity
 
 from datastew.embedding import Vectorizer
-from datastew.process.parsing import DataDictionarySource
+from datastew.io.source import DataDictionarySource
 
 
 def map_dictionary_to_dictionary(

datastew/integrations/ols/__init__.py

@@ -0,0 +1,3 @@
+from .client import OlsClient
+
+__all__ = ["OlsClient"]

datastew/process/ols.py → datastew/integrations/ols/client.py

@@ -10,7 +10,7 @@
 from datastew.repository.model import Concept, Mapping, Terminology
 
 
-class OLSTerminologyImportTask:
+class OlsClient:
 
     def __init__(
         self,

datastew/io/__init__.py

@@ -0,0 +1,4 @@
+from .importer import Importer
+from .source import DataDictionarySource
+
+__all__ = ["Importer", "DataDictionarySource"]

datastew/io/adapters/__init__.py

@@ -0,0 +1,3 @@
+from .jsonl import JsonlAdapter
+
+__all__ = ["JsonlAdapter"]

datastew/process/jsonl_adapter.py → datastew/io/adapters/jsonl.py

@@ -11,7 +11,7 @@
 from datastew.repository.model import Concept, Mapping, Terminology
 
 
-class SQLJsonlConverter:
+class JsonlAdapter:
     def __init__(self, dest_dir: str, buffer_size: int = 1000):
         """Initialize the converter.
 

datastew/process/importer.py → datastew/io/importer.py

@@ -5,14 +5,14 @@
 
 from sqlalchemy import text
 
-from datastew.process.parsing import DataDictionarySource
+from datastew.io.source import DataDictionarySource
 from datastew.repository import PostgreSQLRepository
 from datastew.repository.model import Concept, Mapping, Terminology
 
 logger = logging.getLogger(__name__)
 
 
-class PostgreSQLImporter:
+class Importer:
     def __init__(self, repository: PostgreSQLRepository):
         self.repository = repository
         self.engine = repository.engine

datastew/scripts/mapping_excel_example.py

@@ -23,8 +23,8 @@
 #    result.xlsx — containing matched variables and similarity scores
 """
 
-from datastew.process.mapping import map_dictionary_to_dictionary
-from datastew.process.parsing import DataDictionarySource
+from datastew.harmonization import map_dictionary_to_dictionary
+from datastew.io.source import DataDictionarySource
 
 # --------------------------------------------------------------------
 # 1) Load source and target data dictionaries

datastew/scripts/ohdsi_to_jsonl.py

@@ -11,14 +11,14 @@
 3. The resulting JSONL files will be saved in the specified output directory.
 """
 
-from datastew.process.jsonl_adapter import SQLJsonlConverter
+from datastew.io.adapters import JsonlAdapter
 
 # --------------------------------------------------------------------
 # 1) Initialize the converter
 # --------------------------------------------------------------------
 # The output directory will contain the generated JSONL files.
 output_directory = "resources/results"
-jsonl_converter = SQLJsonlConverter(output_directory)
+jsonl_converter = JsonlAdapter(output_directory)
 
 # --------------------------------------------------------------------
 # 2) Convert the OHDSI CONCEPT.csv file

datastew/scripts/ols_snomed_retrieval.py

@@ -20,7 +20,7 @@
 """
 
 from datastew.embedding import Vectorizer
-from datastew.process.ols import OLSTerminologyImportTask
+from datastew.integrations.ols import OlsClient
 from datastew.repository import PostgreSQLRepository
 
 # --------------------------------------------------------------------
@@ -44,7 +44,7 @@
 # The first argument is the embedding model.
 # The second argument is the OLS display name of the terminology.
 # The third argument is the short identifier used internally.
-task = OLSTerminologyImportTask(vectorizer, "SNOMED CT", "snomed")
+task = OlsClient(vectorizer, "SNOMED CT", "snomed")
 
 # This method fetches, embeds, and uploads the terminology to PostgreSQL.
 task.process_to_repository(repository)

datastew/scripts/tsne_visualization.py

@@ -16,7 +16,7 @@
 """
 
 from datastew.embedding import Vectorizer
-from datastew.process.parsing import DataDictionarySource
+from datastew.io.source import DataDictionarySource
 from datastew.visualisation import plot_embeddings
 
 # --------------------------------------------------------------------

datastew/visualisation.py

@@ -9,7 +9,7 @@
 from sklearn.manifold import TSNE
 
 from datastew.embedding import Vectorizer
-from datastew.process.parsing import DataDictionarySource
+from datastew.io.source import DataDictionarySource
 from datastew.repository import PostgreSQLRepository
 
 

tests/test_harmonization.py

@@ -0,0 +1,22 @@
+import os
+import unittest
+
+from datastew.harmonization import map_dictionary_to_dictionary
+from datastew.io.source import DataDictionarySource
+
+
+class TestHarmonization(unittest.TestCase):
+
+    TEST_DIR_PATH = os.path.dirname(os.path.realpath(__file__))
+
+    data_dictionary_source = DataDictionarySource(
+        os.path.join(TEST_DIR_PATH, "resources", "test_data_dict.csv"), "VAR_1", "DESC"
+    )
+
+    data_dictionary_target = DataDictionarySource(
+        os.path.join(TEST_DIR_PATH, "resources", "test_data_dict.xlsx"), "VAR_1", "DESC"
+    )
+
+    def test_map_dictionary_to_dictionary(self):
+        df = map_dictionary_to_dictionary(self.data_dictionary_source, self.data_dictionary_target, limit=2)
+        print(df)

tests/test_postgresql_import.py → tests/test_importer.py

@@ -6,19 +6,19 @@
 from typing import Any, Literal
 from unittest import TestCase
 
-from datastew.process.importer import PostgreSQLImporter
-from datastew.process.parsing import DataDictionarySource
+from datastew.io.importer import Importer
+from datastew.io.source import DataDictionarySource
 from datastew.repository import PostgreSQLRepository
 
 
-class TestPostgreSQLImporter(TestCase):
+class TestImporter(TestCase):
     def setUp(self) -> None:
 
         POSTGRES_TEST_URL = os.getenv("TEST_POSTGRES_URI", "postgresql://testuser:testpass@localhost/testdb")
         self.TEST_DIR_PATH = os.path.dirname(os.path.realpath(__file__))
         self.repository = PostgreSQLRepository(POSTGRES_TEST_URL)
         self.repository.clear_all()
-        self.importer = PostgreSQLImporter(self.repository)
+        self.importer = Importer(self.repository)
         self.temp_dir = tempfile.mkdtemp()
 
         # Sample data for JSONL files

tests/test_json_adapter.py → tests/test_jsonl_adapter.py

@@ -7,7 +7,7 @@
 import pandas as pd
 
 from datastew.embedding import Vectorizer
-from datastew.process.jsonl_adapter import SQLJsonlConverter
+from datastew.io.adapters import JsonlAdapter
 
 
 class MockVectorizer(Vectorizer):
@@ -17,7 +17,7 @@ def get_embeddings(self, texts):
         return [[0.1, 0.2, 0.3] for _ in texts]
 
 
-class TestSQLJsonlConverter(unittest.TestCase):
+class TestJsonlAdapter(unittest.TestCase):
     """Unit tests for SQLJsonlConverter"""
 
     def setUp(self):
@@ -49,7 +49,7 @@ def setUp(self):
         data.to_csv(self.mock_concept_file, sep="\t", index=False)
 
         # Initialize SQLJsonlConverter
-        self.converter = SQLJsonlConverter(dest_dir=self.temp_dir)
+        self.converter = JsonlAdapter(dest_dir=self.temp_dir)
 
         # Inject Mock Embedding Model
         self.mock_vectorizer_model = MockVectorizer()

tests/test_postgresql_repository.py

@@ -2,7 +2,7 @@
 import unittest
 
 from datastew.embedding import Vectorizer
-from datastew.process.jsonl_adapter import SQLJsonlConverter
+from datastew.io.adapters import JsonlAdapter
 from datastew.repository import PostgreSQLRepository
 from datastew.repository.model import MappingResult
 
@@ -37,7 +37,7 @@ def setUpClass(cls):
 
         cls.repo_args = (cls.POSTGRES_TEST_URL, cls.vectorizer1)
         cls.repository = PostgreSQLRepository(*cls.repo_args)
-        cls.jsonl_converter = SQLJsonlConverter(dest_dir="test_export")
+        cls.jsonl_converter = JsonlAdapter(dest_dir="test_export")
 
     @classmethod
     def tearDownClass(cls):

tests/test_parser.py → tests/test_source.py

@@ -5,10 +5,10 @@
 import numpy as np
 import pandas as pd
 
-from datastew.process.parsing import DataDictionarySource, EmbeddingSource, MappingSource
+from datastew.io.source import DataDictionarySource, EmbeddingSource, MappingSource
 
 
-class TestParsing(TestCase):
+class TestSource(TestCase):
     TEST_DIR = os.path.dirname(os.path.realpath(__file__))
 
     def setUp(self):
@@ -40,7 +40,7 @@ def test_parse_data_dict_excel(self):
         self.assertIn("variable", df.columns)
         self.assertIn("description", df.columns)
 
-    @patch("datastew.process.parsing.Vectorizer")
+    @patch("datastew.io.source.Vectorizer")
     def test_get_embeddings(self, mock_vectorizer_class):
         mock_vectorizer = Mock()
         mock_vectorizer.get_embeddings.return_value = [[0.1 * 5]] * 11

tests/test_version.py

@@ -4,7 +4,7 @@
 import datastew
 
 
-class Test(TestCase):
+class TestVersion(TestCase):
     def test_canonical_version(self):
         version = datastew.__version__
         is_canonical = (

tests/test_visualisation.py

@@ -4,11 +4,11 @@
 import numpy as np
 import pandas as pd
 
-from datastew.process.parsing import DataDictionarySource, MappingSource
+from datastew.io.source import DataDictionarySource, MappingSource
 from datastew.visualisation import bar_chart_average_acc_two_distributions, enrichment_plot, plot_embeddings
 
 
-class Test(TestCase):
+class TestVisualization(TestCase):
     TEST_DIR_PATH = os.path.dirname(os.path.realpath(__file__))
 
     mapping_source = MappingSource(os.path.join(TEST_DIR_PATH, "resources", "test_mapping.xlsx"), "VAR_1", "ID_1")