feat: auto-load datasets from directory in SemanticModel (#208)

Co-authored-by: Suraj Panickar <surajpanickar@Surajs-MacBook-Pro.local>
Co-authored-by: raphael-intugle <raphael.tony@intugle.ai>

Author: 3 people

src/intugle/semantic_model.py

@@ -11,6 +11,7 @@
 from intugle.link_predictor.predictor import LinkPredictor
 from intugle.semantic_search import SemanticSearch
 from intugle.utils.files import update_relationship_file_mtime
+from pathlib import Path
 
 if TYPE_CHECKING:
     from intugle.adapters.adapter import Adapter
@@ -20,7 +21,7 @@
 
 
 class SemanticModel:
-    def __init__(self, data_input: Dict[str, Any] | List[DataSet], domain: str = ""):
+    def __init__(self, data_input: Dict[str, Any] | List[DataSet] | str, domain: str = ""):
         """
         Initialize a SemanticModel to build a semantic layer over your data.
 
@@ -57,13 +58,15 @@ def __init__(self, data_input: Dict[str, Any] | List[DataSet], domain: str = "")
         self.domain = domain
         self._semantic_search_initialized = False
 
-        if isinstance(data_input, dict):
+        if isinstance(data_input, str):
+            self._initialize_from_folder(data_input)
+        elif isinstance(data_input, dict):
             self._initialize_from_dict(data_input)
         elif isinstance(data_input, list):
             self._initialize_from_list(data_input)
         else:
             raise TypeError(
-                "Input must be a dictionary of named dataframes or a list of DataSet objects."
+                "Input must be a dictionary, a list of DataSet objects, or a folder path string."
             )
 
     def _initialize_from_dict(self, data_dict: Dict[str, Any]):
@@ -80,6 +83,56 @@ def _initialize_from_list(self, data_list: List[DataSet]):
                     "DataSet objects provided in a list must have a 'name' attribute."
                 )
             self.datasets[dataset.name] = dataset
+    def _initialize_from_folder(self, folder_path: str):
+        """
+        Initialize datasets by scanning a folder (recursively) for supported data files.
+        Supported formats: CSV, Parquet, Excel.
+        """
+        path = Path(folder_path)
+
+        if not path.exists():
+            raise ValueError(f"Provided path does not exist: {folder_path}")
+
+        if not path.is_dir():
+            raise ValueError(f"Provided path is not a directory: {folder_path}")
+
+        extension_mapping = {
+            ".csv": "csv",
+            ".parquet": "parquet",
+            ".xlsx": "xlsx",
+        }
+
+        found = False
+
+        for file_path in path.rglob("*"):  # Recursive 
+            if not file_path.is_file():
+                continue
+
+            ext = file_path.suffix.lower()
+            if ext not in extension_mapping:
+                continue
+
+            dataset_name = file_path.stem
+
+            if dataset_name in self.datasets:
+                raise ValueError(
+                    f"Duplicate dataset name '{dataset_name}' found while scanning {folder_path}"
+                )
+
+            config = {
+                "path": str(file_path.resolve()),
+                "type": extension_mapping[ext],
+            }
+
+            dataset = DataSet(config, name=dataset_name)
+            self.datasets[dataset_name] = dataset
+            found = True
+
+        if not found:
+            raise ValueError(
+                f"No supported data files (.csv, .parquet, .xlsx) found in directory: {folder_path}"
+            )
+
 
     def profile(self, force_recreate: bool = False):
         """

tests/semantic_model/test_folder_initialization.py

@@ -0,0 +1,63 @@
+import pandas as pd
+import pytest
+
+from intugle.semantic_model import SemanticModel
+
+
+def test_initialize_from_folder_loads_supported_files(tmp_path):
+    """
+    Verify that passing a folder path correctly loads supported files.
+    """
+    csv_file = tmp_path / "users.csv"
+    parquet_file = tmp_path / "orders.parquet"
+
+    pd.DataFrame({"id": [1, 2]}).to_csv(csv_file, index=False)
+    pd.DataFrame({"id": [10, 20]}).to_parquet(parquet_file)
+
+    sm = SemanticModel(str(tmp_path))
+
+    assert "users" in sm.datasets
+    assert "orders" in sm.datasets
+    assert len(sm.datasets) == 2
+
+
+def test_initialize_from_folder_ignores_unsupported_files(tmp_path):
+    """
+    Verify that unsupported files are ignored.
+    """
+    (tmp_path / "readme.txt").write_text("ignore me")
+
+    csv_file = tmp_path / "data.csv"
+    pd.DataFrame({"x": [1]}).to_csv(csv_file, index=False)
+
+    sm = SemanticModel(str(tmp_path))
+
+    assert "data" in sm.datasets
+    assert len(sm.datasets) == 1
+
+
+def test_initialize_from_folder_extension_mapping(tmp_path):
+    """
+    Verify correct mapping of file extensions to DuckDB types.
+    """
+    csv_file = tmp_path / "a.csv"
+    parquet_file = tmp_path / "b.parquet"
+
+    pd.DataFrame({"x": [1]}).to_csv(csv_file, index=False)
+    pd.DataFrame({"y": [2]}).to_parquet(parquet_file)
+
+    sm = SemanticModel(str(tmp_path))
+
+    csv_dataset = sm.datasets["a"]
+    parquet_dataset = sm.datasets["b"]
+
+    assert csv_dataset.data["type"] == "csv"
+    assert parquet_dataset.data["type"] == "parquet"
+
+
+def test_initialize_from_folder_empty_directory_raises(tmp_path):
+    """
+    Verify that an empty directory raises a clear error.
+    """
+    with pytest.raises(ValueError):
+        SemanticModel(str(tmp_path))