refactor: move from deprecated pkg_resources (#1202)

* refactor: move from deprecated pkg_resources

* fix: base str traversal

* fix: to use func

* fix: add missing change

* refactor: move from data_files to package_files

* refactor: resources to be in package

* fix: utils

* fix: path error

* fix: str required

* fix: tests bc of almost

* feat: refactor to pass in a path or string or None

* fix: import for older versions

* fix: Tranversable must be done at runtime

* fix: add test

* fix: accidentally duplicated test on rebase

* fix: remove pass

Author: JGSweets

.pre-commit-config.yaml

@@ -55,7 +55,7 @@ repos:
             python-dateutil>=2.7.5,
             pytz>=2020.1,
             pyarrow>=1.0.1,
-            chardet>=3.0.4,
+            'chardet>=3.0.4,<7.0.0',
             fastavro>=1.0.0.post1,
             python-snappy>=0.7.1,
             charset-normalizer>=1.3.6,

MANIFEST.in

@@ -13,9 +13,10 @@ recursive-include dataprofiler *.parquet
 recursive-include dataprofiler *.py
 recursive-include dataprofiler *.txt
 
-recursive-include resources *.json
-recursive-include resources *.pb
-recursive-include resources *.py
+recursive-include dataprofiler/resources *.json
+recursive-include dataprofiler/resources *.pb
+recursive-include dataprofiler/resources *.py
+recursive-include dataprofiler/resources *.keras
 
 recursive-include dataprofiler/labelers/embeddings *.txt
 include versioneer.py

dataprofiler/labelers/base_data_labeler.py

@@ -1,4 +1,5 @@
 """Contains abstract classes from which labeler classes will inherit."""
+
 from __future__ import annotations
 
 import json
@@ -9,15 +10,14 @@
 
 import numpy as np
 import pandas as pd
-import pkg_resources
 
 from dataprofiler._typing import DataArray
 
 from .. import data_readers
-from . import data_processing
+from . import data_processing, utils
 from .base_model import BaseModel
 
-default_labeler_dir = pkg_resources.resource_filename("resources", "labelers")
+default_labeler_dir = utils.find_resources_dir("labelers")
 
 
 class BaseDataLabeler:
@@ -246,7 +246,8 @@ def set_params(self, params: dict) -> None:
             self._postprocessor.set_params(**params["postprocessor"])
 
         self.check_pipeline(
-            skip_postprocessor=self._postprocessor is None, error_on_mismatch=False
+            skip_postprocessor=self._postprocessor is None,
+            error_on_mismatch=False,
         )
 
     def add_label(self, label: str, same_as: str = None) -> None:
@@ -438,7 +439,9 @@ def get_parameter_overlap_mismatches(
                 messages.append(
                     "Preprocessor and postprocessor value for `{}` do not "
                     "match. {} != {}".format(
-                        param, preprocessor_params[param], postprocessor_params[param]
+                        param,
+                        preprocessor_params[param],
+                        postprocessor_params[param],
                     )
                 )
         if messages:
@@ -490,7 +493,8 @@ def _load_parameters(dirpath: str, load_options: dict = None) -> dict[str, dict]
                     "The load_options preprocessor class does not "
                     "match the required DataLabeler preprocessor."
                     "\n {} != {}".format(
-                        processor_class.__class__.__name__, param_processor_class
+                        processor_class.__class__.__name__,
+                        param_processor_class,
                     )
                 )
             params["preprocessor"]["class"] = load_options.get("preprocessor_class")
@@ -505,7 +509,8 @@ def _load_parameters(dirpath: str, load_options: dict = None) -> dict[str, dict]
                 raise ValueError(
                     "The load_options postprocessor class does not match "
                     "the required DataLabeler postprocessor.\n {} != {}".format(
-                        processor_class.__class__.__name__, param_processor_class
+                        processor_class.__class__.__name__,
+                        param_processor_class,
                     )
                 )
             params["postprocessor"]["class"] = load_options.get("postprocessor_class")

dataprofiler/labelers/data_labelers.py

@@ -1,17 +1,18 @@
 """Module to train and choose between structured and unstructured data labelers."""
+
 from __future__ import annotations
 
 import os
 
 import pandas as pd
-import pkg_resources
 
 from .. import data_readers
+from . import utils
 from .base_data_labeler import BaseDataLabeler, TrainableDataLabeler
 from .base_model import BaseModel
 from .data_processing import BaseDataPostprocessor, BaseDataPreprocessor
 
-default_labeler_dir = pkg_resources.resource_filename("resources", "labelers")
+default_labeler_dir = utils.find_resources_dir("labelers")
 
 
 def train_structured_labeler(

dataprofiler/labelers/data_processing.py

@@ -1,4 +1,5 @@
 """Contains pre-built processors for data labeling/processing."""
+
 from __future__ import annotations
 
 import abc
@@ -15,9 +16,11 @@
 
 import numpy as np
 import numpy.typing as npt
-import pkg_resources
 
-default_labeler_dir = pkg_resources.resource_filename("resources", "labelers")
+from . import utils
+
+default_labeler_dir = utils.find_resources_dir("labelers")
+
 
 Processor = TypeVar("Processor", bound="BaseDataProcessor")
 

dataprofiler/labelers/utils.py

@@ -1,8 +1,18 @@
 """Contains functions for checking for installations/dependencies."""
+
+import importlib.resources
 import sys
 import warnings
+from pathlib import Path
 from typing import Any, Callable, List
 
+try:
+    # Newer Pythons / newer typeshed
+    from importlib.resources.abc import Traversable
+except ModuleNotFoundError:
+    # Older Pythons
+    from importlib.abc import Traversable
+
 
 def warn_missing_module(labeler_function: str, module_name: str) -> None:
     """
@@ -50,3 +60,15 @@ def new_f(*args: Any, **kwds: Any) -> Any:
         return new_f
 
     return check_module
+
+
+def find_resources_dir(resource_path: str | Path | None = None) -> Traversable:
+    """Return the path to the package resources."""
+    resource = importlib.resources.files("dataprofiler") / "resources"
+    if resource_path:
+        resource /= resource_path
+
+    if not (resource.is_file() or resource.is_dir()):
+        raise FileNotFoundError(f"Resource not found: {resource_path}")
+
+    return resource