refactor: move from deprecated pkg_resources

.pre-commit-config.yaml

@@ -55,7 +55,7 @@ repos:
             python-dateutil>=2.7.5,
             pytz>=2020.1,
             pyarrow>=1.0.1,
-            chardet>=3.0.4,
+            'chardet>=3.0.4,<7.0.0',
             fastavro>=1.0.0.post1,
             python-snappy>=0.7.1,
             charset-normalizer>=1.3.6,

MANIFEST.in

@@ -13,9 +13,10 @@ recursive-include dataprofiler *.parquet
 recursive-include dataprofiler *.py
 recursive-include dataprofiler *.txt
 
-recursive-include resources *.json
-recursive-include resources *.pb
-recursive-include resources *.py
+recursive-include dataprofiler/resources *.json
+recursive-include dataprofiler/resources *.pb
+recursive-include dataprofiler/resources *.py
+recursive-include dataprofiler/resources *.keras
 
 recursive-include dataprofiler/labelers/embeddings *.txt
 include versioneer.py

dataprofiler/labelers/base_data_labeler.py

@@ -1,4 +1,5 @@
 """Contains abstract classes from which labeler classes will inherit."""
+
 from __future__ import annotations
 
 import json
@@ -9,15 +10,14 @@
 
 import numpy as np
 import pandas as pd
-import pkg_resources
 
 from dataprofiler._typing import DataArray
 
 from .. import data_readers
-from . import data_processing
+from . import data_processing, utils
 from .base_model import BaseModel
 
-default_labeler_dir = pkg_resources.resource_filename("resources", "labelers")
+default_labeler_dir = utils.find_resources_dir("labelers")
 
 
 class BaseDataLabeler:
@@ -246,7 +246,8 @@ def set_params(self, params: dict) -> None:
             self._postprocessor.set_params(**params["postprocessor"])
 
         self.check_pipeline(
-            skip_postprocessor=self._postprocessor is None, error_on_mismatch=False
+            skip_postprocessor=self._postprocessor is None,
+            error_on_mismatch=False,
         )
 
     def add_label(self, label: str, same_as: str = None) -> None:
@@ -438,7 +439,9 @@ def get_parameter_overlap_mismatches(
                 messages.append(
                     "Preprocessor and postprocessor value for `{}` do not "
                     "match. {} != {}".format(
-                        param, preprocessor_params[param], postprocessor_params[param]
+                        param,
+                        preprocessor_params[param],
+                        postprocessor_params[param],
                     )
                 )
         if messages:
@@ -490,7 +493,8 @@ def _load_parameters(dirpath: str, load_options: dict = None) -> dict[str, dict]
                     "The load_options preprocessor class does not "
                     "match the required DataLabeler preprocessor."
                     "\n {} != {}".format(
-                        processor_class.__class__.__name__, param_processor_class
+                        processor_class.__class__.__name__,
+                        param_processor_class,
                     )
                 )
             params["preprocessor"]["class"] = load_options.get("preprocessor_class")
@@ -505,7 +509,8 @@ def _load_parameters(dirpath: str, load_options: dict = None) -> dict[str, dict]
                 raise ValueError(
                     "The load_options postprocessor class does not match "
                     "the required DataLabeler postprocessor.\n {} != {}".format(
-                        processor_class.__class__.__name__, param_processor_class
+                        processor_class.__class__.__name__,
+                        param_processor_class,
                     )
                 )
             params["postprocessor"]["class"] = load_options.get("postprocessor_class")

dataprofiler/labelers/data_labelers.py

@@ -1,17 +1,18 @@
 """Module to train and choose between structured and unstructured data labelers."""
+
 from __future__ import annotations
 
 import os
 
 import pandas as pd
-import pkg_resources
 
 from .. import data_readers
+from . import utils
 from .base_data_labeler import BaseDataLabeler, TrainableDataLabeler
 from .base_model import BaseModel
 from .data_processing import BaseDataPostprocessor, BaseDataPreprocessor
 
-default_labeler_dir = pkg_resources.resource_filename("resources", "labelers")
+default_labeler_dir = utils.find_resources_dir("labelers")
 
 
 def train_structured_labeler(

dataprofiler/labelers/data_processing.py

@@ -1,4 +1,5 @@
 """Contains pre-built processors for data labeling/processing."""
+
 from __future__ import annotations
 
 import abc
@@ -15,9 +16,11 @@
 
 import numpy as np
 import numpy.typing as npt
-import pkg_resources
 
-default_labeler_dir = pkg_resources.resource_filename("resources", "labelers")
+from . import utils
+
+default_labeler_dir = utils.find_resources_dir("labelers")
+
 
 Processor = TypeVar("Processor", bound="BaseDataProcessor")
 

dataprofiler/labelers/utils.py

@@ -1,8 +1,18 @@
 """Contains functions for checking for installations/dependencies."""
+
+import importlib.resources
 import sys
 import warnings
+from pathlib import Path
 from typing import Any, Callable, List
 
+try:
+    # Newer Pythons / newer typeshed
+    from importlib.resources.abc import Traversable
+except ModuleNotFoundError:
+    # Older Pythons
+    from importlib.abc import Traversable
+
 
 def warn_missing_module(labeler_function: str, module_name: str) -> None:
     """
@@ -50,3 +60,15 @@ def new_f(*args: Any, **kwds: Any) -> Any:
         return new_f
 
     return check_module
+
+
+def find_resources_dir(resource_path: str | Path | None = None) -> Traversable:
+    """Return the path to the package resources."""
+    resource = importlib.resources.files("dataprofiler") / "resources"
+    if resource_path:
+        resource /= resource_path
+
+    if not (resource.is_file() or resource.is_dir()):
+        raise FileNotFoundError(f"Resource not found: {resource_path}")
+
+    return resource

dataprofiler/tests/labelers/test_char_tf_load_model.py

@@ -6,14 +6,13 @@
 
 import numpy as np
 import pandas as pd
-import pkg_resources
 import tensorflow as tf
 
+from dataprofiler.labelers import utils as labeler_utils
 from dataprofiler.labelers.char_load_tf_model import CharLoadTFModel
 
 _file_dir = os.path.dirname(os.path.abspath(__file__))
-_resource_labeler_dir = pkg_resources.resource_filename("resources", "labelers")
-
+default_labeler_dir = labeler_utils.find_resources_dir("labelers")
 
 mock_model_parameters = {
     "model_path": "project/example/path/fake_model.h5",
@@ -303,7 +302,9 @@ def test_param_validation(self, *mocks):
             "fake_extra_param": "fails",
         }
         model = CharLoadTFModel(
-            self.model_path, label_mapping=self.label_mapping, parameters=parameters
+            self.model_path,
+            label_mapping=self.label_mapping,
+            parameters=parameters,
         )
         model._construct_model()
         self.assertDictEqual(parameters, model._parameters)

dataprofiler/tests/labelers/test_character_level_cnn_model.py

@@ -6,16 +6,16 @@
 
 import numpy as np
 import pandas as pd
-import pkg_resources
 import tensorflow as tf
 
+from dataprofiler.labelers import utils as labeler_utils
 from dataprofiler.labelers.character_level_cnn_model import (
     CharacterLevelCnnModel,
     EncodingLayer,
 )
 
 _file_dir = os.path.dirname(os.path.abspath(__file__))
-_resource_labeler_dir = pkg_resources.resource_filename("resources", "labelers")
+_resource_labeler_dir = labeler_utils.find_resources_dir("labelers")
 
 
 mock_model_parameters = {

dataprofiler/tests/labelers/test_column_name_model.py

@@ -6,13 +6,13 @@
 from unittest import mock
 
 import numpy as np
-import pkg_resources
 
 import dataprofiler as dp
+from dataprofiler.labelers import utils as labeler_utils
 from dataprofiler.labelers.column_name_model import ColumnNameModel
 
-_file_dir = os.path.dirname(os.path.abspath(__file__))
-_resource_labeler_dir = pkg_resources.resource_filename("resources", "labelers")
+_resource_labeler_dir = labeler_utils.find_resources_dir("labelers")
+
 
 mock_model_parameters = {
     "true_positive_dict": [

dataprofiler/tests/labelers/test_data_labelers.py

@@ -149,11 +149,12 @@ def test_load_from_library(self, *mocks):
 
     @mock.patch("tensorflow.keras.models.load_model")
     def test_load_from_disk(self, *mocks):
-        import pkg_resources
+        from dataprofiler.labelers import utils as labeler_utils
 
-        default_labeler_dir = pkg_resources.resource_filename(
-            "resources", "labelers/structured_model"
+        default_labeler_dir = str(
+            labeler_utils.find_resources_dir("labelers/structured_model")
         )
+
         data_labeler = dp.DataLabeler.load_from_disk(default_labeler_dir)
         self.assertIsInstance(data_labeler, BaseDataLabeler)
 

dataprofiler/tests/labelers/test_data_processing.py

@@ -7,8 +7,8 @@
 from unittest import mock
 
 import numpy as np
-import pkg_resources
 
+from dataprofiler.labelers import utils as labeler_utils
 from dataprofiler.labelers.data_processing import (
     BaseDataProcessor,
     CharEncodedPreprocessor,
@@ -224,7 +224,9 @@ def test_load_from_library(self, mocked_load, *mocks):
         BaseDataProcessor.load_from_library("default")
 
         # assert called with proper load_processor dirpath
-        default_labeler_dir = pkg_resources.resource_filename("resources", "labelers")
+
+        default_labeler_dir = labeler_utils.find_resources_dir("labelers")
+
         mocked_load.assert_called_with(os.path.join(default_labeler_dir, "default"))
 
     @mock.patch("builtins.open")

dataprofiler/tests/labelers/test_integration_column_name_data_labeler.py

@@ -1,17 +1,17 @@
 import unittest
 
 import numpy as np
-import pkg_resources
 
 import dataprofiler as dp
+from dataprofiler.labelers import utils as labeler_utils
 from dataprofiler.labelers.column_name_model import ColumnNameModel
 from dataprofiler.labelers.data_labelers import BaseDataLabeler
 from dataprofiler.labelers.data_processing import (
     ColumnNameModelPostprocessor,
     DirectPassPreprocessor,
 )
 
-default_labeler_dir = pkg_resources.resource_filename("resources", "labelers")
+default_labeler_dir = labeler_utils.find_resources_dir("labelers")
 
 
 class TestColumnNameDataLabeler(unittest.TestCase):

dataprofiler/tests/labelers/test_integration_regex_data_labeler.py

@@ -2,18 +2,25 @@
 import unittest
 
 import numpy as np
-import pkg_resources
 
+from dataprofiler.labelers import utils as labeler_utils
 from dataprofiler.labelers.data_labelers import BaseDataLabeler
 
-default_labeler_dir = pkg_resources.resource_filename("resources", "labelers")
+default_labeler_dir = labeler_utils.find_resources_dir("labelers")
 
 
 class TestRegexDataLabeler(unittest.TestCase):
     @classmethod
     def setUpClass(cls) -> None:
         cls.data = np.array(
-            ["123 Fake St.", "1/2/2020", "nice.", "4/3/22", "abc", "333-44-2341"]
+            [
+                "123 Fake St.",
+                "1/2/2020",
+                "nice.",
+                "4/3/22",
+                "abc",
+                "333-44-2341",
+            ]
         ).reshape((-1,))
         cls.data_labeler = BaseDataLabeler.load_from_disk(
             os.path.join(default_labeler_dir, "regex_model")

dataprofiler/tests/labelers/test_labeler_utils.py

@@ -1,12 +1,13 @@
 import logging
 import tempfile
 import unittest
+from pathlib import Path
 
 import numpy as np
 import pandas as pd
 import tensorflow as tf
 
-from dataprofiler.labelers import labeler_utils
+from dataprofiler.labelers import labeler_utils, utils
 
 
 class TestEvaluateAccuracy(unittest.TestCase):
@@ -294,3 +295,17 @@ def test_hide_tf_logger_warnings(self):
         # make change and validate updated filter
         labeler_utils.hide_tf_logger_warnings()
         self.assertEqual(1 + num_loggers, len(logger.filters))
+
+
+class TestFindResourcesDir(unittest.TestCase):
+    def test_find_resources_dir(self):
+        resource_dir = utils.find_resources_dir()
+        self.assertTrue(resource_dir.is_dir())
+        self.assertEqual(Path("resources").name, Path(str(resource_dir)).name)
+
+        labeler_dir = utils.find_resources_dir("labelers")
+        self.assertTrue(labeler_dir.is_dir())
+        self.assertEqual(Path("labelers").name, Path(str(labeler_dir)).name)
+
+        with self.assertRaisesRegex(FileNotFoundError, "Resource not found"):
+            utils.find_resources_dir("does/not/exist")

dataprofiler/tests/labelers/test_regex_model.py

@@ -5,12 +5,12 @@
 from unittest import mock
 
 import numpy as np
-import pkg_resources
 
+from dataprofiler.labelers import utils as labeler_utils
 from dataprofiler.labelers.regex_model import RegexModel
 
 _file_dir = os.path.dirname(os.path.abspath(__file__))
-_resource_labeler_dir = pkg_resources.resource_filename("resources", "labelers")
+_resource_labeler_dir = labeler_utils.find_resources_dir("labelers")
 
 
 mock_model_parameters = {
@@ -161,7 +161,8 @@ def test_param_validation(self):
         for invalid_param_set in invalid_parameters:
             with self.assertRaises(ValueError):
                 RegexModel(
-                    label_mapping=self.label_mapping, parameters=invalid_param_set
+                    label_mapping=self.label_mapping,
+                    parameters=invalid_param_set,
                 )
 
     @mock.patch("sys.stdout", new_callable=StringIO)

dataprofiler/tests/profilers/profiler_options/test_datalabeler_options.py

@@ -1,4 +1,3 @@
-pass
 import json
 from unittest import mock
 

dataprofiler/tests/profilers/profiler_options/test_text_options.py

@@ -1,4 +1,3 @@
-pass
 import json
 from unittest import mock