Profiler: add StringColumnProfile.

Author: nikie

docs/profiles.md

@@ -22,4 +22,5 @@ Here are the current supported functionalities of Profiles.
 |  | property: profiles | Done |
 |  | property: numRecords | Done |
 | StandardColumnProfile | StandardColumnProfile(spark_session, column, java_column_profile) | Done |
+| StringColumnProfile | StringColumnProfile(spark_session, column, java_column_profile) | Done |
 | NumericColumnProfile | NumericColumnProfile(spark_session, column, java_column_profile) | Done |

pydeequ/profiles.py

@@ -2,7 +2,9 @@
 """ Profiles file for all the Profiles classes in Deequ"""
 import json
 from collections import namedtuple
+from typing import Optional
 
+from py4j.java_gateway import JavaObject
 from pyspark.sql import DataFrame, SparkSession
 from pydeequ.analyzers import KLLParameters
 from pydeequ.metrics import BucketDistribution
@@ -241,9 +243,8 @@ def __init__(self, spark_session: SparkSession):
         self._numRecords = 0
         self.columnProfileClasses = {
             "StandardColumnProfile": StandardColumnProfile,
-            "StringColumnProfile": StandardColumnProfile,
+            "StringColumnProfile": StringColumnProfile,
             "NumericColumnProfile": NumericColumnProfile,
-
         }
 
     def _columnProfilesFromColumnRunBuilderRun(self, run):
@@ -528,3 +529,36 @@ def approxPercentiles(self):
         """
         return self._approxPercentiles
 
+
+class StringColumnProfile(StandardColumnProfile):
+    """
+    String Column Profile class
+
+    :param SparkSession spark_session: sparkSession
+    :param str column: the designated column of which the profile is run on
+    :param JavaObject java_column_profile: The profile mapped as a Java map
+    """
+
+    def __init__(
+        self, spark_session: SparkSession, column: str, java_column_profile: JavaObject
+    ) -> None:
+        super().__init__(spark_session, column, java_column_profile)
+        self._minLength = get_or_else_none(java_column_profile.minLength())
+        self._maxLength = get_or_else_none(java_column_profile.maxLength())
+        self.all.update(
+            {
+                "minLength": self._minLength,
+                "maxLength": self._maxLength,
+            }
+        )
+
+    @property
+    def minLength(self) -> Optional[int]:
+        return self._minLength
+
+    @property
+    def maxLength(self) -> Optional[int]:
+        return self._maxLength
+
+    def __str__(self) -> str:
+        return f"StringProfiles for column: {self.column}: {json.dumps(self.all, indent=4)}"

tests/test_profiles.py

@@ -1,8 +1,7 @@
 # -*- coding: utf-8 -*-
 import unittest
 from pyspark.sql import Row
-from pydeequ.analyzers import KLLParameters
-from pydeequ.profiles import ColumnProfilerRunBuilder, ColumnProfilerRunner
+from pydeequ.profiles import ColumnProfilerRunBuilder, ColumnProfilerRunner, DistributionValue, StringColumnProfile
 from pydeequ.analyzers import KLLParameters, DataTypeInstances
 from tests.conftest import setup_pyspark
 
@@ -11,7 +10,7 @@ class TestProfiles(unittest.TestCase):
     def setUpClass(cls):
         cls.spark = setup_pyspark().appName("test-profiles-local").getOrCreate()
         cls.sc = cls.spark.sparkContext
-        cls.df = cls.sc.parallelize([Row(a="foo", b=1, c=5), Row(a="bar", b=2, c=6), Row(a="baz", b=3, c=None)]).toDF()
+        cls.df = cls.sc.parallelize([Row(a="foo", b=1, c=5), Row(a="bar", b=2, c=6), Row(a="bazz", b=3, c=None)]).toDF()
 
     @classmethod
     def tearDownClass(cls):
@@ -76,6 +75,26 @@ def test_profile_numRecords(self):
         result = ColumnProfilerRunner(self.spark).onData(self.df).run()
         self.assertEqual(result.numRecords, 3)
 
+    def test_StringColumnProfile(self):
+        result = ColumnProfilerRunner(self.spark).onData(self.df).run()
+        column_profile = result.profiles["a"]
+        self.assertIsInstance(column_profile, StringColumnProfile)
+        self.assertEqual(column_profile.minLength, 3)
+        self.assertEqual(column_profile.maxLength, 4)
+
+        self.assertEqual(column_profile.completeness, 1.0)
+        self.assertEqual(column_profile.approximateNumDistinctValues, 3)
+        self.assertEqual(column_profile.typeCounts["String"], 3)
+        self.assertEqual(column_profile.isDataTypeInferred, False)
+        self.assertListEqual(
+            sorted(column_profile.histogram),
+            [
+                DistributionValue("bar", 1, 1/3),
+                DistributionValue("bazz", 1, 1/3),
+                DistributionValue("foo", 1, 1/3),
+            ]
+        )
+
 
 if __name__ == "__main__":
     unittest.main()