Implement native profiler to replace pydeequ

Author: easel

src/tablespec/__init__.py

@@ -37,7 +37,7 @@
     load_umf_from_yaml,
     save_umf_to_yaml,
 )
-from tablespec.profiling import ColumnProfile, DataFrameProfile, DeequToUmfMapper
+from tablespec.profiling import ColumnProfile, DataFrameProfile, ProfileToGxMapper
 from tablespec.prompts import (
     generate_column_validation_prompt,
     generate_documentation_prompt,
@@ -117,7 +117,7 @@
     # -- Profiling --
     "ColumnProfile",
     "DataFrameProfile",
-    "DeequToUmfMapper",
+    "ProfileToGxMapper",
     # -- LLM Prompt Generation --
     "generate_column_validation_prompt",
     "generate_documentation_prompt",

src/tablespec/cli.py

@@ -21,6 +21,8 @@
 from pathlib import Path
 
 from pydantic import ValidationError
+import os
+
 from rich.console import Console
 from rich.table import Table as RichTable
 import typer
@@ -48,7 +50,7 @@
     name="tablespec",
     help="Work with UMF (Universal Metadata Format) table schemas",
 )
-console = Console()
+console = Console(no_color=bool(os.environ.get("NO_COLOR")))
 
 # Module-level validation context (process lifetime caching) - only when validator is available
 _validation_context = ValidationContext() if _HAS_VALIDATOR else None

src/tablespec/gx_baseline.py

@@ -340,7 +340,7 @@ def generate_baseline_column_expectations(
                                 },
                             }
                         )
-            # If nullable is True (bool from DeequToUmfMapper), column IS nullable — no not-null expectation needed
+            # If nullable is True (bool), column IS nullable — no not-null expectation needed
 
         # 2. Length constraints
         max_length = column.get("max_length") or column.get("length")

src/tablespec/profiling/__init__.py

@@ -1,25 +1,47 @@
 """Schema profiling and mapping utilities for tablespec.
 
-This module provides tools for mapping profiling data and Spark schemas to UMF format.
+This module provides tools for profiling DataFrames and mapping results to UMF format.
 
-Spark-dependent components (SparkToUmfMapper) require installing tablespec[spark]:
+Components:
+    - ``NativeSparkProfiler``: Profiles DataFrames using native SQL (serverless-compatible).
+    - ``ProfileToGxMapper``: Generates GX expectations directly from profiling results.
+    - ``SparkToUmfMapper``: Maps Spark DataFrame schema to UMF (requires pyspark).
+
+Type mappings:
+    - ``SPARK_TO_UMF_TYPE``: Spark DataType class name → UMF data_type string.
+    - ``SQL_TO_UMF_TYPE``: SQL/warehouse type name → UMF data_type string (for dbt, etc.).
+
+Spark-dependent components require installing tablespec[spark]:
     pip install tablespec[spark]
 """
 
-from tablespec.profiling.deequ_mapper import DeequToUmfMapper
+from tablespec.profiling.gx_expectation_builder import ProfileToGxMapper
 from tablespec.profiling.types import ColumnProfile, DataFrameProfile
 
 __all__ = [
     "ColumnProfile",
     "DataFrameProfile",
-    "DeequToUmfMapper",
+    "ProfileToGxMapper",
 ]
 
-# SparkToUmfMapper is available only if pyspark is installed
+# SparkToUmfMapper and type mapping dicts are available only if pyspark is installed
 try:
-    from tablespec.profiling.spark_mapper import SparkToUmfMapper  # noqa: F401
+    from tablespec.profiling.spark_mapper import (  # noqa: F401
+        SPARK_TO_UMF_TYPE,
+        SQL_TO_UMF_TYPE,
+        SparkToUmfMapper,
+    )
 
-    __all__.append("SparkToUmfMapper")
+    __all__.extend(["SparkToUmfMapper", "SPARK_TO_UMF_TYPE", "SQL_TO_UMF_TYPE"])
 except ImportError:
     # pyspark not available - SparkToUmfMapper won't be exported
     pass
+
+# NativeSparkProfiler requires only pyspark (works on Connect/serverless)
+try:
+    from tablespec.profiling.native_profiler import NativeSparkProfiler  # noqa: F401
+
+    __all__.append("NativeSparkProfiler")
+except ImportError:
+    # pyspark not available
+    pass