build: remove ZetaSQL custom validation and modernize dependencies for Python 3.12

This commit updates the TFDV build environment for Python 3.12 compatibility by ripping out ZetaSQL-dependent code and updating Python package constraints.

Specific changes include:
* Strip ZetaSQL: Removed `zetasql` and `six` from `WORKSPACE`. Removed the `custom_validation` cc_library, its pybind11 hook (`CustomValidateStatistics`), and related test targets, as ZetaSQL compilation fails on modern toolchains.
* Modernize PyArrow & TF: Updated `setup.py` to allow `pyarrow>=14,<22` for Python >= 3.11 to avoid building legacy Arrow 10 source code. Relaxed the `tensorflow` constraint to `>=2.16,<2.18` and adjusted `tfx-bsl` / `tensorflow-metadata` base versions.
* Fix Test Dependencies: Added `scikit-learn==1.5.1` and `scipy==1.17.0` to `install_requires` so the mutual information generators and tests can execute properly.
* Cleanup Build Macros: Removed legacy Python 2 pybind11 initialization symbols (`init%s`) from `build_macros.bzl`.

Author: pritamdodeja

WORKSPACE

@@ -71,18 +71,6 @@ http_archive(
     ],
 )
 
-# Needed by abseil-py by zetasql.
-http_archive(
-    name = "six_archive",
-    build_file = "//third_party:six.BUILD",
-    sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a",
-    strip_prefix = "six-1.10.0",
-    urls = [
-        "http://mirror.bazel.build/pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz",
-        "https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz",
-    ],
-)
-
 load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
 
 protobuf_deps()
@@ -112,6 +100,16 @@ http_archive(
     url = "https://github.com/abseil/abseil-cpp/archive/%s.tar.gz" % COM_GOOGLE_ABSL_COMMIT,
 )
 
+
+# re2 required for google tests
+http_archive(
+    name = "com_googlesource_code_re2",
+    #    build_file = "//third_party:re2.BUILD",
+    sha256 = "b90430b2a9240df4459108b3e291be80ae92c68a47bc06ef2dc419c5724de061",
+    strip_prefix = "re2-a276a8c738735a0fe45a6ee590fe2df69bcf4502",
+    urls = ["https://github.com/google/re2/archive/a276a8c738735a0fe45a6ee590fe2df69bcf4502.tar.gz"],
+)
+
 # Will be loaded by workspace.bzl from head
 # TFMD_COMMIT = "404805761e614561cceedc429e67c357c62be26d"  # 1.17.1
 
@@ -218,46 +216,6 @@ load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")  #, "go_repository")
 
 gazelle_dependencies()
 
-################################################################################
-# ZetaSQL                                                                      #
-################################################################################
-
-ZETASQL_COMMIT = "a516c6b26d183efc4f56293256bba92e243b7a61"  # 11/01/2024
-
-http_archive(
-    name = "com_google_zetasql",
-    patch_args = ["-p1"],
-    patches = ["//third_party:zetasql.patch"],
-    sha256 = "1afc2210d4aad371eff0a6bfdd8417ba99e02183a35dff167af2fa6097643f26",
-    strip_prefix = "zetasql-%s" % ZETASQL_COMMIT,
-    urls = ["https://github.com/google/zetasql/archive/%s.tar.gz" % ZETASQL_COMMIT],
-)
-
-load("@com_google_zetasql//bazel:zetasql_deps_step_1.bzl", "zetasql_deps_step_1")
-
-zetasql_deps_step_1()
-
-load("@com_google_zetasql//bazel:zetasql_deps_step_2.bzl", "zetasql_deps_step_2")
-
-zetasql_deps_step_2(
-    analyzer_deps = True,
-    evaluator_deps = True,
-    java_deps = False,
-    testing_deps = False,
-    tools_deps = False,
-)
-
-# No need to run zetasql_deps_step_3 and zetasql_deps_step_4 since all necessary dependencies are
-# already installed.
-
-# load("@com_google_zetasql//bazel:zetasql_deps_step_3.bzl", "zetasql_deps_step_3")
-
-# zetasql_deps_step_3()
-
-# load("@com_google_zetasql//bazel:zetasql_deps_step_4.bzl", "zetasql_deps_step_4")
-
-# zetasql_deps_step_4()
-
 _PLATFORMS_VERSION = "0.0.6"
 
 http_archive(

setup.py

@@ -174,6 +174,8 @@ def select_constraint(default, nightly=None, git_master=None):
     # Make sure to sync the versions of common dependencies (absl-py, numpy,
     # six, and protobuf) with TF.
     install_requires=[
+        'scikit-learn==1.5.1',
+        'scipy==1.17.0',
         'absl-py>=0.9,<2.0.0',
         'apache-beam[gcp]>=2.53,<3;python_version>="3.11"',
         'apache-beam[gcp]>=2.50,<2.51;python_version<"3.11"',
@@ -184,19 +186,20 @@ def select_constraint(default, nightly=None, git_master=None):
         'pandas>=1.0,<2',
         'protobuf>=4.25.2,<6.0.0;python_version>="3.11"',
         'protobuf>=4.21.6,<6.0.0;python_version<"3.11"',
-        'pyarrow>=10,<11',
+        'pyarrow>=10,<11;python_version<"3.11"',
+        'pyarrow>=14,<22;python_version>="3.11"',
         'pyfarmhash>=0.2.2,<0.4',
         'six>=1.12,<2',
-        'tensorflow>=2.17,<2.18',
+        'tensorflow>=2.16,<2.18',
         'tensorflow-metadata'
         + select_constraint(
-            default='>=1.17.1,<1.18',
+            default='>=1.17.0,<1.18',
             nightly='>=1.18.0.dev',
             git_master='@git+https://github.com/tensorflow/metadata@master',
         ),
         'tfx-bsl'
         + select_constraint(
-            default='>=1.17.1,<1.18',
+            default='>=1.17.0,<1.18',
             nightly='>=1.18.0.dev',
             git_master='@git+https://github.com/tensorflow/tfx-bsl@master',
         ),

tensorflow_data_validation/anomalies/BUILD

@@ -425,38 +425,6 @@ cc_test(
     ],
 )
 
-cc_library(
-    name = "custom_validation",
-    srcs = ["custom_validation.cc"],
-    hdrs = ["custom_validation.h"],
-    deps = [
-        ":path",
-        ":schema",
-        ":status_util",
-        "//tensorflow_data_validation/anomalies/proto:custom_validation_config_proto_cc_pb2",
-        "@com_github_tensorflow_metadata//tensorflow_metadata/proto/v0:metadata_v0_proto_cc_pb2",
-        "@com_github_tfx_bsl//tfx_bsl/cc/statistics:sql_util",
-        "@com_google_absl//absl/base:log_severity",
-        "@com_google_absl//absl/container:flat_hash_map",
-        "@com_google_absl//absl/log",
-        "@com_google_absl//absl/status",
-        "@com_google_absl//absl/strings",
-        "@com_google_absl//absl/types:optional",
-    ],
-)
-
-cc_test(
-    name = "custom_validation_test",
-    srcs = ["custom_validation_test.cc"],
-    deps = [
-        ":custom_validation",
-        ":test_util",
-        "@com_github_tensorflow_metadata//tensorflow_metadata/proto/v0:metadata_v0_proto_cc_pb2",
-        "@com_google_absl//absl/types:optional",
-        "@com_google_googletest//:gtest_main",
-    ],
-)
-
 cc_library(
     name = "telemetry",
     srcs = ["telemetry.cc"],

tensorflow_data_validation/anomalies/custom_validation.cc

@@ -22,7 +22,6 @@ limitations under the License.
 #include "tensorflow_data_validation/anomalies/path.h"
 #include "tensorflow_data_validation/anomalies/schema_util.h"
 #include "tensorflow_data_validation/anomalies/status_util.h"
-#include "tfx_bsl/cc/statistics/sql_util.h"
 #include "tensorflow_metadata/proto/v0/anomalies.pb.h"
 #include "tensorflow_metadata/proto/v0/path.pb.h"
 #include "tensorflow_metadata/proto/v0/statistics.pb.h"

tensorflow_data_validation/build_macros.bzl

@@ -25,8 +25,6 @@ def tfdv_pybind_extension(
     so_file = "%s%s.so" % (prefix, sname)
     pyd_file = "%s%s.pyd" % (prefix, sname)
     exported_symbols = [
-        "init%s" % sname,
-        "init_%s" % sname,
         "PyInit_%s" % sname,
     ]
 

tensorflow_data_validation/pywrap/BUILD

@@ -41,7 +41,6 @@ cc_library(
     ],
     features = ["-use_header_modules"],
     deps = [
-        "//tensorflow_data_validation/anomalies:custom_validation",
         "//tensorflow_data_validation/anomalies:feature_statistics_validator",
         "@pybind11",
     ],

tensorflow_data_validation/pywrap/validation_submodule.cc

@@ -13,7 +13,6 @@
 // limitations under the License.
 #include "tensorflow_data_validation/pywrap/validation_submodule.h"
 
-#include "tensorflow_data_validation/anomalies/custom_validation.h"
 #include "tensorflow_data_validation/anomalies/feature_statistics_validator.h"
 #include "include/pybind11/pybind11.h"
 
@@ -77,22 +76,6 @@ void DefineValidationSubmodule(py::module main_module) {
           }
           return py::bytes(anomalies_proto_string);
         });
-  m.def("CustomValidateStatistics",
-        [](const std::string& test_statistics_string,
-           const std::string& base_statistics_string,
-           const std::string& validations_string,
-           const std::string& environment_string) -> py::object {
-          std::string anomalies_proto_string;
-          const absl::Status status =
-              CustomValidateStatisticsWithSerializedInputs(
-                  test_statistics_string, base_statistics_string,
-                  validations_string, environment_string,
-                  &anomalies_proto_string);
-          if (!status.ok()) {
-            throw std::runtime_error(status.ToString());
-          }
-          return py::bytes(anomalies_proto_string);
-        });
 }
 
 }  // namespace data_validation