fix: fix spark4 unit tests

Author: kx79wq

histogrammar/dfinterface/spark_histogrammar.py

@@ -225,7 +225,7 @@ def construct_empty_hist(self, df, features):
         for idx, col in enumerate(revcols):
             # histogram type depends on the data type
             dt = self.var_dtype[col]
-            quant = df[col]
+            quant = f.col(col)
             hist = self.get_hist_bin(hist, features, quant, col, dt)
 
         return hist

histogrammar/util.py

@@ -247,7 +247,7 @@ def __init__(self, expr, name=None):
                 ok = False
             else:
                 if isinstance(expr, Column) and self.name is None:
-                    self.name = str(expr)[7:-1]
+                    self.name = str(expr)[8:-2]
                 ok = True
         if not ok:
             raise TypeError(f"quantity ({expr}) must be a string, function, or SparkSQL Column")

tests/test_spark_histogrammar.py

@@ -53,16 +53,16 @@ def spark_co():
 @pytest.mark.skipif(not spark_found, reason="spark not found")
 @pytest.mark.filterwarnings("ignore:createDataFrame attempted Arrow optimization because")
 def test_get_histograms(spark_co):
-    pytest.age["data"]["name"] = "'age'"
-    pytest.company["data"]["name"] = "'company'"
-    pytest.eyesColor["data"]["name"] = "'eyeColor'"
-    pytest.gender["data"]["name"] = "'gender'"
-    pytest.isActive["data"]["name"] = "'isActive'"
-    pytest.latitude["data"]["name"] = "'latitude'"
-    pytest.longitude["data"]["name"] = "'longitude'"
-    pytest.transaction["data"]["name"] = "'transaction'"
-
-    pytest.latitude_longitude["data"]["name"] = "'latitude:longitude'"
+    pytest.age["data"]["name"] = "age"
+    pytest.company["data"]["name"] = "company"
+    pytest.eyesColor["data"]["name"] = "eyeColor"
+    pytest.gender["data"]["name"] = "gender"
+    pytest.isActive["data"]["name"] = "isActive"
+    pytest.latitude["data"]["name"] = "latitude"
+    pytest.longitude["data"]["name"] = "longitude"
+    pytest.transaction["data"]["name"] = "transaction"
+
+    pytest.latitude_longitude["data"]["name"] = "latitude:longitude"
     pytest.latitude_longitude["data"]["bins:name"] = "unit_func"
 
     spark = spark_co
@@ -113,15 +113,15 @@ def test_get_histograms(spark_co):
 @pytest.mark.skipif(not spark_found, reason="spark not found")
 @pytest.mark.filterwarnings("ignore:createDataFrame attempted Arrow optimization because")
 def test_get_histograms_module(spark_co):
-    pytest.age["data"]["name"] = "'age'"
-    pytest.company["data"]["name"] = "'company'"
-    pytest.eyesColor["data"]["name"] = "'eyeColor'"
-    pytest.gender["data"]["name"] = "'gender'"
-    pytest.isActive["data"]["name"] = "'isActive'"
-    pytest.latitude["data"]["name"] = "'latitude'"
-    pytest.longitude["data"]["name"] = "'longitude'"
-
-    pytest.latitude_longitude["data"]["name"] = "'latitude:longitude'"
+    pytest.age["data"]["name"] = "age"
+    pytest.company["data"]["name"] = "company"
+    pytest.eyesColor["data"]["name"] = "eyeColor"
+    pytest.gender["data"]["name"] = "gender"
+    pytest.isActive["data"]["name"] = "isActive"
+    pytest.latitude["data"]["name"] = "latitude"
+    pytest.longitude["data"]["name"] = "longitude"
+
+    pytest.latitude_longitude["data"]["name"] = "latitude:longitude"
     pytest.latitude_longitude["data"]["bins:name"] = "unit_func"
 
     spark = spark_co
@@ -196,7 +196,7 @@ def test_get_histograms_timestamp(spark_co):
             "bins": {"108": 9.0, "109": 1.0},
             "bins:type": "Count",
             "entries": 10.0,
-            "name": "'dt'",
+            "name": "dt",
             "nanflow": 0.0,
             "nanflow:type": "Count",
             "origin": 1.2625632e18,
@@ -238,7 +238,7 @@ def test_get_histograms_date(spark_co):
             "bins": {"108": 9.0, "109": 1.0},
             "bins:type": "Count",
             "entries": 10.0,
-            "name": "'dt'",
+            "name": "dt",
             "nanflow": 0.0,
             "nanflow:type": "Count",
             "origin": 1.2625632e18,