|
1 | 1 | # -*- coding: utf-8 -*- |
2 | 2 | import unittest |
3 | 3 | from pyspark.sql import Row |
4 | | -from pydeequ.profiles import ColumnProfilerRunBuilder, ColumnProfilerRunner, DistributionValue, StringColumnProfile |
| 4 | +from pydeequ.profiles import ( |
| 5 | + ColumnProfilerRunBuilder, |
| 6 | + ColumnProfilerRunner, |
| 7 | + DistributionValue, |
| 8 | + StringColumnProfile, |
| 9 | +) |
5 | 10 | from pydeequ.analyzers import KLLParameters, DataTypeInstances |
6 | 11 | from tests.conftest import setup_pyspark |
7 | 12 |
|
| 13 | + |
8 | 14 | class TestProfiles(unittest.TestCase): |
9 | 15 | @classmethod |
10 | 16 | def setUpClass(cls): |
11 | 17 | cls.spark = setup_pyspark().appName("test-profiles-local").getOrCreate() |
12 | 18 | cls.sc = cls.spark.sparkContext |
13 | | - cls.df = cls.sc.parallelize([Row(a="foo", b=1, c=5), Row(a="bar", b=2, c=6), Row(a="baz", b=3, c=None)]).toDF() |
| 19 | + cls.df = cls.sc.parallelize( |
| 20 | + [Row(a="foo", b=1, c=5), Row(a="bar", b=2, c=6), Row(a="baz", b=3, c=None)] |
| 21 | + ).toDF() |
14 | 22 |
|
15 | 23 | @classmethod |
16 | 24 | def tearDownClass(cls): |
17 | 25 | cls.spark.sparkContext._gateway.shutdown_callback_server() |
18 | 26 | cls.spark.stop() |
19 | 27 |
|
20 | 28 | def test_setPredefinedTypes(self): |
21 | | - result = ColumnProfilerRunner(self.spark) \ |
22 | | - .onData(self.df) \ |
23 | | - .setPredefinedTypes({'a': DataTypeInstances.Unknown, 'b': DataTypeInstances.String, 'c': DataTypeInstances.Fractional}) \ |
| 29 | + result = ( |
| 30 | + ColumnProfilerRunner(self.spark) |
| 31 | + .onData(self.df) |
| 32 | + .setPredefinedTypes( |
| 33 | + { |
| 34 | + "a": DataTypeInstances.Unknown, |
| 35 | + "b": DataTypeInstances.String, |
| 36 | + "c": DataTypeInstances.Fractional, |
| 37 | + } |
| 38 | + ) |
24 | 39 | .run() |
| 40 | + ) |
25 | 41 | print(result) |
26 | 42 | for col, profile in result.profiles.items(): |
27 | 43 | print("Profiles:", profile) |
|
0 commit comments