fix tests

Author: kevinjqliu

ruff.toml

@@ -63,13 +63,11 @@ ignore = []
 fixable = ["ALL"]
 unfixable = []
 
+per-file-ignores = {}
+
 # Allow unused variables when underscore-prefixed.
 dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
 
-[lint.per-file-ignores]
-"tests/integration/test_inspect_table.py" = ["E501"]
-"tests/io/test_pyarrow.py" = ["E501"]
-
 [lint.pyupgrade]
 # Preserve types, even if a file imports `from __future__ import annotations`.
 keep-runtime-typing = true

tests/catalog/test_hive.py

@@ -970,7 +970,10 @@ def test_rename_table_to_namespace_does_not_exists() -> None:
 
     catalog._client = MagicMock()
     catalog._client.__enter__().alter_table_with_environment_context.side_effect = InvalidOperationException(
-        message="Unable to change partition or table. Database default does not exist Check metastore logs for detailed stack.does_not_exists"  # noqa: E501
+        message=(
+            "Unable to change partition or table. Database default does not exist "
+            "Check metastore logs for detailed stack.does_not_exists"
+        )
     )
 
     with pytest.raises(NoSuchNamespaceError) as exc_info:

tests/catalog/test_rest.py

@@ -156,7 +156,8 @@ def test_no_uri_supplied() -> None:
 
 
 @pytest.mark.filterwarnings(
-    "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"  # noqa: E501
+    "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. "
+    "Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"
 )
 def test_token_200(rest_mock: Mocker) -> None:
     rest_mock.post(
@@ -179,7 +180,8 @@ def test_token_200(rest_mock: Mocker) -> None:
 
 
 @pytest.mark.filterwarnings(
-    "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"  # noqa: E501
+    "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. "
+    "Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"
 )
 def test_token_200_without_optional_fields(rest_mock: Mocker) -> None:
     rest_mock.post(
@@ -198,7 +200,8 @@ def test_token_200_without_optional_fields(rest_mock: Mocker) -> None:
 
 
 @pytest.mark.filterwarnings(
-    "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"  # noqa: E501
+    "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. "
+    "Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"
 )
 def test_token_with_optional_oauth_params(rest_mock: Mocker) -> None:
     mock_request = rest_mock.post(
@@ -223,7 +226,8 @@ def test_token_with_optional_oauth_params(rest_mock: Mocker) -> None:
 
 
 @pytest.mark.filterwarnings(
-    "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"  # noqa: E501
+    "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. "
+    "Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"
 )
 def test_token_with_optional_oauth_params_as_empty(rest_mock: Mocker) -> None:
     mock_request = rest_mock.post(
@@ -246,7 +250,8 @@ def test_token_with_optional_oauth_params_as_empty(rest_mock: Mocker) -> None:
 
 
 @pytest.mark.filterwarnings(
-    "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"  # noqa: E501
+    "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. "
+    "Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"
 )
 def test_token_with_default_scope(rest_mock: Mocker) -> None:
     mock_request = rest_mock.post(
@@ -267,7 +272,8 @@ def test_token_with_default_scope(rest_mock: Mocker) -> None:
 
 
 @pytest.mark.filterwarnings(
-    "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"  # noqa: E501
+    "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. "
+    "Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"
 )
 def test_token_with_custom_scope(rest_mock: Mocker) -> None:
     mock_request = rest_mock.post(
@@ -289,7 +295,8 @@ def test_token_with_custom_scope(rest_mock: Mocker) -> None:
 
 
 @pytest.mark.filterwarnings(
-    "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"  # noqa: E501
+    "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. "
+    "Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"
 )
 def test_token_200_w_oauth2_server_uri(rest_mock: Mocker) -> None:
     rest_mock.post(
@@ -314,7 +321,8 @@ def test_token_200_w_oauth2_server_uri(rest_mock: Mocker) -> None:
 
 
 @pytest.mark.filterwarnings(
-    "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"  # noqa: E501
+    "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. "
+    "Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"
 )
 def test_config_200(requests_mock: Mocker) -> None:
     requests_mock.get(
@@ -402,7 +410,8 @@ def test_config_sets_headers(requests_mock: Mocker) -> None:
 
 
 @pytest.mark.filterwarnings(
-    "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"  # noqa: E501
+    "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. "
+    "Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"
 )
 def test_token_400(rest_mock: Mocker) -> None:
     rest_mock.post(
@@ -418,7 +427,8 @@ def test_token_400(rest_mock: Mocker) -> None:
 
 
 @pytest.mark.filterwarnings(
-    "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"  # noqa: E501
+    "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. "
+    "Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"
 )
 def test_token_401(rest_mock: Mocker) -> None:
     message = "invalid_client"
@@ -664,7 +674,8 @@ def test_list_namespace_with_parent_404(rest_mock: Mocker) -> None:
 
 
 @pytest.mark.filterwarnings(
-    "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"  # noqa: E501
+    "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. "
+    "Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"
 )
 @pytest.mark.parametrize("status_code", [401, 419])
 def test_list_namespaces_token_expired_success_on_retries(rest_mock: Mocker, status_code: int) -> None:
@@ -2017,7 +2028,8 @@ def test_custom_namespace_separator(rest_mock: Mocker) -> None:
 
 
 @pytest.mark.filterwarnings(
-    "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"  # noqa: E501
+    "ignore:Deprecated in 0.8.0, will be removed in 1.0.0. "
+    "Iceberg REST client is missing the OAuth2 server URI:DeprecationWarning"
 )
 def test_auth_header(rest_mock: Mocker) -> None:
     mock_request = rest_mock.post(

tests/cli/test_console.py

@@ -627,9 +627,10 @@ def test_json_schema(catalog: InMemoryCatalog) -> None:
     runner = CliRunner()
     result = runner.invoke(run, ["--output=json", "schema", "default.my_table"])
     assert result.exit_code == 0
-    assert (
-        result.output
-        == """{"type":"struct","fields":[{"id":1,"name":"x","type":"long","required":true},{"id":2,"name":"y","type":"long","required":true,"doc":"comment"},{"id":3,"name":"z","type":"long","required":true}],"schema-id":0,"identifier-field-ids":[]}\n"""  # noqa: E501
+    assert result.output == (
+        '{"type":"struct","fields":[{"id":1,"name":"x","type":"long","required":true},'
+        '{"id":2,"name":"y","type":"long","required":true,"doc":"comment"},'
+        '{"id":3,"name":"z","type":"long","required":true}],"schema-id":0,"identifier-field-ids":[]}\n'
     )
 
 
@@ -819,9 +820,8 @@ def test_json_properties_get_table_specific_property_that_doesnt_exist(catalog:
     runner = CliRunner()
     result = runner.invoke(run, ["--output=json", "properties", "get", "table", "default.my_table", "doesnotexist"])
     assert result.exit_code == 1
-    assert (
-        result.output
-        == """{"type": "NoSuchPropertyException", "message": "Could not find property doesnotexist on table default.my_table"}\n"""  # noqa: E501
+    assert result.output == (
+        '{"type": "NoSuchPropertyException", "message": "Could not find property doesnotexist on table default.my_table"}\n'
     )
 
 

tests/conftest.py

@@ -1188,7 +1188,10 @@ def metadata_location_gz(tmp_path_factory: pytest.TempPathFactory) -> str:
         "status": 1,
         "snapshot_id": 8744736658442914487,
         "data_file": {
-            "file_path": "/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=null/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00001.parquet",  # noqa: E501
+            "file_path": (
+                "/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=null/"
+                "00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00001.parquet"
+            ),
             "file_format": "PARQUET",
             "partition": {"VendorID": 1, "tpep_pickup_day": 1925},
             "record_count": 19513,
@@ -1308,7 +1311,10 @@ def metadata_location_gz(tmp_path_factory: pytest.TempPathFactory) -> str:
         "status": 1,
         "snapshot_id": 8744736658442914487,
         "data_file": {
-            "file_path": "/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=1/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00002.parquet",  # noqa: E501
+            "file_path": (
+                "/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=1/"
+                "00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00002.parquet"
+            ),
             "file_format": "PARQUET",
             "partition": {"VendorID": 1, "tpep_pickup_datetime": None},
             "record_count": 95050,
@@ -2146,7 +2152,10 @@ def adls_fsspec_fileio(request: pytest.FixtureRequest) -> Generator[FsspecFileIO
     azurite_url = request.config.getoption("--adls.endpoint")
     azurite_account_name = request.config.getoption("--adls.account-name")
     azurite_account_key = request.config.getoption("--adls.account-key")
-    azurite_connection_string = f"DefaultEndpointsProtocol=http;AccountName={azurite_account_name};AccountKey={azurite_account_key};BlobEndpoint={azurite_url}/{azurite_account_name};"  # noqa: E501
+    azurite_connection_string = (
+        f"DefaultEndpointsProtocol=http;AccountName={azurite_account_name};"
+        f"AccountKey={azurite_account_key};BlobEndpoint={azurite_url}/{azurite_account_name};"
+    )
     properties = {
         "adls.connection-string": azurite_connection_string,
         "adls.account-name": azurite_account_name,
@@ -2183,7 +2192,10 @@ def pyarrow_fileio_adls(request: pytest.FixtureRequest) -> Generator[Any, None,
 
     azurite_account_name = request.config.getoption("--adls.account-name")
     azurite_account_key = request.config.getoption("--adls.account-key")
-    azurite_connection_string = f"DefaultEndpointsProtocol=http;AccountName={azurite_account_name};AccountKey={azurite_account_key};BlobEndpoint={azurite_url}/{azurite_account_name};"  # noqa: E501
+    azurite_connection_string = (
+        f"DefaultEndpointsProtocol=http;AccountName={azurite_account_name};"
+        f"AccountKey={azurite_account_key};BlobEndpoint={azurite_url}/{azurite_account_name};"
+    )
     properties = {
         ADLS_ACCOUNT_NAME: azurite_account_name,
         ADLS_ACCOUNT_KEY: azurite_account_key,
@@ -2623,7 +2635,8 @@ def spark() -> "SparkSession":
     # Not supported by Spark
     # 'time': [time(1, 22, 0), None, time(19, 25, 0)],
     # Not natively supported by Arrow
-    # 'uuid': [uuid.UUID('00000000-0000-0000-0000-000000000000').bytes, None, uuid.UUID('11111111-1111-1111-1111-111111111111').bytes], # noqa: E501
+    # 'uuid': [uuid.UUID('00000000-0000-0000-0000-000000000000').bytes, None,
+    #          uuid.UUID('11111111-1111-1111-1111-111111111111').bytes],
     "binary": [b"\01", None, b"\22"],
     "fixed": [
         uuid.UUID("00000000-0000-0000-0000-000000000000").bytes,
@@ -2676,7 +2689,8 @@ def arrow_table_with_null(pa_schema: "pa.Schema") -> "pa.Table":
             "long": [1, None, 9],
             "float": [0.0, None, 0.9],
             "double": [0.0, None, 0.9],
-            # 'time': [1_000_000, None, 3_000_000],  # Example times: 1s, none, and 3s past midnight #Spark does not support time fields # noqa: E501
+            # 'time': [1_000_000, None, 3_000_000],  # Example times: 1s, none, and 3s past midnight
+            # Spark does not support time fields
             "timestamp": [datetime(2023, 1, 1, 19, 25, 00), None, datetime(2023, 3, 1, 19, 25, 00)],
             "timestamptz": [
                 datetime(2023, 1, 1, 19, 25, 00, tzinfo=timezone.utc),
@@ -2687,7 +2701,8 @@ def arrow_table_with_null(pa_schema: "pa.Schema") -> "pa.Table":
             # Not supported by Spark
             # 'time': [time(1, 22, 0), None, time(19, 25, 0)],
             # Not natively supported by Arrow
-            # 'uuid': [uuid.UUID('00000000-0000-0000-0000-000000000000').bytes, None, uuid.UUID('11111111-1111-1111-1111-111111111111').bytes], # noqa: E501
+            # 'uuid': [uuid.UUID('00000000-0000-0000-0000-000000000000').bytes, None,
+            #          uuid.UUID('11111111-1111-1111-1111-111111111111').bytes],
             "binary": [b"\01", None, b"\22"],
             "fixed": [
                 uuid.UUID("00000000-0000-0000-0000-000000000000").bytes,

tests/expressions/test_evaluator.py

@@ -1427,7 +1427,9 @@ def test_strict_integer_in(strict_data_file_schema: Schema, strict_data_file_1:
 
 
 def test_strict_integer_not_in(strict_data_file_schema: Schema, strict_data_file_1: DataFile) -> None:
-    # should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotIn("id", {INT_MIN_VALUE - 25, INT_MIN_VALUE - 24})).eval(strict_data_file_1) # noqa: E501
+    # should_read = _StrictMetricsEvaluator(
+    #     strict_data_file_schema, NotIn("id", {INT_MIN_VALUE - 25, INT_MIN_VALUE - 24})
+    # ).eval(strict_data_file_1)
     # assert should_read, "Should match: all values != 5 and != 6"
 
     should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotIn("id", {INT_MIN_VALUE - 1, INT_MIN_VALUE})).eval(

tests/expressions/test_visitors.py

@@ -261,9 +261,10 @@ def test_bind_visitor_already_bound(table_schema_simple: Schema) -> None:
     with pytest.raises(TypeError) as exc_info:
         visit(bound, visitor=BindVisitor(schema=table_schema_simple, case_sensitive=True))
     assert (
-        "Found already bound predicate: BoundEqualTo(term=BoundReference(field=NestedField(field_id=1, name='foo', field_type=StringType(), required=False), accessor=Accessor(position=0,inner=None)), literal=literal('hello'))"  # noqa: E501
-        == str(exc_info.value)
-    )
+        "Found already bound predicate: BoundEqualTo(term=BoundReference("
+        "field=NestedField(field_id=1, name='foo', field_type=StringType(), required=False), "
+        "accessor=Accessor(position=0,inner=None)), literal=literal('hello'))"
+    ) == str(exc_info.value)
 
 
 def test_visit_bound_visitor_unknown_predicate() -> None:

tests/integration/test_add_files.py

@@ -470,8 +470,8 @@ def test_add_files_to_bucket_partitioned_table_fails(spark: SparkSession, sessio
     with pytest.raises(ValueError) as exc_info:
         tbl.add_files(file_paths=file_paths)
     assert (
-        "Cannot infer partition value from parquet metadata for a non-linear Partition Field: baz_bucket_3 with transform bucket[3]"  # noqa: E501
-        in str(exc_info.value)
+        "Cannot infer partition value from parquet metadata for a non-linear Partition Field: "
+        "baz_bucket_3 with transform bucket[3]" in str(exc_info.value)
     )
 
 
@@ -518,8 +518,8 @@ def test_add_files_to_partitioned_table_fails_with_lower_and_upper_mismatch(
     with pytest.raises(ValueError) as exc_info:
         tbl.add_files(file_paths=file_paths)
     assert (
-        "Cannot infer partition value from parquet metadata as there are more than one partition values for Partition Field: baz. lower_value=123, upper_value=124"  # noqa: E501
-        in str(exc_info.value)
+        "Cannot infer partition value from parquet metadata as there are more than one partition values "
+        "for Partition Field: baz. lower_value=123, upper_value=124" in str(exc_info.value)
     )
 
 
@@ -754,8 +754,8 @@ def test_add_files_with_timestamp_tz_ns_fails(session_catalog: Catalog, format_v
     exception_cause = exc_info.value.__cause__
     assert isinstance(exception_cause, TypeError)
     assert (
-        "Iceberg does not yet support 'ns' timestamp precision. Use 'downcast-ns-timestamp-to-us-on-write' configuration property to automatically downcast 'ns' to 'us' on write."  # noqa: E501
-        in exception_cause.args[0]
+        "Iceberg does not yet support 'ns' timestamp precision. Use 'downcast-ns-timestamp-to-us-on-write' "
+        "configuration property to automatically downcast 'ns' to 'us' on write." in exception_cause.args[0]
     )
 
 

tests/integration/test_inspect_table.py

@@ -1096,13 +1096,22 @@ def test_inspect_files_format_version_3(spark: SparkSession, session_catalog: Ca
         },
     )
 
-    # fmt: off
+    row1 = (
+        "(false, 'a', 'aaaaaaaaaaaaaaaaaaaaaa', 1, 1, 0.0, 0.0, "
+        "TIMESTAMP('2023-01-01 19:25:00'), TIMESTAMP('2023-01-01 19:25:00+00:00'), "
+        "DATE('2023-01-01'), X'01', X'00000000000000000000000000000000')"
+    )
+    row2 = "(NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)"
+    row3 = (
+        "(true, 'z', 'zzzzzzzzzzzzzzzzzzzzzz', 9, 9, 0.9, 0.9, "
+        "TIMESTAMP('2023-03-01 19:25:00'), TIMESTAMP('2023-03-01 19:25:00+00:00'), "
+        "DATE('2023-03-01'), X'12', X'11111111111111111111111111111111')"
+    )
     insert_data_sql = f"""INSERT INTO {identifier} VALUES
-        (false, 'a', 'aaaaaaaaaaaaaaaaaaaaaa', 1, 1, 0.0, 0.0, TIMESTAMP('2023-01-01 19:25:00'), TIMESTAMP('2023-01-01 19:25:00+00:00'), DATE('2023-01-01'), X'01', X'00000000000000000000000000000000'),
-        (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-        (true, 'z', 'zzzzzzzzzzzzzzzzzzzzzz', 9, 9, 0.9, 0.9, TIMESTAMP('2023-03-01 19:25:00'), TIMESTAMP('2023-03-01 19:25:00+00:00'), DATE('2023-03-01'), X'12', X'11111111111111111111111111111111');
+        {row1},
+        {row2},
+        {row3};
     """
-    # fmt: on
 
     spark.sql(insert_data_sql)
     spark.sql(insert_data_sql)