Skip to content

Commit 8523b3a

Browse files
author
Sreesh Maheshwar
committed
Add test for object storage data file locations
1 parent 9999cbb commit 8523b3a

File tree

1 file changed

+23
-0
lines changed

1 file changed

+23
-0
lines changed

tests/integration/test_writes/test_writes.py

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,7 @@ def test_data_files(spark: SparkSession, session_catalog: Catalog, arrow_table_w
273273
# Since we don't rewrite, this should produce a new manifest with an ADDED entry
274274
tbl.append(arrow_table_with_null)
275275

276+
276277
rows = spark.sql(
277278
f"""
278279
SELECT added_data_files_count, existing_data_files_count, deleted_data_files_count
@@ -285,6 +286,28 @@ def test_data_files(spark: SparkSession, session_catalog: Catalog, arrow_table_w
285286
assert [row.deleted_data_files_count for row in rows] == [0, 1, 0, 0, 0]
286287

287288

289+
@pytest.mark.integration
290+
@pytest.mark.parametrize("format_version", [1, 2])
291+
def test_object_storage_data_files(spark: SparkSession, session_catalog: Catalog, arrow_table_with_null: pa.Table, format_version: int) -> None:
292+
# TODO: What to do about "tbl.add_files()"?
293+
identifier = "default.object_stored_table"
294+
295+
tbl = _create_table(session_catalog, identifier, {"format-version": format_version, "write.object-storage.enabled": True}, [])
296+
tbl.append(arrow_table_with_null)
297+
298+
paths = tbl.inspect.entries().to_pydict()["data_file"]
299+
assert len(paths) == 1
300+
location = paths[0]["file_path"]
301+
302+
parts = location.split("/")
303+
assert len(parts) == 11
304+
305+
assert location.startswith("s3://warehouse/default/object_stored_table/data/")
306+
for i in range(6, 10):
307+
assert len(parts[i]) == (8 if i == 9 else 4)
308+
assert all(c in "01" for c in parts[i])
309+
310+
288311
@pytest.mark.integration
289312
def test_python_writes_with_spark_snapshot_reads(
290313
spark: SparkSession, session_catalog: Catalog, arrow_table_with_null: pa.Table

0 commit comments

Comments
 (0)