From 0ad2e35eda6c5e1ed5e5c88ae124a8f3a4407a31 Mon Sep 17 00:00:00 2001 From: Victor Sotero Date: Mon, 22 Jun 2026 11:12:03 +0200 Subject: [PATCH] docs(iceberg): use Iceberg Compat V3 Signed-off-by: Victor Sotero --- experimental/databricks-iceberg/SKILL.md | 8 ++-- .../references/1-managed-iceberg-tables.md | 5 +-- .../references/2-uniform-and-compatibility.md | 38 +++++++++---------- tests/iceberg_content_test.py | 23 +++++++++++ 4 files changed, 48 insertions(+), 26 deletions(-) create mode 100644 tests/iceberg_content_test.py diff --git a/experimental/databricks-iceberg/SKILL.md b/experimental/databricks-iceberg/SKILL.md index eb442bee..702269a2 100644 --- a/experimental/databricks-iceberg/SKILL.md +++ b/experimental/databricks-iceberg/SKILL.md @@ -33,7 +33,7 @@ Databricks provides multiple ways to work with Apache Iceberg: native managed Ic | **External Iceberg Reads (Uniform)** | Delta table that auto-generates Iceberg metadata — read as Iceberg externally, write as Delta internally | | **Compatibility Mode** | UniForm variant for streaming tables and materialized views in SDP pipelines | | **Iceberg REST Catalog (IRC)** | Unity Catalog's built-in REST endpoint implementing the Iceberg REST Catalog spec — lets external engines (Spark, PyIceberg, Snowflake) access UC-managed Iceberg data | -| **Iceberg v3** | Next-gen format (Beta, DBR 17.3+) — deletion vectors, VARIANT type, row lineage | +| **Iceberg v3** | Next-gen format (Beta, DBR 18.0+) — deletion vectors, VARIANT type, row lineage | --- @@ -78,7 +78,7 @@ AS SELECT * FROM raw_events; ALTER TABLE my_catalog.my_schema.customers SET TBLPROPERTIES ( 'delta.columnMapping.mode' = 'name', - 'delta.enableIcebergCompatV2' = 'true', + 'delta.enableIcebergCompatV3' = 'true', 'delta.universalFormat.enabledFormats' = 'iceberg' ); ``` @@ -127,7 +127,7 @@ SET TBLPROPERTIES ( | **UniForm async delay** | Iceberg metadata generation is asynchronous. After a write, there may be a brief delay before external engines see the latest data. Check status with `DESCRIBE EXTENDED table_name`. | | **Compression codec change** | Managed Iceberg tables use `zstd` compression by default (not `snappy`). Older Iceberg readers that don't support zstd will fail. Verify reader compatibility or set `write.parquet.compression-codec` to `snappy`. | | **Snowflake 1000-commit limit** | Snowflake's Iceberg catalog integration can only see the last 1000 Iceberg commits. High-frequency writers must compact metadata or Snowflake will lose visibility of older data. | -| **Deletion vectors with UniForm** | UniForm requires deletion vectors to be disabled (`delta.enableDeletionVectors = false`). If your table has deletion vectors enabled, disable them before enabling UniForm. | +| **Deletion vectors with UniForm** | Use Iceberg v3 (`delta.enableIcebergCompatV3 = true`) when deletion vectors must stay enabled; Iceberg v2 UniForm requires deletion vectors to be disabled first. | | **No shallow clone for Iceberg** | `SHALLOW CLONE` is not supported for Iceberg tables. Use `DEEP CLONE` or `CREATE TABLE ... AS SELECT` instead. | | **Version mismatch with external engines** | Ensure external engines use an Iceberg library version compatible with the format version of your tables. Iceberg v3 tables require Iceberg library 1.9.0+. | @@ -148,5 +148,5 @@ SET TBLPROPERTIES ( - **[UniForm](https://docs.databricks.com/delta/uniform.html)** — Delta Universal Format - **[Iceberg REST Catalog](https://docs.databricks.com/external-access/iceberg)** — IRC endpoint and external engine access - **[Compatibility Mode](https://docs.databricks.com/external-access/compatibility-mode)** — UniForm for streaming tables and MVs -- **[Iceberg v3](https://docs.databricks.com/iceberg/iceberg-v3)** — next-gen format features (Beta) +- **[Iceberg v3](https://docs.databricks.com/iceberg/iceberg-v3)** — next-gen format features (Beta, DBR 18.0+) - **[Foreign Tables](https://docs.databricks.com/query-data/foreign-tables.html)** — reading external catalog data diff --git a/experimental/databricks-iceberg/references/1-managed-iceberg-tables.md b/experimental/databricks-iceberg/references/1-managed-iceberg-tables.md index a0f3f06e..5b85b043 100644 --- a/experimental/databricks-iceberg/references/1-managed-iceberg-tables.md +++ b/experimental/databricks-iceberg/references/1-managed-iceberg-tables.md @@ -2,7 +2,7 @@ Managed Iceberg tables are native Apache Iceberg tables created and stored within Unity Catalog. They support full read/write operations in Databricks and are accessible to external engines via the UC Iceberg REST Catalog (IRC) endpoint. -**Requirements**: Unity Catalog, DBR 16.4 LTS+ (Managed Iceberg v2), DBR 17.3+ (Managed Iceberg v3 Beta) +**Requirements**: Unity Catalog, DBR 16.4 LTS+ (Managed Iceberg v2), DBR 18.0+ (Managed Iceberg v3 Beta) --- @@ -171,7 +171,7 @@ ANALYZE TABLE my_catalog.my_schema.events COMPUTE STATISTICS FOR ALL COLUMNS; ## Iceberg v3 (Beta) -**Requires**: DBR 17.3+ +**Requires**: DBR 18.0+ Iceberg v3 introduces new capabilities on top of v2: @@ -259,4 +259,3 @@ CLUSTER BY (event_date) AS SELECT * FROM foreign_catalog.foreign_schema.events; ``` - diff --git a/experimental/databricks-iceberg/references/2-uniform-and-compatibility.md b/experimental/databricks-iceberg/references/2-uniform-and-compatibility.md index 02ad367f..430863f6 100644 --- a/experimental/databricks-iceberg/references/2-uniform-and-compatibility.md +++ b/experimental/databricks-iceberg/references/2-uniform-and-compatibility.md @@ -6,7 +6,7 @@ UniForm and Compatibility Mode make Delta tables readable as Iceberg by external ## External Iceberg Reads (fka UniForm) (GA) -**Requirements**: Unity Catalog, DBR 14.3+, column mapping enabled, deletion vectors disabled, the Delta table must have a minReaderVersion >= 2 and minWriterVersion >= 7, both managed and external tables supported. +**Requirements**: Unity Catalog, DBR 18.0+ for Iceberg v3, column mapping enabled, the Delta table must have a minReaderVersion >= 2 and minWriterVersion >= 7, both managed and external tables supported. Iceberg v3 supports deletion vectors; only Iceberg v2 UniForm requires deletion vectors to be disabled. UniForm adds automatic Iceberg metadata generation to regular Delta tables. The table remains Delta internally but is readable as Iceberg externally. @@ -21,7 +21,7 @@ CREATE TABLE my_catalog.my_schema.customers ( ) TBLPROPERTIES ( 'delta.columnMapping.mode' = 'name', - 'delta.enableIcebergCompatV2' = 'true', + 'delta.enableIcebergCompatV3' = 'true', 'delta.universalFormat.enabledFormats' = 'iceberg' ); ``` @@ -32,11 +32,21 @@ TBLPROPERTIES ( ALTER TABLE my_catalog.my_schema.customers SET TBLPROPERTIES ( 'delta.columnMapping.mode' = 'name', - 'delta.enableIcebergCompatV2' = 'true', + 'delta.enableIcebergCompatV3' = 'true', 'delta.universalFormat.enabledFormats' = 'iceberg' ); ``` +### Upgrading an Existing UniForm Table from Iceberg v2 to v3 + +```sql +ALTER TABLE my_catalog.my_schema.customers +SET TBLPROPERTIES ( + 'delta.enableIcebergCompatV3' = 'true', + 'delta.enableIcebergCompatV2' = 'false' +); +``` + ### Requirements and Prerequisites UniForm requires the following properties to be set explicitly: @@ -44,27 +54,17 @@ UniForm requires the following properties to be set explicitly: | Requirement | Details | |-------------|---------| | **Unity Catalog** | Table must be registered in UC | -| **DBR 14.3+** | Minimum runtime version | -| **Deletion vectors disabled** | Set `delta.enableDeletionVectors = false` before enabling UniForm | -| **No column mapping conflicts** | If table uses `id` mode, migrate to `name` mode first | +| **DBR 18.0+ for Iceberg v3** | Use Iceberg v3 for new UniForm examples and feature coverage | +| **Column mapping enabled** | Use `delta.columnMapping.mode`; Databricks recommends `id` for most new-table compatibility, while `name` is suitable for existing tables and pipeline compatibility mode examples | +| **Deletion vector behavior** | Iceberg v3 supports deletion vectors. Disable deletion vectors only when staying on Iceberg v2. | -If deletion vectors are currently enabled: +If an existing UniForm table is still on Iceberg v2 and you want to keep deletion vectors enabled, upgrade the table to Iceberg v3: ```sql --- Disable deletion vectors first -ALTER TABLE my_catalog.my_schema.customers -SET TBLPROPERTIES ('delta.enableDeletionVectors' = 'false'); - --- Rewrite to remove existing deletion vectors -REORG TABLE my_catalog.my_schema.customers -APPLY (PURGE); - --- Then enable UniForm ALTER TABLE my_catalog.my_schema.customers SET TBLPROPERTIES ( - 'delta.columnMapping.mode' = 'name', - 'delta.enableIcebergCompatV2' = 'true', - 'delta.universalFormat.enabledFormats' = 'iceberg' + 'delta.enableIcebergCompatV3' = 'true', + 'delta.enableIcebergCompatV2' = 'false' ); ``` diff --git a/tests/iceberg_content_test.py b/tests/iceberg_content_test.py new file mode 100644 index 00000000..132b3996 --- /dev/null +++ b/tests/iceberg_content_test.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Content guardrails for the experimental Iceberg skill.""" + +import unittest +from pathlib import Path + + +_REPO = Path(__file__).resolve().parent.parent +_ICEBERG_DIR = _REPO / "experimental" / "databricks-iceberg" + + +class IcebergContentTest(unittest.TestCase): + def test_uniform_examples_recommend_iceberg_v3(self): + text = "\n".join(path.read_text() for path in _ICEBERG_DIR.rglob("*.md")) + + self.assertIn("delta.enableIcebergCompatV3", text) + self.assertNotIn("delta.enableIcebergCompatV2' = 'true", text) + self.assertNotIn("delta.enableIcebergCompatV2\" = \"true", text) + self.assertNotIn("ICEBERG_COMPAT_VERSION=2", text) + + +if __name__ == "__main__": + unittest.main()