Kevinjqliu/tmp

.github/dependabot.yml

@@ -22,9 +22,9 @@ updates:
   - package-ecosystem: "pip"
     directory: "/"
     schedule:
-      interval: "daily"
+      interval: "weekly"
     open-pull-requests-limit: 50
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
-      interval: "daily"
+      interval: "weekly"

.github/workflows/pypi-build-artifacts.yml

@@ -62,7 +62,7 @@ jobs:
         if: startsWith(matrix.os, 'ubuntu')
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.23.1
+        uses: pypa/cibuildwheel@v2.23.2
         with:
           output-dir: wheelhouse
           config-file: "pyproject.toml"

.github/workflows/svn-build-artifacts.yml

@@ -57,7 +57,7 @@ jobs:
         if: startsWith(matrix.os, 'ubuntu')
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.23.1
+        uses: pypa/cibuildwheel@v2.23.2
         with:
           output-dir: wheelhouse
           config-file: "pyproject.toml"

Makefile

@@ -19,7 +19,7 @@
 help:  ## Display this help
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
-POETRY_VERSION = 2.0.1
+POETRY_VERSION = 2.1.1
 install-poetry:  ## Ensure Poetry is installed and the correct version is being used.
 	@if ! command -v poetry &> /dev/null; then \
 		echo "Poetry could not be found. Installing..."; \

dev/Dockerfile

@@ -39,20 +39,20 @@ WORKDIR ${SPARK_HOME}
 # Remember to also update `tests/conftest`'s spark setting
 ENV SPARK_VERSION=3.5.4
 ENV ICEBERG_SPARK_RUNTIME_VERSION=3.5_2.12
-ENV ICEBERG_VERSION=1.8.0
+ENV ICEBERG_VERSION=1.9.0-SNAPSHOT
 ENV PYICEBERG_VERSION=0.9.0
 
 RUN curl --retry 5 -s -C - https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \
  && tar xzf spark-${SPARK_VERSION}-bin-hadoop3.tgz --directory /opt/spark --strip-components 1 \
  && rm -rf spark-${SPARK_VERSION}-bin-hadoop3.tgz
 
 # Download iceberg spark runtime
-RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar \
+RUN curl --retry 5 -s https://repository.apache.org/content/groups/snapshots/org/apache/iceberg/iceberg-spark-runtime-3.5_2.12/1.9.0-SNAPSHOT/iceberg-spark-runtime-3.5_2.12-1.9.0-20250409.001855-44.jar \
       -Lo /opt/spark/jars/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar
 
 
 # Download AWS bundle
-RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar \
+RUN curl --retry 5 -s https://repository.apache.org/content/groups/snapshots/org/apache/iceberg/iceberg-aws-bundle/1.9.0-SNAPSHOT/iceberg-aws-bundle-1.9.0-20250409.002731-88.jar \
       -Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar
 
 COPY spark-defaults.conf /opt/spark/conf

dev/provision.py

@@ -14,6 +14,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+import math
 
 from pyspark.sql import SparkSession
 from pyspark.sql.functions import current_date, date_add, expr
@@ -113,89 +114,99 @@
     """
     )
 
-    spark.sql(
-        f"""
-    CREATE OR REPLACE TABLE {catalog_name}.default.test_positional_mor_deletes (
-        dt     date,
-        number integer,
-        letter string
-    )
-    USING iceberg
-    TBLPROPERTIES (
-        'write.delete.mode'='merge-on-read',
-        'write.update.mode'='merge-on-read',
-        'write.merge.mode'='merge-on-read',
-        'format-version'='2'
-    );
-    """
-    )
+    # Merge on read has been implemented in version ≥2:
+    #   v2: Using positional deletes
+    #   v3: Using deletion vectors
 
-    spark.sql(
-        f"""
-    INSERT INTO {catalog_name}.default.test_positional_mor_deletes
-    VALUES
-        (CAST('2023-03-01' AS date), 1, 'a'),
-        (CAST('2023-03-02' AS date), 2, 'b'),
-        (CAST('2023-03-03' AS date), 3, 'c'),
-        (CAST('2023-03-04' AS date), 4, 'd'),
-        (CAST('2023-03-05' AS date), 5, 'e'),
-        (CAST('2023-03-06' AS date), 6, 'f'),
-        (CAST('2023-03-07' AS date), 7, 'g'),
-        (CAST('2023-03-08' AS date), 8, 'h'),
-        (CAST('2023-03-09' AS date), 9, 'i'),
-        (CAST('2023-03-10' AS date), 10, 'j'),
-        (CAST('2023-03-11' AS date), 11, 'k'),
-        (CAST('2023-03-12' AS date), 12, 'l');
-    """
-    )
+    for format_version in [2, 3]:
+        identifier = f'{catalog_name}.default.test_positional_mor_deletes_v{format_version}'
+        spark.sql(
+            f"""
+        CREATE OR REPLACE TABLE {identifier} (
+            dt     date,
+            number integer,
+            letter string
+        )
+        USING iceberg
+        TBLPROPERTIES (
+            'write.delete.mode'='merge-on-read',
+            'write.update.mode'='merge-on-read',
+            'write.merge.mode'='merge-on-read',
+            'format-version'='{format_version}'
+        );
+        """
+        )
+
+        spark.sql(
+            f"""
+        INSERT INTO {identifier}
+        VALUES
+            (CAST('2023-03-01' AS date), 1, 'a'),
+            (CAST('2023-03-02' AS date), 2, 'b'),
+            (CAST('2023-03-03' AS date), 3, 'c'),
+            (CAST('2023-03-04' AS date), 4, 'd'),
+            (CAST('2023-03-05' AS date), 5, 'e'),
+            (CAST('2023-03-06' AS date), 6, 'f'),
+            (CAST('2023-03-07' AS date), 7, 'g'),
+            (CAST('2023-03-08' AS date), 8, 'h'),
+            (CAST('2023-03-09' AS date), 9, 'i'),
+            (CAST('2023-03-10' AS date), 10, 'j'),
+            (CAST('2023-03-11' AS date), 11, 'k'),
+            (CAST('2023-03-12' AS date), 12, 'l');
+        """
+        )
 
-    spark.sql(f"ALTER TABLE {catalog_name}.default.test_positional_mor_deletes CREATE TAG tag_12")
+        spark.sql(f"ALTER TABLE {identifier} CREATE TAG tag_12")
 
-    spark.sql(f"ALTER TABLE {catalog_name}.default.test_positional_mor_deletes CREATE BRANCH without_5")
+        spark.sql(f"ALTER TABLE {identifier} CREATE BRANCH without_5")
 
-    spark.sql(f"DELETE FROM {catalog_name}.default.test_positional_mor_deletes.branch_without_5 WHERE number = 5")
+        spark.sql(f"DELETE FROM {identifier}.branch_without_5 WHERE number = 5")
 
-    spark.sql(f"DELETE FROM {catalog_name}.default.test_positional_mor_deletes WHERE number = 9")
+        spark.sql(f"DELETE FROM {identifier} WHERE number = 9")
 
-    spark.sql(
-        f"""
-      CREATE OR REPLACE TABLE {catalog_name}.default.test_positional_mor_double_deletes (
-        dt     date,
-        number integer,
-        letter string
-      )
-      USING iceberg
-      TBLPROPERTIES (
-        'write.delete.mode'='merge-on-read',
-        'write.update.mode'='merge-on-read',
-        'write.merge.mode'='merge-on-read',
-        'format-version'='2'
-      );
-    """
-    )
+        identifier = f'{catalog_name}.default.test_positional_mor_double_deletes_v{format_version}'
 
-    spark.sql(
-        f"""
-    INSERT INTO {catalog_name}.default.test_positional_mor_double_deletes
-    VALUES
-        (CAST('2023-03-01' AS date), 1, 'a'),
-        (CAST('2023-03-02' AS date), 2, 'b'),
-        (CAST('2023-03-03' AS date), 3, 'c'),
-        (CAST('2023-03-04' AS date), 4, 'd'),
-        (CAST('2023-03-05' AS date), 5, 'e'),
-        (CAST('2023-03-06' AS date), 6, 'f'),
-        (CAST('2023-03-07' AS date), 7, 'g'),
-        (CAST('2023-03-08' AS date), 8, 'h'),
-        (CAST('2023-03-09' AS date), 9, 'i'),
-        (CAST('2023-03-10' AS date), 10, 'j'),
-        (CAST('2023-03-11' AS date), 11, 'k'),
-        (CAST('2023-03-12' AS date), 12, 'l');
-    """
-    )
+        spark.sql(
+            f"""
+          CREATE OR REPLACE TABLE {identifier} (
+            dt     date,
+            number integer,
+            letter string
+          )
+          USING iceberg
+          TBLPROPERTIES (
+            'write.delete.mode'='merge-on-read',
+            'write.update.mode'='merge-on-read',
+            'write.merge.mode'='merge-on-read',
+            'format-version'='2'
+          );
+        """
+        )
 
-    spark.sql(f"DELETE FROM {catalog_name}.default.test_positional_mor_double_deletes WHERE number = 9")
+        spark.sql(
+            f"""
+        INSERT INTO {identifier}
+        VALUES
+            (CAST('2023-03-01' AS date), 1, 'a'),
+            (CAST('2023-03-02' AS date), 2, 'b'),
+            (CAST('2023-03-03' AS date), 3, 'c'),
+            (CAST('2023-03-04' AS date), 4, 'd'),
+            (CAST('2023-03-05' AS date), 5, 'e'),
+            (CAST('2023-03-06' AS date), 6, 'f'),
+            (CAST('2023-03-07' AS date), 7, 'g'),
+            (CAST('2023-03-08' AS date), 8, 'h'),
+            (CAST('2023-03-09' AS date), 9, 'i'),
+            (CAST('2023-03-10' AS date), 10, 'j'),
+            (CAST('2023-03-11' AS date), 11, 'k'),
+            (CAST('2023-03-12' AS date), 12, 'l');
+        """
+        )
 
-    spark.sql(f"DELETE FROM {catalog_name}.default.test_positional_mor_double_deletes WHERE letter == 'f'")
+        # Perform two deletes, should produce:
+        #   v2: two positional delete files in v2
+        #   v3: one deletion vector since they are merged
+        spark.sql(f"DELETE FROM {identifier} WHERE number = 9")
+        spark.sql(f"DELETE FROM {identifier} WHERE letter == 'f'")
 
     all_types_dataframe = (
         spark.range(0, 5, 1, 5)
@@ -328,6 +339,7 @@
     CREATE TABLE {catalog_name}.default.test_table_empty_list_and_map (
         col_list             array<int>,
         col_map              map<int, int>,
+        col_struct           struct<test:int>,
         col_list_with_struct array<struct<test:int>>
     )
     USING iceberg
@@ -340,8 +352,8 @@
     spark.sql(
         f"""
     INSERT INTO {catalog_name}.default.test_table_empty_list_and_map
-    VALUES (null, null, null),
-           (array(), map(), array(struct(1)))
+    VALUES (null, null, null, null),
+           (array(), map(), struct(1), array(struct(1)))
     """
     )
 

mkdocs/docs/api.md

@@ -215,6 +215,17 @@ static_table = StaticTable.from_metadata(
 
 The static-table is considered read-only.
 
+Alternatively, if your table metadata directory contains a `version-hint.text` file, you can just specify
+the table root path, and the latest metadata file will be picked automatically.
+
+```python
+from pyiceberg.table import StaticTable
+
+static_table = StaticTable.from_metadata(
+    "s3://warehouse/wh/nyc.db/taxis
+)
+```
+
 ## Check if a table exists
 
 To check whether the `bids` table exists:

mkdocs/docs/configuration.md

@@ -189,7 +189,7 @@ PyIceberg uses [S3FileSystem](https://arrow.apache.org/docs/python/generated/pya
 | s3.access-key-id     | admin                      | Configure the static access key id used to access the FileIO.                                                                                                                                                                                             |
 | s3.secret-access-key | password                   | Configure the static secret access key used to access the FileIO.                                                                                                                                                                                         |
 | s3.session-token     | AQoDYXdzEJr...             | Configure the static session token used to access the FileIO.                                                                                                                                                                                             |
-| s3.force-virtual-addressing   | True                       | Whether to use virtual addressing of buckets. This must be set to True as OSS can only be accessed with virtual hosted style address.                                                                                                                                                                                                        |
+| s3.force-virtual-addressing   | True                       | Whether to use virtual addressing of buckets. This is set to `True` by default as OSS can only be accessed with virtual hosted style address.                                                                                                                                                                                                        |
 
 <!-- markdown-link-check-enable-->