Merge branch 'apache:main' into fix-3008-sigv4-retries

Author: mrutunjay-kinagi

.github/workflows/codeql.yml

@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+  schedule:
+    - cron: '16 4 * * 1'
+
+jobs:
+  analyze:
+    name: Analyze Actions
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+      packages: read
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v4
+      with:
+        languages: actions
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v4
+      with:
+        category: "/language:actions"

mkdocs/docs/cli.md

@@ -26,7 +26,7 @@ hide:
 
 Pyiceberg comes with a CLI that's available after installing the `pyiceberg` package.
 
-You can pass the path to the Catalog using the `--uri` and `--credential` argument, but it is recommended to setup a `~/.pyiceberg.yaml` config as described in the [Catalog](configuration.md) section.
+You can pass the path to the Catalog using `--uri` and `--credential`. For REST catalogs that require a path prefix, you can also set `--prefix`. It is still recommended to set up a `~/.pyiceberg.yaml` config as described in the [Catalog](configuration.md) section.
 
 ```sh
 ➜  pyiceberg --help
@@ -39,6 +39,7 @@ Options:
   --ugi TEXT
   --uri TEXT
   --credential TEXT
+  --prefix TEXT
   --help                Show this message and exit.
 
 Commands:

pyiceberg/catalog/rest/__init__.py

@@ -96,6 +96,11 @@ class HttpMethod(str, Enum):
     HEAD = "HEAD"
     POST = "POST"
     DELETE = "DELETE"
+    PUT = "PUT"
+    CONNECT = "CONNECT"
+    OPTIONS = "OPTIONS"
+    TRACE = "TRACE"
+    PATCH = "PATCH"
 
 
 class Endpoint(IcebergBaseModel):
@@ -370,7 +375,7 @@ def _create_session(self) -> Session:
 
         # Sets the client side and server side SSL cert verification, if provided as properties.
         if ssl_config := self.properties.get(SSL):
-            if ssl_ca_bundle := ssl_config.get(CA_BUNDLE):
+            if (ssl_ca_bundle := ssl_config.get(CA_BUNDLE)) is not None:
                 session.verify = ssl_ca_bundle
             if ssl_client := ssl_config.get(CLIENT):
                 if all(k in ssl_client for k in (CERT, KEY)):

pyiceberg/catalog/rest/response.py

@@ -28,6 +28,7 @@
     RESTError,
     ServerError,
     ServiceUnavailableError,
+    TooManyRequestsError,
     UnauthorizedError,
 )
 from pyiceberg.typedef import IcebergBaseModel
@@ -79,6 +80,8 @@ def _handle_non_200_response(exc: HTTPError, error_handler: dict[int, type[Excep
         exception = RESTError
     elif code == 419:
         exception = AuthorizationExpiredError
+    elif code == 429:
+        exception = TooManyRequestsError
     elif code == 501:
         exception = NotImplementedError
     elif code == 503:

pyiceberg/cli/console.py

@@ -66,6 +66,7 @@ def wrapper(*args: Any, **kwargs: Any):  # type: ignore
 @click.option("--ugi")
 @click.option("--uri")
 @click.option("--credential")
+@click.option("--prefix")
 @click.pass_context
 def run(
     ctx: Context,
@@ -76,6 +77,7 @@ def run(
     ugi: str | None,
     uri: str | None,
     credential: str | None,
+    prefix: str | None,
 ) -> None:
     logging.basicConfig(
         level=getattr(logging, log_level.upper()),
@@ -89,6 +91,8 @@ def run(
         properties[URI] = uri
     if credential:
         properties["credential"] = credential
+    if prefix:
+        properties["prefix"] = prefix
 
     ctx.ensure_object(dict)
     if output == "text":

pyiceberg/exceptions.py

@@ -84,6 +84,10 @@ class AuthorizationExpiredError(RESTError):
     """When the credentials are expired when performing an action on the REST catalog."""
 
 
+class TooManyRequestsError(RESTError):
+    """Raises when too many requests error is returned by the REST catalog."""
+
+
 class OAuthError(RESTError):
     """Raises when there is an error with the OAuth call."""
 

pyiceberg/table/update/__init__.py

@@ -212,7 +212,7 @@ class RemoveSchemasUpdate(IcebergBaseModel):
 
 class SetPartitionStatisticsUpdate(IcebergBaseModel):
     action: Literal["set-partition-statistics"] = Field(default="set-partition-statistics")
-    partition_statistics: PartitionStatisticsFile
+    partition_statistics: PartitionStatisticsFile = Field(alias="partition-statistics")
 
 
 class RemovePartitionStatisticsUpdate(IcebergBaseModel):

pyproject.toml

@@ -126,7 +126,7 @@ docs = [
     "mkdocs==1.6.1",
     "griffe==2.0.0",
     "jinja2==3.1.6",
-    "mkdocstrings==1.0.2",
+    "mkdocstrings==1.0.3",
     "mkdocstrings-python==2.0.2",
     "mkdocs-literate-nav==0.6.2",
     "mkdocs-autorefs==1.4.4",

tests/catalog/test_rest.py

@@ -37,6 +37,8 @@
     SIGV4_MAX_RETRIES,
     SNAPSHOT_LOADING_MODE,
     Capability,
+    Endpoint,
+    HttpMethod,
     RestCatalog,
 )
 from pyiceberg.exceptions import (
@@ -1704,6 +1706,19 @@ def test_update_namespace_properties_invalid_namespace(rest_mock: Mocker) -> Non
     assert "Empty namespace identifier" in str(e.value)
 
 
+def test_with_disabled_ssl_ca_bundle(rest_mock: Mocker) -> None:
+    # Given
+    catalog_properties = {
+        "uri": TEST_URI,
+        "token": TEST_TOKEN,
+        "ssl": {
+            "cabundle": False,
+        },
+    }
+    catalog = RestCatalog("rest", **catalog_properties)  # type: ignore
+    assert catalog._session.verify is False
+
+
 def test_request_session_with_ssl_ca_bundle(monkeypatch: pytest.MonkeyPatch) -> None:
     # Given
     catalog_properties = {
@@ -2414,3 +2429,27 @@ def test_table_uuid_check_on_refresh(rest_mock: Mocker, example_table_metadata_v
     assert "Table UUID does not match" in str(exc_info.value)
     assert f"current={original_uuid}" in str(exc_info.value)
     assert f"refreshed={different_uuid}" in str(exc_info.value)
+
+
+def test_endpoint_parsing_from_string_with_valid_http_method() -> None:
+    test_cases = [
+        ("GET /v1/resource", HttpMethod.GET, "/v1/resource"),
+        ("HEAD /v1/resource", HttpMethod.HEAD, "/v1/resource"),
+        ("POST /v1/resource", HttpMethod.POST, "/v1/resource"),
+        ("DELETE /v1/resource", HttpMethod.DELETE, "/v1/resource"),
+        ("PUT /v1/resource", HttpMethod.PUT, "/v1/resource"),
+        ("CONNECT /v1/resource", HttpMethod.CONNECT, "/v1/resource"),
+        ("OPTIONS /v1/resource", HttpMethod.OPTIONS, "/v1/resource"),
+        ("TRACE /v1/resource", HttpMethod.TRACE, "/v1/resource"),
+        ("PATCH /v1/resource", HttpMethod.PATCH, "/v1/resource"),
+    ]
+
+    for raw_string, http_method, path in test_cases:
+        endpoint = Endpoint.from_string(raw_string)
+        assert endpoint.http_method == http_method
+        assert endpoint.path == path
+
+
+def test_endpoint_parsing_from_string_with_invalid_http_method() -> None:
+    with pytest.raises(ValueError, match="not a valid HttpMethod"):
+        Endpoint.from_string("INVALID /v1/resource")

tests/cli/test_console.py

@@ -1029,3 +1029,18 @@ def test_log_level_cli_overrides_env(mocker: MockFixture) -> None:
     mock_basicConfig.assert_called_once()
     call_kwargs = mock_basicConfig.call_args[1]
     assert call_kwargs["level"] == logging.ERROR
+
+
+def test_prefix_cli_option_forwarded_to_catalog(mocker: MockFixture) -> None:
+    mock_basicConfig = mocker.patch("logging.basicConfig")
+    mock_catalog = MagicMock(spec=InMemoryCatalog)
+    mock_catalog.list_tables.return_value = []
+    mock_catalog.list_namespaces.return_value = []
+    mock_load_catalog = mocker.patch("pyiceberg.cli.console.load_catalog", return_value=mock_catalog)
+
+    runner = CliRunner()
+    result = runner.invoke(run, ["--catalog", "rest", "--uri", "https://example.invalid", "--prefix", "v1/ws", "list"])
+
+    assert result.exit_code == 0
+    mock_basicConfig.assert_called_once()
+    mock_load_catalog.assert_called_once_with("rest", uri="https://example.invalid", prefix="v1/ws")