Align SigV4 auth with Java: sign relocated Authorization, validate config, add docs

Author: plusplusjiajia

mkdocs/docs/configuration.md

@@ -384,6 +384,8 @@ Legacy OAuth2 Properties will be removed in PyIceberg 1.0 in place of pluggable
 | rest.signing-region | us-east-1                        | The region to use when SigV4 signing a request                                                     |
 | rest.signing-name   | execute-api                      | The service signing name to use when SigV4 signing a request                                       |
 
+SigV4 can also be enabled as `auth.type: sigv4`, which additionally lets you choose the wrapped header-based auth (see the AuthManager section below).
+
 ##### Pluggable Authentication via AuthManager
 
 The RESTCatalog supports pluggable authentication via the `auth` configuration block. This allows you to specify which how the access token will be fetched and managed for use with the HTTP requests to the RESTCatalog server. The authentication method is selected by setting the `auth.type` property, and additional configuration can be provided as needed for each method.
@@ -396,6 +398,7 @@ The RESTCatalog supports pluggable authentication via the `auth` configuration b
 - `custom`: Custom authentication manager (requires `auth.impl`).
 - `google`: Google Authentication support
 - `entra`: Microsoft Entra ID (Azure AD) authentication support
+- `sigv4`: AWS SigV4 request signing, optionally wrapping a delegate auth type.
 
 ###### Configuration Properties
 
@@ -424,6 +427,7 @@ catalog:
 | `auth.custom`    | If type is `custom` | Block containing configuration for the custom AuthManager.                          |
 | `auth.google`    | If type is `google` | Block containing `credentials_path` to a service account file (if using). Will default to using Application Default Credentials. |
 | `auth.entra`     | If type is `entra` | Block containing Entra ID configuration. Will default to using DefaultAzureCredential. |
+| `auth.sigv4`     | If type is `sigv4` | Block containing an optional `delegate` auth block whose `Authorization` header is preserved as `Original-Authorization` after signing. Signing region/name come from `rest.signing-region`/`rest.signing-name`; AWS credentials from `client.*` or the standard boto3 chain. |
 
 ###### Examples
 
@@ -469,6 +473,24 @@ auth:
     property2: value2
 ```
 
+SigV4 Signing (wrapping OAuth2):
+
+```yaml
+auth:
+  type: sigv4
+  sigv4:
+    delegate:
+      type: oauth2
+      oauth2:
+        client_id: my-client-id
+        client_secret: my-client-secret
+        token_url: https://auth.example.com/oauth/token
+rest.signing-region: us-east-1
+rest.signing-name: execute-api
+client.access-key-id: my-access-key
+client.secret-access-key: my-secret-key
+```
+
 ###### Notes
 
 - If `auth.type` is `custom`, you **must** specify `auth.impl` with the full class path to your custom AuthManager.

pyiceberg/catalog/rest/__init__.py

@@ -466,6 +466,12 @@ def _build_auth_manager(self, session: Session) -> AuthManager:
         """Build the AuthManager, wrapping the delegate in SigV4 when enabled."""
         delegate = self._build_delegate_auth_manager(session)
         if self._is_sigv4_enabled():
+            if property_as_bool(self.properties, SIGV4, False):
+                deprecation_message(
+                    deprecated_in="0.11.0",
+                    removed_in="1.0.0",
+                    help_message=f"The property {SIGV4} is deprecated. Please use auth.type={SIGV4_AUTH_TYPE} instead",
+                )
             return self._build_sigv4_auth_manager(delegate)
         return delegate
 
@@ -477,13 +483,17 @@ def _build_delegate_auth_manager(self, session: Session) -> AuthManager:
                 raise ValueError("auth.type must be defined")
 
             if auth_type == SIGV4_AUTH_TYPE:
+                if auth_config.get("impl"):
+                    raise ValueError("auth.impl can only be specified when using custom auth.type")
                 # The delegate is configured under auth.sigv4.delegate.*
                 sigv4_config = auth_config.get(SIGV4_AUTH_TYPE, {})
                 delegate_config = sigv4_config.get("delegate")
                 if not delegate_config or "type" not in delegate_config:
                     # No delegate configured: SigV4-only auth, with no header-based delegate.
                     return NoopAuthManager()
                 delegate_type = delegate_config["type"]
+                if delegate_type == SIGV4_AUTH_TYPE:
+                    raise ValueError("Cannot delegate a SigV4 auth manager to another SigV4 auth manager")
                 return AuthManagerFactory.create(delegate_type, delegate_config.get(delegate_type, {}))
 
             auth_type_config = auth_config.get(auth_type, {})

pyiceberg/catalog/rest/auth.py

@@ -415,6 +415,9 @@ def sign_request(self, request: PreparedRequest) -> PreparedRequest:
             content_sha256_header = EMPTY_BODY_SHA256
 
         signing_headers = dict(request.headers)
+        # Relocate Authorization before signing so it lands in SignedHeaders, like Java.
+        if "Authorization" in signing_headers:
+            signing_headers["Original-Authorization"] = signing_headers.pop("Authorization")
         signing_headers["x-amz-content-sha256"] = content_sha256_header
 
         aws_request = AWSRequest(method=request.method, url=url, params=params, data=request.body, headers=signing_headers)

tests/catalog/test_rest.py

@@ -586,6 +586,9 @@ def test_list_tables_page_size(rest_mock: Mocker) -> None:
     ]
 
 
+@pytest.mark.filterwarnings(
+    "ignore:Deprecated in 0.11.0, will be removed in 1.0.0. The property rest.sigv4-enabled is deprecated:DeprecationWarning"
+)
 def test_list_tables_200_sigv4(rest_mock: Mocker) -> None:
     namespace = "examples"
     # SigV4 signing replaces the bearer Authorization header with an AWS4-HMAC-SHA256
@@ -610,6 +613,9 @@ def test_list_tables_200_sigv4(rest_mock: Mocker) -> None:
     assert rest_mock.called
 
 
+@pytest.mark.filterwarnings(
+    "ignore:Deprecated in 0.11.0, will be removed in 1.0.0. The property rest.sigv4-enabled is deprecated:DeprecationWarning"
+)
 def test_sigv4_adapter_default_retry_config(rest_mock: Mocker) -> None:
     catalog = RestCatalog(
         "rest",
@@ -628,6 +634,9 @@ def test_sigv4_adapter_default_retry_config(rest_mock: Mocker) -> None:
     assert adapter.max_retries.total == SIGV4_MAX_RETRIES_DEFAULT
 
 
+@pytest.mark.filterwarnings(
+    "ignore:Deprecated in 0.11.0, will be removed in 1.0.0. The property rest.sigv4-enabled is deprecated:DeprecationWarning"
+)
 def test_sigv4_adapter_override_retry_config(rest_mock: Mocker) -> None:
     catalog = RestCatalog(
         "rest",
@@ -804,6 +813,9 @@ def test_list_views_invalid_page_size(rest_mock: Mocker) -> None:
     assert str(e.value) == "rest-page-size must be a positive integer"
 
 
+@pytest.mark.filterwarnings(
+    "ignore:Deprecated in 0.11.0, will be removed in 1.0.0. The property rest.sigv4-enabled is deprecated:DeprecationWarning"
+)
 def test_list_views_200_sigv4(rest_mock: Mocker) -> None:
     namespace = "examples"
     # SigV4 signing replaces the bearer Authorization header with an AWS4-HMAC-SHA256
@@ -2687,6 +2699,9 @@ def test_catalog_close(self, rest_mock: Mocker) -> None:
         # Second close should not raise any exception
         catalog.close()
 
+    @pytest.mark.filterwarnings(
+        "ignore:Deprecated in 0.11.0, will be removed in 1.0.0. The property rest.sigv4-enabled is deprecated:DeprecationWarning"
+    )
     def test_rest_catalog_close_sigv4(self, rest_mock: Mocker) -> None:
         catalog = None
         rest_mock.get(
@@ -2729,6 +2744,9 @@ def test_rest_catalog_context_manager_with_exception(self, rest_mock: Mocker) ->
         assert catalog is not None and hasattr(catalog, "_session")
         assert len(catalog._session.adapters) == self.EXPECTED_ADAPTERS
 
+    @pytest.mark.filterwarnings(
+        "ignore:Deprecated in 0.11.0, will be removed in 1.0.0. The property rest.sigv4-enabled is deprecated:DeprecationWarning"
+    )
     def test_rest_catalog_context_manager_with_exception_sigv4(self, rest_mock: Mocker) -> None:
         """Test RestCatalog context manager properly closes with exceptions."""
         catalog = None

tests/catalog/test_rest_auth.py

@@ -323,8 +323,13 @@ def test_sigv4_auth_manager_relocates_delegate_authorization() -> None:
     # SigV4 owns Authorization; the delegate's Basic header is relocated.
     assert prepared.headers["Authorization"].startswith("AWS4-HMAC-SHA256 Credential=")
     assert prepared.headers["Original-Authorization"].startswith("Basic ")
+    # Relocated header is signed (in SignedHeaders), matching Iceberg Java.
+    assert "original-authorization" in prepared.headers["Authorization"]
 
 
+@pytest.mark.filterwarnings(
+    "ignore:Deprecated in 0.11.0, will be removed in 1.0.0. The property rest.sigv4-enabled is deprecated:DeprecationWarning"
+)
 def test_sigv4_legacy_config_builds_sigv4_auth_manager(rest_mock: Mocker) -> None:
     """Legacy rest.sigv4-enabled config produces a SigV4AuthManager."""
     from pyiceberg.catalog.rest.auth import SigV4AuthManager
@@ -359,6 +364,54 @@ def test_sigv4_auth_type_config_builds_sigv4_auth_manager(rest_mock: Mocker) ->
     assert isinstance(catalog._auth_manager, SigV4AuthManager)
 
 
+def test_sigv4_auth_type_rejects_auth_impl(rest_mock: Mocker) -> None:
+    """auth.impl is only valid with auth.type=custom, not sigv4."""
+    with pytest.raises(ValueError, match="auth.impl can only be specified when using custom auth.type"):
+        RestCatalog(
+            "rest",
+            **{  # type: ignore
+                "uri": TEST_URI,
+                "auth": {"type": "sigv4", "impl": "my.custom.AuthManager"},
+                "rest.signing-region": "us-east-1",
+                "client.access-key-id": "id",
+                "client.secret-access-key": "secret",
+            },
+        )
+
+
+def test_sigv4_rejects_sigv4_delegate(rest_mock: Mocker) -> None:
+    """A SigV4 delegate cannot itself be sigv4, matching Iceberg Java's AuthManagers check."""
+    with pytest.raises(ValueError, match="Cannot delegate a SigV4 auth manager to another SigV4 auth manager"):
+        RestCatalog(
+            "rest",
+            **{  # type: ignore
+                "uri": TEST_URI,
+                "auth": {"type": "sigv4", "sigv4": {"delegate": {"type": "sigv4"}}},
+                "rest.signing-region": "us-east-1",
+                "client.access-key-id": "id",
+                "client.secret-access-key": "secret",
+            },
+        )
+
+
+def test_sigv4_legacy_flag_emits_deprecation_warning(rest_mock: Mocker) -> None:
+    """The legacy rest.sigv4-enabled flag warns and points at auth.type=sigv4, matching Iceberg Java."""
+    with pytest.warns(DeprecationWarning, match="rest.sigv4-enabled is deprecated"):
+        RestCatalog(
+            "rest",
+            **{
+                "uri": TEST_URI,
+                "rest.sigv4-enabled": "true",
+                "rest.signing-region": "us-east-1",
+                "client.access-key-id": "id",
+                "client.secret-access-key": "secret",
+            },
+        )
+
+
+@pytest.mark.filterwarnings(
+    "ignore:Deprecated in 0.11.0, will be removed in 1.0.0. The property rest.sigv4-enabled is deprecated:DeprecationWarning"
+)
 def test_sigv4_sign_request_without_body(rest_mock: Mocker) -> None:
     from pyiceberg.catalog.rest.auth import EMPTY_BODY_SHA256
 
@@ -391,6 +444,9 @@ def test_sigv4_sign_request_without_body(rest_mock: Mocker) -> None:
     assert "x-amz-content-sha256" in auth_header
 
 
+@pytest.mark.filterwarnings(
+    "ignore:Deprecated in 0.11.0, will be removed in 1.0.0. The property rest.sigv4-enabled is deprecated:DeprecationWarning"
+)
 def test_sigv4_sign_request_with_body(rest_mock: Mocker) -> None:
     existing_token = "existing_token"
 
@@ -429,6 +485,9 @@ def test_sigv4_sign_request_with_body(rest_mock: Mocker) -> None:
     assert "x-amz-content-sha256" in auth_header
 
 
+@pytest.mark.filterwarnings(
+    "ignore:Deprecated in 0.11.0, will be removed in 1.0.0. The property rest.sigv4-enabled is deprecated:DeprecationWarning"
+)
 def test_sigv4_content_sha256_with_bytes_body(rest_mock: Mocker) -> None:
     existing_token = "existing_token"
 
@@ -460,6 +519,9 @@ def test_sigv4_content_sha256_with_bytes_body(rest_mock: Mocker) -> None:
     assert content_sha256 == expected_sha256
 
 
+@pytest.mark.filterwarnings(
+    "ignore:Deprecated in 0.11.0, will be removed in 1.0.0. The property rest.sigv4-enabled is deprecated:DeprecationWarning"
+)
 def test_sigv4_conflicting_sigv4_headers(rest_mock: Mocker) -> None:
     from pyiceberg.catalog.rest.auth import EMPTY_BODY_SHA256
 
@@ -493,6 +555,9 @@ def test_sigv4_conflicting_sigv4_headers(rest_mock: Mocker) -> None:
     assert "X-Amz-Date" in prepared.headers
 
 
+@pytest.mark.filterwarnings(
+    "ignore:Deprecated in 0.11.0, will be removed in 1.0.0. The property rest.sigv4-enabled is deprecated:DeprecationWarning"
+)
 def test_sigv4_canonical_request_uses_hex_payload(rest_mock: Mocker) -> None:
     """Verify that the canonical request uses hex-encoded payload hash, not the base64 header value."""
     from typing import Any
@@ -542,6 +607,9 @@ def capturing_add_auth(self: Any, request: Any) -> None:
     assert prepared.headers["x-amz-content-sha256"] == base64.b64encode(hashlib.sha256(body_content).digest()).decode()
 
 
+@pytest.mark.filterwarnings(
+    "ignore:Deprecated in 0.11.0, will be removed in 1.0.0. The property rest.sigv4-enabled is deprecated:DeprecationWarning"
+)
 def test_sigv4_content_sha256_matches_iceberg_java_reference(rest_mock: Mocker) -> None:
     """Pin byte-for-byte equivalence with Iceberg Java TestRESTSigV4AuthSession (L121, L177)."""
     java_reference_body = b'{"namespace":["ns"],"properties":{}}'
@@ -596,6 +664,9 @@ def test_sigv4_unsupported_body_type_raises() -> None:
         manager.sign_request(prepared)
 
 
+@pytest.mark.filterwarnings(
+    "ignore:Deprecated in 0.11.0, will be removed in 1.0.0. The property rest.sigv4-enabled is deprecated:DeprecationWarning"
+)
 def test_sigv4_uses_client_profile_name(rest_mock: Mocker) -> None:
     import boto3