add Java cross-language reference vector, harden body type handling

Author: plusplusjiajia

pyiceberg/catalog/rest/__init__.py

@@ -773,21 +773,18 @@ def _init_sigv4(self, session: Session) -> None:
         from requests.adapters import HTTPAdapter
 
         class _IcebergSigV4Auth(SigV4Auth):
-            def canonical_request(self, request: Any) -> str:
-                # Reuses the logic from botocore's SigV4Auth.canonical_request
-                # (https://github.com/boto/botocore/blob/develop/botocore/auth.py)
-                # but always uses self.payload(request) for the body checksum.
-                # Validated against botocore <= 1.42.x
-                # (https://github.com/boto/botocore/blob/1.42.85/botocore/auth.py#L622-L637)
+            def canonical_request(self, request: AWSRequest) -> str:
+                # Override forces hex payload hash in the canonical request even when
+                # x-amz-content-sha256 header is base64 (see body-hash block below).
+                # Mirrors botocore <=1.42.x SigV4Auth.canonical_request layout:
+                # https://github.com/boto/botocore/blob/1.42.85/botocore/auth.py#L622-L637
                 cr = [request.method.upper()]
                 path = self._normalize_url_path(parse.urlsplit(request.url).path)
                 cr.append(path)
                 cr.append(self.canonical_query_string(request))
                 headers_to_sign = self.headers_to_sign(request)
                 cr.append(self.canonical_headers(headers_to_sign) + "\n")
                 cr.append(self.signed_headers(headers_to_sign))
-                # Always use hex-encoded payload hash per SigV4 spec,
-                # regardless of the x-amz-content-sha256 header value (which may be base64).
                 cr.append(self.payload(request))
                 return "\n".join(cr)
 
@@ -818,11 +815,20 @@ def add_headers(self, request: PreparedRequest, **kwargs: Any) -> None:  # pylin
                 if "connection" in request.headers:
                     del request.headers["connection"]
 
-                # Compute the x-amz-content-sha256 header to match Iceberg Java SDK:
-                #   - empty body → hex (EMPTY_BODY_SHA256)
-                #   - non-empty body → base64
+                # Match Iceberg Java's AWS SDK v2 flexible-checksum signing:
+                # x-amz-content-sha256 header is base64 for non-empty bodies, hex for empty.
+                # The SigV4 canonical request still uses hex (enforced in _IcebergSigV4Auth above).
+                # Ref: https://github.com/apache/iceberg/blob/main/aws/src/main/java/org/apache/iceberg/aws/RESTSigV4AuthSession.java
                 if request.body:
-                    body_bytes = request.body.encode("utf-8") if isinstance(request.body, str) else request.body
+                    if isinstance(request.body, str):
+                        body_bytes = request.body.encode("utf-8")
+                    elif isinstance(request.body, (bytes, bytearray)):
+                        body_bytes = request.body
+                    else:
+                        raise TypeError(
+                            f"Unsupported request body type for SigV4 signing: "
+                            f"{type(request.body).__name__}; expected str or bytes."
+                        )
                     content_sha256_header = base64.b64encode(hashlib.sha256(body_bytes).digest()).decode()
                 else:
                     content_sha256_header = EMPTY_BODY_SHA256

pyproject.toml

@@ -92,7 +92,7 @@ sql-postgres = [
 ]
 sql-sqlite = ["sqlalchemy>=2.0.18,<3"]
 gcsfs = ["gcsfs>=2023.1.0"]
-rest-sigv4 = ["boto3>=1.24.59"]
+rest-sigv4 = ["boto3>=1.24.59", "botocore<2"]
 hf = ["huggingface-hub>=0.24.0"]
 pyiceberg-core = ["pyiceberg-core>=0.5.1,<0.10.0"]
 datafusion = ["datafusion>=52,<53"]

tests/catalog/test_rest.py

@@ -793,6 +793,59 @@ def capturing_add_auth(self: Any, request: Any) -> None:
     assert prepared.headers["x-amz-content-sha256"] == base64.b64encode(hashlib.sha256(body_content).digest()).decode()
 
 
+def test_sigv4_content_sha256_matches_iceberg_java_reference(rest_mock: Mocker) -> None:
+    """Pin byte-for-byte equivalence with Iceberg Java TestRESTSigV4AuthSession (L121, L177)."""
+    java_reference_body = b'{"namespace":["ns"],"properties":{}}'
+    java_reference_base64 = "yc5oAKPWjHY4sW8XQq0l/3aNrrXJKBycVFNnDEGMfww="
+    java_reference_empty_hex = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+
+    catalog = RestCatalog(
+        "rest",
+        **{
+            "uri": TEST_URI,
+            "rest.sigv4-enabled": "true",
+            "rest.signing-region": "us-east-1",
+            "client.access-key-id": "id",
+            "client.secret-access-key": "secret",
+        },
+    )
+    adapter = catalog._session.adapters[catalog.uri]
+    assert isinstance(adapter, HTTPAdapter)
+
+    # Non-empty body: must match Java's base64 reference value exactly
+    prepared_with_body = catalog._session.prepare_request(Request("POST", f"{TEST_URI}v1/namespaces", data=java_reference_body))
+    adapter.add_headers(prepared_with_body)
+    assert prepared_with_body.headers["x-amz-content-sha256"] == java_reference_base64
+
+    # Empty body: must match Java's hex reference value exactly
+    prepared_empty = catalog._session.prepare_request(Request("GET", f"{TEST_URI}v1/config"))
+    adapter.add_headers(prepared_empty)
+    assert prepared_empty.headers["x-amz-content-sha256"] == java_reference_empty_hex
+
+
+def test_sigv4_unsupported_body_type_raises(rest_mock: Mocker) -> None:
+    """Unsupported body types (e.g. file-like) raise a clear error rather than crashing in hashlib."""
+    catalog = RestCatalog(
+        "rest",
+        **{
+            "uri": TEST_URI,
+            "rest.sigv4-enabled": "true",
+            "rest.signing-region": "us-east-1",
+            "client.access-key-id": "id",
+            "client.secret-access-key": "secret",
+        },
+    )
+    adapter = catalog._session.adapters[catalog.uri]
+    assert isinstance(adapter, HTTPAdapter)
+
+    prepared = catalog._session.prepare_request(Request("POST", f"{TEST_URI}v1/namespaces"))
+    # Inject an unsupported body type (a list — not str/bytes)
+    prepared.body = ["not", "a", "valid", "body"]  # type: ignore[assignment]
+
+    with pytest.raises(TypeError, match="Unsupported request body type for SigV4 signing"):
+        adapter.add_headers(prepared)
+
+
 def test_sigv4_adapter_default_retry_config(rest_mock: Mocker) -> None:
     catalog = RestCatalog(
         "rest",